From f17c280e1aa8d5727475d0cd171e36b780b5d027 Mon Sep 17 00:00:00 2001 From: BlivionIaG <15323295+BlivionIaG@users.noreply.github.com> Date: Sun, 25 Oct 2026 01:13:08 +0200 Subject: [PATCH 01/39] lfm2: keep a drafter's tapped rows on every forward The lane core can put an external drafter in front of a target. The drafter reads the target's hidden rows after some layers (its taps). LFM2-MoE had no taps, so it could draft only with the core's own heads. prepare_features names the layers. From then on, each forward copies each row's output of those layers into the scratch (lfm2_tap, a plain copy of bf16 values). features() gives a stream's rows of the latest verify or one-token round. prefill_features hands each prompt chunk's rows to the drafter before the chunk's end is kept for reuse. The prompt's last row is kept beside the stream's caches for its first draft. A drafting stream resumes only a kept state that also holds its drafter's context (Snap.draft). Without a drafter nothing changes: no taps are kept and every forward runs as before. lfm2_tap registers with the kernel contract (metal_lfm2_tap.zig): the third of five taps' columns at 1, 10, 32 and 2,048 rows must hold the layer's bits and the other columns what the call found. Gate 5 checks its launch against every Apple chip we serve and this GPU. test-lfm2-dspark checks the same copy through the family's own kernels. --- zig/build/metal_targets.zig | 2 + zig/kernels/metal/lfm2/lfm2.metal | 8 ++ zig/src/families/lfm2_moe/backend.zig | 82 ++++++++++++- zig/src/families/lfm2_moe/dspark_test.zig | 77 ++++++++++++ zig/src/families/lfm2_moe/forward.zig | 14 +++ zig/src/families/lfm2_moe/kernels.zig | 1 + zig/src/families/lfm2_moe/snapshot.zig | 3 +- zig/src/families/lfm2_moe/state.zig | 12 ++ zig/src/tensorfold.zig | 1 + zig/tests/contract/metal_lfm2_tap.zig | 143 ++++++++++++++++++++++ zig/tests/contract/metal_main.zig | 17 +++ 11 files changed, 354 insertions(+), 6 deletions(-) create mode 100644 zig/src/families/lfm2_moe/dspark_test.zig create mode 100644 zig/tests/contract/metal_lfm2_tap.zig diff --git a/zig/build/metal_targets.zig b/zig/build/metal_targets.zig index 6e50add80d..5c01545d3f 100644 --- a/zig/build/metal_targets.zig +++ b/zig/build/metal_targets.zig @@ -458,6 +458,8 @@ pub fn metalTargets(b: *std.Build, target: std.Build.ResolvedTarget, optimize: s test_step.dependOn(&b.addRunArtifact(b.addTest(.{ .root_module = engine })).step); const lfm2_tests = b.addRunArtifact(b.addTest(.{ .root_module = engine, .filters = &.{"lfm2_moe"} })); b.step("test-lfm2", "The LFM2-MoE family's tests: config, draws, state and its shared rounds on this Mac's GPU").dependOn(&lfm2_tests.step); + const lfm2_dspark = b.addRunArtifact(b.addTest(.{ .root_module = engine, .filters = &.{"lfm2 dspark:"} })); + b.step("test-lfm2-dspark", "LFM2's taps for a drafter: the tap kernel on this Mac's GPU").dependOn(&lfm2_dspark.step); test_step.dependOn(&b.addRunArtifact(b.addTest(.{ .root_module = lanes })).step); // the fake fabric maps its windows with mmap: the Macs' RDMA, not Windows if (target.result.os.tag != .windows) test_step.dependOn(&b.addRunArtifact(b.addTest(.{ .root_module = fabric })).step); diff --git a/zig/kernels/metal/lfm2/lfm2.metal b/zig/kernels/metal/lfm2/lfm2.metal index af6a2a7372..b2772cf64e 100644 --- a/zig/kernels/metal/lfm2/lfm2.metal +++ b/zig/kernels/metal/lfm2/lfm2.metal @@ -568,3 +568,11 @@ constant constexpr uint SORT_CHUNKS = 16; // thread (e, c) counts expert e's pi best[2 * row + 1] = bad; } } + +// A drafter's tap: H's rows [rows, D] copied into TAPS at column `at.y` of rows `at.x` values apart, 8 a thread. +[[kernel]] void lfm2_tap(const device bfloat* h [[buffer(0)]], device bfloat* taps [[buffer(1)]], constant uint2& at [[buffer(2)]], + uint2 i [[thread_position_in_grid]]) { + if (i.x >= D / 8) return; + const device vec* from = (const device vec*)(h + size_t(i.y) * D); + ((device vec*)(taps + size_t(i.y) * at.x + at.y))[i.x] = from[i.x]; +} diff --git a/zig/src/families/lfm2_moe/backend.zig b/zig/src/families/lfm2_moe/backend.zig index d713c9ec24..44427d435a 100644 --- a/zig/src/families/lfm2_moe/backend.zig +++ b/zig/src/families/lfm2_moe/backend.zig @@ -23,6 +23,8 @@ pub const Metal = struct { /// Forward ms by window width, timed at load: the lane core weighs drafts against them. costs: [st.window_rows]lanes.config.Cost = undefined, timed: usize = 0, + /// Forwards run so far: a stream's tapped rows in the scratch are current while their gen matches. + gen: u64 = 0, pub fn init(gpa: std.mem.Allocator, model: *Model, options: Options) !*Metal { const out = try gpa.create(Metal); @@ -88,7 +90,7 @@ pub const Metal = struct { } pub fn backend(self: *Metal) be.Backend { - return .{ .ptr = self, .vtable = &.{ .prefill = prefillFn, .first = firstFn, .first_masked = firstMaskedFn, .queue = queueFn, .read = readFn, .verify = verifyFn, .keep = keepFn, .draft = draftFn, .release = releaseFn } }; + return .{ .ptr = self, .vtable = &.{ .prefill = prefillFn, .prepare_features = prepareFn, .features = featuresFn, .prefill_features = prefillFeaturesFn, .first = firstFn, .first_masked = firstMaskedFn, .queue = queueFn, .read = readFn, .verify = verifyFn, .keep = keepFn, .draft = draftFn, .release = releaseFn } }; } fn cast(ptr: *anyopaque) *Metal { @@ -107,8 +109,65 @@ pub const Metal = struct { return cache; } + /// One forward on the scratch; a stream's tapped rows from earlier forwards are stale after it. + fn run(self: *Metal, segments: []const fwd.Segment, tokens: []const u32, record: bool, head: fwd.Head) !void { + self.gen += 1; + try fwd.run(self.model, &self.scratch, segments, tokens, record, head); + } + fn prefillFn(ptr: *anyopaque, stream: *lanes.Stream) !void { + return cast(ptr).prefillWith(stream, null); + } + + /// The prompt pass, `sink` taking each chunk's tapped rows before the chunk's end is kept for reuse. + fn prefillFeaturesFn(ptr: *anyopaque, stream: *lanes.Stream, layers: []const u32, sink: be.FeatureSink) !void { + const self = cast(ptr); + const t = self.scratch.taps orelse return error.UnpreparedLfmTaps; + if (layers.len != t.count) return error.UnpreparedLfmTaps; + return self.prefillWith(stream, sink); + } + + /// A tapped row's bytes: every tap's hidden row, bf16. + fn tapBytes(self: *const Metal) usize { + return if (self.scratch.taps) |t| t.count * c.hidden * 2 else 0; + } + + /// The drafter's layers: every forward from now on keeps their outputs for each row. + fn prepareFn(ptr: *anyopaque, layers: []const u32) !void { const self = cast(ptr); + if (layers.len == 0 or layers.len > st.max_taps) return error.BadLfmTaps; + for (layers) |l| if (l >= c.layers) return error.BadLfmTaps; + if (self.scratch.taps) |t| return if (std.mem.eql(u32, t.layers[0..t.count], layers)) {} else error.BadLfmTaps; + var t: st.Taps = .{ .count = @intCast(layers.len), .buf = undefined }; + @memcpy(t.layers[0..layers.len], layers); + t.buf = try self.scratch.memory.alloc(self.scratch.rows * layers.len * c.hidden * 2); + self.scratch.taps = t; + } + + /// Cache rows [start, start + count) of the stream's latest tapped rows, on the device. + fn featuresFn(ptr: *anyopaque, stream: *lanes.Stream, layers: []const u32, start: u64, count: u32) !be.Features { + const self = cast(ptr); + const t = self.scratch.taps orelse return error.UnpreparedLfmTaps; + if (layers.len != t.count) return error.UnpreparedLfmTaps; + const cache = try self.cacheOf(stream); + const held = cache.tap orelse return error.LfmTapsNotHeld; + if (count == 0 or start < held.start or start + count > held.start + held.rows) return error.LfmTapsNotHeld; + const width = self.tapBytes(); + const buf = if (held.gen == 0) cache.last_tap.? else if (held.gen == self.gen) t.buf else return error.StaleLfmTaps; + const row = held.row + @as(usize, @intCast(start - held.start)); + return .{ .buffer = @intFromPtr(buf.id), .offset = row * width, .rows = count, .row_bytes = @intCast(width), .space = .device, .dtype = .bf16 }; + } + + /// The prompt's last row's taps, copied beside the stream's caches for its first draft (gen 0: kept there). + fn keepLastTap(self: *Metal, cache: *st.Cache, row: usize, position: usize) !void { + const width = self.tapBytes(); + if (width == 0) return; + if (cache.last_tap == null) cache.last_tap = try cache.memory.alloc(width); + @memcpy(cache.last_tap.?.contents()[0..width], (self.scratch.taps.?.buf.contents() + row * width)[0..width]); + cache.tap = .{ .start = position, .row = 0, .rows = 1, .gen = 0 }; + } + + fn prefillWith(self: *Metal, stream: *lanes.Stream, sink: ?be.FeatureSink) !void { const ids = stream.prompt(); if (ids.len == 0 or ids.len + stream.max_new > self.options.capacity or self.caches.contains(stream)) return error.InvalidLfmPrompt; const capacity = @min(self.options.capacity, ids.len + stream.max_new + st.window_rows); @@ -121,7 +180,9 @@ pub const Metal = struct { stream.cached = 0; if (stream.reuse.saved) |saved| { // a kept state of this prompt's prefix: the pass starts there const snap: *snapshot.Snap = @ptrCast(@alignCast(saved)); - if (snap.at < ids.len) if (snapshot.restore(cache, snap)) |_| { + // a drafting stream resumes only a state kept with its drafter's context + const drafter_less = sink != null and snap.draft == null; + if (snap.at < ids.len and !drafter_less) if (snapshot.restore(cache, snap)) |_| { at = snap.at; } else |_| {}; stream.cached = @intCast(at); @@ -136,7 +197,13 @@ pub const Metal = struct { }; while (mark < stream.reuse.marks.len and stream.reuse.marks[mark] <= at) mark += 1; if (mark < stream.reuse.marks.len) end = @min(end, stream.reuse.marks[mark]); - try fwd.run(self.model, &self.scratch, &.{.{ .cache = cache, .rows = end - at }}, ids[at..end], false, .last); + try self.run(&.{.{ .cache = cache, .rows = end - at }}, ids[at..end], false, .last); + if (sink) |k| { + const width = self.tapBytes(); + const f: be.Features = .{ .buffer = @intFromPtr(self.scratch.taps.?.buf.id), .rows = @intCast(end - at), .row_bytes = @intCast(width), .space = .device, .dtype = .bf16 }; + try k.take(k.ptr, stream, f, at); + if (end == ids.len) try self.keepLastTap(cache, end - 1 - at, end - 1); + } at = end; if (stream.reuse.hook) |hk| if (std.mem.indexOfScalar(u32, stream.reuse.marks, @intCast(at)) != null) hk.at(hk.ptr, stream, @intCast(at)); @@ -165,7 +232,8 @@ pub const Metal = struct { .value => |v| v, .handle => |h| @intCast(h), }; - try fwd.run(self.model, &self.scratch, &.{.{ .cache = cache, .rows = 1 }}, &.{token}, false, .all); + try self.run(&.{.{ .cache = cache, .rows = 1 }}, &.{token}, false, .all); + if (self.scratch.taps != null) cache.tap = .{ .start = cache.len - 1, .row = 0, .rows = 1, .gen = self.gen }; return try draw(self.gpa, cache.logits.slice(u16, c.vocab), stream.sampling, position, &.{}); } @@ -191,7 +259,11 @@ pub const Metal = struct { @memcpy(ids[total + 1 ..][0..w.tokens.len], w.tokens); total += rows; } - try fwd.run(self.model, &self.scratch, segments[0..windows.len], ids[0..total], true, .all); + try self.run(segments[0..windows.len], ids[0..total], true, .all); + if (self.scratch.taps != null) for (segments[0..windows.len]) |seg| { + const last = seg.cache.last.?; + seg.cache.tap = .{ .start = last.start, .row = last.base, .rows = last.rows, .gen = self.gen }; + }; const logits = self.scratch.logits.slice(u16, total * c.vocab); const best = self.scratch.best.slice(u32, 2 * total); var base: usize = 0; diff --git a/zig/src/families/lfm2_moe/dspark_test.zig b/zig/src/families/lfm2_moe/dspark_test.zig new file mode 100644 index 0000000000..d220b9ea66 --- /dev/null +++ b/zig/src/families/lfm2_moe/dspark_test.zig @@ -0,0 +1,77 @@ +//! The DSpark drafter's kernels on the GPU against host references: the target's taps. +const std = @import("std"); +const mtl = @import("metal"); +const c = @import("config.zig"); +const kernels = @import("kernels.zig"); + +const shared = mtl.ResourceOptions.shared; + +const Rig = struct { + device: mtl.Device, + queue: mtl.Queue, + k: kernels.Kernels, + + fn init() !Rig { + const device = mtl.Device.init() catch return error.SkipZigTest; + errdefer device.deinit(); + const queue = try device.queue(); + errdefer queue.deinit(); + return .{ .device = device, .queue = queue, .k = try kernels.load(device) }; + } + + fn deinit(r: *Rig) void { + r.k.deinit(); + r.queue.deinit(); + r.device.deinit(); + } + + fn buffer(r: *Rig, comptime T: type, n: usize) !mtl.Buffer { + const b = try r.device.buffer(n * @sizeOf(T), shared); + @memset(b.slice(u8, n * @sizeOf(T)), 0); + return b; + } + + fn run(r: *Rig, comptime key: []const u8, bufs: []const mtl.Buffer, value: anytype, at: usize, grid: [2]usize) !void { + const cb = r.queue.commandBuffer(); + const e = cb.compute(.serial); + e.setPipeline(r.k.get(key)); + for (bufs, 0..) |b, i| e.setBuffer(b, 0, if (i >= at) i + 1 else i); + e.setValue(value, at); + e.dispatchThreads(mtl.Size.of(grid[0], grid[1], 1), mtl.Size.of(@min(grid[0], 256), 1, 1)); + e.end(); + cb.commit(); + cb.wait(); + if (cb.failure()) |_| return error.GpuFailed; + } +}; + +/// Round to nearest even, as MLX casts. +fn toBf16(f: f32) u16 { + const b: u32 = @bitCast(f); + return @truncate((b + 0x7fff + ((b >> 16) & 1)) >> 16); +} + +fn noise(seed: u64, out: []u16, scale: f32) void { + var prng = std.Random.DefaultPrng.init(seed); + for (out) |*v| v.* = toBf16((prng.random().float(f32) * 2 - 1) * scale); +} + +test "lfm2 dspark: the tap kernel copies each row's hidden values into its tap's columns, the rest untouched" { + var r = try Rig.init(); + defer r.deinit(); + const rows = 5; + const taps = 3; + const h = try r.buffer(u16, rows * c.hidden); + defer h.deinit(); + const out = try r.buffer(u16, rows * taps * c.hidden); + defer out.deinit(); + noise(7, h.slice(u16, rows * c.hidden), 4); + try r.run("lfm2_tap", &.{ h, out }, [2]u32{ taps * c.hidden, c.hidden }, 2, .{ c.hidden / 8, rows }); + const want = h.slice(u16, rows * c.hidden); + const got = out.slice(u16, rows * taps * c.hidden); + for (0..rows) |row| { + try std.testing.expectEqualSlices(u16, want[row * c.hidden ..][0..c.hidden], got[(row * taps + 1) * c.hidden ..][0..c.hidden]); + for (got[row * taps * c.hidden ..][0..c.hidden]) |v| try std.testing.expectEqual(@as(u16, 0), v); + for (got[(row * taps + 2) * c.hidden ..][0..c.hidden]) |v| try std.testing.expectEqual(@as(u16, 0), v); + } +} diff --git a/zig/src/families/lfm2_moe/forward.zig b/zig/src/families/lfm2_moe/forward.zig index 212b5978bc..7b63833192 100644 --- a/zig/src/families/lfm2_moe/forward.zig +++ b/zig/src/families/lfm2_moe/forward.zig @@ -146,6 +146,7 @@ pub fn run(m: *Model, s: *st.Scratch, segments: []const Segment, tokens: []const for (m.weights.blocks, 0..) |block, i| { enc.norm(s, block.operator_norm, i == 0, total, combine); combine = false; + if (i > 0) tap(enc, s, i - 1, total); switch (block.mixer) { .conv => |cv| conv(enc, s, cv, i, segments, total), .attention => |a| attention(enc, s, a, &m.config, segments, i, total), @@ -162,6 +163,7 @@ pub fn run(m: *Model, s: *st.Scratch, segments: []const Segment, tokens: []const } } enc.norm(s, m.weights.norm, false, total, combine); + tap(enc, s, c.layers - 1, total); if (head == .all) { enc.projection(embedding, s.x, 0, s.logits, 0, total); enc.pipe("lfm2_argmax"); @@ -187,6 +189,18 @@ pub fn run(m: *Model, s: *st.Scratch, segments: []const Segment, tokens: []const } } +/// Layer `layer`'s output rows (h, just summed by the next norm) into each tap of it, when a drafter reads them. +fn tap(enc: Encoder, s: *st.Scratch, layer: usize, total: usize) void { + const taps = s.taps orelse return; + for (taps.layers[0..taps.count], 0..) |l, t| if (l == layer) { + enc.pipe("lfm2_tap"); + enc.e.setBuffer(s.h, 0, 0); + enc.e.setBuffer(taps.buf, 0, 1); + enc.e.setValue([2]u32{ taps.count * c.hidden, @intCast(t * c.hidden) }, 2); + enc.threads(.{ c.hidden / 8, total, 1 }, .{ 256, 1, 1 }); + }; +} + fn conv(enc: Encoder, s: *st.Scratch, cv: wts.Conv, layer: usize, segments: []const Segment, total: usize) void { const e = enc.e; enc.projection(cv.in, s.x, 0, s.bcx, 0, total); diff --git a/zig/src/families/lfm2_moe/kernels.zig b/zig/src/families/lfm2_moe/kernels.zig index aa1e798fc0..61af45f4c8 100644 --- a/zig/src/families/lfm2_moe/kernels.zig +++ b/zig/src/families/lfm2_moe/kernels.zig @@ -27,6 +27,7 @@ const names = [_][:0]const u8{ "lfm2_qk_segs", "lfm2_attn_segs", "lfm2_attn_merge_segs", + "lfm2_tap", }; pub const Kernels = struct { diff --git a/zig/src/families/lfm2_moe/snapshot.zig b/zig/src/families/lfm2_moe/snapshot.zig index d1c1a8b3ae..d7289331b7 100644 --- a/zig/src/families/lfm2_moe/snapshot.zig +++ b/zig/src/families/lfm2_moe/snapshot.zig @@ -4,7 +4,7 @@ const c = @import("config.zig"); const st = @import("state.zig"); /// One state: conv states by conv layer, then keys and values [kv head][at][head dim] by attention layer. -pub const Snap = struct { at: u32, bytes: []u8 }; +pub const Snap = struct { at: u32, bytes: []u8, draft: ?[]u8 = null }; /// A row's keys or values for every kv head of one attention layer. const row_bytes = c.kv_heads * c.head_dim * 2; @@ -56,6 +56,7 @@ pub fn restore(cache: *st.Cache, snap: *const Snap) !void { } pub fn drop(gpa: std.mem.Allocator, snap: *Snap) void { + if (snap.draft) |d| gpa.free(d); // the drafter's context at `at`, when one was kept with the state gpa.free(snap.bytes); gpa.destroy(snap); } diff --git a/zig/src/families/lfm2_moe/state.zig b/zig/src/families/lfm2_moe/state.zig index 3242d8bf26..1bc749ae95 100644 --- a/zig/src/families/lfm2_moe/state.zig +++ b/zig/src/families/lfm2_moe/state.zig @@ -51,6 +51,12 @@ pub const Storage = struct { }; pub const Attention = struct { keys: mtl.Buffer, values: mtl.Buffer }; +/// Layers a drafter reads at most (each layer's output, hidden wide). +pub const max_taps = 8; +/// The drafter's layers in its order and every forward row's outputs of them, [rows][count][hidden] bf16. +pub const Taps = struct { layers: [max_taps]u32 = @splat(0), count: u32 = 0, buf: mtl.Buffer }; +/// Where a stream's latest tapped rows sit in the scratch: cache rows [start, start + rows) from scratch row `row`. +pub const TapRows = struct { start: usize, row: usize, rows: usize, gen: u64 }; pub const Mixer = union(enum) { conv: mtl.Buffer, attention: Attention }; pub const Cache = struct { @@ -60,6 +66,9 @@ pub const Cache = struct { last: ?struct { start: usize, base: usize, rows: usize } = null, blocks: [c.layers]Mixer, logits: mtl.Buffer, + /// The stream's latest tapped rows in the scratch, and its prompt's last row's taps (kept for the first draft). + tap: ?TapRows = null, + last_tap: ?mtl.Buffer = null, pub fn init(gpa: std.mem.Allocator, device: mtl.Device, attention: *const [c.layers]bool, capacity: usize) !Cache { var out: Cache = .{ .memory = .{ .gpa = gpa, .device = device }, .capacity = capacity, .blocks = undefined, .logits = undefined }; @@ -148,6 +157,8 @@ pub const Scratch = struct { segs: mtl.Buffer, rowseg: mtl.Buffer, tiles: mtl.Buffer, + /// A drafter's taps of every row, once the backend is told its layers (null: none kept). + taps: ?Taps = null, pub fn init(gpa: std.mem.Allocator, device: mtl.Device, attention: *const [c.layers]bool, rows: usize) !Scratch { var out: Scratch = undefined; @@ -155,6 +166,7 @@ pub const Scratch = struct { errdefer out.memory.deinit(); out.rows = rows; out.snapshots = @splat(null); + out.taps = null; const mem = &out.memory; const slots = rows * c.topk; out.ids = try mem.alloc(rows * 4); diff --git a/zig/src/tensorfold.zig b/zig/src/tensorfold.zig index 4fa4fbe147..6148579bf6 100644 --- a/zig/src/tensorfold.zig +++ b/zig/src/tensorfold.zig @@ -88,6 +88,7 @@ test { _ = @import("families/lfm2_moe/config.zig"); _ = @import("families/lfm2_moe/backend.zig"); _ = @import("families/lfm2_moe/state.zig"); + _ = @import("families/lfm2_moe/dspark_test.zig"); } pub const lane_projection = @import("core/lane_projection.zig"); diff --git a/zig/tests/contract/metal_lfm2_tap.zig b/zig/tests/contract/metal_lfm2_tap.zig new file mode 100644 index 0000000000..871ea8a8a2 --- /dev/null +++ b/zig/tests/contract/metal_lfm2_tap.zig @@ -0,0 +1,143 @@ +//! LFM2's tap copy for an external drafter (lfm2_tap): a layer's rows into their tap's columns, the others kept. +const std = @import("std"); +const mtl = @import("metal"); +const contract = @import("contract"); +const sources = @import("kernel_sources"); +const c = contract.spec; +const g = contract.gates; +const rig_ = @import("rig.zig"); + +const hidden = 2048; +const taps = 5; +pub const name = "metal lfm2 tap copy"; + +/// rows: one decode row, a verify window, a shared round's 32 and a 2,048-row prompt chunk. +const shapes = [_]c.Shape{ .{ .rows = 1, .n = taps * hidden, .k = 1 }, .{ .rows = 10, .n = taps * hidden, .k = 1 }, .{ .rows = 32, .n = taps * hidden, .k = 1 }, .{ .rows = 2048, .n = taps * hidden, .k = 1 } }; + +fn none(_: c.Shape) usize { + return 0; +} + +pub fn spec() c.Spec { + return .{ .name = name, .accum = .none, .depth = none, .out = .bf16, .served = &shapes, .sums = .{ .f16 = 0, .bf16 = 0, .why = "a copy moves values: no sums" } }; +} + +/// A bf16 value for element `i` over signs and binades; the nonfinite fill plants NaN and infinities. +fn value(i: usize, seed: u64, fill: g.Fill) u16 { + if (fill == .nonfinite) switch ((@as(u64, i) *% 2654435761 +% seed) % 9973) { + 0 => return 0x7fc0, + 1 => return 0x7f80, + 2 => return 0xff80, + else => {}, + }; + const h: u64 = (@as(u64, i) *% 0x9e3779b97f4a7c15 +% seed *% 0xbf58476d1ce4e5b9) >> 40; + const mant: f32 = @as(f32, @floatFromInt(h % 1024)) / 1024 + 1; + const exp: i32 = @as(i32, @intCast((h >> 10) % 8)) - 4; + const v = std.math.ldexp(mant, exp) * @as(f32, if (((h >> 15) & 1) == 0) 1 else -1); + return c.toBf16(v); +} + +pub const Probe = struct { + a: std.mem.Allocator, + rig: rig_.Rig, + pipe: mtl.Pipeline, + rows: usize = 0, + src: []u16 = &.{}, // H rows + prior: []u16 = &.{}, // the taps rows as the call finds them (the other taps' columns must stay) + args: [2]rig_.Ref = undefined, + y: rig_.Ref = undefined, + + pub fn init(a: std.mem.Allocator, device: mtl.Device, queue: mtl.Queue) !Probe { + return .{ .a = a, .rig = .{ .a = a, .device = device, .queue = queue }, .pipe = try rig_.pipeline(device, sources.lfm2, "lfm2_tap") }; + } + + fn drop(p: *Probe) void { + p.a.free(p.src); + p.a.free(p.prior); + p.src = &.{}; + p.prior = &.{}; + p.rig.reset(); + } + + pub fn deinit(p: *Probe) void { + p.drop(); + p.rig.deinit(); + p.pipe.deinit(); + } + + const vt = g.hooks(Probe); + pub fn kernel(p: *Probe) g.Kernel { + return .{ .ctx = p, .vt = &vt, .spec = spec() }; + } + + fn fill(out: []u16, seed: u64, f: g.Fill) void { + for (out, 0..) |*v, i| v.* = value(i, seed, f); + } + + pub fn prepare(p: *Probe, s: c.Shape, f: g.Fill, seed: u64) !bool { + p.drop(); + if (s.n != taps * hidden or s.rows == 0) return false; + p.rows = s.rows; + p.src = try p.a.alloc(u16, s.rows * hidden); + fill(p.src, seed, f); + p.prior = try p.a.alloc(u16, s.rows * taps * hidden); + fill(p.prior, seed +% 3, .random); + p.args[0] = try p.rig.input(std.mem.sliceAsBytes(p.src)); + const at = [2]u32{ taps * hidden, 2 * hidden }; // the third tap's columns + p.args[1] = try p.rig.input(std.mem.asBytes(&at)); + p.y = try p.rig.output(p.count() * 2); + return true; + } + + pub fn count(p: *Probe) usize { + return p.rows * taps * hidden; + } + + pub fn run(p: *Probe) !void { + @memcpy(p.rig.bytes(p.y, p.count() * 2), std.mem.sliceAsBytes(p.prior)); + const pass = p.rig.begin(); + const e = pass.e; + e.setPipeline(p.pipe); + e.setBuffer(p.args[0].buf, p.args[0].off, 0); + e.setBuffer(p.y.buf, p.y.off, 1); + e.setBuffer(p.args[1].buf, p.args[1].off, 2); + e.dispatchThreads(mtl.Size.of(hidden / 8, p.rows, 1), mtl.Size.of(256, 1, 1)); + return p.rig.finish(pass); + } + + pub fn reference(p: *Probe, want: []f64, mag: []f64) !void { + for (want, mag, p.prior) |*w, *m, v| { // the other taps' columns keep what the call found + w.* = c.fromBf16(v); + m.* = @abs(w.*); + } + for (0..p.rows) |r| for (0..hidden) |i| { + const v = c.fromBf16(p.src[r * hidden + i]); + want[r * taps * hidden + 2 * hidden + i] = v; + mag[r * taps * hidden + 2 * hidden + i] = @abs(v); + }; + } + + /// The copy is exact: the third tap's columns hold H's bits, the others what the call found. + pub fn check(p: *Probe, _: g.Gate) !?[]const u8 { + const out = std.mem.bytesAsSlice(u16, p.rig.bytes(p.y, p.count() * 2)); + for (out, p.prior, 0..) |got, before, i| { + const r = i / (taps * hidden); + const col = i % (taps * hidden); + const want = if (col >= 2 * hidden and col < 3 * hidden) p.src[r * hidden + col - 2 * hidden] else before; + if (got != want) return "an output differs from the copy's bits"; + } + return null; + } +}; + +/// Gate 5 for the tap's launch of 256 threads and no threadgroup memory: every Apple chip we serve, then this GPU. +pub fn limits(a: std.mem.Allocator, device: mtl.Device, only: ?[]const u8) !usize { + if (only) |s| if (std.mem.indexOf(u8, name, s) == null) return 0; + const d = contract.limits.MetalLaunch{ .name = name, .threads = 256, .tg_memory = 0, .offline = false }; + var notes: std.ArrayList(u8) = .empty; + defer notes.deinit(a); + var problems: usize = 0; + for (contract.limits.apple) |chip| problems += try contract.limits.metal(d, chip, ¬es, a); + std.debug.print("{s}", .{notes.items}); + return problems + try @import("metal_limits.zig").onThisGpu(a, device, .{ .decl = d, .function = "lfm2_tap", .here = sources.lfm2, .pre_m5 = "" }, false); +} diff --git a/zig/tests/contract/metal_main.zig b/zig/tests/contract/metal_main.zig index 46801e171c..7f44fbb441 100644 --- a/zig/tests/contract/metal_main.zig +++ b/zig/tests/contract/metal_main.zig @@ -26,6 +26,7 @@ const gemma_embed = @import("metal_gemma4_embed.zig"); const gemma_mixed = @import("metal_gemma4_mixed.zig"); const lfm2_argmax = @import("metal_lfm2_argmax.zig"); const lfm2_streams = @import("metal_lfm2_streams.zig"); +const lfm2_tap = @import("metal_lfm2_tap.zig"); const lfm2_controls = @import("metal_lfm2_controls.zig"); const g = contract.gates; const usage = "usage: tf-kernel-contract [--only TEXT] [--no-gates] [--no-limits] [--offline DIR] [--seed N] [--repro] [--rotary-long] [--rotary-accuracy FILE]"; @@ -191,6 +192,22 @@ pub fn main(init: std.process.Init) !u8 { try kernels.append(a, p.kernel()); } } + var tap_list: std.ArrayList(*lfm2_tap.Probe) = .empty; + defer { + for (tap_list.items) |p| { + p.deinit(); + a.destroy(p); + } + tap_list.deinit(a); + } + for ([_]lfm2_controls.Mutation{ .none, .tap_packed_rows, .tap_clobbers_column }) |m| { + if (!wanted(o, if (m == .none) lfm2_tap.name else m.name())) continue; + const p = try a.create(lfm2_tap.Probe); + errdefer a.destroy(p); + p.* = try lfm2_tap.Probe.init(a, device, queue, m); + try tap_list.append(a, p); + try kernels.append(a, p.kernel()); + } var rows_list: std.ArrayList(*rows_k.Row) = .empty; var proj_list: std.ArrayList(*affine_k.Proj) = .empty; defer for (rows_list.items) |r| r.deinit(); From 3b88fa3b759ab99cb8dda87159b2e5e3d696cf17 Mon Sep 17 00:00:00 2001 From: BlivionIaG <15323295+BlivionIaG@users.noreply.github.com> Date: Mon, 26 Oct 2026 09:02:12 +0100 Subject: [PATCH 02/39] lanes: a lone stream's pipelined steps move to pipelined.zig engine.zig is 592 lines and the shared-round guard needs room in it. pipelinedRound and landInflight move unchanged to pipelined.zig as round and land; Engine.familyRound and Result become public for them. No behaviour change. --- zig/src/core/lanes/engine.zig | 67 +++--------------------------- zig/src/core/lanes/pipelined.zig | 70 ++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 62 deletions(-) create mode 100644 zig/src/core/lanes/pipelined.zig diff --git a/zig/src/core/lanes/engine.zig b/zig/src/core/lanes/engine.zig index 5d357c646d..a90dd3166f 100644 --- a/zig/src/core/lanes/engine.zig +++ b/zig/src/core/lanes/engine.zig @@ -16,6 +16,7 @@ const lone_costs = @import("lone_costs.zig"); const open = @import("open.zig"); const handover = @import("handover.zig"); const trim = @import("trim.zig"); +const pipelined = @import("pipelined.zig"); const Stream = sm.Stream; const Feed = be.Feed; const LogRow = @import("logprob.zig").Row; @@ -24,7 +25,7 @@ const f = ev.f; const str = trail.str; const int = trail.int; const Outcome = struct { got: []const u32, path: []const u32, cut: ?usize, follow: []const u32 }; -const Result = struct { rows: u32, keep: u32, got: []const u32 }; +pub const Result = struct { rows: u32, keep: u32, got: []const u32 }; const Unread = struct { event: usize, handle: u64 }; pub const Engine = struct { const pieces_ = @import("pieces.zig"); @@ -124,7 +125,7 @@ pub const Engine = struct { if (live.items.len > 1 and e.cfg.family_streams) { // a shared round is synchronous: a stream that ran a step ahead lands its queued token first for (live.items) |s| { - if (s.inflight != null) try e.landInflight(s); + if (s.inflight != null) try pipelined.land(e, s); } var kept: usize = 0; for (live.items) |s| { @@ -147,7 +148,7 @@ pub const Engine = struct { for (live.items) |s| { // logprobs and grammars take verify windows: rows' logits are read, or bits made, before a forward const windowed = (e.cfg.family_mtp and s.drafts) or !e.cfg.pipelined or s.logprobs != null or s.grammar != null; - const r = if (windowed) try e.familyRound(s, null) else try e.pipelinedRound(s); + const r = if (windowed) try e.familyRound(s, null) else try pipelined.round(e, s); s.min_rows = if (s.min_rows == 0) r.rows else @min(s.min_rows, r.rows); } } @@ -195,7 +196,7 @@ pub const Engine = struct { } /// One stream's round: verify pending and drafted tokens together, keep drafts to the first mismatch. - fn familyRound(e: *Engine, s: *Stream, copied: ?win.Copy) !Result { + pub fn familyRound(e: *Engine, s: *Stream, copied: ?win.Copy) !Result { e.clock.start(); const priced = s.priced; s.priced = null; @@ -490,64 +491,6 @@ pub const Engine = struct { } } - /// One token, the next forward queued first; a copied continuation ahead switches to verify windows. - fn pipelinedRound(e: *Engine, s: *Stream) !Result { - const before = s.mode; - // an adopted stream holds neither a mode nor a queued step: it starts where a drain step leaves one - const mode: sm.Mode = s.mode orelse if (s.inflight == null) .verify else .pipe; - var result: Result = .{ .rows = 1, .keep = 1, .got = &.{} }; - if (mode == .verify or mode == .exit) { - const verifying = mode == .verify and e.cfg.family_width >= 2; - const proposal: win.Copy = if (verifying) try win.copyProposal(e, s) else .{}; - if (proposal.tokens.len > 0) { - const copied = try e.arena.allocator().dupe(u32, proposal.tokens); - result = try e.familyRound(s, .{ .tokens = copied, .recalled = proposal.recalled }); - if (result.keep == 1 or s.force.items.len > 0) s.mode = .exit; - } else { - // nothing to copy: back to steps queued ahead (this one lands next round) - s.mode = .pipe; - try e.queueNext(s, .{ .value = s.pending.? }); - } - try trail.pipe(e, s, before, result.got); - return result; - } - var current: Feed = .{ .handle = s.inflight.? }; - s.inflight = null; - if (try e.forcedNext(s)) |t| current = .{ .value = t }; - var token: u32 = undefined; - if (mode == .drain) { - token = try e.readFeed(current); // the last queued step: no new one - s.mode = .verify; - } else { - try e.queueNext(s, current); // the GPU starts the next step first - token = try e.readFeed(current); - } - s.rounds += 1; - const landed = try s.commit(e.gpa, &.{token}, &.{}); - s.pending = token; - if (s.recall) |r| r.noteHead(1); // a step lands one token: the bar a copy must beat - if (e.cfg.root_copies) s.root_copy_miss = false; - const piping = e.cfg.family_width >= 2 and (s.mode orelse .pipe) == .pipe and !s.finished and s.drafts; - if (piping and (try win.copyProposal(e, s)).tokens.len > 0) - s.mode = .drain; // a copy window is ahead: land the queued step - result.got = try e.arena.allocator().dupe(u32, s.emitted()[s.emitted().len - landed ..]); - try trail.pipe(e, s, before, result.got); - return result; - } - - /// Land the queued token without queuing another (a shared round is synchronous). - fn landInflight(e: *Engine, s: *Stream) !void { - var current: Feed = .{ .handle = s.inflight.? }; - s.inflight = null; - if (try e.forcedNext(s)) |t| current = .{ .value = t }; - const token = try e.readFeed(current); - s.rounds += 1; - const landed = try s.commit(e.gpa, &.{token}, &.{}); - s.pending = token; - s.mode = .verify; - try trail.event(e, &.{ f("ev", str("land")), f("stream", str(s.id)), f("got", .{ .u32s = s.emitted()[s.emitted().len - landed ..] }) }); - } - /// The first token's draw: among the grammar's allowed tokens when the stream has one that constrains it. pub fn drawFirst(e: *Engine, s: *Stream, position: u64) !u64 { const g = s.grammar orelse return e.backend.first(s, position); diff --git a/zig/src/core/lanes/pipelined.zig b/zig/src/core/lanes/pipelined.zig new file mode 100644 index 0000000000..99de33304b --- /dev/null +++ b/zig/src/core/lanes/pipelined.zig @@ -0,0 +1,70 @@ +//! A lone stream's pipelined steps: the next forward queued before the last one's token is read. +const be = @import("backend.zig"); +const sm = @import("stream.zig"); +const ev = @import("events.zig"); +const win = @import("windows.zig"); +const trail = @import("trail.zig"); +const Engine = @import("engine.zig").Engine; +const Result = @import("engine.zig").Result; +const Stream = sm.Stream; +const Feed = be.Feed; +const f = ev.f; +const str = trail.str; + +/// One token, the next forward queued first; a copied continuation ahead switches to verify windows. +pub fn round(e: *Engine, s: *Stream) !Result { + const before = s.mode; + // an adopted stream holds neither a mode nor a queued step: it starts where a drain step leaves one + const mode: sm.Mode = s.mode orelse if (s.inflight == null) .verify else .pipe; + var result: Result = .{ .rows = 1, .keep = 1, .got = &.{} }; + if (mode == .verify or mode == .exit) { + const verifying = mode == .verify and e.cfg.family_width >= 2; + const proposal: win.Copy = if (verifying) try win.copyProposal(e, s) else .{}; + if (proposal.tokens.len > 0) { + const copied = try e.arena.allocator().dupe(u32, proposal.tokens); + result = try e.familyRound(s, .{ .tokens = copied, .recalled = proposal.recalled }); + if (result.keep == 1 or s.force.items.len > 0) s.mode = .exit; + } else { + // nothing to copy: back to steps queued ahead (this one lands next round) + s.mode = .pipe; + try e.queueNext(s, .{ .value = s.pending.? }); + } + try trail.pipe(e, s, before, result.got); + return result; + } + var current: Feed = .{ .handle = s.inflight.? }; + s.inflight = null; + if (try e.forcedNext(s)) |t| current = .{ .value = t }; + var token: u32 = undefined; + if (mode == .drain) { + token = try e.readFeed(current); // the last queued step: no new one + s.mode = .verify; + } else { + try e.queueNext(s, current); // the GPU starts the next step first + token = try e.readFeed(current); + } + s.rounds += 1; + const landed = try s.commit(e.gpa, &.{token}, &.{}); + s.pending = token; + if (s.recall) |r| r.noteHead(1); // a step lands one token: the bar a copy must beat + if (e.cfg.root_copies) s.root_copy_miss = false; + const piping = e.cfg.family_width >= 2 and (s.mode orelse .pipe) == .pipe and !s.finished and s.drafts; + if (piping and (try win.copyProposal(e, s)).tokens.len > 0) + s.mode = .drain; // a copy window is ahead: land the queued step + result.got = try e.arena.allocator().dupe(u32, s.emitted()[s.emitted().len - landed ..]); + try trail.pipe(e, s, before, result.got); + return result; +} + +/// Land the queued token without queuing another (a shared round is synchronous). +pub fn land(e: *Engine, s: *Stream) !void { + var current: Feed = .{ .handle = s.inflight.? }; + s.inflight = null; + if (try e.forcedNext(s)) |t| current = .{ .value = t }; + const token = try e.readFeed(current); + s.rounds += 1; + const landed = try s.commit(e.gpa, &.{token}, &.{}); + s.pending = token; + s.mode = .verify; + try trail.event(e, &.{ f("ev", str("land")), f("stream", str(s.id)), f("got", .{ .u32s = s.emitted()[s.emitted().len - landed ..] }) }); +} From 474105af9dcb4fb7307d269a3cf38aaaaa94b734 Mon Sep 17 00:00:00 2001 From: BlivionIaG <15323295+BlivionIaG@users.noreply.github.com> Date: Mon, 26 Oct 2026 09:30:40 +0100 Subject: [PATCH 03/39] lfm2: draft with LiquidAI's DSpark drafter (--drafter) LiquidAI/LFM2.5-24B-A2B-DSpark is LiquidAI's speculative-decoding drafter for LFM2.5-24B-A2B. `tensorfold serve` now takes it with --drafter DIR for lfm2_moe checkpoints. The lane core verifies every draft, so a drafted reply equals its "draft": false reply. The drafter reads the target's rows after layers 2, 10, 18, 26 and 34 (the taps). fc and hidden_norm turn each row into the context keys and values of five Qwen3 layers. A block of the pending token and mask tokens runs the layers with non-causal attention, then the target's head. The Markov head adds W2 W1[previous token] to each row's logits and takes the argmax, one row after the other. A block is as long as the deepest chain its pass holds, at most the nine it was trained on. The drafter runs on the family's own kernels and core/tile_qmm (on GPUs with tensor units, core/affine_mm for absorb passes past the twin's rows): its bf16 linears are quantized to 8-bit MLX affine codes at load, and the q and k rows are reordered so that the target's half-split RoPE kernel gives the drafter's interleaved RoPE. New kernels: - lfm2_markov_scan and lfm2_markov_pick (lfm2_dspark.metal): a Markov step's W1 gather, W2 dot over 8-bit codes, bias add and argmax, ties to the lowest token, a NaN score flagged; - lfm2_attn_block and lfm2_attn_block_merge: the target's attention and merge kernels with a FULL template flag, every block row seeing the keys [0, len + 1). The target's own instantiations keep their causal count. Both register with the kernel contract (metal_lfm2_dspark.zig, metal_lfm2_attn.zig), gate 5 against every Apple chip we serve and this GPU. Rows reach the drafter in passes: a prompt chunk in one pass of up to 2,048 rows; a round's verified rows are staged on the host (128 a lane) and absorbed by the next hold, so a round without holds does no drafter GPU work. A hold pass takes as many streams as their blocks fit in 32 rows. The drafter sets Facts.shared_guard, so shared rounds draft only while drafted rounds measure faster than plain ones at that stream count; a lone stream keeps the depth rule's own plain guard. The drafter keeps its context with each kept prompt state, so a drafting stream resumes a prompt as before. Its lane's bytes join the streams the load limit admits. --drafter-bits 0 is refused for this drafter, and a refused open frees the model it loaded. Tests: test-lfm2-dspark checks the config's admission and refusals, the quantizer, the RoPE reordering through lfm2_qk, the Markov step and the block attention on the GPU. tf-lfm2-dspark runs the real drafter against an fp32 host reference of the released checkpoint and times its passes. --- zig/build/metal_targets.zig | 3 +- zig/kernels/metal/lfm2/lfm2.metal | 19 +- zig/kernels/metal/lfm2/lfm2_dspark.metal | 99 +++++ zig/kernels/metal/sources.zig | 2 +- zig/src/families/lfm2_moe/dspark.zig | 386 +++++++++++++++++++ zig/src/families/lfm2_moe/dspark_block.zig | 271 +++++++++++++ zig/src/families/lfm2_moe/dspark_config.zig | 136 +++++++ zig/src/families/lfm2_moe/dspark_quant.zig | 118 ++++++ zig/src/families/lfm2_moe/dspark_test.zig | 147 ++++++- zig/src/families/lfm2_moe/dspark_weights.zig | 144 +++++++ zig/src/families/lfm2_moe/kernels.zig | 4 + zig/src/families/lfm2_moe/lfm2_moe.zig | 4 + zig/src/native/lfm2_moe.zig | 80 +++- zig/src/native/metal.zig | 4 +- zig/src/tensorfold.zig | 2 + zig/tests/contract/metal_lfm2_attn.zig | 182 +++++++++ zig/tests/contract/metal_lfm2_dspark.zig | 221 +++++++++++ zig/tests/contract/metal_main.zig | 19 + zig/tests/lfm2_dspark.zig | 301 +++++++++++++++ 19 files changed, 2114 insertions(+), 28 deletions(-) create mode 100644 zig/kernels/metal/lfm2/lfm2_dspark.metal create mode 100644 zig/src/families/lfm2_moe/dspark.zig create mode 100644 zig/src/families/lfm2_moe/dspark_block.zig create mode 100644 zig/src/families/lfm2_moe/dspark_config.zig create mode 100644 zig/src/families/lfm2_moe/dspark_quant.zig create mode 100644 zig/src/families/lfm2_moe/dspark_weights.zig create mode 100644 zig/tests/contract/metal_lfm2_attn.zig create mode 100644 zig/tests/contract/metal_lfm2_dspark.zig create mode 100644 zig/tests/lfm2_dspark.zig diff --git a/zig/build/metal_targets.zig b/zig/build/metal_targets.zig index 5c01545d3f..ea81d7ea7a 100644 --- a/zig/build/metal_targets.zig +++ b/zig/build/metal_targets.zig @@ -292,6 +292,7 @@ pub fn metalTargets(b: *std.Build, target: std.Build.ResolvedTarget, optimize: s .{ .name = "tf-lfm2-forward", .path = "zig/tests/lfm2_forward.zig", .about = "LFM2-MoE native teacher-forced logits" }, .{ .name = "tf-lfm2-bench", .path = "zig/tests/lfm2_bench.zig", .about = "LFM2-MoE prompt-chunk and one-row decode timing" }, .{ .name = "tf-lfm2-exact", .path = "zig/tests/lfm2_exact.zig", .about = "LFM2-MoE prompt chunks, windows and committed-state exactness" }, + .{ .name = "tf-lfm2-dspark", .path = "zig/tests/lfm2_dspark.zig", .about = "LFM2's DSpark drafter on the GPU against an fp32 host reference of the released drafter" }, .{ .name = "tf-lfm2-fp64", .path = "zig/tests/lfm2_fp64.zig", .about = "LFM2 projection classes and attention on the tensor units, the tiles and the twin, each against float64" }, .{ .name = "tensorfold", .path = "zig/src/main.zig", .about = "The native engine's command line" }, .{ .name = "tf-nemotron-fixtures", .path = "zig/tests/nemotron_fixtures.zig", .about = "Nemotron kernels against the Python engine's captured ops" }, @@ -459,7 +460,7 @@ pub fn metalTargets(b: *std.Build, target: std.Build.ResolvedTarget, optimize: s const lfm2_tests = b.addRunArtifact(b.addTest(.{ .root_module = engine, .filters = &.{"lfm2_moe"} })); b.step("test-lfm2", "The LFM2-MoE family's tests: config, draws, state and its shared rounds on this Mac's GPU").dependOn(&lfm2_tests.step); const lfm2_dspark = b.addRunArtifact(b.addTest(.{ .root_module = engine, .filters = &.{"lfm2 dspark:"} })); - b.step("test-lfm2-dspark", "LFM2's taps for a drafter: the tap kernel on this Mac's GPU").dependOn(&lfm2_dspark.step); + b.step("test-lfm2-dspark", "LFM2's DSpark drafter and its taps: config, quantizer and kernels on this Mac's GPU").dependOn(&lfm2_dspark.step); test_step.dependOn(&b.addRunArtifact(b.addTest(.{ .root_module = lanes })).step); // the fake fabric maps its windows with mmap: the Macs' RDMA, not Windows if (target.result.os.tag != .windows) test_step.dependOn(&b.addRunArtifact(b.addTest(.{ .root_module = fabric })).step); diff --git a/zig/kernels/metal/lfm2/lfm2.metal b/zig/kernels/metal/lfm2/lfm2.metal index b2772cf64e..76c223805d 100644 --- a/zig/kernels/metal/lfm2/lfm2.metal +++ b/zig/kernels/metal/lfm2/lfm2.metal @@ -232,7 +232,7 @@ inline void lfm2_stage(const thread uint4* pre, threadgroup uint* ks, threadgrou } // Every row's attention partials (one kernel for prompt and decode): a threadgroup a (chunk, kv head, 8 * RB rows). -template // a simdgroup's RB row blocks share each staged K and V fragment, each block's arithmetic its own +template // RB row blocks share each staged K and V fragment; FULL: all see [0, len + 1) inline void lfm2_attn_rows(const device bfloat* queries, const device bfloat* keys, const device bfloat* values, lfm2_attn_args p, device float* pm, device float* pl, device float* po, uint rows, uint3 tg, uint tid, uint sg, uint lane, threadgroup uint (*ks)[HD * (KT / 2)], @@ -242,8 +242,8 @@ inline void lfm2_attn_rows(const device bfloat* queries, const device bfloat* ke const uint kh = tg.y, r0 = tg.x * 8 * RB, head = kh * (QH / KVH) + sg; const uint qid = lane / 4, fm = (qid & 4) + ((lane / 2) % 4), fn = (qid & 2) * 2 + (lane % 2) * 2; uint count[RB]; // this lane's rows' visible keys - for (int b = 0; b < RB; ++b) count[b] = p.len + min(r0 + 8 * b + fm, rows - 1) + 1; - const uint top = p.len + min(r0 + 8 * RB - 1, rows - 1) + 1; // the tile's most + for (int b = 0; b < RB; ++b) count[b] = FULL ? p.len + 1 : p.len + min(r0 + 8 * b + fm, rows - 1) + 1; + const uint top = FULL ? p.len + 1 : p.len + min(r0 + 8 * RB - 1, rows - 1) + 1; // the tile's most if (chunk * PER + sub0 * SUB >= top) return; vec q[RB][8]; for (int b = 0; b < RB; ++b) @@ -340,7 +340,7 @@ inline void lfm2_attn_rows(const device bfloat* queries, const device bfloat* ke for (int d = 0; d < 8; ++d) *(device float2*)(po + at * HD + 8 * d + fn) = O[b][d]; } } -template +template [[kernel]] void lfm2_attn_mma(const device bfloat* queries [[buffer(0)]], const device bfloat* keys [[buffer(1)]], const device bfloat* values [[buffer(2)]], constant lfm2_attn_args& p [[buffer(3)]], device float* pm [[buffer(4)]], device float* pl [[buffer(5)]], device float* po [[buffer(6)]], @@ -349,15 +349,17 @@ template uint lane [[thread_index_in_simdgroup]]) { threadgroup uint ks[2][HD * (KT / 2)]; threadgroup bfloat vs[2][KB * KP]; - lfm2_attn_rows(queries, keys, values, p, pm, pl, po, rows, tg, tid, sg, lane, ks, vs); + lfm2_attn_rows(queries, keys, values, p, pm, pl, po, rows, tg, tid, sg, lane, ks, vs); } template [[host_name("lfm2_attn_mma")]] [[kernel]] decltype(lfm2_attn_mma<1>) lfm2_attn_mma<1>; template [[host_name("lfm2_attn_mma_r2")]] [[kernel]] decltype(lfm2_attn_mma<2>) lfm2_attn_mma<2>; +template [[host_name("lfm2_attn_block")]] [[kernel]] decltype(lfm2_attn_mma<1, true>) lfm2_attn_mma<1, true>; // The chunks merged in chunk order for (head, row): Y [rows, QH*HD] bf16. +template inline void lfm2_merge_row(const device float* pm, const device float* pl, const device float* po, lfm2_attn_args p, device bfloat* y, uint head, uint r, uint lane) { - const uint count = p.len + r + 1; + const uint count = FULL ? p.len + 1 : p.len + r + 1; const uint per = PER; const uint chunks = (count + per - 1) / per; const size_t at = (size_t(r) * QH + head) * (CHUNKS * p.subs); @@ -391,12 +393,15 @@ inline void lfm2_merge_row(const device float* pm, const device float* pl, const out[lane] = static_cast(o0 / l); out[lane + 32] = static_cast(o1 / l); } +template [[kernel]] void lfm2_attn_merge(const device float* pm [[buffer(0)]], const device float* pl [[buffer(1)]], const device float* po [[buffer(2)]], constant lfm2_attn_args& p [[buffer(3)]], device bfloat* y [[buffer(4)]], uint2 tg [[threadgroup_position_in_grid]], uint lane [[thread_index_in_simdgroup]]) { - lfm2_merge_row(pm, pl, po, p, y, tg.x, tg.y, lane); + lfm2_merge_row(pm, pl, po, p, y, tg.x, tg.y, lane); } +template [[host_name("lfm2_attn_merge")]] [[kernel]] decltype(lfm2_attn_merge) lfm2_attn_merge; +template [[host_name("lfm2_attn_block_merge")]] [[kernel]] decltype(lfm2_attn_merge) lfm2_attn_merge; // Router logits [rows, E] in fp32: a simdgroup an expert, lane l over inputs 4l..4l+3 of each 128, then simd_sum. [[kernel]] void lfm2_route_logits(const device bfloat* x [[buffer(0)]], const device float* router [[buffer(1)]], diff --git a/zig/kernels/metal/lfm2/lfm2_dspark.metal b/zig/kernels/metal/lfm2/lfm2_dspark.metal new file mode 100644 index 0000000000..995d63a20a --- /dev/null +++ b/zig/kernels/metal/lfm2/lfm2_dspark.metal @@ -0,0 +1,99 @@ +// DSpark's Markov step, compiled after lfm2.metal's text (its includes and constants). + +// DSpark's Markov scan: a threadgroup's best of 512 tokens of a row, bf16(base + bf16(W2 . W1[prev])), W2 8-bit codes. +struct lfm2_markov_args { uint stride, rank, vocab, prev_stride; }; +constant constexpr uint MARKOV_SPAN = 512; +[[kernel]] void lfm2_markov_scan(const device bfloat* logits [[buffer(0)]], const device bfloat* w1 [[buffer(1)]], + const device uint8_t* w2 [[buffer(2)]], const device uint* prev [[buffer(3)]], + constant lfm2_markov_args& p [[buffer(4)]], device float* best [[buffer(5)]], + device uint* at [[buffer(6)]], const device bfloat* w2s [[buffer(7)]], + const device bfloat* w2b [[buffer(8)]], uint2 tg [[threadgroup_position_in_grid]], + uint2 grid [[threadgroups_per_grid]], uint sg [[simdgroup_index_in_threadgroup]], + uint lane [[thread_index_in_simdgroup]]) { + threadgroup float e[256]; + threadgroup float sv[8]; + threadgroup uint si[8]; + const uint s = tg.y, t = sg * 32 + lane, id = prev[size_t(s) * p.prev_stride]; + for (uint i = t; i < p.rank; i += 256) e[i] = id < p.vocab ? float(w1[size_t(id) * p.rank + i]) : 0.0f; + threadgroup_barrier(mem_flags::mem_threadgroup); + const device bfloat* row = logits + size_t(s) * p.stride; + float mv = -INFINITY; + uint mi = 0xffffffffu; + bool nan = false; + const uint first = tg.x * MARKOV_SPAN, last = min(first + MARKOV_SPAN, p.vocab); + for (uint v = first + sg; v < last; v += 8) { + float d = 0.0f; + for (uint i = 8 * lane; i < p.rank; i += 256) { + const size_t g = (size_t(v) * p.rank + i) / 64; + const float sc = w2s[g], bi = w2b[g]; + const uchar4 lo = *(const device uchar4*)(w2 + size_t(v) * p.rank + i); + const uchar4 hi = *(const device uchar4*)(w2 + size_t(v) * p.rank + i + 4); + for (int j = 0; j < 4; ++j) d = fma(fma(sc, float(lo[j]), bi), e[i + j], d); + for (int j = 0; j < 4; ++j) d = fma(fma(sc, float(hi[j]), bi), e[i + 4 + j], d); + } + d = simd_sum(d); + const float sc = float(static_cast(float(row[v]) + float(static_cast(d)))); + nan = nan || isnan(sc); + if (sc > mv || (mi == 0xffffffffu && sc == mv)) { + mv = sc; + mi = v; + } + } + if (lane == 0) { + sv[sg] = nan ? NAN : mv; + si[sg] = mi; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + if (t != 0) return; + float bv = sv[0]; + uint bi = si[0]; + for (uint g = 1; g < 8; ++g) { + if (isnan(bv) || isnan(sv[g])) bv = NAN; + else if (sv[g] > bv || (sv[g] == bv && si[g] < bi)) { + bv = sv[g]; + bi = si[g]; + } + } + best[size_t(s) * grid.x + tg.x] = bv; + at[size_t(s) * grid.x + tg.x] = bi; +} + +// DSpark's Markov step pick: the scan's best over a stream's threadgroups (ties to the lowest token), as a Pick. +[[kernel]] void lfm2_markov_pick(const device float* best [[buffer(0)]], const device uint* at [[buffer(1)]], + constant uint& groups [[buffer(2)]], device uint* picks [[buffer(3)]], + uint s [[threadgroup_position_in_grid]], uint t [[thread_index_in_threadgroup]]) { + threadgroup float tv[256]; + threadgroup uint ti[256]; + threadgroup uint tn[256]; + float bv = -INFINITY; + uint bi = 0xffffffffu, nan = 0; + for (uint g = t; g < groups; g += 256) { + const float v = best[size_t(s) * groups + g]; + const uint i = at[size_t(s) * groups + g]; + if (isnan(v)) nan = 1; + else if (v > bv || (v == bv && i < bi)) { + bv = v; + bi = i; + } + } + tv[t] = bv; + ti[t] = bi; + tn[t] = nan; + threadgroup_barrier(mem_flags::mem_threadgroup); + for (uint w = 128; w > 0; w >>= 1) { + if (t < w) { + const float v = tv[t + w]; + const uint i = ti[t + w]; + if (v > tv[t] || (v == tv[t] && i < ti[t])) { + tv[t] = v; + ti[t] = i; + } + tn[t] |= tn[t + w]; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + } + if (t == 0) { + picks[2 * s] = ti[0]; + picks[2 * s + 1] = tn[0]; + } +} diff --git a/zig/kernels/metal/sources.zig b/zig/kernels/metal/sources.zig index 2506fb4905..9f4604f678 100644 --- a/zig/kernels/metal/sources.zig +++ b/zig/kernels/metal/sources.zig @@ -19,7 +19,7 @@ pub const gemma4_prompt_attention = @embedFile("gemma4/prompt_attention.metal"); pub const gemma4_vision = @embedFile("prefill/gemm_nax.metal") ++ @embedFile("gemma4/vision.metal"); /// LFM2-MoE: embedding, norms, the short conv, attention, the route and the experts' glue. -pub const lfm2 = @embedFile("lfm2/lfm2.metal") ++ @embedFile("lfm2/lfm2_segments.metal"); +pub const lfm2 = @embedFile("lfm2/lfm2.metal") ++ @embedFile("lfm2/lfm2_dspark.metal") ++ @embedFile("lfm2/lfm2_segments.metal"); /// LFM2's prompt attention on the tensor units (nax.h inlined at load, after lfm2.metal's text). pub const lfm2_attn_nax = @embedFile("lfm2/lfm2_attn_nax.metal"); diff --git a/zig/src/families/lfm2_moe/dspark.zig b/zig/src/families/lfm2_moe/dspark.zig new file mode 100644 index 0000000000..339071ac6c --- /dev/null +++ b/zig/src/families/lfm2_moe/dspark.zig @@ -0,0 +1,386 @@ +//! LiquidAI's DSpark drafter on LFM2's taps: a stream's context keys, a block of rows, a Markov chain of drafts. +const std = @import("std"); +const mtl = @import("metal"); +const lanes = @import("lanes"); +const c = @import("config.zig"); +const dc = @import("dspark_config.zig"); +const dw = @import("dspark_weights.zig"); +const block = @import("dspark_block.zig"); +const st = @import("state.zig"); +const snapshot = @import("snapshot.zig"); +const Model = @import("model.zig").Model; + +const dr = lanes.drafter; +const Stream = lanes.Stream; +const shared = mtl.ResourceOptions.shared | mtl.ResourceOptions.untracked; + +/// Block rows one hold pass takes (the decode twin's rows), and context rows one absorb pass takes. +pub const pass_rows = 32; +pub const absorb_rows = 2048; +/// Verified rows a lane keeps on the host until its next hold absorbs them (a round's rows for no GPU work). +pub const stage_rows = 128; +const kw = c.kv_heads * c.head_dim; + +/// `bits`: the width the bf16 linears are quantized to (8 keeps the fp32 reference's drafts; 4 does not). +pub const Options = struct { capacity: usize, bits: u8 = 8 }; + +/// One stream's context keys and values a layer ([kv head][cap][head dim], as the target's attention reads them). +pub const Lane = struct { + keys: [8]mtl.Buffer, + values: [8]mtl.Buffer, + cap: usize, // positions each holds: the context and one block past it + end: u64 = 0, // context rows absorbed: positions [0, end) + stage: mtl.Buffer, // tapped rows [end, end + staged), not absorbed yet (shared storage) + staged: u32 = 0, + held: [16]u32 = @splat(0), + held_len: u32 = 0, +}; + +/// The passes' working buffers, one pass at a time on the target's queue. +pub const Scratch = struct { + ids: mtl.Buffer, + h: mtl.Buffer, + x: mtl.Buffer, + r: mtl.Buffer, + y: mtl.Buffer, + q: mtl.Buffer, + k: mtl.Buffer, + v: mtl.Buffer, + queries: mtl.Buffer, + pm: mtl.Buffer, + pl: mtl.Buffer, + po: mtl.Buffer, + gate: mtl.Buffer, + up: mtl.Buffer, + act: mtl.Buffer, + logits: mtl.Buffer, + sums: mtl.Buffer, + fcx: mtl.Buffer, + hctx: mtl.Buffer, + best: mtl.Buffer, // the Markov scan's best score and token of each threadgroup, by stream + at: mtl.Buffer, + picks: mtl.Buffer, +}; + +pub const Drafter = struct { + gpa: std.mem.Allocator, + io: std.Io, + target: *Model, + config: dc.Config, + w: dw.Weights, + options: Options, + s: Scratch, + owned: std.ArrayList(mtl.Buffer) = .empty, + lanes: std.AutoHashMapUnmanaged(*Stream, *Lane) = .empty, + /// One stream's hold wall ms at full depth, and that over the block (the depth rule's cost a draft). + hold_ms: f64 = 0, + step_ms: f64 = 0, + + pub fn init(gpa: std.mem.Allocator, io: std.Io, target: *Model, dir: []const u8, options: Options) !*Drafter { + const d = try gpa.create(Drafter); + errdefer gpa.destroy(d); + d.* = .{ .gpa = gpa, .io = io, .target = target, .config = try dc.Config.read(gpa, io, dir), .w = undefined, .options = options, .s = undefined }; + d.w = try dw.load(gpa, target.device, io, dir, &d.config, options.bits); + errdefer d.w.deinit(gpa); + errdefer for (d.owned.items) |b| b.deinit(); + errdefer d.owned.deinit(gpa); + try d.scratch(); + try d.warm(); + return d; + } + + pub fn deinit(d: *Drafter) void { + var it = d.lanes.valueIterator(); + while (it.next()) |l| d.dropLane(l.*); + d.lanes.deinit(d.gpa); + for (d.owned.items) |b| b.deinit(); + d.owned.deinit(d.gpa); + d.w.deinit(d.gpa); + d.gpa.destroy(d); + } + + fn buffer(d: *Drafter, bytes: usize) !mtl.Buffer { + const b = try d.target.device.buffer(@max(bytes, 16), shared); + errdefer b.deinit(); + try d.owned.append(d.gpa, b); + return b; + } + + fn scratch(d: *Drafter) !void { + const r = pass_rows; + const wide = @max(pass_rows, absorb_rows); + const i = d.config.intermediate; + const partials = @max(st.split_rows, d.config.block) * st.subs * c.query_heads * st.chunks; + const sums = @max(absorb_rows * (d.config.tap_count * c.hidden / 64), r * (i / 64)); + d.s = .{ + .ids = try d.buffer(r * 4), + .h = try d.buffer(r * c.hidden * 2), + .x = try d.buffer(wide * c.hidden * 2), + .r = try d.buffer(r * c.hidden * 2), + .y = try d.buffer(r * c.hidden * 2), + .q = try d.buffer(r * c.hidden * 2), + .k = try d.buffer(wide * kw * 2), + .v = try d.buffer(wide * kw * 2), + .queries = try d.buffer((r + st.attn_rows) * c.hidden * 2), + .pm = try d.buffer(partials * 4), + .pl = try d.buffer(partials * 4), + .po = try d.buffer(partials * c.head_dim * 4), + .gate = try d.buffer(r * i * 2), + .up = try d.buffer(r * i * 2), + .act = try d.buffer(r * i * 2), + .logits = try d.buffer(r * c.vocab * 2), + .sums = try d.buffer(sums * 4), + .fcx = try d.buffer(absorb_rows * c.hidden * 2), + .hctx = try d.buffer(absorb_rows * c.hidden * 2), + .best = try d.buffer(r * block.scan_groups * 4), + .at = try d.buffer(r * block.scan_groups * 4), + .picks = try d.buffer(d.config.block * r * 8), + }; + } + + /// The drafter's buffers that every round reads (weights and scratch), for the server's residency set. + pub fn buffers(d: *const Drafter, gpa: std.mem.Allocator) ![]mtl.Buffer { + return std.mem.concat(gpa, mtl.Buffer, &.{ d.w.buffers(), d.owned.items }); + } + + pub fn drafter(d: *Drafter) dr.Drafter { + return .{ .ptr = d, .vtable = &.{ .taps = tapsFn, .facts = factsFn, .open = openFn, .absorb = absorbFn, .hold = holdFn, .held = heldFn, .release = releaseFn } }; + } + + fn cast(ptr: *anyopaque) *Drafter { + return @ptrCast(@alignCast(ptr)); + } + + fn tapsFn(ptr: *anyopaque) []const u32 { + const d = cast(ptr); + return d.config.taps[0..d.config.tap_count]; + } + + /// Acceptance a depth until a stream has its own: the card's 5.16 tokens a step at block 9 is 0.84 a depth. + const prior: [16]f64 = @splat(0.84); + + fn factsFn(ptr: *anyopaque) dr.Facts { + const d = cast(ptr); + return .{ .depth = d.config.block, .step_ms = d.step_ms, .prior = prior[0..d.config.block], .batched = true, .shared_guard = true, .stream_bytes = d.laneBytes(d.options.capacity) }; + } + + pub fn laneBytes(d: *const Drafter, capacity: usize) usize { + return 2 * d.config.layers * (capacity + d.config.block) * kw * 2 + stage_rows * d.tapBytes(); + } + + /// A tapped row's bytes: every tap's hidden row, bf16. + pub fn tapBytes(d: *const Drafter) usize { + return d.config.tap_count * c.hidden * 2; + } + + /// The lane's staged rows as an absorb piece (null: none staged). + pub fn stagedPiece(l: *Lane) ?block.Piece { + if (l.staged == 0) return null; + return .{ .lane = l, .src = .{ .buf = l.stage }, .rows = l.staged, .p0 = l.end }; + } + + /// The lane's staged rows absorbed now, in a command buffer of their own. + fn flush(d: *Drafter, l: *Lane) !void { + const pc = stagedPiece(l) orelse return; + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const run = block.begin(d); + try block.absorbPass(d, run.e, &.{pc}); + try block.end(run.cb, run.e); + l.end += l.staged; + l.staged = 0; + } + + fn openFn(ptr: *anyopaque, s: *Stream) anyerror!void { + const d = cast(ptr); + if (d.lanes.contains(s)) return error.DraftStreamOpen; + const want = s.prompt_len + s.max_new + 2 * d.config.block; + const l = try d.newLane(@min(d.options.capacity, (want + 63) / 64 * 64)); + errdefer d.dropLane(l); + try d.lanes.put(d.gpa, s, l); + } + + pub fn newLane(d: *Drafter, capacity: usize) !*Lane { + const l = try d.gpa.create(Lane); + errdefer d.gpa.destroy(l); + l.* = .{ .keys = undefined, .values = undefined, .stage = try d.target.device.buffer(stage_rows * d.tapBytes(), shared), .cap = capacity + d.config.block }; + errdefer l.stage.deinit(); + var made: usize = 0; + errdefer for (0..made) |i| { + l.keys[i].deinit(); + l.values[i].deinit(); + }; + for (0..d.config.layers) |i| { + l.keys[i] = try d.target.device.buffer(l.cap * kw * 2, shared); + l.values[i] = d.target.device.buffer(l.cap * kw * 2, shared) catch |e| { + l.keys[i].deinit(); + return e; + }; + made += 1; + } + return l; + } + + pub fn dropLane(d: *Drafter, l: *Lane) void { + l.stage.deinit(); + for (0..d.config.layers) |i| { + l.keys[i].deinit(); + l.values[i].deinit(); + } + d.gpa.destroy(l); + } + + fn releaseFn(ptr: *anyopaque, s: *Stream) void { + const d = cast(ptr); + if (d.lanes.fetchRemove(s)) |e| d.dropLane(e.value); + } + + fn heldFn(ptr: *anyopaque, streams: []const *Stream, out: []const []u32) anyerror!void { + const d = cast(ptr); + for (streams, out) |s, o| { + const l = d.lanes.get(s) orelse return error.DraftStreamUnknown; + if (o.len > l.held_len) return error.DraftHeldFewer; + @memcpy(o, l.held[0..o.len]); + } + } + + /// Bytes a lane's context at `at` takes beside a kept prompt state. + pub fn keptBytes(d: *const Drafter, at: usize) u64 { + return 2 * @as(u64, d.config.layers) * at * kw * 2; + } + + /// Stream `s`'s context copied into `snap` when its lane stands at the snapshot's position (a prompt chunk's end). + pub fn keepLane(d: *Drafter, s: *Stream, snap: *snapshot.Snap) !void { + const l = d.lanes.get(s) orelse return; + try d.flush(l); + if (l.end != snap.at) return; + const bytes = try d.gpa.alloc(u8, @intCast(d.keptBytes(snap.at))); + copyLane(d, l, snap.at, bytes, true); + snap.draft = bytes; + } + + /// The lane takes the context kept with the state its stream's prompt pass resumed at `start`. + fn resumeLane(d: *Drafter, s: *Stream, l: *Lane, start: u64) !void { + const saved = s.reuse.saved orelse return error.DraftContextGap; + const snap: *const snapshot.Snap = @ptrCast(@alignCast(saved)); + const bytes = snap.draft orelse return error.DraftContextGap; + if (s.cached != start or snap.at != start or l.end != 0 or start + d.config.block > l.cap) return error.DraftContextGap; + copyLane(d, l, snap.at, bytes, false); + l.end = start; + } + + /// Rows [0, at) of every layer's keys and values and `bytes` (`out`: the lane's into bytes), kv head by kv head. + fn copyLane(d: *const Drafter, l: *const Lane, at: usize, bytes: []u8, out: bool) void { + const n = at * c.head_dim * 2; + var off: usize = 0; + for (0..d.config.layers) |i| for ([_]mtl.Buffer{ l.keys[i], l.values[i] }) |buf| for (0..c.kv_heads) |h| { + const live = (buf.contents() + h * l.cap * c.head_dim * 2)[0..n]; + if (out) @memcpy(bytes[off..][0..n], live) else @memcpy(live, bytes[off..][0..n]); + off += n; + }; + } + + /// Rows from `start` replace the context there: verified rows wait on the host for the next hold, others run now. + fn absorbFn(ptr: *anyopaque, items: []const dr.Absorb) anyerror!void { + const d = cast(ptr); + const width = d.tapBytes(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + var run: ?@TypeOf(block.begin(d)) = null; + var pieces: [32]block.Piece = undefined; + var count: usize = 0; + var rows: usize = 0; + for (items) |it| { + const l = d.lanes.get(it.stream) orelse return error.DraftStreamUnknown; + const f = it.features; + if (f.space != .device or f.dtype != .bf16 or f.row_bytes != width or it.rows != null) return error.DraftFeaturesUnsupported; + if (it.start > l.end + l.staged) try d.resumeLane(it.stream, l, it.start); // a prompt pass that resumed + if (it.start + f.rows + d.config.block > l.cap) return error.DraftContextGap; + l.held_len = 0; + const src = mtl.Buffer{ .id = @ptrFromInt(f.buffer) }; + // rows from `start` replace what was staged past it; rows before the absorbed end go to the GPU now + const keep: u32 = if (it.start >= l.end) @intCast(it.start - l.end) else 0; + l.staged = @min(l.staged, keep); + if (!it.prompt and it.start >= l.end and l.staged + f.rows <= stage_rows) { + const from = (src.contents() + f.offset)[0 .. f.rows * width]; + @memcpy((l.stage.contents() + l.staged * width)[0 .. f.rows * width], from); + l.staged += f.rows; + continue; + } + if (run == null) run = block.begin(d); + const e = run.?.e; + if (stagedPiece(l)) |pc| { + if (count == pieces.len or rows + pc.rows > absorb_rows) { + try block.absorbPass(d, e, pieces[0..count]); + count = 0; + rows = 0; + } + pieces[count] = pc; + count += 1; + rows += pc.rows; + } + var at: usize = 0; + while (at < f.rows) { + if (rows == absorb_rows or count == pieces.len) { // the pass is full: encode it + try block.absorbPass(d, e, pieces[0..count]); + count = 0; + rows = 0; + } + const n = @min(absorb_rows - rows, f.rows - at); + pieces[count] = .{ .lane = l, .src = .{ .buf = src, .off = f.offset + at * width }, .rows = n, .p0 = it.start + at }; + count += 1; + rows += n; + at += n; + } + l.end = it.start + f.rows; + l.staged = 0; + } + const r = run orelse return; + if (count > 0) try block.absorbPass(d, r.e, pieces[0..count]); + try block.end(r.cb, r.e); + } + + /// Each stream's chain of `depth` drafts after `pending` at `position`, its block's rows in a pass of pass_rows. + fn holdFn(ptr: *anyopaque, items: []const dr.Hold) anyerror!void { + const d = cast(ptr); + var at: usize = 0; + while (at < items.len) { + // as many streams as their blocks' rows fit a pass: each block is as long as the pass's deepest chain + var deepest: usize = 0; + var end = at; + while (end < items.len) : (end += 1) { + const rows = @max(deepest, items[end].depth); + if ((end - at + 1) * rows > pass_rows) break; + deepest = rows; + } + if (end == at) return error.DraftDepth; + var blocks: [pass_rows]block.Block = undefined; + const group = items[at..end]; + at = end; + for (group, blocks[0..group.len]) |it, *b| { + const l = d.lanes.get(it.stream) orelse return error.DraftStreamUnknown; + if (it.depth == 0 or it.depth > d.config.block or it.position == 0) return error.DraftDepth; + const anchor = it.position - 1; + if (l.end + l.staged != anchor or anchor + d.config.block > l.cap) return error.DraftContextGap; + b.* = .{ .lane = l, .pending = it.pending, .anchor = anchor, .depth = it.depth }; + } + try block.hold(d, blocks[0..group.len]); + } + } + + /// Holds on an empty context: the pipelines compiled, then one-draft and full-depth holds timed (best of three). + fn warm(d: *Drafter) !void { + const l = try d.newLane(64); + defer d.dropLane(l); + var best: [2]f64 = @splat(std.math.floatMax(f64)); + for (0..4) |i| for ([_]u32{ 1, d.config.block }, &best) |depth, *b| { + const t0 = std.Io.Clock.awake.now(d.io); + try block.hold(d, &.{.{ .lane = l, .pending = 1, .anchor = 0, .depth = depth }}); + const ms = @as(f64, @floatFromInt(t0.durationTo(std.Io.Clock.awake.now(d.io)).toNanoseconds())) / 1e6; + if (i > 0) b.* = @min(b.*, ms); + }; + d.hold_ms = best[1]; + d.step_ms = best[1] / @as(f64, @floatFromInt(d.config.block)); + std.log.info("lfm2 dspark: a 1-draft hold takes {d:.2} ms, a {d}-draft hold {d:.2} ms", .{ best[0], d.config.block, best[1] }); + } +}; diff --git a/zig/src/families/lfm2_moe/dspark_block.zig b/zig/src/families/lfm2_moe/dspark_block.zig new file mode 100644 index 0000000000..22676dca6e --- /dev/null +++ b/zig/src/families/lfm2_moe/dspark_block.zig @@ -0,0 +1,271 @@ +//! DSpark's passes on the target's queue: context rows into a lane's keys, and a block's rows into Markov drafts. +const std = @import("std"); +const mtl = @import("metal"); +const c = @import("config.zig"); +const st = @import("state.zig"); +const fwd = @import("forward.zig"); +const tile = @import("../../core/tile_qmm.zig"); +const am = @import("../../core/affine_mm.zig"); +const round = @import("tree_round_gpu"); +const ds = @import("dspark.zig"); +const Drafter = ds.Drafter; +const Lane = ds.Lane; + +const kw = c.kv_heads * c.head_dim; +const QkArgs = extern struct { len: u32, cap: u32, eps: f32, log2_theta: f32 }; +const AttnArgs = extern struct { len: u32, cap: u32, scale: f32, subs: u32 }; +const Pick = round.contract.Pick; +const MarkovArgs = extern struct { stride: u32, rank: u32, vocab: u32, prev_stride: u32 }; +/// Tokens a Markov scan threadgroup takes (the kernel's MARKOV_SPAN), and threadgroups a stream's row takes. +pub const scan_span = 512; +pub const scan_groups = (c.vocab + scan_span - 1) / scan_span; + +/// A buffer and byte offset. +pub const Ref = struct { buf: mtl.Buffer, off: usize = 0 }; + +/// One stream's block in a hold pass: rows [pending, mask x (block - 1)] at positions anchor .. anchor + block - 1. +pub const Block = struct { lane: *Lane, pending: u32, anchor: u64, depth: u32 }; + +const Enc = struct { + d: *Drafter, + e: mtl.ComputeEncoder, + + fn pipe(x: Enc, comptime key: []const u8) void { + x.e.setPipeline(x.d.target.kernels.get(key)); + } + + /// y[rows] = x[rows] W^T: the twin up to its rows, past them the M5's tensor-unit kernels, else the tiles. + fn project(x: Enc, w: tile.Matrix, in: Ref, out: Ref, rows: usize) !void { + const t = &x.d.target.kernels.tiles; + if (rows > tile.twin_rows) if (x.d.target.kernels.naxPipes(w.bits)) |p| { + const sums = Ref{ .buf = x.d.s.sums }; + am.rowSums(x.e, p, c.group, in, sums, @intCast(rows), @intCast(w.k)); + const q = .{ .w = Ref{ .buf = w.w, .off = w.w_off }, .s = Ref{ .buf = w.scales, .off = w.s_off }, .b = Ref{ .buf = w.biases, .off = w.b_off }, .n = @as(u32, @intCast(w.n)), .k = @as(u32, @intCast(w.k)) }; + return am.dense(x.e, p, in, sums, q, out, @intCast(rows)); + }; + if (rows <= tile.twin_rows and x.d.target.kernels.twin) return tile.encodeTwin(x.e, t, w, in.buf, in.off, out.buf, out.off, rows); + tile.encodeSums(x.e, t, w.k, in.buf, in.off, x.d.s.sums, rows); + try tile.encode(x.e, t, w, in.buf, in.off, x.d.s.sums, 0, out.buf, out.off, rows); + } + + /// h += r when `add`, then out = norm(h) * gain: LFM2's own norm (the drafter's width and eps are the target's). + fn norm(x: Enc, h: mtl.Buffer, r: mtl.Buffer, gain: mtl.Buffer, add: bool, out: mtl.Buffer, rows: usize) void { + x.pipe("lfm2_norm"); + x.e.setBuffer(h, 0, 0); + x.e.setBuffer(r, 0, 1); + x.e.setBuffer(gain, 0, 2); + x.e.setValue(c.eps, 3); + x.e.setValue(@as(u32, @intFromBool(add)), 4); + x.e.setBuffer(out, 0, 5); + x.e.dispatchGroups(mtl.Size.of(rows, 1, 1), mtl.Size.of(c.hidden / 4, 1, 1)); + } + + /// Rows' queries normed and rotated into `queries`, keys and values into the lane's layer at positions from `len`. + fn qk(x: Enc, q: Ref, k: Ref, v: Ref, layer: usize, lane: *const Lane, len: u64, queries: Ref, rows: usize) void { + const ly = &x.d.w.layers[layer]; + x.pipe("lfm2_qk"); + x.e.setBuffer(q.buf, q.off, 0); + x.e.setBuffer(k.buf, k.off, 1); + x.e.setBuffer(v.buf, v.off, 2); + x.e.setBuffer(ly.q_norm, 0, 3); + x.e.setBuffer(ly.k_norm, 0, 4); + const args = QkArgs{ .len = @intCast(len), .cap = @intCast(lane.cap), .eps = c.eps, .log2_theta = std.math.log2(x.d.config.theta) }; + x.e.setBytes(std.mem.asBytes(&args), 5); + x.e.setBuffer(queries.buf, queries.off, 6); + x.e.setBuffer(lane.keys[layer], 0, 7); + x.e.setBuffer(lane.values[layer], 0, 8); + x.e.dispatchGroups(mtl.Size.of(c.query_heads + c.kv_heads, rows, 1), mtl.Size.of(32, 1, 1)); + } + + /// Query rows [row, row + rows) against the lane's keys [0, keys), all of them visible to every row. + fn attend(x: Enc, layer: usize, lane: *const Lane, row: usize, rows: usize, keys: u64) void { + const s = &x.d.s; + const args = AttnArgs{ .len = @intCast(keys - 1), .cap = @intCast(lane.cap), .scale = 1.0 / @sqrt(@as(f32, c.head_dim)), .subs = st.subs }; + const used = (keys + st.chunk_keys - 1) / st.chunk_keys * st.subs; + x.pipe("lfm2_attn_block"); + x.e.setBuffer(s.queries, row * c.hidden * 2, 0); + x.e.setBuffer(lane.keys[layer], 0, 1); + x.e.setBuffer(lane.values[layer], 0, 2); + x.e.setBytes(std.mem.asBytes(&args), 3); + x.e.setBuffer(s.pm, 0, 4); + x.e.setBuffer(s.pl, 0, 5); + x.e.setBuffer(s.po, 0, 6); + x.e.setValue(@as(u32, @intCast(rows)), 7); + x.e.dispatchGroups(mtl.Size.of((rows + 7) / 8, c.kv_heads, @intCast(used)), mtl.Size.of(128, 1, 1)); + x.pipe("lfm2_attn_block_merge"); + x.e.setBuffer(s.pm, 0, 0); + x.e.setBuffer(s.pl, 0, 1); + x.e.setBuffer(s.po, 0, 2); + x.e.setBytes(std.mem.asBytes(&args), 3); + x.e.setBuffer(s.y, row * c.hidden * 2, 4); + x.e.dispatchGroups(mtl.Size.of(c.query_heads, rows, 1), mtl.Size.of(32, 1, 1)); + } +}; + +fn failed(msg: [*:0]const u8) anyerror { + std.log.err("lfm2 dspark: command buffer failed: {s}", .{msg}); + return error.GpuFailed; +} + +fn finish(cb: mtl.CommandBuffer, e: mtl.ComputeEncoder) !void { + e.end(); + cb.commit(); + cb.wait(); + if (cb.failure()) |msg| return failed(msg); +} + +/// One stream's tapped rows in an absorb pass: `rows` rows at `src`, the context from position `p0`. +pub const Piece = struct { lane: *Lane, src: Ref, rows: usize, p0: u64 }; + +/// Each piece's rows into its lane's keys and values (fc, hidden_norm, k, v, k_norm, RoPE) in passes on `e`. +pub fn absorbPass(d: *Drafter, e: mtl.ComputeEncoder, pieces: []const Piece) !void { + const s = &d.s; + const x = Enc{ .d = d, .e = e }; + var rows: usize = 0; + for (pieces) |pc| { + try x.project(d.w.fc, pc.src, .{ .buf = s.fcx, .off = rows * c.hidden * 2 }, pc.rows); + rows += pc.rows; + } + if (rows == 0 or rows > ds.absorb_rows) return error.DraftContextGap; + x.norm(s.fcx, s.fcx, d.w.hidden_norm, false, s.hctx, rows); + for (0..d.config.layers) |i| { + const ly = &d.w.layers[i]; + try x.project(ly.k, .{ .buf = s.hctx }, .{ .buf = s.k }, rows); + try x.project(ly.v, .{ .buf = s.hctx }, .{ .buf = s.v }, rows); + var at: usize = 0; + for (pieces) |pc| { + // the kernel rotates query heads too: they read the context rows and land in x, which no one reads + const q = Ref{ .buf = s.hctx, .off = at * c.hidden * 2 }; + x.qk(q, .{ .buf = s.k, .off = at * kw * 2 }, .{ .buf = s.v, .off = at * kw * 2 }, i, pc.lane, pc.p0, .{ .buf = s.x }, pc.rows); + at += pc.rows; + } + } +} + +/// A command buffer for absorb passes, and its wait. +pub fn begin(d: *Drafter) struct { cb: mtl.CommandBuffer, e: mtl.ComputeEncoder } { + const cb = d.target.queue.commandBuffer(); + return .{ .cb = cb, .e = cb.compute(.serial) }; +} + +pub fn end(cb: mtl.CommandBuffer, e: mtl.ComputeEncoder) !void { + return finish(cb, e); +} + +/// The blocks' drafts into their lanes' held chains: the backbone over every block's rows, then the Markov chain. +pub fn hold(d: *Drafter, blocks: []const Block) !void { + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const s = &d.s; + var depth: u32 = 0; + for (blocks) |b| depth = @max(depth, b.depth); + // a block of the deepest chain's rows, at most the 9 the drafter was trained on + const n: u32 = depth; + const rows = blocks.len * n; + if (rows == 0 or rows > ds.pass_rows) return error.DraftTooManyStreams; + const ids = s.ids.slice(u32, ds.pass_rows); + for (0..blocks.len) |i| { + ids[i * n] = blocks[i].pending; + @memset(ids[i * n + 1 ..][0 .. n - 1], d.config.mask); + } + const steps = depth; + // past the last layer's attention a lone block carries only the rows its chain reads; several keep them all + const tail = if (blocks.len == 1) steps else rows; + const cb = d.target.queue.commandBuffer(); + const x = Enc{ .d = d, .e = cb.compute(.serial) }; + const e = x.e; + // the blocks' staged context rows first, in the same command buffer + var staged: [ds.pass_rows]Piece = undefined; + var count: usize = 0; + var held: usize = 0; + for (blocks) |b| if (ds.Drafter.stagedPiece(b.lane)) |pc| { + if (held + pc.rows > ds.absorb_rows) { + try absorbPass(d, e, staged[0..count]); + count = 0; + held = 0; + } + staged[count] = pc; + count += 1; + held += pc.rows; + }; + if (count > 0) try absorbPass(d, e, staged[0..count]); + for (blocks) |b| { + b.lane.end += b.lane.staged; + b.lane.staged = 0; + } + const t = d.target; + const emb = t.weights.embedding; + e.setPipeline(t.kernels.embed(emb.bits)); + e.setBuffer(s.ids, 0, 0); + e.setBuffer(emb.w, emb.w_off, 1); + e.setBuffer(emb.scales, emb.s_off, 2); + e.setBuffer(emb.biases, emb.b_off, 3); + e.setBuffer(s.h, 0, 4); + const units = c.hidden / @as(usize, if (emb.bits == 6) 4 else 8 / emb.bits); + e.dispatchThreads(mtl.Size.of(units, rows, 1), mtl.Size.of(256, 1, 1)); + for (0..d.config.layers) |i| { + const ly = &d.w.layers[i]; + const out = if (i + 1 == d.config.layers) tail else rows; + x.norm(s.h, s.r, ly.input_norm, i > 0, s.x, rows); + try x.project(ly.q, .{ .buf = s.x }, .{ .buf = s.q }, rows); + try x.project(ly.k, .{ .buf = s.x }, .{ .buf = s.k }, rows); + try x.project(ly.v, .{ .buf = s.x }, .{ .buf = s.v }, rows); + for (blocks, 0..) |b, bi| { + const at = bi * n; + x.qk(.{ .buf = s.q, .off = at * c.hidden * 2 }, .{ .buf = s.k, .off = at * kw * 2 }, .{ .buf = s.v, .off = at * kw * 2 }, i, b.lane, b.anchor, .{ .buf = s.queries, .off = at * c.hidden * 2 }, n); + x.attend(i, b.lane, at, if (blocks.len == 1) out else n, b.anchor + n); + } + try x.project(ly.o, .{ .buf = s.y }, .{ .buf = s.r }, out); + x.norm(s.h, s.r, ly.post_norm, true, s.x, out); + try x.project(ly.gate, .{ .buf = s.x }, .{ .buf = s.gate }, out); + try x.project(ly.up, .{ .buf = s.x }, .{ .buf = s.up }, out); + x.pipe("lfm2_swiglu"); + e.setBuffer(s.gate, 0, 0); + e.setBuffer(s.up, 0, 1); + e.setBuffer(s.act, 0, 2); + e.setValue(@as(u32, @intCast(out * d.config.intermediate)), 3); + e.dispatchThreads(mtl.Size.of(out * d.config.intermediate / 8, 1, 1), mtl.Size.of(256, 1, 1)); + try x.project(ly.down, .{ .buf = s.act }, .{ .buf = s.r }, out); + } + x.norm(s.h, s.r, d.w.norm, true, s.x, tail); + try x.project(fwd.matrix(emb), .{ .buf = s.x }, .{ .buf = s.logits }, tail); + try markov(x, blocks.len, n, steps); + try finish(cb, e); + const picks = s.picks.slice(Pick, d.config.block * ds.pass_rows); + for (blocks, 0..) |b, bi| { + for (b.lane.held[0..steps], 0..) |*h, i| { + const p = picks[i * ds.pass_rows + bi]; + if (p.nonfinite != 0 or p.token >= c.vocab) return error.DraftNonfinite; + h.* = p.token; + } + b.lane.held_len = b.depth; + } +} + +/// Step i of each stream: its row i's logits plus W2 W1[previous draft] (the pending token first), then the argmax. +fn markov(x: Enc, streams: usize, n: u32, depth: u32) !void { + const d = x.d; + const s = &d.s; + const e = x.e; + for (0..depth) |i| { + // step i's previous token: the pending one (each block's first id), then step i - 1's pick + x.pipe("lfm2_markov_scan"); + e.setBuffer(s.logits, i * c.vocab * 2, 0); + e.setBuffer(d.w.w1, 0, 1); + e.setBuffer(d.w.w2.w, d.w.w2.w_off, 2); + if (i == 0) e.setBuffer(s.ids, 0, 3) else e.setBuffer(s.picks, (i - 1) * ds.pass_rows * @sizeOf(Pick), 3); + const args = MarkovArgs{ .stride = n * c.vocab, .rank = d.config.rank, .vocab = c.vocab, .prev_stride = if (i == 0) n else @sizeOf(Pick) / 4 }; + e.setBytes(std.mem.asBytes(&args), 4); + e.setBuffer(s.best, 0, 5); + e.setBuffer(s.at, 0, 6); + e.setBuffer(d.w.w2.scales, d.w.w2.s_off, 7); + e.setBuffer(d.w.w2.biases, d.w.w2.b_off, 8); + e.dispatchGroups(mtl.Size.of(scan_groups, streams, 1), mtl.Size.of(256, 1, 1)); + x.pipe("lfm2_markov_pick"); + e.setBuffer(s.best, 0, 0); + e.setBuffer(s.at, 0, 1); + e.setValue(@as(u32, scan_groups), 2); + e.setBuffer(s.picks, i * ds.pass_rows * @sizeOf(Pick), 3); + e.dispatchGroups(mtl.Size.of(streams, 1, 1), mtl.Size.of(256, 1, 1)); + } +} diff --git a/zig/src/families/lfm2_moe/dspark_config.zig b/zig/src/families/lfm2_moe/dspark_config.zig new file mode 100644 index 0000000000..7b3b628a02 --- /dev/null +++ b/zig/src/families/lfm2_moe/dspark_config.zig @@ -0,0 +1,136 @@ +//! LiquidAI's DSpark drafter for LFM2.5-24B-A2B: its config admitted, any other shape refused with its reason. +const std = @import("std"); +const c = @import("config.zig"); + +/// The geometry the drafter's kernels take: the target's attention shape, five taps at most eight layers. +pub const Config = struct { + taps: [8]u32 = @splat(0), + tap_count: u32, + layers: u32, + intermediate: u32, + /// Rows a block takes: the pending token and block - 1 mask tokens, one draft a row. + block: u32, + mask: u32, + rank: u32, + theta: f32, + + pub fn read(gpa: std.mem.Allocator, io: std.Io, dir: []const u8) !Config { + const path = try std.fs.path.join(gpa, &.{ dir, "config.json" }); + defer gpa.free(path); + const bytes = try std.Io.Dir.cwd().readFileAlloc(io, path, gpa, .limited(1 << 20)); + defer gpa.free(bytes); + return parse(gpa, bytes); + } + + pub fn parse(gpa: std.mem.Allocator, bytes: []const u8) !Config { + var parsed = try std.json.parseFromSlice(std.json.Value, gpa, bytes, .{}); + defer parsed.deinit(); + const o = if (parsed.value == .object) parsed.value.object else return error.BadDraftConfig; + const archs = o.get("architectures") orelse return error.BadDraftConfig; + if (archs != .array or archs.array.items.len != 1 or archs.array.items[0] != .string) return error.BadDraftConfig; + if (!std.mem.eql(u8, archs.array.items[0].string, "Lfm2DSparkDraftModel")) return error.NotLfmDSpark; + const d = if (o.get("dflash_config")) |v| (if (v == .object) v.object else return error.BadDraftConfig) else return error.BadDraftConfig; + var out: Config = .{ + .tap_count = 0, + .layers = try small(o, "num_hidden_layers"), + .intermediate = try small(o, "intermediate_size"), + .block = try small(o, "block_size"), + .mask = try small(d, "mask_token_id"), + .rank = try small(o, "markov_rank"), + .theta = @floatCast(try number(o, "rope_theta")), + }; + // the target's widths: the drafter shares its embedding, head and attention kernels + if (try small(o, "hidden_size") != c.hidden or try small(o, "vocab_size") != c.vocab) return error.DraftForAnotherTarget; + if (try small(d, "num_target_layers") != c.layers) return error.DraftForAnotherTarget; + const heads = try small(o, "num_attention_heads"); + const kv = try small(o, "num_key_value_heads"); + if (heads != c.query_heads or kv != c.kv_heads or try small(o, "head_dim") != c.head_dim) return error.UnsupportedDraftShape; + if (@as(f32, @floatCast(try number(o, "rms_norm_eps"))) != c.eps) return error.UnsupportedDraftShape; + const taps = d.get("target_layer_ids") orelse return error.BadDraftConfig; + if (taps != .array or taps.array.items.len == 0 or taps.array.items.len > out.taps.len) return error.BadDraftConfig; + for (taps.array.items, 0..) |t, i| { + if (t != .integer or t.integer < 0) return error.BadDraftConfig; + if (t.integer >= c.layers) return error.DraftForAnotherTarget; + out.taps[i] = @intCast(t.integer); + } + out.tap_count = @intCast(taps.array.items.len); + try kinds(o, out.layers); + // the vendored defaults decide the block: non-causal attention, interleaved RoPE, a plain Markov head + if (o.get("is_causal") != null) return error.UnsupportedDraftMask; + const neox = o.get("rope_is_neox_style") orelse return error.BadDraftConfig; + if (neox != .bool or neox.bool) return error.UnsupportedDraftRope; + const head = o.get("markov_head_type") orelse return error.BadDraftConfig; + if (head != .string or !std.mem.eql(u8, head.string, "vanilla")) return error.UnsupportedMarkovHead; + if (o.get("sample_from_anchor")) |v| if (v != .bool or !v.bool) return error.UnsupportedDraftAnchor; + if (out.layers == 0 or out.layers > 8 or out.block < 2 or out.block > 16) return error.UnsupportedDraftShape; + if (out.intermediate == 0 or out.intermediate % 64 != 0 or out.rank == 0 or out.rank % 64 != 0) return error.UnsupportedDraftShape; + if (out.mask >= c.vocab or !(out.theta > 0)) return error.BadDraftConfig; + return out; + } +}; + +/// Every layer full attention (the block's mask has no sliding window). +fn kinds(o: std.json.ObjectMap, layers: u32) !void { + const v = o.get("layer_types") orelse return; + if (v != .array or v.array.items.len != layers) return error.BadDraftConfig; + for (v.array.items) |k| if (k != .string or !std.mem.eql(u8, k.string, "full_attention")) return error.UnsupportedDraftMask; +} + +fn small(o: std.json.ObjectMap, key: []const u8) !u32 { + const v = o.get(key) orelse return error.BadDraftConfig; + if (v != .integer or v.integer < 0 or v.integer > std.math.maxInt(u32)) return error.BadDraftConfig; + return @intCast(v.integer); +} + +fn number(o: std.json.ObjectMap, key: []const u8) !f64 { + const v = o.get(key) orelse return error.BadDraftConfig; + return switch (v) { + .integer => |i| @floatFromInt(i), + .float => |f| f, + else => error.BadDraftConfig, + }; +} + +/// LiquidAI/LFM2.5-24B-A2B-DSpark's config.json at e052e5b1dd10, as released. +pub const stock = + \\{"architectures":["Lfm2DSparkDraftModel"],"model_type":"qwen3","dtype":"bfloat16","hidden_size":2048, + \\"num_hidden_layers":5,"num_attention_heads":32,"num_key_value_heads":8,"head_dim":64,"intermediate_size":6144, + \\"hidden_act":"silu","rms_norm_eps":1e-05,"vocab_size":128000,"rope_theta":10000000.0, + \\"max_position_embeddings":131072,"layer_types":["full_attention","full_attention","full_attention", + \\"full_attention","full_attention"],"block_size":9,"dflash_config":{"mask_token_id":125017, + \\"target_layer_ids":[2,10,18,26,34],"num_target_layers":40},"markov_rank":256,"rope_is_neox_style":false, + \\"enable_confidence_head":true,"markov_head_type":"vanilla"} +; + +fn edited(from: []const u8, to: []const u8) !Config { + const a = std.testing.allocator; + const text = try std.mem.replaceOwned(u8, a, stock, from, to); + defer a.free(text); + return Config.parse(a, text); +} + +test "lfm2 dspark: the released config is admitted with its taps, block, mask token and Markov rank" { + const cfg = try Config.parse(std.testing.allocator, stock); + try std.testing.expectEqualSlices(u32, &.{ 2, 10, 18, 26, 34 }, cfg.taps[0..cfg.tap_count]); + try std.testing.expectEqual(@as(u32, 5), cfg.layers); + try std.testing.expectEqual(@as(u32, 9), cfg.block); + try std.testing.expectEqual(@as(u32, 125017), cfg.mask); + try std.testing.expectEqual(@as(u32, 256), cfg.rank); + try std.testing.expectEqual(@as(u32, 6144), cfg.intermediate); + try std.testing.expectEqual(@as(f32, 1e7), cfg.theta); +} + +test "lfm2 dspark: other drafters and shapes are refused with their reason" { + try std.testing.expectError(error.NotLfmDSpark, edited("Lfm2DSparkDraftModel", "DFlashDraftModel")); + try std.testing.expectError(error.DraftForAnotherTarget, edited("\"hidden_size\":2048", "\"hidden_size\":4096")); + try std.testing.expectError(error.DraftForAnotherTarget, edited("\"num_target_layers\":40", "\"num_target_layers\":24")); + try std.testing.expectError(error.DraftForAnotherTarget, edited("26,34]", "26,44]")); + try std.testing.expectError(error.UnsupportedDraftShape, edited("\"head_dim\":64", "\"head_dim\":128")); + try std.testing.expectError(error.UnsupportedDraftRope, edited("\"rope_is_neox_style\":false", "\"rope_is_neox_style\":true")); + try std.testing.expectError(error.UnsupportedMarkovHead, edited("\"vanilla\"", "\"gated\"")); + try std.testing.expectError(error.UnsupportedDraftMask, edited("\"full_attention\"]", "\"sliding_attention\"]")); + try std.testing.expectError(error.UnsupportedDraftMask, edited("\"block_size\"", "\"is_causal\":true,\"block_size\"")); + try std.testing.expectError(error.UnsupportedDraftAnchor, edited("\"block_size\"", "\"sample_from_anchor\":false,\"block_size\"")); + try std.testing.expectError(error.UnsupportedDraftShape, edited("\"block_size\":9", "\"block_size\":17")); + try std.testing.expectError(error.BadDraftConfig, edited("\"mask_token_id\":125017", "\"mask_token_id\":128000")); +} diff --git a/zig/src/families/lfm2_moe/dspark_quant.zig b/zig/src/families/lfm2_moe/dspark_quant.zig new file mode 100644 index 0000000000..a6a5baccd6 --- /dev/null +++ b/zig/src/families/lfm2_moe/dspark_quant.zig @@ -0,0 +1,118 @@ +//! BF16 drafter rows quantized at load to MLX's affine codes in groups of 64, the layout core/tile_qmm reads. +const std = @import("std"); + +/// MLX's affine codes of 64 bf16 values at `bits`: bias the larger edge, scale |edge| / round(|edge| / step). +pub fn group(x: []const u16, bits: u8, codes: []u8, scale: *u16, bias: *u16) void { + var lo: f32 = std.math.inf(f32); + var hi: f32 = -std.math.inf(f32); + for (x) |v| { + lo = @min(lo, toF32(v)); + hi = @max(hi, toF32(v)); + } + const bins: f32 = @floatFromInt((@as(u32, 1) << @intCast(bits)) - 1); + const step = @max(hi - lo, 1e-7) / bins; + const from_min = @abs(lo) > @abs(hi); + const edge = if (from_min) lo else hi; + const q0 = @round(@abs(edge) / step); + const s_f: f32 = if (q0 == 0) step else (if (from_min) @abs(edge) else -@abs(edge)) / q0; + scale.* = toBf16(s_f); + bias.* = toBf16(if (q0 == 0) lo else edge); + const s = toF32(scale.*); + const b = toF32(bias.*); + @memset(codes, 0); + for (x, 0..) |v, i| { + const q: u8 = @intFromFloat(std.math.clamp(@round((toF32(v) - b) / s), 0, bins)); + if (bits == 8) codes[i] = q else codes[i / 2] |= q << @intCast((i % 2) * 4); + } +} + +pub fn toF32(v: u16) f32 { + return @bitCast(@as(u32, v) << 16); +} + +/// Round to nearest even, as MLX casts. +pub fn toBf16(f: f32) u16 { + const b: u32 = @bitCast(f); + if ((b & 0x7f80_0000) == 0x7f80_0000) return @truncate(b >> 16); + return @truncate((b + 0x7fff + ((b >> 16) & 1)) >> 16); +} + +/// A matrix's codes [n][k * bits / 8], scales and biases [n][k / 64], each written by one thread's rows. +pub const Job = struct { + src: []const u16, + k: usize, + bits: u8, + codes: []u8, + scales: []u16, + biases: []u16, + fn run(j: Job, from: usize, to: usize) void { + const groups = j.k / 64; + const per = 64 * @as(usize, j.bits) / 8; + for (from..to) |r| for (0..groups) |g| { + const at = r * groups + g; + group(j.src[r * j.k + g * 64 ..][0..64], j.bits, j.codes[at * per ..][0..per], &j.scales[at], &j.biases[at]); + }; + } + + /// The job over its rows on every CPU (at most 16 threads). + pub fn all(j: Job, gpa: std.mem.Allocator, rows: usize) !void { + const threads = @min(std.Thread.getCpuCount() catch 1, 16); + const per = (rows + threads - 1) / threads; + const handles = try gpa.alloc(std.Thread, threads); + defer gpa.free(handles); + var spawned: usize = 0; + defer for (handles[0..spawned]) |h| h.join(); + var at: usize = 0; + while (at < rows) : (at += per) { + handles[spawned] = try std.Thread.spawn(.{}, run, .{ j, at, @min(rows, at + per) }); + spawned += 1; + } + } +}; + +/// Head `h`'s interleaved RoPE pairs (2i, 2i + 1) as half-split pairs (i, i + dim / 2): row h * dim + j's source. +pub fn halfSplit(row: usize, dim: usize) usize { + const h = row / dim; + const j = row % dim; + const half = dim / 2; + return h * dim + if (j < half) 2 * j else 2 * (j - half) + 1; +} + +test "lfm2 dspark: 4- and 8-bit codes keep the larger edge exact, as MLX quantizes" { + var x: [64]u16 = undefined; + for (&x, 0..) |*v, i| v.* = toBf16(@as(f32, @floatFromInt(i)) * 0.125 - 2.0); + for ([_]u8{ 4, 8 }) |bits| { + var codes: [64]u8 = undefined; + var s: u16 = undefined; + var b: u16 = undefined; + group(&x, bits, codes[0 .. 64 * bits / 8], &s, &b); + try std.testing.expect(toF32(s) < 0 and toF32(b) == 5.875); // the max is the edge: codes descend from it + const last: u8 = if (bits == 8) codes[63] else codes[31] >> 4; + try std.testing.expectEqual(@as(u8, 0), last); + for (x, 0..) |v, i| { + const q: u8 = if (bits == 8) codes[i] else (codes[i / 2] >> @intCast((i % 2) * 4)) & 0xf; + const back = toF32(s) * @as(f32, @floatFromInt(q)) + toF32(b); + try std.testing.expect(@abs(back - toF32(v)) <= @abs(toF32(s)) * 0.51); + } + } +} + +test "lfm2 dspark: half-split rows rotated by the half-split rule equal interleaved pairs rotated, row for row" { + const dim = 64; + var x: [2 * dim]f32 = undefined; + for (&x, 0..) |*v, i| v.* = @sin(@as(f32, @floatFromInt(i)) * 0.37) * 3; + var moved: [2 * dim]f32 = undefined; + for (&moved, 0..) |*v, r| v.* = x[halfSplit(r, dim)]; + const pos: f32 = 1234; + for (0..2) |h| for (0..dim / 2) |i| { + const angle = pos * std.math.pow(f32, 1e7, -@as(f32, @floatFromInt(i)) / (dim / 2)); + const cs = @cos(angle); + const sn = @sin(angle); + const a = x[h * dim + 2 * i]; + const b = x[h * dim + 2 * i + 1]; + const l = moved[h * dim + i]; + const r = moved[h * dim + i + dim / 2]; + try std.testing.expectEqual(a * cs - b * sn, l * cs - r * sn); + try std.testing.expectEqual(a * sn + b * cs, l * sn + r * cs); + }; +} diff --git a/zig/src/families/lfm2_moe/dspark_test.zig b/zig/src/families/lfm2_moe/dspark_test.zig index d220b9ea66..9ee31d868e 100644 --- a/zig/src/families/lfm2_moe/dspark_test.zig +++ b/zig/src/families/lfm2_moe/dspark_test.zig @@ -1,8 +1,9 @@ -//! The DSpark drafter's kernels on the GPU against host references: the target's taps. +//! The DSpark drafter's kernels on the GPU against host references: taps, the Markov step, half-split RoPE. const std = @import("std"); const mtl = @import("metal"); const c = @import("config.zig"); const kernels = @import("kernels.zig"); +const quant = @import("dspark_quant.zig"); const shared = mtl.ResourceOptions.shared; @@ -45,15 +46,9 @@ const Rig = struct { } }; -/// Round to nearest even, as MLX casts. -fn toBf16(f: f32) u16 { - const b: u32 = @bitCast(f); - return @truncate((b + 0x7fff + ((b >> 16) & 1)) >> 16); -} - fn noise(seed: u64, out: []u16, scale: f32) void { var prng = std.Random.DefaultPrng.init(seed); - for (out) |*v| v.* = toBf16((prng.random().float(f32) * 2 - 1) * scale); + for (out) |*v| v.* = quant.toBf16((prng.random().float(f32) * 2 - 1) * scale); } test "lfm2 dspark: the tap kernel copies each row's hidden values into its tap's columns, the rest untouched" { @@ -75,3 +70,139 @@ test "lfm2 dspark: the tap kernel copies each row's hidden values into its tap's for (got[(row * taps + 2) * c.hidden ..][0..c.hidden]) |v| try std.testing.expectEqual(@as(u16, 0), v); } } + +test "lfm2 dspark: a Markov step picks the argmax of bf16(base + bf16(W2 . W1[prev])) over every threadgroup" { + var r = try Rig.init(); + defer r.deinit(); + const rank = 256; + const vocab = 1300; // three scan threadgroups, the last one short + const streams = 4; + const groups = (vocab + 511) / 512; + const w1 = try r.buffer(u16, vocab * rank); + defer w1.deinit(); + noise(11, w1.slice(u16, vocab * rank), 1); + const codes = try r.buffer(u8, vocab * rank); + defer codes.deinit(); + for (codes.slice(u8, vocab * rank), 0..) |*q, i| q.* = @truncate(i *% 2654435761 >> 7); + const sc = try r.buffer(u16, vocab * rank / 64); + defer sc.deinit(); + const bi = try r.buffer(u16, vocab * rank / 64); + defer bi.deinit(); + noise(12, sc.slice(u16, vocab * rank / 64), 0.01); + noise(13, bi.slice(u16, vocab * rank / 64), 0.5); + // a token every second word (the picks' layout): inside W1, its last row, past it (a zero row), and a NaN row + const ids = try r.buffer(u32, 2 * streams); + defer ids.deinit(); + @memcpy(ids.slice(u32, 2 * streams), &[_]u32{ 17, 9, vocab - 1, 9, vocab + 5, 0, 3, 3 }); + const logits = try r.buffer(u16, streams * 2 * vocab); + defer logits.deinit(); + noise(14, logits.slice(u16, streams * 2 * vocab), 4); + logits.slice(u16, streams * 2 * vocab)[3 * 2 * vocab + 700] = 0x7fc0; + const best = try r.buffer(f32, streams * groups); + defer best.deinit(); + const at = try r.buffer(u32, streams * groups); + defer at.deinit(); + const picks = try r.buffer(u32, 2 * streams); + defer picks.deinit(); + const cb = r.queue.commandBuffer(); + const e = cb.compute(.serial); + e.setPipeline(r.k.get("lfm2_markov_scan")); + for ([_]mtl.Buffer{ logits, w1, codes, ids }, 0..) |b, i| e.setBuffer(b, 0, i); + e.setValue([4]u32{ 2 * vocab, rank, vocab, 2 }, 4); + for ([_]mtl.Buffer{ best, at, sc, bi }, 5..) |b, i| e.setBuffer(b, 0, i); + e.dispatchGroups(mtl.Size.of(groups, streams, 1), mtl.Size.of(256, 1, 1)); + e.setPipeline(r.k.get("lfm2_markov_pick")); + e.setBuffer(best, 0, 0); + e.setBuffer(at, 0, 1); + e.setValue(@as(u32, groups), 2); + e.setBuffer(picks, 0, 3); + e.dispatchGroups(mtl.Size.of(streams, 1, 1), mtl.Size.of(256, 1, 1)); + e.end(); + cb.commit(); + cb.wait(); + const got = picks.slice(u32, 2 * streams); + const base = logits.slice(u16, streams * 2 * vocab); + const id_list = ids.slice(u32, 2 * streams); + for (0..streams) |s| { + if (s == 3) { // the planted NaN: flagged + try std.testing.expectEqual(@as(u32, 1), got[2 * s + 1]); + continue; + } + try std.testing.expectEqual(@as(u32, 0), got[2 * s + 1]); + const id = id_list[2 * s]; + var top: f64 = -std.math.inf(f64); + var mine: f64 = 0; + for (0..vocab) |v| { + var dot: f64 = 0; + for (0..rank) |i| { + const g = (v * rank + i) / 64; + const w = @as(f64, quant.toF32(sc.slice(u16, vocab * rank / 64)[g])) * @as(f64, @floatFromInt(codes.slice(u8, vocab * rank)[v * rank + i])) + quant.toF32(bi.slice(u16, vocab * rank / 64)[g]); + dot += w * (if (id < vocab) quant.toF32(w1.slice(u16, vocab * rank)[id * rank + i]) else 0); + } + const score: f64 = quant.toF32(quant.toBf16(quant.toF32(base[s * 2 * vocab + v]) + quant.toF32(quant.toBf16(@floatCast(dot))))); + top = @max(top, score); + if (v == got[2 * s]) mine = score; + } + // the pick is the best score, or within two bf16 steps of it where fp32 sums round apart + try std.testing.expect(got[2 * s] < vocab and mine >= top - @abs(top) * 0x1p-7); + } +} + +test "lfm2 dspark: lfm2_qk on half-split rows and gains gives interleaved RoPE's keys, reordered" { + var r = try Rig.init(); + defer r.deinit(); + const hd = c.head_dim; + const pos: u32 = 777; + const theta: f32 = 1e7; + var k_il: [c.kv_heads * hd]u16 = undefined; + var gain_il: [hd]u16 = undefined; + noise(21, &k_il, 2); + noise(22, &gain_il, 1.5); + const q = try r.buffer(u16, c.query_heads * hd); + defer q.deinit(); + const k = try r.buffer(u16, c.kv_heads * hd); + defer k.deinit(); + const v = try r.buffer(u16, c.kv_heads * hd); + defer v.deinit(); + const qn = try r.buffer(u16, hd); + defer qn.deinit(); + const kn = try r.buffer(u16, hd); + defer kn.deinit(); + for (k.slice(u16, c.kv_heads * hd), 0..) |*x, i| x.* = k_il[quant.halfSplit(i, hd)]; + for (kn.slice(u16, hd), 0..) |*x, i| x.* = gain_il[quant.halfSplit(i, hd)]; + @memset(qn.slice(u16, hd), quant.toBf16(1)); + const cap = 1024; + const queries = try r.buffer(u16, c.query_heads * hd); + defer queries.deinit(); + const keys = try r.buffer(u16, c.kv_heads * cap * hd); + defer keys.deinit(); + const values = try r.buffer(u16, c.kv_heads * cap * hd); + defer values.deinit(); + const Args = extern struct { len: u32, cap: u32, eps: f32, log2_theta: f32 }; + const cb = r.queue.commandBuffer(); + const e = cb.compute(.serial); + e.setPipeline(r.k.get("lfm2_qk")); + for ([_]mtl.Buffer{ q, k, v, qn, kn }, 0..) |b, i| e.setBuffer(b, 0, i); + const args = Args{ .len = pos, .cap = cap, .eps = c.eps, .log2_theta = std.math.log2(theta) }; + e.setBytes(std.mem.asBytes(&args), 5); + for ([_]mtl.Buffer{ queries, keys, values }, 6..) |b, i| e.setBuffer(b, 0, i); + e.dispatchGroups(mtl.Size.of(c.query_heads + c.kv_heads, 1, 1), mtl.Size.of(32, 1, 1)); + e.end(); + cb.commit(); + cb.wait(); + const got = keys.slice(u16, c.kv_heads * cap * hd); + for (0..c.kv_heads) |h| { + var ss: f32 = 0; + for (0..hd) |i| ss += quant.toF32(k_il[h * hd + i]) * quant.toF32(k_il[h * hd + i]); + const inv = 1 / @sqrt(ss / hd + c.eps); + for (0..hd / 2) |i| { + const angle = @as(f32, pos) * std.math.pow(f32, theta, -@as(f32, @floatFromInt(i)) / (hd / 2)); + const a = quant.toF32(k_il[h * hd + 2 * i]) * inv * quant.toF32(gain_il[2 * i]); + const b = quant.toF32(k_il[h * hd + 2 * i + 1]) * inv * quant.toF32(gain_il[2 * i + 1]); + const want = [2]f32{ a * @cos(angle) - b * @sin(angle), a * @sin(angle) + b * @cos(angle) }; + const at = (h * cap + pos) * hd; + const have = [2]f32{ quant.toF32(got[at + i]), quant.toF32(got[at + i + hd / 2]) }; + for (want, have) |w, x| try std.testing.expect(@abs(w - x) <= 0.02 * @max(1, @abs(w))); + } + } +} diff --git a/zig/src/families/lfm2_moe/dspark_weights.zig b/zig/src/families/lfm2_moe/dspark_weights.zig new file mode 100644 index 0000000000..81dc727e56 --- /dev/null +++ b/zig/src/families/lfm2_moe/dspark_weights.zig @@ -0,0 +1,144 @@ +//! The DSpark drafter's weights: bf16 linears quantized to MLX affine codes at load, norms and W1 kept as stored. +const std = @import("std"); +const mtl = @import("metal"); +const c = @import("config.zig"); +const dc = @import("dspark_config.zig"); +const quant = @import("dspark_quant.zig"); +const ckpt = @import("../../core/checkpoint_metal.zig"); +const shards = @import("../../core/checkpoint.zig"); +const tile = @import("../../core/tile_qmm.zig"); + +const shared = mtl.ResourceOptions.shared | mtl.ResourceOptions.untracked; + +/// A bf16 vector in its own buffer (norm gains; RoPE-paired ones in the half-split order the kernels rotate). +pub const Vector = mtl.Buffer; + +pub const Layer = struct { + q: tile.Matrix, + k: tile.Matrix, + v: tile.Matrix, + o: tile.Matrix, + gate: tile.Matrix, + up: tile.Matrix, + down: tile.Matrix, + input_norm: Vector, + post_norm: Vector, + q_norm: Vector, + k_norm: Vector, +}; + +pub const Weights = struct { + fc: tile.Matrix, + hidden_norm: Vector, + norm: Vector, + layers: [8]Layer, + /// The Markov head: W1 [vocab, rank] bf16 as stored (rows gathered by token), W2 [vocab, rank] as 8-bit codes. + w1: mtl.Buffer, + w2: tile.Matrix, + owned: std.ArrayList(mtl.Buffer) = .empty, + + pub fn deinit(w: *Weights, gpa: std.mem.Allocator) void { + for (w.owned.items) |b| b.deinit(); + w.owned.deinit(gpa); + } + + /// Every buffer the weights hold, for a residency set. + pub fn buffers(w: *const Weights) []const mtl.Buffer { + return w.owned.items; + } +}; + +const Loader = struct { + gpa: std.mem.Allocator, + device: mtl.Device, + check: *const ckpt.Checkpoint, + owned: *std.ArrayList(mtl.Buffer), + bits: u8, + + fn buffer(l: Loader, bytes: usize) !mtl.Buffer { + const b = try l.device.buffer(bytes, shared); + errdefer b.deinit(); + try l.owned.append(l.gpa, b); + return b; + } + + fn tensor(l: Loader, name: []const u8, shape: []const usize) !ckpt.Tensor { + const t = try l.check.get(name); + if (t.dtype != .bf16 or t.rank != shape.len or !std.mem.eql(usize, t.shape[0..t.rank], shape)) { + std.log.err("{s}: unexpected DSpark tensor dtype or shape", .{name}); + return error.BadDraftWeight; + } + return t; + } + + /// A bf16 vector copied out, its pairs reordered for half-split RoPE when `paired`. + fn vector(l: Loader, name: []const u8, len: usize, paired: bool) !Vector { + const src = (try l.tensor(name, &.{len})).host(u16); + const b = try l.buffer(len * 2); + const dst = b.slice(u16, len); + for (dst, 0..) |*v, i| v.* = src[if (paired) quant.halfSplit(i, len) else i]; + return b; + } + + /// An [n, k] bf16 matrix as affine codes at the loader's width; rows reordered for half-split RoPE when `paired`. + fn matrix(l: Loader, name: []const u8, n: usize, k: usize, paired: bool) !tile.Matrix { + const t = try l.tensor(name, &.{ n, k }); + var rows = t.host(u16); + var moved: ?[]u16 = null; + defer if (moved) |m| l.gpa.free(m); + if (paired) { + const m = try l.gpa.alloc(u16, n * k); + moved = m; + for (0..n) |r| @memcpy(m[r * k ..][0..k], rows[quant.halfSplit(r, c.head_dim) * k ..][0..k]); + rows = m; + } + const codes = try l.buffer(n * k * l.bits / 8); + const scales = try l.buffer(n * k / 64 * 2); + const biases = try l.buffer(n * k / 64 * 2); + const job: quant.Job = .{ .src = rows, .k = k, .bits = l.bits, .codes = codes.slice(u8, n * k * l.bits / 8), .scales = scales.slice(u16, n * k / 64), .biases = biases.slice(u16, n * k / 64) }; + try job.all(l.gpa, n); + return .{ .w = codes, .scales = scales, .biases = biases, .n = n, .k = k, .bits = l.bits }; + } +}; + +fn part(buf: []u8, layer: usize, name: []const u8) ![]const u8 { + return std.fmt.bufPrint(buf, "layers.{d}.{s}.weight", .{ layer, name }); +} + +/// The drafter's checkpoint read and quantized at `bits` (4 or 8); the checkpoint's own buffers are released after. +pub fn load(gpa: std.mem.Allocator, device: mtl.Device, io: std.Io, dir: []const u8, cfg: *const dc.Config, bits: u8) !Weights { + if (bits != 4 and bits != 8) return error.UnsupportedDraftBits; + var check = ckpt.Checkpoint.init(gpa); + defer check.deinit(); + const files = try shards.shardFiles(gpa, io, dir); + defer shards.freeShardFiles(gpa, files); + for (files) |path| try check.addFile(device, path, ""); + var w: Weights = undefined; + w.owned = .empty; + errdefer w.deinit(gpa); + const l: Loader = .{ .gpa = gpa, .device = device, .check = &check, .owned = &w.owned, .bits = bits }; + const qw = c.query_heads * c.head_dim; + const kw = c.kv_heads * c.head_dim; + w.fc = try l.matrix("fc.weight", c.hidden, cfg.tap_count * c.hidden, false); + w.hidden_norm = try l.vector("hidden_norm.weight", c.hidden, false); + w.norm = try l.vector("norm.weight", c.hidden, false); + var name: [128]u8 = undefined; + for (w.layers[0..cfg.layers], 0..) |*ly, i| { + ly.q = try l.matrix(try part(&name, i, "self_attn.q_proj"), qw, c.hidden, true); + ly.k = try l.matrix(try part(&name, i, "self_attn.k_proj"), kw, c.hidden, true); + ly.v = try l.matrix(try part(&name, i, "self_attn.v_proj"), kw, c.hidden, false); + ly.o = try l.matrix(try part(&name, i, "self_attn.o_proj"), c.hidden, qw, false); + ly.gate = try l.matrix(try part(&name, i, "mlp.gate_proj"), cfg.intermediate, c.hidden, false); + ly.up = try l.matrix(try part(&name, i, "mlp.up_proj"), cfg.intermediate, c.hidden, false); + ly.down = try l.matrix(try part(&name, i, "mlp.down_proj"), c.hidden, cfg.intermediate, false); + ly.input_norm = try l.vector(try part(&name, i, "input_layernorm"), c.hidden, false); + ly.post_norm = try l.vector(try part(&name, i, "post_attention_layernorm"), c.hidden, false); + ly.q_norm = try l.vector(try part(&name, i, "self_attn.q_norm"), c.head_dim, true); + ly.k_norm = try l.vector(try part(&name, i, "self_attn.k_norm"), c.head_dim, true); + } + const w1 = (try l.tensor("markov_head.markov_w1.weight", &.{ c.vocab, cfg.rank })).host(u16); + w.w1 = try l.buffer(c.vocab * cfg.rank * 2); + @memcpy(w.w1.slice(u16, c.vocab * cfg.rank), w1); + w.w2 = try l.matrix("markov_head.markov_w2.weight", c.vocab, cfg.rank, false); + return w; +} diff --git a/zig/src/families/lfm2_moe/kernels.zig b/zig/src/families/lfm2_moe/kernels.zig index 61af45f4c8..357500c04d 100644 --- a/zig/src/families/lfm2_moe/kernels.zig +++ b/zig/src/families/lfm2_moe/kernels.zig @@ -16,6 +16,8 @@ const names = [_][:0]const u8{ "lfm2_attn_mma", "lfm2_attn_mma_r2", "lfm2_attn_merge", + "lfm2_attn_block", + "lfm2_attn_block_merge", "lfm2_route_logits", "lfm2_route_select", "lfm2_swiglu", @@ -28,6 +30,8 @@ const names = [_][:0]const u8{ "lfm2_attn_segs", "lfm2_attn_merge_segs", "lfm2_tap", + "lfm2_markov_scan", + "lfm2_markov_pick", }; pub const Kernels = struct { diff --git a/zig/src/families/lfm2_moe/lfm2_moe.zig b/zig/src/families/lfm2_moe/lfm2_moe.zig index 97f9cbd5d8..8f861e31b4 100644 --- a/zig/src/families/lfm2_moe/lfm2_moe.zig +++ b/zig/src/families/lfm2_moe/lfm2_moe.zig @@ -7,3 +7,7 @@ pub const state = @import("state.zig"); pub const forward = @import("forward.zig"); pub const backend = @import("backend.zig"); pub const snapshot = @import("snapshot.zig"); +pub const dspark = @import("dspark.zig"); +pub const dspark_config = @import("dspark_config.zig"); +pub const dspark_block = @import("dspark_block.zig"); +pub const dspark_quant = @import("dspark_quant.zig"); diff --git a/zig/src/native/lfm2_moe.zig b/zig/src/native/lfm2_moe.zig index 5340d807de..05f0260dcc 100644 --- a/zig/src/native/lfm2_moe.zig +++ b/zig/src/native/lfm2_moe.zig @@ -1,4 +1,4 @@ -//! LFM2-MoE served by the existing lane host and scheduler. +//! LFM2-MoE served by the existing lane host and scheduler, with LiquidAI's DSpark drafter from --drafter. const std = @import("std"); const mtl = @import("metal"); const api = @import("engine_api"); @@ -8,11 +8,16 @@ const lanes = tf.lanes; const cache_fit = @import("cache_fit.zig"); const metal_admit = @import("metal_admit.zig"); +/// --drafter loads LiquidAI's DSpark drafter (LiquidAI/LFM2.5-24B-A2B-DSpark) in front of the engine. +pub const takes_drafter = true; + const Host = struct { resident: ?mtl.resident.Resident = null, gpa: std.mem.Allocator, model: *q.Model, metal: *q.backend.Metal, + draft: ?*q.dspark.Drafter = null, + wrap: ?lanes.drafted.Drafted = null, warm: mtl.keepalive.Target = undefined, // the model's queue, for the lane host's idle ticker config: lanes.Config, clock: lanes.backend.WallClock, @@ -25,7 +30,9 @@ const Host = struct { h.host.stop(); if (h.cache) |*store| store.deinit(); h.core.deinit(); + if (h.wrap) |*w| w.deinit(); mtl.resident.streams.drop(h); + if (h.draft) |d| d.deinit(); h.config.deinit(h.gpa); h.metal.deinit(); h.model.deinit(); @@ -35,6 +42,13 @@ const Host = struct { const cannot = "the native LFM2 engine cannot load this checkpoint ({s})"; +/// A refused open's backend, model and host, freed before it reports why. +fn release(h: *Host) void { + h.metal.deinit(); + h.model.deinit(); + h.gpa.destroy(h); +} + pub fn open(a: std.mem.Allocator, gpa: std.mem.Allocator, io: std.Io, o: api.Open, problem: *[]const u8) !?api.Opened { const pool = mtl.objc.Pool.push(); defer pool.pop(); @@ -65,18 +79,36 @@ pub fn open(a: std.mem.Allocator, gpa: std.mem.Allocator, io: std.Io, o: api.Ope const chunk = 2048; h.metal = try q.backend.Metal.init(gpa, model, .{ .capacity = window, .chunk = chunk, .streams = @max(o.lanes, 2) }); errdefer h.metal.deinit(); - const per_stream = q.state.cacheBytes(&model.config.attention, window); + h.draft = null; + h.wrap = null; + if (if (o.drafts) o.drafter else null) |dir| { + if (o.drafter_bits == 0) { + problem.* = "--drafter-bits 0: the LFM2 DSpark drafter runs its bf16 linears as 8-bit codes, quantized at load"; + release(h); + return null; + } + h.draft = q.dspark.Drafter.init(gpa, io, model, dir, .{ .capacity = window }) catch |err| { + problem.* = try std.fmt.allocPrint(a, "the native LFM2 drafter cannot load {s} ({s}): it takes LiquidAI/LFM2.5-24B-A2B-DSpark", .{ dir, @errorName(err) }); + release(h); + return null; + }; + } + errdefer if (h.draft) |d| d.deinit(); + // a stream's caches, and its drafter lane when one drafts + const per_stream = q.state.cacheBytes(&model.config.attention, window) + if (h.draft) |d| d.laneBytes(window) else 0; const served = try metal_admit.streams(a, "lfm2", per_stream, metal_admit.room(model.device), o, window, problem) orelse { - h.metal.deinit(); - model.deinit(); - gpa.destroy(h); + if (h.draft) |d| d.deinit(); + release(h); return null; }; h.metal.options.streams = @max(served, 2); - h.config = try lanes.Config.init(gpa, h.metal.facts(), q.state.window_rows, q.state.window_rows - 1); + if (h.draft) |d| h.wrap = try lanes.drafted.Drafted.init(gpa, h.metal.backend(), d.drafter()); + errdefer if (h.wrap) |*w| w.deinit(); + const facts = if (h.wrap) |*w| w.facts(h.metal.facts()) else h.metal.facts(); + h.config = try lanes.Config.init(gpa, facts, q.state.window_rows, q.state.window_rows - 1); errdefer h.config.deinit(gpa); h.clock = .{ .io = io }; - h.core = lanes.Engine.init(gpa, &h.config, h.metal.backend(), h.clock.clock()); + h.core = lanes.Engine.init(gpa, &h.config, if (h.wrap) |*w| w.backend() else h.metal.backend(), h.clock.clock()); errdefer h.core.deinit(); h.host = api.LaneHost.init(gpa, io, &h.core, .{ .lanes = served, .context_window = @intCast(window), .prefill_step = chunk, .structures = true }); h.cache = null; @@ -97,8 +129,8 @@ pub fn open(a: std.mem.Allocator, gpa: std.mem.Allocator, io: std.Io, o: api.Ope if (plan.budget_bytes > 0) { const S = q.backend.Snaps; h.cache = api.prompt_cache.Store.init(gpa, .{ - .ptr = h.metal, - .vtable = &.{ .bytes = S.bytes, .save = S.save, .restore = S.restore, .drop = S.drop }, + .ptr = h, + .vtable = &.{ .bytes = Snaps.bytes, .save = Snaps.save, .restore = S.restore, .drop = Snaps.drop }, }, .{ .chunk = chunk }, plan.budget_bytes); h.host.cache = &h.cache.?; h.host.info_.prompt_cache = true; @@ -106,7 +138,37 @@ pub fn open(a: std.mem.Allocator, gpa: std.mem.Allocator, io: std.Io, o: api.Ope errdefer if (h.cache) |*store| store.deinit(); try mtl.resident.streams.hold(h, io, "lfm2 moe"); errdefer mtl.resident.streams.drop(h); + if (h.draft) |d| if (h.resident) |*r| { + const held = try d.buffers(gpa); + defer gpa.free(held); + r.add(held); + }; h.host.keepalive_target = .{ .ctx = &h.warm, .tick = mtl.keepalive.Target.tick }; try h.host.start(); + if (h.draft) |d| std.log.info("lfm2: DSpark drafts from --drafter, {d} a round at most, {d}-bit codes", .{ d.config.block, d.options.bits }); return .{ .engine = h.host.engine(), .close = Host.close, .ctx = h }; } + +/// The prompt cache's copies of a stream: the target's state at a mark (backend.zig), and the drafter's context there. +const Snaps = struct { + const S = q.backend.Snaps; + fn host(ptr: *anyopaque) *Host { + return @ptrCast(@alignCast(ptr)); + } + fn bytes(ptr: *anyopaque, at: u32) u64 { + const h = host(ptr); + return S.bytes(h.metal, at) + if (h.draft) |d| d.keptBytes(at) else 0; + } + fn save(ptr: *anyopaque, owner: ?*anyopaque, at: u32) anyerror!*anyopaque { + const h = host(ptr); + const saved = try S.save(h.metal, owner, at); + if (h.draft) |d| d.keepLane(@ptrCast(@alignCast(owner.?)), @ptrCast(@alignCast(saved))) catch |err| { + S.drop(h.metal, saved); + return err; + }; + return saved; + } + fn drop(ptr: *anyopaque, saved: *anyopaque) void { + S.drop(host(ptr).metal, saved); + } +}; diff --git a/zig/src/native/metal.zig b/zig/src/native/metal.zig index a42272fe56..104bbfc22b 100644 --- a/zig/src/native/metal.zig +++ b/zig/src/native/metal.zig @@ -176,8 +176,8 @@ pub fn open(a: Allocator, gpa: Allocator, io: std.Io, o: api.Open, problem: *[]c problem.* = why; return null; } - if (o.drafter != null and !dense and !isGemma4(o.model_type)) { - problem.* = "--drafter is supported for Qwen3.8-27B-class checkpoints (qwen3_5 with its own output head) and Gemma 4 dense"; + if (o.drafter != null and !dense and !isGemma4(o.model_type) and !std.mem.eql(u8, o.model_type, "lfm2_moe")) { + problem.* = "--drafter is supported for Qwen3.8-27B-class checkpoints (qwen3_5 with its own output head), Gemma 4 dense and LFM2.5-24B-A2B"; return null; } if (dense) return openQwen27(a, gpa, io, o, problem); diff --git a/zig/src/tensorfold.zig b/zig/src/tensorfold.zig index 6148579bf6..99591ca724 100644 --- a/zig/src/tensorfold.zig +++ b/zig/src/tensorfold.zig @@ -88,6 +88,8 @@ test { _ = @import("families/lfm2_moe/config.zig"); _ = @import("families/lfm2_moe/backend.zig"); _ = @import("families/lfm2_moe/state.zig"); + _ = @import("families/lfm2_moe/dspark_config.zig"); + _ = @import("families/lfm2_moe/dspark_quant.zig"); _ = @import("families/lfm2_moe/dspark_test.zig"); } diff --git a/zig/tests/contract/metal_lfm2_attn.zig b/zig/tests/contract/metal_lfm2_attn.zig new file mode 100644 index 0000000000..799b337c59 --- /dev/null +++ b/zig/tests/contract/metal_lfm2_attn.zig @@ -0,0 +1,182 @@ +//! LFM2's block attention (lfm2_attn_block, then lfm2_attn_block_merge): every row of a drafter's block sees all keys. +const std = @import("std"); +const mtl = @import("metal"); +const contract = @import("contract"); +const sources = @import("kernel_sources"); +const c = contract.spec; +const g = contract.gates; +const rig_ = @import("rig.zig"); + +const qh = 32; +const kvh = 8; +const hd = 64; +const subs = 8; // the drafter's sub-spans a 4,096-key chunk (state.subs) +const chunks = 32; +const chunk_keys = 4096; +pub const name = "metal lfm2 block attention"; + +/// rows: a block's rows; k: the keys every row sees (a context and the block past it). +const shapes = [_]c.Shape{ .{ .rows = 1, .n = qh * hd, .k = 9 }, .{ .rows = 3, .n = qh * hd, .k = 700 }, .{ .rows = 9, .n = qh * hd, .k = 4500 } }; + +fn depth(s: c.Shape) usize { + return s.k; +} + +pub fn spec() c.Spec { + return .{ .name = name, .accum = .none, .depth = depth, .out = .bf16, .served = &shapes, .nan = .{ .stated = "a NaN key of a kv head makes its query heads' outputs NaN, the others stay finite" }, .sums = .{ .f16 = 0, .bf16 = 0, .why = "the check bounds each output by its bf16 probabilities' rounding, not a sum's bits" } }; +} + +fn value(i: usize, seed: u64, scale: f32) u16 { + const h: u64 = (@as(u64, i) *% 0x9e3779b97f4a7c15 +% seed *% 0xbf58476d1ce4e5b9) >> 40; + return c.toBf16((@as(f32, @floatFromInt(h % 2048)) / 1024 - 1) * scale); +} + +pub const Probe = struct { + a: std.mem.Allocator, + rig: rig_.Rig, + pipes: [2]mtl.Pipeline, + shape: c.Shape = undefined, + cap: usize = 0, + nan: bool = false, + q: []u16 = &.{}, + k: []u16 = &.{}, + v: []u16 = &.{}, + args: [5]rig_.Ref = undefined, + y: rig_.Ref = undefined, + + pub fn init(a: std.mem.Allocator, device: mtl.Device, queue: mtl.Queue) !Probe { + const first = try rig_.pipeline(device, sources.lfm2, "lfm2_attn_block"); + errdefer first.deinit(); + return .{ .a = a, .rig = .{ .a = a, .device = device, .queue = queue }, .pipes = .{ first, try rig_.pipeline(device, sources.lfm2, "lfm2_attn_block_merge") } }; + } + + fn drop(p: *Probe) void { + inline for (.{ "q", "k", "v" }) |f| { + p.a.free(@field(p, f)); + @field(p, f) = &.{}; + } + p.rig.reset(); + } + + pub fn deinit(p: *Probe) void { + p.drop(); + p.rig.deinit(); + for (p.pipes) |x| x.deinit(); + } + + const vt = g.hooks(Probe); + pub fn kernel(p: *Probe) g.Kernel { + return .{ .ctx = p, .vt = &vt, .spec = spec() }; + } + + pub fn prepare(p: *Probe, s: c.Shape, f: g.Fill, seed: u64) !bool { + if (s.n != qh * hd or s.rows == 0 or s.rows > 16 or s.k < s.rows or s.k > chunks * chunk_keys) return false; + p.drop(); + p.shape = s; + p.nan = f == .nonfinite; + p.cap = (s.k + 63) / 64 * 64 + 32; // keys past `k` hold finite values, as a lane's cache does + const padded = (s.rows + 7) / 8 * 8 + 8; // the kernel reads a tile's 8 rows whatever its row count + p.q = try p.a.alloc(u16, padded * qh * hd); + p.k = try p.a.alloc(u16, kvh * p.cap * hd); + p.v = try p.a.alloc(u16, kvh * p.cap * hd); + for (p.q, 0..) |*x, i| x.* = value(i, seed, 1); + for (p.k, 0..) |*x, i| x.* = value(i, seed +% 1, if (f == .stress) 4 else 1); + for (p.v, 0..) |*x, i| x.* = value(i, seed +% 2, 2); + if (p.nan) p.k[(0 * p.cap + s.k / 2) * hd + 5] = 0x7fc0; // kv head 0, a visible key + p.args[0] = try p.rig.input(std.mem.sliceAsBytes(p.q)); + p.args[1] = try p.rig.input(std.mem.sliceAsBytes(p.k)); + p.args[2] = try p.rig.input(std.mem.sliceAsBytes(p.v)); + const Args = extern struct { len: u32, cap: u32, scale: f32, subs: u32 }; + const args = Args{ .len = @intCast(s.k - 1), .cap = @intCast(p.cap), .scale = 1.0 / @sqrt(@as(f32, hd)), .subs = subs }; + p.args[3] = try p.rig.input(std.mem.asBytes(&args)); + const n: u32 = @intCast(s.rows); + p.args[4] = try p.rig.input(std.mem.asBytes(&n)); + p.y = try p.rig.output(p.count() * 2); + return true; + } + + pub fn count(p: *Probe) usize { + return p.shape.rows * qh * hd; + } + + pub fn run(p: *Probe) !void { + const rows = p.shape.rows; + const partials = @max(rows, 8) * subs * qh * chunks; + const pm = try p.rig.scratch(partials * 4); + const pl = try p.rig.scratch(partials * 4); + const po = try p.rig.scratch(partials * hd * 4); + const pass = p.rig.begin(); + const e = pass.e; + e.setPipeline(p.pipes[0]); + for (p.args[0..4], 0..) |r, i| e.setBuffer(r.buf, r.off, i); + for ([_]rig_.Ref{ pm, pl, po, p.args[4] }, 4..) |r, i| e.setBuffer(r.buf, r.off, i); + const used = (p.shape.k + chunk_keys - 1) / chunk_keys * subs; + e.dispatchGroups(mtl.Size.of((rows + 7) / 8, kvh, used), mtl.Size.of(128, 1, 1)); + e.setPipeline(p.pipes[1]); + for ([_]rig_.Ref{ pm, pl, po, p.args[3], p.y }, 0..) |r, i| e.setBuffer(r.buf, r.off, i); + e.dispatchGroups(mtl.Size.of(qh, rows, 1), mtl.Size.of(32, 1, 1)); + try p.rig.finish(pass); + } + + pub fn reference(_: *Probe, want: []f64, mag: []f64) !void { + @memset(want, 0); // the check computes its own references + @memset(mag, 0); + } + + /// Every output against float64 attention: within 2^-7 of its values' largest magnitude and 2^-8 of itself. + pub fn check(p: *Probe, _: g.Gate) !?[]const u8 { + const s = p.shape; + const out = std.mem.bytesAsSlice(u16, p.rig.bytes(p.y, p.count() * 2)); + const w = try p.a.alloc(f64, s.k); + defer p.a.free(w); + for (0..s.rows) |r| for (0..qh) |h| { + const kv = h / (qh / kvh); + const nan_head = p.nan and kv == 0; + var top: f64 = -std.math.inf(f64); + for (0..s.k) |j| { + var dot: f64 = 0; + for (0..hd) |d| dot += c.fromBf16(p.q[(r * qh + h) * hd + d]) * c.fromBf16(p.k[(kv * p.cap + j) * hd + d]); + w[j] = dot / @sqrt(@as(f64, hd)); + top = @max(top, w[j]); + } + var sum: f64 = 0; + for (w) |*x| { + x.* = @exp(x.* - top); + sum += x.*; + } + for (0..hd) |d| { + var o: f64 = 0; + var big: f64 = 0; + for (0..s.k) |j| { + const vv: f64 = c.fromBf16(p.v[(kv * p.cap + j) * hd + d]); + o += w[j] / sum * vv; + big = @max(big, @abs(vv)); + } + const got: f64 = c.fromBf16(out[(r * qh + h) * hd + d]); + if (nan_head) { + if (!std.math.isNan(got)) return "a head behind a NaN key gave a number"; + continue; + } + if (!std.math.isFinite(got)) return "a finite head gave a non-finite output"; + if (@abs(got - o) > big * 0x1p-7 + @abs(o) * 0x1p-8) return "an output is past its float64 bound"; + } + }; + return null; + } +}; + +/// Gate 5 for the block attention's two launches on every Apple chip we serve and this GPU. +pub fn limits(a: std.mem.Allocator, device: mtl.Device, only: ?[]const u8) !usize { + var problems: usize = 0; + const launches = [_]struct { []const u8, []const u8, u32, u32 }{ .{ "metal lfm2 block attention", "lfm2_attn_block", 128, 18688 }, .{ "metal lfm2 block attention merge", "lfm2_attn_block_merge", 32, 0 } }; + for (launches) |l| { + if (only) |s| if (std.mem.indexOf(u8, l[0], s) == null) continue; + const d = contract.limits.MetalLaunch{ .name = l[0], .threads = l[2], .tg_memory = l[3], .offline = false }; + var notes: std.ArrayList(u8) = .empty; + defer notes.deinit(a); + for (contract.limits.apple) |chip| problems += try contract.limits.metal(d, chip, ¬es, a); + std.debug.print("{s}", .{notes.items}); + problems += try @import("metal_limits.zig").onThisGpu(a, device, .{ .decl = d, .function = l[1], .here = sources.lfm2, .pre_m5 = "" }, false); + } + return problems; +} diff --git a/zig/tests/contract/metal_lfm2_dspark.zig b/zig/tests/contract/metal_lfm2_dspark.zig new file mode 100644 index 0000000000..3f2ae29907 --- /dev/null +++ b/zig/tests/contract/metal_lfm2_dspark.zig @@ -0,0 +1,221 @@ +//! LFM2's DSpark Markov step (scan then pick): each block row's draft from its base logits and the previous token. +const std = @import("std"); +const mtl = @import("metal"); +const contract = @import("contract"); +const sources = @import("kernel_sources"); +const c = contract.spec; +const g = contract.gates; +const rig_ = @import("rig.zig"); + +pub const Which = enum { markov_step }; +const rank = 256; +const vocab = 128000; +const span = 512; // tokens a scan threadgroup takes (the kernel's MARKOV_SPAN) +const groups = (vocab + span - 1) / span; +const table = 1000; // W1's rows in the check: ids are inside it or past the vocabulary (a zero row) +const names = [_][]const u8{"metal lfm2 markov step"}; + +const step_shapes = [_]c.Shape{ .{ .rows = 1, .n = vocab, .k = rank }, .{ .rows = 4, .n = vocab, .k = rank }, .{ .rows = 9, .n = vocab, .k = rank } }; + +fn none(_: c.Shape) usize { + return 0; +} + +fn depth(_: c.Shape) usize { + return rank; +} + +pub fn spec(w: Which) c.Spec { + return switch (w) { + .markov_step => .{ .name = names[0], .accum = .none, .depth = depth, .out = .raw, .served = &step_shapes, .nan = .{ .stated = "a row with a NaN score is flagged nonfinite" }, .sums = .{ .f16 = 0, .bf16 = 0, .why = "picks are tokens: the check bounds each pick's score by gamma(256), not sum bits" } }, + }; +} + +/// A bf16 value for element `i` over signs and binades; the nonfinite fill plants NaN and infinities. +fn value(i: usize, seed: u64, fill: g.Fill) u16 { + if (fill == .nonfinite) switch ((@as(u64, i) *% 2654435761 +% seed) % 9973) { + 0 => return 0x7fc0, + 1 => return 0x7f80, + 2 => return 0xff80, + else => {}, + }; + const h: u64 = (@as(u64, i) *% 0x9e3779b97f4a7c15 +% seed *% 0xbf58476d1ce4e5b9) >> 40; + const mant: f32 = @as(f32, @floatFromInt(h % 1024)) / 1024 + 1; + const exp: i32 = @as(i32, @intCast((h >> 10) % 8)) - 4; + const v = std.math.ldexp(mant, exp) * @as(f32, if (((h >> 15) & 1) == 0) 1 else -1); + return c.toBf16(v); +} + +pub const Probe = struct { + a: std.mem.Allocator, + rig: rig_.Rig, + which: Which, + pipes: [2]mtl.Pipeline, + rows: usize = 0, + src: []u16 = &.{}, // the base logits rows + w1: []u16 = &.{}, + codes: []u8 = &.{}, + scales: []u16 = &.{}, + biases: []u16 = &.{}, + ids: []u32 = &.{}, + args: [8]rig_.Ref = undefined, + y: rig_.Ref = undefined, + + pub fn init(a: std.mem.Allocator, device: mtl.Device, queue: mtl.Queue, w: Which) !Probe { + const kernels: [2][:0]const u8 = switch (w) { + .markov_step => .{ "lfm2_markov_scan", "lfm2_markov_pick" }, + }; + const first = try rig_.pipeline(device, sources.lfm2, kernels[0]); + errdefer first.deinit(); + const second = try rig_.pipeline(device, sources.lfm2, kernels[1]); + return .{ .a = a, .rig = .{ .a = a, .device = device, .queue = queue }, .which = w, .pipes = .{ first, second } }; + } + + fn drop(p: *Probe) void { + inline for (.{ "src", "w1", "codes", "scales", "biases", "ids" }) |f| { + p.a.free(@field(p, f)); + @field(p, f) = &.{}; + } + p.rig.reset(); + } + + pub fn deinit(p: *Probe) void { + p.drop(); + p.rig.deinit(); + for (p.pipes) |x| x.deinit(); + } + + const vt = g.hooks(Probe); + pub fn kernel(p: *Probe) g.Kernel { + return .{ .ctx = p, .vt = &vt, .spec = spec(p.which) }; + } + + fn fill(out: []u16, seed: u64, f: g.Fill) void { + for (out, 0..) |*v, i| v.* = value(i, seed, f); + } + + pub fn prepare(p: *Probe, s: c.Shape, f: g.Fill, seed: u64) !bool { + p.drop(); + p.rows = s.rows; + if (s.n != vocab or s.k != rank or s.rows == 0) return false; + // each stream's step row sits a block (two rows here) apart, as a hold's rows of one step do + p.src = try p.a.alloc(u16, 2 * s.rows * vocab); + fill(p.src, seed, f); + p.w1 = try p.a.alloc(u16, table * rank); + fill(p.w1, seed +% 5, .random); + p.codes = try p.a.alloc(u8, vocab * rank); + for (p.codes, 0..) |*q, i| q.* = @truncate((@as(u64, i) *% 2654435761 +% seed) >> 11); + p.scales = try p.a.alloc(u16, vocab * rank / 64); + p.biases = try p.a.alloc(u16, vocab * rank / 64); + for (p.scales, p.biases, 0..) |*sc, *bi, i| { + sc.* = c.toBf16(@as(f32, @floatFromInt(i % 7 + 1)) / 4096); + bi.* = c.toBf16(@as(f32, @floatFromInt(i % 5)) / 64 - 0.03); + } + // the previous token every second word (the picks' layout): inside W1, its last row, or past the vocabulary + p.ids = try p.a.alloc(u32, 2 * s.rows); + for (p.ids, 0..) |*id, i| id.* = if (i % 2 == 1) 0xdead else switch ((i / 2) % 3) { + 0 => @intCast((@as(u64, i) * 37 + seed) % table), + 1 => table - 1, + else => vocab + @as(u32, @intCast(i)), + }; + p.args[0] = try p.rig.input(std.mem.sliceAsBytes(p.src)); + p.args[1] = try p.rig.input(std.mem.sliceAsBytes(p.w1)); + p.args[2] = try p.rig.input(p.codes); + p.args[3] = try p.rig.input(std.mem.sliceAsBytes(p.ids)); + const args = [4]u32{ 2 * vocab, rank, vocab, 2 }; + p.args[4] = try p.rig.input(std.mem.asBytes(&args)); + p.args[5] = try p.rig.input(std.mem.sliceAsBytes(p.scales)); + p.args[6] = try p.rig.input(std.mem.sliceAsBytes(p.biases)); + const n: u32 = groups; + p.args[7] = try p.rig.input(std.mem.asBytes(&n)); + p.y = try p.rig.output(p.count()); + return true; + } + + pub fn count(p: *Probe) usize { + return switch (p.which) { + .markov_step => p.rows * 8, + }; + } + + pub fn run(p: *Probe) !void { + const pass = p.rig.begin(); + const e = pass.e; + const best = try p.rig.scratch(p.rows * groups * 4); + const at = try p.rig.scratch(p.rows * groups * 4); + e.setPipeline(p.pipes[0]); + for (p.args[0..5], 0..) |r, i| e.setBuffer(r.buf, r.off, i); + e.setBuffer(best.buf, best.off, 5); + e.setBuffer(at.buf, at.off, 6); + e.setBuffer(p.args[5].buf, p.args[5].off, 7); + e.setBuffer(p.args[6].buf, p.args[6].off, 8); + e.dispatchGroups(mtl.Size.of(groups, p.rows, 1), mtl.Size.of(256, 1, 1)); + e.setPipeline(p.pipes[1]); + e.setBuffer(best.buf, best.off, 0); + e.setBuffer(at.buf, at.off, 1); + e.setBuffer(p.args[7].buf, p.args[7].off, 2); + e.setBuffer(p.y.buf, p.y.off, 3); + e.dispatchGroups(mtl.Size.of(p.rows, 1, 1), mtl.Size.of(256, 1, 1)); + try p.rig.finish(pass); + } + + pub fn reference(_: *Probe, want: []f64, mag: []f64) !void { + @memset(mag, 0); + @memset(want, 0); // the step and the ids check themselves (check) + } + + /// Each stream's pick against float64 scores with the kernel's bf16 roundings: the best, but for near ties. + pub fn check(p: *Probe, _: g.Gate) !?[]const u8 { + const out = p.rig.bytes(p.y, p.count()); + for (0..p.rows) |s| { + const token = std.mem.readInt(u32, out[8 * s ..][0..4], .little); + const flag = std.mem.readInt(u32, out[8 * s + 4 ..][0..4], .little); + const id = p.ids[2 * s]; + var top: f64 = -std.math.inf(f64); + var mine: ?f64 = null; + var nan = false; + var bound: f64 = 0; + for (0..vocab) |v| { + var dot: f64 = 0; + var size: f64 = 0; + if (id < table) for (0..rank) |i| { + const k = (v * rank + i) / 64; + const w = @as(f64, c.fromBf16(p.scales[k])) * @as(f64, @floatFromInt(p.codes[v * rank + i])) + c.fromBf16(p.biases[k]); + dot += w * c.fromBf16(p.w1[id * rank + i]); + size += @abs(w * c.fromBf16(p.w1[id * rank + i])); + }; + const base: f32 = c.fromBf16(p.src[2 * s * vocab + v]); + const score: f64 = c.fromBf16(c.toBf16(base + c.fromBf16(c.toBf16(@floatCast(dot))))); + if (std.math.isNan(score)) nan = true; + if (score > top) top = score; + if (v == token) mine = score; + // two bf16 steps of the score and the fp32 sum's own error + bound = @max(bound, @abs(score) * 0x1p-7 + c.gamma(rank, 0x1p-24) * size); + } + if (nan) { + if (flag != 1) return "a row with a NaN score is not flagged"; + continue; + } + if (flag != 0) return "a finite row is flagged nonfinite"; + const got = mine orelse return "the pick is past the vocabulary"; + if (got < top - bound) return "the pick's score is below the row's best by more than its bound"; + } + return null; + } +}; + +/// Gate 5 for the scan and pick on every Apple chip and this GPU: 256 threads each, 1,088 and 3,072 threadgroup bytes. +pub fn limits(a: std.mem.Allocator, device: mtl.Device, only: ?[]const u8) !usize { + var problems: usize = 0; + const launches = [_]struct { []const u8, []const u8, u32, u32 }{ .{ "metal lfm2 markov scan", "lfm2_markov_scan", 1088, 256 }, .{ "metal lfm2 markov pick", "lfm2_markov_pick", 3072, 256 } }; + for (launches) |l| { + if (only) |s| if (std.mem.indexOf(u8, l[0], s) == null) continue; + const d = contract.limits.MetalLaunch{ .name = l[0], .threads = l[3], .tg_memory = l[2], .offline = false }; + var notes: std.ArrayList(u8) = .empty; + defer notes.deinit(a); + for (contract.limits.apple) |chip| problems += try contract.limits.metal(d, chip, ¬es, a); + std.debug.print("{s}", .{notes.items}); + problems += try @import("metal_limits.zig").onThisGpu(a, device, .{ .decl = d, .function = l[1], .here = sources.lfm2, .pre_m5 = "" }, false); + } + return problems; +} diff --git a/zig/tests/contract/metal_main.zig b/zig/tests/contract/metal_main.zig index 7f44fbb441..96020f9ff3 100644 --- a/zig/tests/contract/metal_main.zig +++ b/zig/tests/contract/metal_main.zig @@ -28,6 +28,8 @@ const lfm2_argmax = @import("metal_lfm2_argmax.zig"); const lfm2_streams = @import("metal_lfm2_streams.zig"); const lfm2_tap = @import("metal_lfm2_tap.zig"); const lfm2_controls = @import("metal_lfm2_controls.zig"); +const lfm2_dspark = @import("metal_lfm2_dspark.zig"); +const lfm2_attn = @import("metal_lfm2_attn.zig"); const g = contract.gates; const usage = "usage: tf-kernel-contract [--only TEXT] [--no-gates] [--no-limits] [--offline DIR] [--seed N] [--repro] [--rotary-long] [--rotary-accuracy FILE]"; @@ -208,6 +210,21 @@ pub fn main(init: std.process.Init) !u8 { try tap_list.append(a, p); try kernels.append(a, p.kernel()); } + var dspark_list: [1]lfm2_dspark.Probe = undefined; + var lfm2_made: usize = 0; + defer for (dspark_list[0..lfm2_made]) |*p| p.deinit(); + for (std.enums.values(lfm2_dspark.Which)) |w| { + if (!wanted(o, lfm2_dspark.spec(w).name)) continue; + dspark_list[lfm2_made] = try lfm2_dspark.Probe.init(a, device, queue, w); + lfm2_made += 1; + } + for (dspark_list[0..lfm2_made]) |*p| try kernels.append(a, p.kernel()); + var block_attn: ?lfm2_attn.Probe = null; + defer if (block_attn) |*p| p.deinit(); + if (wanted(o, lfm2_attn.name)) { + block_attn = try lfm2_attn.Probe.init(a, device, queue); + try kernels.append(a, block_attn.?.kernel()); + } var rows_list: std.ArrayList(*rows_k.Row) = .empty; var proj_list: std.ArrayList(*affine_k.Proj) = .empty; defer for (rows_list.items) |r| r.deinit(); @@ -399,6 +416,8 @@ pub fn main(init: std.process.Init) !u8 { lim += try gemma.limits(a, device, o.only); lim += try gemma_embed.limits(a, device, o.only); lim += try gemma_mixed.limits(a, device, o.only); + lim += try lfm2_dspark.limits(a, device, o.only); + lim += try lfm2_attn.limits(a, device, o.only); } const t2 = std.Io.Clock.awake.now(init.io); const ms = struct { diff --git a/zig/tests/lfm2_dspark.zig b/zig/tests/lfm2_dspark.zig new file mode 100644 index 0000000000..09c7f68a52 --- /dev/null +++ b/zig/tests/lfm2_dspark.zig @@ -0,0 +1,301 @@ +//! LFM2's DSpark drafter on the GPU against an fp32 host reference of the released model, on the real weights. +const std = @import("std"); +const mtl = @import("metal"); +const tf = @import("tensorfold"); +const q = tf.lfm2_moe; +const c = q.config; +const st = q.state; +const ck = tf.checkpoint; +const quant = q.dspark_quant; + +const hd = c.head_dim; +const kw = c.kv_heads * hd; +const qw = c.query_heads * hd; + +/// The drafter's tensors as fp32 on the host, in the checkpoint's own layout (interleaved RoPE pairs). +const Ref = struct { + gpa: std.mem.Allocator, + check: ck.Checkpoint, + cfg: q.dspark_config.Config, + + fn get(r: *const Ref, name: []const u8) ![]f32 { + const t = try r.check.get(name); + const src = t.host(u16); + const out = try r.gpa.alloc(f32, src.len); + for (out, src) |*o, v| o.* = quant.toF32(v); + return out; + } + + fn layer(r: *const Ref, buf: []u8, i: usize, part: []const u8) ![]f32 { + return r.get(try std.fmt.bufPrint(buf, "layers.{d}.{s}.weight", .{ i, part })); + } +}; + +/// y[n] = W[n, k] x[k] for each of `rows` rows, on every core. +fn matmul(gpa: std.mem.Allocator, w: []const f32, x: []const f32, y: []f32, n: usize, k: usize, rows: usize) !void { + const Job = struct { + fn run(w_: []const f32, x_: []const f32, y_: []f32, n_: usize, k_: usize, rows_: usize, from: usize, to: usize) void { + for (0..rows_) |r| for (from..to) |col| { + var s: f32 = 0; + const wr = w_[col * k_ ..][0..k_]; + const xr = x_[r * k_ ..][0..k_]; + for (wr, xr) |a, b| s += a * b; + y_[r * n_ + col] = s; + }; + } + }; + const threads = @min(std.Thread.getCpuCount() catch 1, 16); + const per = (n + threads - 1) / threads; + const hs = try gpa.alloc(std.Thread, threads); + defer gpa.free(hs); + var made: usize = 0; + var at: usize = 0; + while (at < n) : (at += per) { + hs[made] = try std.Thread.spawn(.{}, Job.run, .{ w, x, y, n, k, rows, at, @min(n, at + per) }); + made += 1; + } + for (hs[0..made]) |h| h.join(); +} + +fn rms(x: []f32, gain: []const f32) void { + var ss: f32 = 0; + for (x) |v| ss += v * v; + const inv = 1 / @sqrt(ss / @as(f32, @floatFromInt(x.len)) + c.eps); + for (x, gain) |*v, g| v.* = v.* * inv * g; +} + +/// Interleaved RoPE at `pos` over each head of `x` (pairs 2i, 2i + 1). +fn rope(x: []f32, pos: u64, theta: f32) void { + var h: usize = 0; + while (h < x.len) : (h += hd) for (0..hd / 2) |i| { + const angle = @as(f32, @floatFromInt(pos)) * std.math.pow(f32, theta, -@as(f32, @floatFromInt(i)) / (hd / 2)); + const a = x[h + 2 * i]; + const b = x[h + 2 * i + 1]; + x[h + 2 * i] = a * @cos(angle) - b * @sin(angle); + x[h + 2 * i + 1] = a * @sin(angle) + b * @cos(angle); + }; +} + +/// The target's embedding row `id`, dequantized from its 6-bit codes (MLX's scale * q + bias). +fn embedRow(m: *q.Model, id: u32, out: []f32) void { + const e = m.weights.embedding; + const codes = (e.w.contents() + e.w_off)[@as(usize, id) * c.hidden * 6 / 8 ..][0 .. c.hidden * 6 / 8]; + const sc = @as([*]const u16, @ptrCast(@alignCast(e.scales.contents() + e.s_off)))[@as(usize, id) * c.hidden / 64 ..][0 .. c.hidden / 64]; + const bi = @as([*]const u16, @ptrCast(@alignCast(e.biases.contents() + e.b_off)))[@as(usize, id) * c.hidden / 64 ..][0 .. c.hidden / 64]; + for (0..c.hidden / 4) |u| { + const w = codes[u * 3 ..][0..3]; + const qs = [4]u32{ w[0] & 0x3f, (w[0] >> 6) | ((@as(u32, w[1]) & 0x0f) << 2), (w[1] >> 4) | ((@as(u32, w[2]) & 0x03) << 4), w[2] >> 2 }; + for (qs, 0..) |qv, j| { + const g = (u * 4 + j) / 64; + out[u * 4 + j] = quant.toF32(sc[g]) * @as(f32, @floatFromInt(qv)) + quant.toF32(bi[g]); + } + } +} + +/// GPU bf16 row against the reference's row + bias: cosine and worst difference. +fn compare(label: []const u8, gpu: []const u16, row: []const f32, bias: []const f32) void { + var dot: f64 = 0; + var gg: f64 = 0; + var rr: f64 = 0; + var worst: f32 = 0; + for (gpu, row, bias) |g, b, m2| { + const x = quant.toF32(g); + const y = b + m2; + dot += x * y; + gg += x * x; + rr += y * y; + worst = @max(worst, @abs(x - y)); + } + std.debug.print("{s}: cosine {d:.5}, worst difference {d:.3}\n", .{ label, dot / @sqrt(gg * rr), worst }); +} + +fn absorb(d: *q.dspark.Drafter, piece: q.dspark_block.Piece) !void { + const run = q.dspark_block.begin(d); + try q.dspark_block.absorbPass(d, run.e, &.{piece}); + try q.dspark_block.end(run.cb, run.e); +} + +pub fn main(init: std.process.Init) !void { + const gpa = init.gpa; + const args = try init.minimal.args.toSlice(init.arena.allocator()); + if (args.len < 4 or args.len > 6) return error.ExpectedModelDrafterPromptLengthRowsBits; + const bits: u8 = if (args.len == 6) try std.fmt.parseInt(u8, args[5], 10) else 8; + const length = try std.fmt.parseInt(usize, args[3], 10); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const m = try q.Model.load(gpa, init.io, args[1]); + defer m.deinit(); + const d = try q.dspark.Drafter.init(gpa, init.io, m, args[2], .{ .capacity = length + 64, .bits = bits }); + defer d.deinit(); + const cfg = d.config; + // the compared block's rows (a hold of depth d runs a d-row block) + const block: u32 = if (args.len >= 5) try std.fmt.parseInt(u32, args[4], 10) else cfg.block; + if (block == 0 or block > cfg.block) return error.BadBlockRows; + var scratch = try st.Scratch.init(gpa, m.device, &m.config.attention, length); + defer scratch.deinit(); + const width = cfg.tap_count * c.hidden; + var taps: st.Taps = .{ .count = cfg.tap_count, .buf = try scratch.memory.alloc(length * width * 2) }; + @memcpy(taps.layers[0..cfg.tap_count], cfg.taps[0..cfg.tap_count]); + scratch.taps = taps; + var cache = try st.Cache.init(gpa, m.device, &m.config.attention, length + 16); + defer cache.deinit(); + const ids = try gpa.alloc(u32, length); + defer gpa.free(ids); + for (ids, 0..) |*id, i| id.* = @intCast((i * 7919 + 13) % 120000); + try q.forward.run(m, &scratch, &.{.{ .cache = &cache, .rows = length }}, ids, false, .last); + // the context: rows [0, length - 1); the pending token is the last id, at position length - 1 + const ctx = length - 1; + const lane = try d.newLane(length + 32); + defer d.dropLane(lane); + var at: usize = 0; + while (at < ctx) : (at += q.dspark.absorb_rows) { + const rows = @min(q.dspark.absorb_rows, ctx - at); + try absorb(d, .{ .lane = lane, .src = .{ .buf = taps.buf, .off = at * width * 2 }, .rows = rows, .p0 = at }); + } + lane.end = ctx; + try q.dspark_block.hold(d, &.{.{ .lane = lane, .pending = ids[ctx], .anchor = ctx, .depth = block }}); + var held: [16]u32 = undefined; + @memcpy(held[0..block], lane.held[0..block]); + const gpu = held[0..block]; + for ([_]usize{ 1, 4, 16 }) |rows| { + var best: f64 = std.math.floatMax(f64); + for (0..5) |_| { + const t0 = std.Io.Clock.awake.now(init.io); + try absorb(d, .{ .lane = lane, .src = .{ .buf = taps.buf, .off = (ctx - rows) * width * 2 }, .rows = rows, .p0 = ctx - rows }); + best = @min(best, @as(f64, @floatFromInt(t0.durationTo(std.Io.Clock.awake.now(init.io)).toNanoseconds())) / 1e6); + } + std.debug.print("absorb of {d} rows: {d:.2} ms (best of 5)\n", .{ rows, best }); + } + const depths = [_]u32{ 1, 3, cfg.block }; + for (depths) |depth| { + var best: f64 = std.math.floatMax(f64); + for (0..5) |_| { + const t0 = std.Io.Clock.awake.now(init.io); + try q.dspark_block.hold(d, &.{.{ .lane = lane, .pending = ids[ctx], .anchor = ctx, .depth = depth }}); + best = @min(best, @as(f64, @floatFromInt(t0.durationTo(std.Io.Clock.awake.now(init.io)).toNanoseconds())) / 1e6); + } + std.debug.print("hold at depth {d}: {d:.2} ms (best of 5)\n", .{ depth, best }); + } + try q.dspark_block.hold(d, &.{.{ .lane = lane, .pending = ids[ctx], .anchor = ctx, .depth = block }}); + // the host reference, fp32 throughout + var r: Ref = .{ .gpa = gpa, .check = ck.Checkpoint.init(gpa), .cfg = cfg }; + defer r.check.deinit(); + const path = try std.fs.path.joinZ(gpa, &.{ args[2], "model.safetensors" }); + defer gpa.free(path); + try r.check.addFile(m.device, path, ""); + var arena = std.heap.ArenaAllocator.init(gpa); + defer arena.deinit(); + r.gpa = arena.allocator(); + const a = r.gpa; + const n = block; + const tap_rows = taps.buf.slice(u16, length * width); + const feats = try a.alloc(f32, ctx * width); + for (feats, tap_rows[0 .. ctx * width]) |*f, v| f.* = quant.toF32(v); + const hctx = try a.alloc(f32, ctx * c.hidden); + try matmul(gpa, try r.get("fc.weight"), feats, hctx, c.hidden, width, ctx); + const hn = try r.get("hidden_norm.weight"); + for (0..ctx) |j| rms(hctx[j * c.hidden ..][0..c.hidden], hn); + const h = try a.alloc(f32, n * c.hidden); + for (0..n) |i| embedRow(m, if (i == 0) ids[ctx] else cfg.mask, h[i * c.hidden ..][0..c.hidden]); + const keys = ctx + n; + var name: [96]u8 = undefined; + const x = try a.alloc(f32, n * c.hidden); + for (0..cfg.layers) |li| { + const kmat = try r.layer(&name, li, "self_attn.k_proj"); + const vmat = try r.layer(&name, li, "self_attn.v_proj"); + const qn = try r.layer(&name, li, "self_attn.q_norm"); + const kn = try r.layer(&name, li, "self_attn.k_norm"); + const kk = try a.alloc(f32, keys * kw); + const vv = try a.alloc(f32, keys * kw); + try matmul(gpa, kmat, hctx, kk, kw, c.hidden, ctx); + try matmul(gpa, vmat, hctx, vv, kw, c.hidden, ctx); + @memcpy(x, h); + for (0..n) |i| rms(x[i * c.hidden ..][0..c.hidden], try r.layer(&name, li, "input_layernorm")); + const qq = try a.alloc(f32, n * qw); + try matmul(gpa, try r.layer(&name, li, "self_attn.q_proj"), x, qq, qw, c.hidden, n); + try matmul(gpa, kmat, x, kk[ctx * kw ..], kw, c.hidden, n); + try matmul(gpa, vmat, x, vv[ctx * kw ..], kw, c.hidden, n); + for (0..keys) |j| for (0..c.kv_heads) |kh| { + const row = kk[j * kw + kh * hd ..][0..hd]; + rms(row, kn); + rope(row, j, cfg.theta); + }; + for (0..n) |i| for (0..c.query_heads) |qh| { + const row = qq[i * qw + qh * hd ..][0..hd]; + rms(row, qn); + rope(row, ctx + i, cfg.theta); + }; + const o = try a.alloc(f32, n * qw); + const sc = try a.alloc(f32, keys); + for (0..n) |i| for (0..c.query_heads) |qh| { + const kh = qh / (c.query_heads / c.kv_heads); + var top: f32 = -std.math.inf(f32); + for (0..keys) |j| { + var s: f32 = 0; + for (0..hd) |t| s += qq[i * qw + qh * hd + t] * kk[j * kw + kh * hd + t]; + sc[j] = s / @sqrt(@as(f32, hd)); + top = @max(top, sc[j]); + } + var sum: f32 = 0; + for (sc) |*s| { + s.* = @exp(s.* - top); + sum += s.*; + } + for (0..hd) |t| { + var acc: f32 = 0; + for (0..keys) |j| acc += sc[j] * vv[j * kw + kh * hd + t]; + o[i * qw + qh * hd + t] = acc / sum; + } + }; + const y = try a.alloc(f32, n * c.hidden); + try matmul(gpa, try r.layer(&name, li, "self_attn.o_proj"), o, y, c.hidden, qw, n); + for (h, y) |*v, w| v.* += w; + @memcpy(x, h); + for (0..n) |i| rms(x[i * c.hidden ..][0..c.hidden], try r.layer(&name, li, "post_attention_layernorm")); + const gate = try a.alloc(f32, n * cfg.intermediate); + const up = try a.alloc(f32, n * cfg.intermediate); + try matmul(gpa, try r.layer(&name, li, "mlp.gate_proj"), x, gate, cfg.intermediate, c.hidden, n); + try matmul(gpa, try r.layer(&name, li, "mlp.up_proj"), x, up, cfg.intermediate, c.hidden, n); + for (gate, up) |*g, u| g.* = g.* / (1 + @exp(-g.*)) * u; + try matmul(gpa, try r.layer(&name, li, "mlp.down_proj"), gate, y, c.hidden, cfg.intermediate, n); + for (h, y) |*v, w| v.* += w; + } + const fin = try r.get("norm.weight"); + for (0..n) |i| rms(h[i * c.hidden ..][0..c.hidden], fin); + const head = try a.alloc(f32, c.vocab * c.hidden); + for (0..c.vocab) |v| embedRow(m, @intCast(v), head[v * c.hidden ..][0..c.hidden]); + const base = try a.alloc(f32, n * c.vocab); + try matmul(gpa, head, h, base, c.vocab, c.hidden, n); + const w1 = try r.get("markov_head.markov_w1.weight"); + const w2 = try r.get("markov_head.markov_w2.weight"); + const bias = try a.alloc(f32, c.vocab); + var prev = ids[ctx]; + var same: usize = 0; + var worst: f32 = 0; + for (0..n) |i| { + try matmul(gpa, w2, w1[@as(usize, prev) * cfg.rank ..][0..cfg.rank], bias, c.vocab, cfg.rank, 1); + const row = base[i * c.vocab ..][0..c.vocab]; + var best: usize = 0; + var second: usize = 1; + for (row, bias, 0..) |b, m2, v| { + const s = b + m2; + if (s > row[best] + bias[best]) { + second = best; + best = v; + } else if (v != best and s > row[second] + bias[second]) second = v; + } + const margin = (row[best] + bias[best]) - (row[second] + bias[second]); + if (i == 0) { // the GPU keeps the base logits (the Markov step adds its bias on the way to the pick) + const zero = try a.alloc(f32, c.vocab); + @memset(zero, 0); + compare("row 0 base logits", d.s.logits.slice(u16, c.vocab), row, zero); + } + const ok = gpu[i] == best; + same += @intFromBool(ok); + std.debug.print("step {d}: reference {d} (margin {d:.3}, second {d}), GPU {d} {s}\n", .{ i, best, margin, second, gpu[i], if (ok) "equal" else "DIFFERENT" }); + if (!ok) worst = @max(worst, margin); + prev = gpu[i]; // teacher-forced: step i + 1 follows the GPU's pick, so each step is compared alone + } + std.debug.print("the reference's largest margin over a different GPU pick: {d:.3}\n", .{worst}); + std.debug.print("{d} of {d} drafts equal the fp32 reference's picks on the same chain ({d}-token context)\n", .{ same, n, ctx }); +} From 2ecd7f1f4a9118fe91e67b303ab3dd7c66c38a4e Mon Sep 17 00:00:00 2001 From: BlivionIaG <15323295+BlivionIaG@users.noreply.github.com> Date: Mon, 26 Oct 2026 09:30:52 +0100 Subject: [PATCH 04/39] lfm2: a DSpark hold is not waited on; the verify copies its drafts on the GPU A hold that fits in one pass signals a GPU event and returns without a wait. The next verify waits on that event on the GPU, and a new kernel, lfm2_draft_ids, copies each window's drafts from the drafter's picks into the verify's ids. The host reads the picks only after the verify, or when a grammar, an absorb, a keep or a release needs them. Multi-pass holds stay synchronous. The LFM2 backend takes held windows (Window.held) from a HeldSource the drafter gives, and Drafted.held_on_target passes windows without fillers to it as they are. lfm2_draft_ids registers with the kernel contract (metal lfm2 draft ids): each stream's first pick, or 0 for a token past the vocabulary, copied exactly; gate 5 against every Apple chip we serve and this GPU. --- zig/kernels/metal/lfm2/lfm2_dspark.metal | 8 ++ zig/src/families/lfm2_moe/backend.zig | 34 ++++++- zig/src/families/lfm2_moe/dspark.zig | 73 +++++++++++++- zig/src/families/lfm2_moe/dspark_block.zig | 27 ++++- zig/src/families/lfm2_moe/forward.zig | 18 ++++ zig/src/families/lfm2_moe/kernels.zig | 1 + zig/src/native/lfm2_moe.zig | 7 +- zig/tests/contract/metal_lfm2_dspark.zig | 44 +++++++- zig/tests/contract/metal_main.zig | 111 +++++++++++++++------ zig/tests/lfm2_dspark.zig | 6 +- 10 files changed, 280 insertions(+), 49 deletions(-) diff --git a/zig/kernels/metal/lfm2/lfm2_dspark.metal b/zig/kernels/metal/lfm2/lfm2_dspark.metal index 995d63a20a..050fd5e8cc 100644 --- a/zig/kernels/metal/lfm2/lfm2_dspark.metal +++ b/zig/kernels/metal/lfm2/lfm2_dspark.metal @@ -97,3 +97,11 @@ constant constexpr uint MARKOV_SPAN = 512; picks[2 * s + 1] = tn[0]; } } + +// A verify window's held drafts from the drafter's picks: picks[i * stride] into ids[i], a token past the vocabulary 0. +[[kernel]] void lfm2_draft_ids(const device uint* picks [[buffer(0)]], constant uint4& p [[buffer(1)]], + device uint* ids [[buffer(2)]], uint i [[thread_position_in_grid]]) { + if (i >= p.y) return; + const uint token = picks[size_t(i) * p.x]; + ids[i] = token < p.z ? token : 0u; +} diff --git a/zig/src/families/lfm2_moe/backend.zig b/zig/src/families/lfm2_moe/backend.zig index 44427d435a..8af90dc43b 100644 --- a/zig/src/families/lfm2_moe/backend.zig +++ b/zig/src/families/lfm2_moe/backend.zig @@ -10,6 +10,14 @@ const snapshot = @import("snapshot.zig"); const Model = @import("model.zig").Model; const be = lanes.backend; +/// Where a drafter holds a stream's drafts on the device, for verify windows that take them there (Window.held). +pub const HeldSource = struct { + ptr: *anyopaque, + get: *const fn (ptr: *anyopaque, s: *lanes.Stream) ?Device, + host: *const fn (ptr: *anyopaque, s: *lanes.Stream, out: []u32) anyerror!void, // where `get` gives none +}; +pub const Device = struct { src: mtl.Buffer, off: usize, stride: u32, wait: fwd.Wait }; + pub const Options = struct { capacity: usize, chunk: usize = 128, streams: u32 = 8 }; pub const Metal = struct { @@ -25,6 +33,8 @@ pub const Metal = struct { timed: usize = 0, /// Forwards run so far: a stream's tapped rows in the scratch are current while their gen matches. gen: u64 = 0, + /// A drafter whose drafts verify windows read on the device (null: windows carry host tokens only). + held_source: ?HeldSource = null, pub fn init(gpa: std.mem.Allocator, model: *Model, options: Options) !*Metal { const out = try gpa.create(Metal); @@ -246,20 +256,34 @@ pub const Metal = struct { const self = cast(ptr); var segments: [st.batch_rows]fwd.Segment = undefined; var ids: [st.batch_rows]u32 = undefined; + var held: [st.batch_rows]fwd.Held = undefined; + var count: usize = 0; + var wait: ?fwd.Wait = null; var total: usize = 0; if (windows.len > segments.len or windows.len != out.len) return error.InvalidLfmWindows; for (windows, 0..) |w, i| { const rows = w.rows(); - if (w.parents != null or w.held != 0 or rows > st.window_rows or total + rows > st.batch_rows or w.positions.len != rows) return error.UnsupportedLfmWindow; + if (w.held != 0) { // the drafter's picks, copied into the window's rows on the GPU + const source = self.held_source orelse return error.UnsupportedLfmWindow; + if (total + rows > st.batch_rows) return error.UnsupportedLfmWindow; + if (source.get(source.ptr, w.stream)) |dev| { + held[count] = .{ .row = total + 1, .src = dev.src, .off = dev.off, .stride = dev.stride, .count = w.held }; + count += 1; + wait = if (wait) |x| (if (x.value >= dev.wait.value) x else dev.wait) else dev.wait; + @memset(ids[total + 1 ..][0..w.held], 0); + } else try source.host(source.ptr, w.stream, ids[total + 1 ..][0..w.held]); + } + if (w.parents != null or rows > st.window_rows or total + rows > st.batch_rows or w.positions.len != rows) return error.UnsupportedLfmWindow; if (w.masks.len != 0 and w.masks.len != rows * mask_words) return error.InvalidLfmMasks; const cache = try self.cacheOf(w.stream); for (w.positions, 0..) |p, r| if (p != cache.len + r + 1) return error.InvalidLfmPosition; segments[i] = .{ .cache = cache, .rows = rows }; ids[total] = w.pending; - @memcpy(ids[total + 1 ..][0..w.tokens.len], w.tokens); + @memcpy(ids[total + 1 + w.held ..][0..w.tokens.len], w.tokens); total += rows; } - try self.run(segments[0..windows.len], ids[0..total], true, .all); + self.gen += 1; + try fwd.runHeld(self.model, &self.scratch, segments[0..windows.len], ids[0..total], true, .all, held[0..count], wait); if (self.scratch.taps != null) for (segments[0..windows.len]) |seg| { const last = seg.cache.last.?; seg.cache.tap = .{ .start = last.start, .row = last.base, .rows = last.rows, .gen = self.gen }; @@ -268,7 +292,7 @@ pub const Metal = struct { const best = self.scratch.best.slice(u32, 2 * total); var base: usize = 0; for (windows, out) |w, o| { - if (o.sampled.len != w.rows() or o.drafts.len != w.tokens.len) return error.InvalidLfmOutputs; + if (o.sampled.len != w.rows() or o.drafts.len != w.rows() - 1) return error.InvalidLfmOutputs; for (o.sampled, w.positions, 0..) |*t, p, r| { const mask = if (w.masks.len > 0) w.masks[r * mask_words ..][0..mask_words] else &.{}; // A greedy row takes the GPU's pick, the host's draw on the same logits; a nonfinite row errors there. @@ -279,7 +303,7 @@ pub const Metal = struct { } t.* = try draw(self.gpa, logits[(base + r) * c.vocab ..][0..c.vocab], w.stream.sampling, p, mask); } - @memcpy(o.drafts, w.tokens); + @memcpy(o.drafts, self.scratch.ids.slice(u32, base + w.rows())[base + 1 ..][0 .. w.rows() - 1]); base += w.rows(); } } diff --git a/zig/src/families/lfm2_moe/dspark.zig b/zig/src/families/lfm2_moe/dspark.zig index 339071ac6c..850c1ce34c 100644 --- a/zig/src/families/lfm2_moe/dspark.zig +++ b/zig/src/families/lfm2_moe/dspark.zig @@ -9,6 +9,7 @@ const block = @import("dspark_block.zig"); const st = @import("state.zig"); const snapshot = @import("snapshot.zig"); const Model = @import("model.zig").Model; +const backend = @import("backend.zig"); const dr = lanes.drafter; const Stream = lanes.Stream; @@ -34,6 +35,7 @@ pub const Lane = struct { staged: u32 = 0, held: [16]u32 = @splat(0), held_len: u32 = 0, + slot: ?u32 = null, // its block's place in the last signalled hold, whose picks still hold its drafts }; /// The passes' working buffers, one pass at a time on the target's queue. @@ -75,11 +77,19 @@ pub const Drafter = struct { /// One stream's hold wall ms at full depth, and that over the block (the depth rule's cost a draft). hold_ms: f64 = 0, step_ms: f64 = 0, + event: mtl.Event = undefined, + signalled: u64 = 0, + pending: ?mtl.CommandBuffer = null, + pass: [pass_rows]*Lane = undefined, // the signalled hold's lanes and depths, in its blocks' order + depths: [pass_rows]u32 = undefined, + pass_len: usize = 0, pub fn init(gpa: std.mem.Allocator, io: std.Io, target: *Model, dir: []const u8, options: Options) !*Drafter { const d = try gpa.create(Drafter); errdefer gpa.destroy(d); d.* = .{ .gpa = gpa, .io = io, .target = target, .config = try dc.Config.read(gpa, io, dir), .w = undefined, .options = options, .s = undefined }; + d.event = try target.device.event(); + errdefer d.event.deinit(); d.w = try dw.load(gpa, target.device, io, dir, &d.config, options.bits); errdefer d.w.deinit(gpa); errdefer for (d.owned.items) |b| b.deinit(); @@ -90,6 +100,8 @@ pub const Drafter = struct { } pub fn deinit(d: *Drafter) void { + d.settle() catch {}; + d.event.deinit(); var it = d.lanes.valueIterator(); while (it.next()) |l| d.dropLane(l.*); d.lanes.deinit(d.gpa); @@ -232,11 +244,13 @@ pub const Drafter = struct { fn releaseFn(ptr: *anyopaque, s: *Stream) void { const d = cast(ptr); + d.unpass() catch {}; if (d.lanes.fetchRemove(s)) |e| d.dropLane(e.value); } fn heldFn(ptr: *anyopaque, streams: []const *Stream, out: []const []u32) anyerror!void { const d = cast(ptr); + try d.settle(); for (streams, out) |s, o| { const l = d.lanes.get(s) orelse return error.DraftStreamUnknown; if (o.len > l.held_len) return error.DraftHeldFewer; @@ -252,6 +266,7 @@ pub const Drafter = struct { /// Stream `s`'s context copied into `snap` when its lane stands at the snapshot's position (a prompt chunk's end). pub fn keepLane(d: *Drafter, s: *Stream, snap: *snapshot.Snap) !void { const l = d.lanes.get(s) orelse return; + try d.settle(); try d.flush(l); if (l.end != snap.at) return; const bytes = try d.gpa.alloc(u8, @intCast(d.keptBytes(snap.at))); @@ -283,6 +298,7 @@ pub const Drafter = struct { /// Rows from `start` replace the context there: verified rows wait on the host for the next hold, others run now. fn absorbFn(ptr: *anyopaque, items: []const dr.Absorb) anyerror!void { const d = cast(ptr); + try d.settle(); const width = d.tapBytes(); const pool = mtl.objc.Pool.push(); defer pool.pop(); @@ -297,6 +313,7 @@ pub const Drafter = struct { if (it.start > l.end + l.staged) try d.resumeLane(it.stream, l, it.start); // a prompt pass that resumed if (it.start + f.rows + d.config.block > l.cap) return error.DraftContextGap; l.held_len = 0; + l.slot = null; const src = mtl.Buffer{ .id = @ptrFromInt(f.buffer) }; // rows from `start` replace what was staged past it; rows before the absorbed end go to the GPU now const keep: u32 = if (it.start >= l.end) @intCast(it.start - l.end) else 0; @@ -343,6 +360,7 @@ pub const Drafter = struct { /// Each stream's chain of `depth` drafts after `pending` at `position`, its block's rows in a pass of pass_rows. fn holdFn(ptr: *anyopaque, items: []const dr.Hold) anyerror!void { const d = cast(ptr); + try d.unpass(); var at: usize = 0; while (at < items.len) { // as many streams as their blocks' rows fit a pass: each block is as long as the pass's deepest chain @@ -364,8 +382,59 @@ pub const Drafter = struct { if (l.end + l.staged != anchor or anchor + d.config.block > l.cap) return error.DraftContextGap; b.* = .{ .lane = l, .pending = it.pending, .anchor = anchor, .depth = it.depth }; } - try block.hold(d, blocks[0..group.len]); + if (group.len == items.len) { // one pass: signalled, not waited on + d.signalled += 1; + d.pending = try block.hold(d, blocks[0..group.len], .{ .event = d.event, .value = d.signalled }); + for (blocks[0..group.len], 0..) |b, bi| { + d.pass[bi] = b.lane; + d.depths[bi] = b.depth; + b.lane.slot = @intCast(bi); + b.lane.held_len = 0; + } + d.pass_len = group.len; + continue; + } + _ = try block.hold(d, blocks[0..group.len], null); + } + } + + /// The signalled hold waited on, its picks read into its lanes' held chains (nothing pending: nothing done). + fn settle(d: *Drafter) !void { + const cb = d.pending orelse return; + d.pending = null; + defer cb.release(); + cb.wait(); + if (cb.failure()) |msg| { + std.log.err("lfm2 dspark: a hold failed on the GPU: {s}", .{msg}); + return error.DraftGpuFailed; + } + block.readPicks(d, d.pass[0..d.pass_len], d.depths[0..d.pass_len]); + } + + /// The signalled hold settled and its lanes' picks forgotten (a lane may be dropped, or a hold may overwrite). + fn unpass(d: *Drafter) !void { + defer { + for (d.pass[0..d.pass_len]) |l| l.slot = null; + d.pass_len = 0; } + try d.settle(); + } + + /// Where stream `s`'s drafts from the last signalled hold lie in the picks, for a verify to copy on the GPU. + pub fn device(ptr: *anyopaque, s: *Stream) ?backend.Device { + const d = cast(ptr); + const l = d.lanes.get(s) orelse return null; + const slot = l.slot orelse return null; + return .{ .src = d.s.picks, .off = slot * 8, .stride = pass_rows * 2, .wait = .{ .event = d.event, .value = d.signalled } }; + } + + /// The target's verify reading the held drafts on the device, or as host tokens where it cannot. + pub fn heldSource(d: *Drafter) backend.HeldSource { + return .{ .ptr = d, .get = device, .host = hostHeld }; + } + + fn hostHeld(ptr: *anyopaque, s: *Stream, out: []u32) anyerror!void { + return heldFn(ptr, &.{s}, &.{out}); } /// Holds on an empty context: the pipelines compiled, then one-draft and full-depth holds timed (best of three). @@ -375,7 +444,7 @@ pub const Drafter = struct { var best: [2]f64 = @splat(std.math.floatMax(f64)); for (0..4) |i| for ([_]u32{ 1, d.config.block }, &best) |depth, *b| { const t0 = std.Io.Clock.awake.now(d.io); - try block.hold(d, &.{.{ .lane = l, .pending = 1, .anchor = 0, .depth = depth }}); + _ = try block.hold(d, &.{.{ .lane = l, .pending = 1, .anchor = 0, .depth = depth }}, null); const ms = @as(f64, @floatFromInt(t0.durationTo(std.Io.Clock.awake.now(d.io)).toNanoseconds())) / 1e6; if (i > 0) b.* = @min(b.*, ms); }; diff --git a/zig/src/families/lfm2_moe/dspark_block.zig b/zig/src/families/lfm2_moe/dspark_block.zig index 22676dca6e..d3f6cbabb8 100644 --- a/zig/src/families/lfm2_moe/dspark_block.zig +++ b/zig/src/families/lfm2_moe/dspark_block.zig @@ -152,8 +152,11 @@ pub fn end(cb: mtl.CommandBuffer, e: mtl.ComputeEncoder) !void { return finish(cb, e); } -/// The blocks' drafts into their lanes' held chains: the backbone over every block's rows, then the Markov chain. -pub fn hold(d: *Drafter, blocks: []const Block) !void { +/// A GPU event a hold signals instead of being waited on, and the value it sets. +pub const Signal = struct { event: mtl.Event, value: u64 }; + +/// The blocks' drafts (backbone, then Markov chain) into the lanes' held chains; with `signal`, left for readPicks. +pub fn hold(d: *Drafter, blocks: []const Block, signal: ?Signal) !?mtl.CommandBuffer { const pool = mtl.objc.Pool.push(); defer pool.pop(); const s = &d.s; @@ -230,6 +233,13 @@ pub fn hold(d: *Drafter, blocks: []const Block) !void { x.norm(s.h, s.r, d.w.norm, true, s.x, tail); try x.project(fwd.matrix(emb), .{ .buf = s.x }, .{ .buf = s.logits }, tail); try markov(x, blocks.len, n, steps); + if (signal) |sg| { // committed unwaited, the command buffer returned retained + e.end(); + cb.signalEvent(sg.event, sg.value); + const kept = cb.retain(); + cb.commit(); + return kept; + } try finish(cb, e); const picks = s.picks.slice(Pick, d.config.block * ds.pass_rows); for (blocks, 0..) |b, bi| { @@ -240,6 +250,19 @@ pub fn hold(d: *Drafter, blocks: []const Block) !void { } b.lane.held_len = b.depth; } + return null; +} + +/// A finished signalled hold's picks into the lanes' held chains (a pick past the vocabulary reads as token 0). +pub fn readPicks(d: *Drafter, lanes: []const *Lane, depths: []const u32) void { + const picks = d.s.picks.slice(Pick, d.config.block * ds.pass_rows); + for (lanes, depths, 0..) |l, depth, bi| { + for (l.held[0..depth], 0..) |*h, i| { + const p = picks[i * ds.pass_rows + bi]; + h.* = if (p.token < c.vocab) p.token else 0; + } + l.held_len = depth; + } } /// Step i of each stream: its row i's logits plus W2 W1[previous draft] (the pending token first), then the argmax. diff --git a/zig/src/families/lfm2_moe/forward.zig b/zig/src/families/lfm2_moe/forward.zig index 7b63833192..10040c0d77 100644 --- a/zig/src/families/lfm2_moe/forward.zig +++ b/zig/src/families/lfm2_moe/forward.zig @@ -118,7 +118,17 @@ pub fn matrix(w: row.Weights) tile.Matrix { return .{ .w = w.w, .w_off = w.w_off, .scales = w.scales, .s_off = w.s_off, .biases = w.biases, .b_off = w.b_off, .n = w.n, .k = w.k, .bits = w.bits }; } +/// A window's held drafts on the device: `count` tokens at `src` + `off`, `stride` words apart, into ids from `row`. +pub const Held = struct { row: usize, src: Buffer, off: usize, stride: u32, count: u32 }; +/// A GPU event the forward waits on before it reads held drafts, and the value it waits for. +pub const Wait = struct { event: mtl.Event, value: u64 }; + pub fn run(m: *Model, s: *st.Scratch, segments: []const Segment, tokens: []const u32, record: bool, head: Head) !void { + return runHeld(m, s, segments, tokens, record, head, &.{}, null); +} + +/// `run` with held drafts copied into the rows' ids on the GPU first (after `wait`): their host ids are ignored. +pub fn runHeld(m: *Model, s: *st.Scratch, segments: []const Segment, tokens: []const u32, record: bool, head: Head, held: []const Held, wait: ?Wait) !void { if (tokens.len == 0 or tokens.len > s.rows or (record and tokens.len > st.batch_rows)) return error.LfmRowsExceeded; var total: usize = 0; for (segments) |seg| { @@ -131,10 +141,18 @@ pub fn run(m: *Model, s: *st.Scratch, segments: []const Segment, tokens: []const const pool = mtl.objc.Pool.push(); defer pool.pop(); const cb = m.queue.commandBuffer(); + if (wait) |w| cb.waitForEvent(w.event, w.value); const prompt = (!record and head == .last) or total >= tile_rows; const e = cb.compute(.serial); const plan = if (segs.batched(record, segments.len)) segs.prepare(e, s, segments) else null; const enc = Encoder{ .e = e, .m = m, .s = s, .prompt = prompt, .record = record, .plan = plan }; + for (held) |h| { + enc.pipe("lfm2_draft_ids"); + e.setBuffer(h.src, h.off, 0); + e.setValue([4]u32{ h.stride, h.count, c.vocab, 0 }, 1); + e.setBuffer(s.ids, h.row * 4, 2); + enc.threads(.{ h.count, 1, 1 }, .{ 32, 1, 1 }); + } const embedding = m.weights.embedding; e.setPipeline(m.kernels.embed(embedding.bits)); e.setBuffer(s.ids, 0, 0); diff --git a/zig/src/families/lfm2_moe/kernels.zig b/zig/src/families/lfm2_moe/kernels.zig index 357500c04d..936ba33197 100644 --- a/zig/src/families/lfm2_moe/kernels.zig +++ b/zig/src/families/lfm2_moe/kernels.zig @@ -32,6 +32,7 @@ const names = [_][:0]const u8{ "lfm2_tap", "lfm2_markov_scan", "lfm2_markov_pick", + "lfm2_draft_ids", }; pub const Kernels = struct { diff --git a/zig/src/native/lfm2_moe.zig b/zig/src/native/lfm2_moe.zig index 05f0260dcc..a78d67d97b 100644 --- a/zig/src/native/lfm2_moe.zig +++ b/zig/src/native/lfm2_moe.zig @@ -102,7 +102,12 @@ pub fn open(a: std.mem.Allocator, gpa: std.mem.Allocator, io: std.Io, o: api.Ope return null; }; h.metal.options.streams = @max(served, 2); - if (h.draft) |d| h.wrap = try lanes.drafted.Drafted.init(gpa, h.metal.backend(), d.drafter()); + if (h.draft) |d| { + h.wrap = try lanes.drafted.Drafted.init(gpa, h.metal.backend(), d.drafter()); + // the verify copies the drafts on the GPU after the unwaited hold + h.metal.held_source = d.heldSource(); + h.wrap.?.held_on_target = true; + } errdefer if (h.wrap) |*w| w.deinit(); const facts = if (h.wrap) |*w| w.facts(h.metal.facts()) else h.metal.facts(); h.config = try lanes.Config.init(gpa, facts, q.state.window_rows, q.state.window_rows - 1); diff --git a/zig/tests/contract/metal_lfm2_dspark.zig b/zig/tests/contract/metal_lfm2_dspark.zig index 3f2ae29907..63d7bcf590 100644 --- a/zig/tests/contract/metal_lfm2_dspark.zig +++ b/zig/tests/contract/metal_lfm2_dspark.zig @@ -1,4 +1,4 @@ -//! LFM2's DSpark Markov step (scan then pick): each block row's draft from its base logits and the previous token. +//! LFM2's DSpark kernels: the Markov step (scan then pick), and held drafts into a verify's ids. const std = @import("std"); const mtl = @import("metal"); const contract = @import("contract"); @@ -7,14 +7,16 @@ const c = contract.spec; const g = contract.gates; const rig_ = @import("rig.zig"); -pub const Which = enum { markov_step }; +pub const Which = enum { markov_step, draft_ids }; const rank = 256; const vocab = 128000; const span = 512; // tokens a scan threadgroup takes (the kernel's MARKOV_SPAN) const groups = (vocab + span - 1) / span; const table = 1000; // W1's rows in the check: ids are inside it or past the vocabulary (a zero row) -const names = [_][]const u8{"metal lfm2 markov step"}; +const names = [_][]const u8{ "metal lfm2 markov step", "metal lfm2 draft ids" }; +const stride = 64; // words between a stream's picks: a hold pass's 32 blocks, a token and a flag each +const ids_shapes = [_]c.Shape{ .{ .rows = 1, .n = 1, .k = 1 }, .{ .rows = 4, .n = 1, .k = 1 }, .{ .rows = 15, .n = 1, .k = 1 } }; const step_shapes = [_]c.Shape{ .{ .rows = 1, .n = vocab, .k = rank }, .{ .rows = 4, .n = vocab, .k = rank }, .{ .rows = 9, .n = vocab, .k = rank } }; fn none(_: c.Shape) usize { @@ -27,6 +29,7 @@ fn depth(_: c.Shape) usize { pub fn spec(w: Which) c.Spec { return switch (w) { + .draft_ids => .{ .name = names[1], .accum = .none, .depth = none, .out = .raw, .served = &ids_shapes, .sums = .{ .f16 = 0, .bf16 = 0, .why = "a copy moves tokens: no sums" } }, .markov_step => .{ .name = names[0], .accum = .none, .depth = depth, .out = .raw, .served = &step_shapes, .nan = .{ .stated = "a row with a NaN score is flagged nonfinite" }, .sums = .{ .f16 = 0, .bf16 = 0, .why = "picks are tokens: the check bounds each pick's score by gamma(256), not sum bits" } }, }; } @@ -64,6 +67,7 @@ pub const Probe = struct { pub fn init(a: std.mem.Allocator, device: mtl.Device, queue: mtl.Queue, w: Which) !Probe { const kernels: [2][:0]const u8 = switch (w) { .markov_step => .{ "lfm2_markov_scan", "lfm2_markov_pick" }, + .draft_ids => .{ "lfm2_draft_ids", "lfm2_draft_ids" }, }; const first = try rig_.pipeline(device, sources.lfm2, kernels[0]); errdefer first.deinit(); @@ -97,6 +101,20 @@ pub const Probe = struct { pub fn prepare(p: *Probe, s: c.Shape, f: g.Fill, seed: u64) !bool { p.drop(); p.rows = s.rows; + if (p.which == .draft_ids) { // picks a stride apart: tokens inside the vocabulary, its last, and past it + if (s.n != 1 or s.rows == 0 or s.rows > 16) return false; + p.ids = try p.a.alloc(u32, s.rows * stride); + for (p.ids, 0..) |*id, i| id.* = switch ((i / stride) % 3) { + 0 => @intCast((@as(u64, i) * 7919 + seed) % vocab), + 1 => vocab - 1, + else => vocab + @as(u32, @intCast(i % 5)), + }; + p.args[0] = try p.rig.input(std.mem.sliceAsBytes(p.ids)); + const at = [4]u32{ stride, @intCast(s.rows), vocab, 0 }; + p.args[1] = try p.rig.input(std.mem.asBytes(&at)); + p.y = try p.rig.output(p.count()); + return true; + } if (s.n != vocab or s.k != rank or s.rows == 0) return false; // each stream's step row sits a block (two rows here) apart, as a hold's rows of one step do p.src = try p.a.alloc(u16, 2 * s.rows * vocab); @@ -135,12 +153,21 @@ pub const Probe = struct { pub fn count(p: *Probe) usize { return switch (p.which) { .markov_step => p.rows * 8, + .draft_ids => p.rows * 4, }; } pub fn run(p: *Probe) !void { const pass = p.rig.begin(); const e = pass.e; + if (p.which == .draft_ids) { + e.setPipeline(p.pipes[0]); + e.setBuffer(p.args[0].buf, p.args[0].off, 0); + e.setBuffer(p.args[1].buf, p.args[1].off, 1); + e.setBuffer(p.y.buf, p.y.off, 2); + e.dispatchThreads(mtl.Size.of(p.rows, 1, 1), mtl.Size.of(32, 1, 1)); + return p.rig.finish(pass); + } const best = try p.rig.scratch(p.rows * groups * 4); const at = try p.rig.scratch(p.rows * groups * 4); e.setPipeline(p.pipes[0]); @@ -167,6 +194,13 @@ pub const Probe = struct { /// Each stream's pick against float64 scores with the kernel's bf16 roundings: the best, but for near ties. pub fn check(p: *Probe, _: g.Gate) !?[]const u8 { const out = p.rig.bytes(p.y, p.count()); + if (p.which == .draft_ids) { // exact: each stream's first pick, or 0 for a token past the vocabulary + for (0..p.rows) |r| { + const token = p.ids[r * stride]; + if (std.mem.readInt(u32, out[4 * r ..][0..4], .little) != (if (token < vocab) token else 0)) return "an id differs from its pick"; + } + return null; + } for (0..p.rows) |s| { const token = std.mem.readInt(u32, out[8 * s ..][0..4], .little); const flag = std.mem.readInt(u32, out[8 * s + 4 ..][0..4], .little); @@ -204,10 +238,10 @@ pub const Probe = struct { } }; -/// Gate 5 for the scan and pick on every Apple chip and this GPU: 256 threads each, 1,088 and 3,072 threadgroup bytes. +/// Gate 5 for the three launches on every Apple chip and this GPU: ids 32 threads, others 256, the scan 1,088 bytes. pub fn limits(a: std.mem.Allocator, device: mtl.Device, only: ?[]const u8) !usize { var problems: usize = 0; - const launches = [_]struct { []const u8, []const u8, u32, u32 }{ .{ "metal lfm2 markov scan", "lfm2_markov_scan", 1088, 256 }, .{ "metal lfm2 markov pick", "lfm2_markov_pick", 3072, 256 } }; + const launches = [_]struct { []const u8, []const u8, u32, u32 }{ .{ "metal lfm2 markov scan", "lfm2_markov_scan", 1088, 256 }, .{ "metal lfm2 markov pick", "lfm2_markov_pick", 3072, 256 }, .{ "metal lfm2 draft ids", "lfm2_draft_ids", 0, 32 } }; for (launches) |l| { if (only) |s| if (std.mem.indexOf(u8, l[0], s) == null) continue; const d = contract.limits.MetalLaunch{ .name = l[0], .threads = l[3], .tg_memory = l[2], .offline = false }; diff --git a/zig/tests/contract/metal_main.zig b/zig/tests/contract/metal_main.zig index 96020f9ff3..4007a7566a 100644 --- a/zig/tests/contract/metal_main.zig +++ b/zig/tests/contract/metal_main.zig @@ -20,16 +20,17 @@ const glm_rows_k = @import("metal_glm_rows.zig"); const flash_pool = @import("metal_flashnext_pool.zig"); const flash = @import("metal_flashnext.zig"); -const fsr = @import("metal_fsr.zig"); const gemma = @import("metal_gemma4.zig"); const gemma_embed = @import("metal_gemma4_embed.zig"); const gemma_mixed = @import("metal_gemma4_mixed.zig"); const lfm2_argmax = @import("metal_lfm2_argmax.zig"); const lfm2_streams = @import("metal_lfm2_streams.zig"); const lfm2_tap = @import("metal_lfm2_tap.zig"); -const lfm2_controls = @import("metal_lfm2_controls.zig"); const lfm2_dspark = @import("metal_lfm2_dspark.zig"); +const lfm2_bf16 = @import("metal_lfm2_bf16.zig"); const lfm2_attn = @import("metal_lfm2_attn.zig"); +const lfm2_conf = @import("metal_lfm2_confidence.zig"); +const lfm2_controls = @import("metal_lfm2_controls.zig"); const g = contract.gates; const usage = "usage: tf-kernel-contract [--only TEXT] [--no-gates] [--no-limits] [--offline DIR] [--seed N] [--repro] [--rotary-long] [--rotary-accuracy FILE]"; @@ -210,20 +211,84 @@ pub fn main(init: std.process.Init) !u8 { try tap_list.append(a, p); try kernels.append(a, p.kernel()); } - var dspark_list: [1]lfm2_dspark.Probe = undefined; - var lfm2_made: usize = 0; - defer for (dspark_list[0..lfm2_made]) |*p| p.deinit(); - for (std.enums.values(lfm2_dspark.Which)) |w| { - if (!wanted(o, lfm2_dspark.spec(w).name)) continue; - dspark_list[lfm2_made] = try lfm2_dspark.Probe.init(a, device, queue, w); - lfm2_made += 1; + var drafter_list: std.ArrayList(*lfm2_dspark.Probe) = .empty; + var block_list: std.ArrayList(*lfm2_attn.Probe) = .empty; + var bf16_list: std.ArrayList(*lfm2_bf16.Probe) = .empty; + defer { + for (drafter_list.items) |p| { + p.deinit(); + a.destroy(p); + } + for (block_list.items) |p| { + p.deinit(); + a.destroy(p); + } + for (bf16_list.items) |p| { + p.deinit(); + a.destroy(p); + } + drafter_list.deinit(a); + block_list.deinit(a); + bf16_list.deinit(a); } - for (dspark_list[0..lfm2_made]) |*p| try kernels.append(a, p.kernel()); - var block_attn: ?lfm2_attn.Probe = null; - defer if (block_attn) |*p| p.deinit(); - if (wanted(o, lfm2_attn.name)) { - block_attn = try lfm2_attn.Probe.init(a, device, queue); - try kernels.append(a, block_attn.?.kernel()); + for (std.enums.values(lfm2_controls.Mutation)) |m| { + const drafter_which: ?lfm2_dspark.Which = switch (m) { + .markov_nan_dropped, .markov_first_prev => .markov_step, + .markov_bf16_nan_dropped, .markov_bf16_first_prev => .markov_step_bf16, + .draft_ids_unzeroed => .draft_ids, + else => null, + }; + if (m == .none or drafter_which != null) { + for (std.enums.values(lfm2_dspark.Which)) |w| { + if (m != .none and w != drafter_which.?) continue; + if (!wanted(o, if (m == .none) lfm2_dspark.spec(w).name else m.name())) continue; + const p = try a.create(lfm2_dspark.Probe); + errdefer a.destroy(p); + p.* = try lfm2_dspark.Probe.init(a, device, queue, w, m); + try drafter_list.append(a, p); + try kernels.append(a, p.kernel()); + } + } + const bf16_which: ?lfm2_bf16.Which = switch (m) { + .bf16_rows_short_k, .bf16_rows_last_row => .rows, + .bf16_tile_untransposed, .bf16_tile_last_column => .tile, + else => null, + }; + if (m == .none or bf16_which != null) { + for (std.enums.values(lfm2_bf16.Which)) |w| { + if (m != .none and w != bf16_which.?) continue; + if (!wanted(o, if (m == .none) lfm2_bf16.spec(w).name else m.name())) continue; + const p = try a.create(lfm2_bf16.Probe); + errdefer a.destroy(p); + p.* = try lfm2_bf16.Probe.init(a, device, queue, w, m); + try bf16_list.append(a, p); + try kernels.append(a, p.kernel()); + } + } + if (m == .none or m == .block_attention_causal) { + if (!wanted(o, if (m == .none) lfm2_attn.name else m.name())) continue; + const p = try a.create(lfm2_attn.Probe); + errdefer a.destroy(p); + p.* = try lfm2_attn.Probe.init(a, device, queue, m); + try block_list.append(a, p); + try kernels.append(a, p.kernel()); + } + } + var conf_list: std.ArrayList(*lfm2_conf.Probe) = .empty; + defer { + for (conf_list.items) |p| { + p.deinit(); + a.destroy(p); + } + conf_list.deinit(a); + } + for ([_]lfm2_controls.Mutation{ .none, .confidence_no_w1 }) |m| { + if (!wanted(o, if (m == .none) lfm2_conf.name else m.name())) continue; + const p = try a.create(lfm2_conf.Probe); + errdefer a.destroy(p); + p.* = try lfm2_conf.Probe.init(a, device, queue, m); + try conf_list.append(a, p); + try kernels.append(a, p.kernel()); } var rows_list: std.ArrayList(*rows_k.Row) = .empty; var proj_list: std.ArrayList(*affine_k.Proj) = .empty; @@ -260,15 +325,6 @@ pub fn main(init: std.process.Init) !u8 { try kernels.append(a, flash_list[flash_made - 1].kernel()); } }; - var fsr_probes: [4]fsr.Probe = undefined; - var fsr_made: usize = 0; - defer for (fsr_probes[0..fsr_made]) |*probe| probe.deinit(); - for ([_]usize{ 4, 6 }) |bits| for ([_]bool{ false, true }) |control| { - if (!wanted(o, fsr.Probe.name(bits, control))) continue; - fsr_probes[fsr_made] = .{ .a = a, .device = device, .queue = queue, .bits = bits, .control = control }; - fsr_made += 1; - try kernels.append(a, fsr_probes[fsr_made - 1].kernel()); - }; var d3norm: ?drama3_k.Norm = null; defer if (d3norm) |*n| n.deinit(); if (wanted(o, drama3_k.spec().name)) { @@ -360,9 +416,6 @@ pub fn main(init: std.process.Init) !u8 { var glm_windows: glm_rows_k.Suite = .{ .a = a, .device = device, .queue = queue, .kernels = &kernels }; defer glm_windows.deinit(); try glm_windows.collect(o.only); - var nemotron_experts: @import("metal_nemotron_experts.zig").Suite = .{ .a = a, .device = device, .queue = queue, .kernels = &kernels }; - defer nemotron_experts.deinit(); - try nemotron_experts.collect(o.only); const weight_only = @import("metal_weight_only.zig"); const weight_source = try weight_only.text(a); defer a.free(weight_source); @@ -401,8 +454,6 @@ pub fn main(init: std.process.Init) !u8 { defer native_ple.destroy(); const native_gdn = try @import("metal_qwen4_gdn.zig").Suite.create(a, device, queue, &kernels, o.only); defer native_gdn.destroy(); - const moss_image = try @import("metal_moss_image.zig").Suite.create(a, device, queue, &kernels, o.only); - defer moss_image.destroy(); var t: g.Tally = .{}; const t0 = std.Io.Clock.awake.now(init.io); if (o.gates) { @@ -416,8 +467,6 @@ pub fn main(init: std.process.Init) !u8 { lim += try gemma.limits(a, device, o.only); lim += try gemma_embed.limits(a, device, o.only); lim += try gemma_mixed.limits(a, device, o.only); - lim += try lfm2_dspark.limits(a, device, o.only); - lim += try lfm2_attn.limits(a, device, o.only); } const t2 = std.Io.Clock.awake.now(init.io); const ms = struct { diff --git a/zig/tests/lfm2_dspark.zig b/zig/tests/lfm2_dspark.zig index 09c7f68a52..f95ce615e5 100644 --- a/zig/tests/lfm2_dspark.zig +++ b/zig/tests/lfm2_dspark.zig @@ -153,7 +153,7 @@ pub fn main(init: std.process.Init) !void { try absorb(d, .{ .lane = lane, .src = .{ .buf = taps.buf, .off = at * width * 2 }, .rows = rows, .p0 = at }); } lane.end = ctx; - try q.dspark_block.hold(d, &.{.{ .lane = lane, .pending = ids[ctx], .anchor = ctx, .depth = block }}); + _ = try q.dspark_block.hold(d, &.{.{ .lane = lane, .pending = ids[ctx], .anchor = ctx, .depth = block }}, null); var held: [16]u32 = undefined; @memcpy(held[0..block], lane.held[0..block]); const gpu = held[0..block]; @@ -171,12 +171,12 @@ pub fn main(init: std.process.Init) !void { var best: f64 = std.math.floatMax(f64); for (0..5) |_| { const t0 = std.Io.Clock.awake.now(init.io); - try q.dspark_block.hold(d, &.{.{ .lane = lane, .pending = ids[ctx], .anchor = ctx, .depth = depth }}); + _ = try q.dspark_block.hold(d, &.{.{ .lane = lane, .pending = ids[ctx], .anchor = ctx, .depth = depth }}, null); best = @min(best, @as(f64, @floatFromInt(t0.durationTo(std.Io.Clock.awake.now(init.io)).toNanoseconds())) / 1e6); } std.debug.print("hold at depth {d}: {d:.2} ms (best of 5)\n", .{ depth, best }); } - try q.dspark_block.hold(d, &.{.{ .lane = lane, .pending = ids[ctx], .anchor = ctx, .depth = block }}); + _ = try q.dspark_block.hold(d, &.{.{ .lane = lane, .pending = ids[ctx], .anchor = ctx, .depth = block }}, null); // the host reference, fp32 throughout var r: Ref = .{ .gpa = gpa, .check = ck.Checkpoint.init(gpa), .cfg = cfg }; defer r.check.deinit(); From fca43798a8c4e347c512b3c6fc607cd213f9288a Mon Sep 17 00:00:00 2001 From: BlivionIaG <15323295+BlivionIaG@users.noreply.github.com> Date: Mon, 26 Oct 2026 10:06:08 +0100 Subject: [PATCH 05/39] docs: the LFM2 recipe's DSpark drafts with LiquidAI's drafter The recipe says where the drafter comes from (LiquidAI/LFM2.5-24B-A2B-DSpark at its revision), that its weights are under the LFM Open License v1.0 and not shipped with TensorFold, how to serve with --drafter, what the drafter runs and which kernels and checks cover it. The README row and the recipe index name the drafts. --- README.md | 2 +- docs/recipes/README.md | 2 +- docs/recipes/lfm2.5-24b-a2b.md | 41 ++++++++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3d616f7630..0847d13abd 100644 --- a/README.md +++ b/README.md @@ -173,7 +173,7 @@ Conversions are explicitly marked: their upstream weights need the linked conver | GLM-5.3-Flash, FP4-Local | `tensorfold pull nvidia/GLM-5.3-Flash-FP4-Local@76348ddb2be2e35e39ecfc00e75dc07703f4f3d5` | Two GB10s at TP=2, or three at TP=3 over a ring of direct links: @Bizuayeu loaded it as stored and served it at both sizes. 165.3 GiB of weights, 13% less than NVFP4's 190.4 GiB. | [Native CUDA](docs/recipes/glm-5.3-flash-cuda.md), in 1.1.5 (#1050). NVIDIA's mixed-precision checkpoint: NVFP4 routed experts with compressed block scales, MXFP8 and 128×128-block FP8 for most of the rest. As close to BF16 as NVFP4 at its smaller size: at TP=2 its NLL is 1.11% above BF16's against NVFP4's 1.37% ([contributed receipt](docs/receipts/glm-fp4local-80517fc3/README.md) on #1050's head, not rerun on main), a difference the per-document intervals in [XSFP4's card](https://huggingface.co/Bizuayeu/GLM-5.3-Flash-XSFP4#measured) do not resolve. | | GLM-5.3-Flash, XSFP4 | `tensorfold pull Bizuayeu/GLM-5.3-Flash-XSFP4@88736f916f8939607fde8a4b881dcae749f7551d` | Two GB10s at TP=2, or three at TP=3 over a ring of direct links: on main `8086c205c` @Bizuayeu loaded it in 370.7 s at TP=2 (rank 0's model 96.97 GiB at a 524,288-token window) and in 313.8 s at TP=3 (63.60 GiB at 1,048,576), and both served. 165.7 GiB of weights, 0.4 GiB more than FP4-Local's. | [Native CUDA](docs/recipes/glm-5.3-flash-cuda.md), in 1.1.5 through FP4-Local's path. FP4-Local with its routed experts' NVFP4 blocks fitted again from the BF16 weights, a smoothing folded into them, so it runs on the same kernels. Closer to BF16 than FP4-Local and NVFP4 at TP=2 by all three measures in [its card](https://huggingface.co/Bizuayeu/GLM-5.3-Flash-XSFP4#measured), on #1050's code: top-1 agreement 93.26%, NLL 0.85% above and a KL bound of 0.0431, against FP4-Local's 92.82%, 1.11% and 0.0510. | | GLM-5.3, EXLR8 `home` | `tensorfold pull kindlingai/glm-5.3-exlr8-k2-k3-k4@f459f5d0272a529403dd9b3fd32f771f4250a0d4 --exlr8 home --node 0/2 --dir ~/models/glm-5.3-exlr8-home` | Two M5 Ultra Macs, 256 GB each; about 160 GB resident per Mac at 8k context / four streams. | [Two-Mac recipe](docs/glm-5.3-exlr8.md). Use `--node 1/2` on the other Mac; Thunderbolt 5 and MCDMA required. No CUDA backend. | -| LFM2.5-24B-A2B, MLX 4 / 6 / 8-bit | `tensorfold pull LiquidAI/LFM2.5-24B-A2B-MLX-4bit@667408d494bde191c2cb9dd4f4549827816b9275`
`tensorfold pull LiquidAI/LFM2.5-24B-A2B-MLX-6bit@1deec53eb49571e21cdcbf82fd5839248f07ed88`
`tensorfold pull LiquidAI/LFM2.5-24B-A2B-MLX-8bit@67ab417dd02d0c9545802a726df668620eae049d` | One Mac; M4 Pro checked. | [Native recipe](docs/recipes/lfm2.5-24b-a2b.md), [release history](CHANGELOG.md). | +| LFM2.5-24B-A2B, MLX 4 / 6 / 8-bit | `tensorfold pull LiquidAI/LFM2.5-24B-A2B-MLX-4bit@667408d494bde191c2cb9dd4f4549827816b9275`
`tensorfold pull LiquidAI/LFM2.5-24B-A2B-MLX-6bit@1deec53eb49571e21cdcbf82fd5839248f07ed88`
`tensorfold pull LiquidAI/LFM2.5-24B-A2B-MLX-8bit@67ab417dd02d0c9545802a726df668620eae049d` | One Mac; M4 Pro checked. | [Native recipe](docs/recipes/lfm2.5-24b-a2b.md), [release history](CHANGELOG.md). LiquidAI's DSpark drafter with `--drafter`. | | Qwen3.6-35B-A3B, MLX with MTP | `tensorfold pull TensorFold/Qwen3.6-35B-A3B-MLX-4bit-MTP@81169a9bc511a27c1b4eedb77a2cd98ced431847` | One GB10; 20.9 GB checkpoint. | [Qwen3.6](docs/recipes/qwen3.6-moe.md). Native CUDA; the recipe's Metal path belongs to Python. | | Gemma 4 31B, MLX 4-bit | `tensorfold pull mlx-community/gemma-4-31B-it-4bit@696d436c404745a59f30e4939a658162b0a9e57f`
`tensorfold pull z-lab/gemma-4-31B-it-DFlash@eabd648301ce28583cc14757912e5e0f84e152e1` | One M5 Mac; M5 Ultra checked. M1-M4 refused. | [Native 31B](docs/recipes/gemma-4-native.md). Name the drafter with `--drafter`; images with `--vision`. | | Gemma 4 12B, converted MLX 4-bit | `hf download google/gemma-4-12B-it --revision 707f0a3b8a3c7ad586ed01e27eafbad8a27dd0f7 --local-dir ~/models/gemma-4-12b-hf`
`hf download google/gemma-4-12B-it-assistant --revision 46d4c6f13f0ac0ad827b915669b8df9b81c64c51 --local-dir ~/models/gemma-4-12b-assistant` | One M5 Mac; converted text weights 6.3 GB, image weights 100 MB, plus assistant/caches. M1-M4 refused. | [Convert and serve](docs/recipes/gemma-4-12b-native.md). Upstream download needs `hf`; conversion needs Python, mlx and mlx-lm. Serving is native. | diff --git a/docs/recipes/README.md b/docs/recipes/README.md index 8017124388..0cfdf94f1b 100644 --- a/docs/recipes/README.md +++ b/docs/recipes/README.md @@ -22,7 +22,7 @@ For agent-assisted contributions, read the [TensorFold contributor skill](SKILL. | Qwen3.6-35B-A3B | [MLX with MTP drafts, one-GPU CUDA](qwen3.6-moe.md) | | Step-5-Preview | [Native Metal on one 256 GB Mac: 2-bit experts, MTP drafts, images](step-5-preview.md) | | Kolibri-1 | [Native CUDA, block FP8, research checkpoint](kolibri-1.md) | -| LFM2.5-24B-A2B | [Native Metal, MLX 4-bit, shared rounds](lfm2.5-24b-a2b.md) | +| LFM2.5-24B-A2B | [Native Metal, MLX 4-bit, shared rounds, DSpark drafts](lfm2.5-24b-a2b.md) | | Spark-X2.6-4B | [Native CUDA](spark-x2.6-4b-cuda.md), [native Metal, the bf16 release, contexts to 1M](spark-x2.6-4b-metal.md) | ## Capability floor diff --git a/docs/recipes/lfm2.5-24b-a2b.md b/docs/recipes/lfm2.5-24b-a2b.md index 7b8b2cdf4b..f4272b59a5 100644 --- a/docs/recipes/lfm2.5-24b-a2b.md +++ b/docs/recipes/lfm2.5-24b-a2b.md @@ -41,3 +41,44 @@ the shared-round kernels on it. The M4 Pro is not yet measured on a quiet machin Every reply was token for token equal before and after, and equal to its solo run and to `"draft": false`. Cold prefill at 2k, 8k and 32k tokens and peak memory did not change. + +## DSpark drafts + +LiquidAI's DSpark drafter for this model is +[`LiquidAI/LFM2.5-24B-A2B-DSpark`](https://huggingface.co/LiquidAI/LFM2.5-24B-A2B-DSpark/tree/e052e5b1dd103c38cdd4f8ddc068d7dea4586f55) +at revision `e052e5b1dd103c38cdd4f8ddc068d7dea4586f55` (327.7M bf16 parameters, 655 MB). Its weights are under the +[LFM Open License v1.0](https://huggingface.co/LiquidAI/LFM2.5-24B-A2B-DSpark/blob/e052e5b1dd103c38cdd4f8ddc068d7dea4586f55/LICENSE); +TensorFold does not ship them, so read the license before use. + +```bash +hf download LiquidAI/LFM2.5-24B-A2B-DSpark --revision e052e5b1dd103c38cdd4f8ddc068d7dea4586f55 \ + config.json model.safetensors LICENSE README.md --local-dir "$HOME/models/LFM2.5-24B-A2B-DSpark" +zig-out/native/bin/tensorfold-native serve "$MODEL_DIR" --port 8080 --parallel 16 \ + --drafter "$HOME/models/LFM2.5-24B-A2B-DSpark" +``` + +Without `--drafter` the engine serves as before. `"draft": false` on a request, or `--no-drafts`, turns the drafter +off; the lane core verifies every draft, so a drafted reply equals its `"draft": false` reply token for token. + +The drafter has no embedding and no head of its own: + +- Context: the target's rows after layers 2, 10, 18, 26 and 34 (the backend's taps, `lfm2_tap`), through `fc` and + `hidden_norm`, give each of its five layers' context keys and values (k_norm, then RoPE at the row's position). +- Block: the pending token and mask tokens run five Qwen3 layers with non-causal attention over the context and the + block, then the target's head. A block is as long as the deepest chain its pass holds, at most the nine rows the + drafter was trained on. +- Markov head: row i's logits plus `W2 W1[previous token]` (rank 256); draft i is their argmax. + +Its bf16 linears are quantized at load to 8-bit MLX affine codes in groups of 64 and run on the family's kernels +and `core/tile_qmm`; the q and k rows are reordered so that the target's half-split RoPE kernel gives the +checkpoint's interleaved RoPE. Its own kernels are `lfm2_tap`, `lfm2_attn_block` and `lfm2_attn_block_merge` +(non-causal block attention, the target's attention kernels with a template flag), `lfm2_markov_scan` and +`lfm2_markov_pick` (a Markov step) and `lfm2_draft_ids` (a hold's drafts into the next verify's rows on the GPU, so +the host does not wait for a hold that fits one pass). Each kept prompt state keeps the drafter's context too, so a +drafting stream resumes prompts. Shared rounds take the lane core's shared-round guard: they draft only while +drafted rounds measure faster than plain ones at that stream count. + +`tf-kernel-contract --only lfm2` checks these kernels. `tf-lfm2-dspark MODEL DRAFTER TOKENS [ROWS]` runs the drafter +against an fp32 host reference of the released checkpoint (the same taps and the target's embedding and head), +prints each draft against the reference's pick with the margin between the reference's two best tokens, and times +the drafter's passes; a near tie can pick the other token. From ea9b796fc24be0210eb4342e3480e06b5c58df77 Mon Sep 17 00:00:00 2001 From: BlivionIaG <15323295+BlivionIaG@users.noreply.github.com> Date: Mon, 26 Oct 2026 09:06:32 +0100 Subject: [PATCH 06/39] lanes: an external drafter's chain chances price verify rows, and its holds follow the grants A drafter that gives each held chain draft's chance (Drafter.VTable.chances) may set Facts.chances. Drafted.facts then sets Model.draft_probabilities and Model.granted_depth, so the window allocator prices each stream's verify rows by those chances, as it does a head's node chances. With Facts.confidence_floor it instead verifies the drafts at least that likely (a fixed window when 0). With granted_depth a stream's next hold is as deep as its last grant plus a slack: one more draft when the grant took every held draft, none when it stopped short of them (Stream.grant_short), and after a grant of none a probe of draft_slack drafts at the depth rule's probe interval. Facts.slack sets one slack for all three. A shared round's granted depths are cut so the drafting streams' blocks fit one hold pass of batch_rows. When the shared-round guard starts a drafted stretch, its first round grants every stream one draft, so the stretch measures drafted rounds rather than plain ones left by grants of none. All of it is behind Facts.chances and Model.granted_depth, off by default; windows.allocate records grant_short only for granted depths. Tests: chances price the chain's rows and drafted replies equal one-token decode alone and at 1, 4 and 8 shared streams, with the scheduled verify ragged across streams and verifying fewer rows than a fixed window; the facts reach the lane config; granted depths, their slacks and the one-pass cut; holds as wide as the grants. --- zig/src/core/lanes/config.zig | 11 +- zig/src/core/lanes/depth.zig | 62 ++++- zig/src/core/lanes/drafted.zig | 7 + zig/src/core/lanes/drafted_chances_test.zig | 266 ++++++++++++++++++++ zig/src/core/lanes/drafter.zig | 3 + zig/src/core/lanes/engine.zig | 1 + zig/src/core/lanes/lanes.zig | 1 + zig/src/core/lanes/stream.zig | 2 + zig/src/core/lanes/windows.zig | 7 +- 9 files changed, 355 insertions(+), 5 deletions(-) create mode 100644 zig/src/core/lanes/drafted_chances_test.zig diff --git a/zig/src/core/lanes/config.zig b/zig/src/core/lanes/config.zig index a6721fcf57..71c455637a 100644 --- a/zig/src/core/lanes/config.zig +++ b/zig/src/core/lanes/config.zig @@ -36,6 +36,8 @@ pub const Model = struct { max_streams: u32 = 0, // a shared forward's streams at most (0: 32) shared_costs: []const Cost = &.{}, // shared forwards' ms by total rows (timed at load) draft_probabilities: bool = false, // the head gives each drafted node's chance of landing + granted_depth: bool = false, // with draft_probabilities: every stream drafts its last grant plus a slack + draft_slack: u32 = 0, // drafts past a stream's last grant, short or not (0: the Constants' own) draft_streams: bool = false, // the head drafts every stream of a shared round in one batch head_trees: bool = false, // the head drafts trees of lanes (DraftRequest.lanes) and holds them join_tail: bool = false, // a prompt in pieces leaves its last token for the next shared round's window (Backend.join) @@ -66,6 +68,7 @@ pub const Config = struct { batch_rows: u32, batch_streams: u32, node_probabilities: bool, + granted_depth: bool = false, // Model.granted_depth fill_lanes: u32 = 0, // rows a window may grow to with suffix-match branches (0: off; lanes/fill.zig) fill_match: u32 = 8, // context tokens a suffix-match branch must match (shorter ones are coincidental phrases) fill_floor: f64 = 0.35, // the share of grafted rows a stream must keep to go on reading its chain back every round @@ -149,6 +152,7 @@ pub const Config = struct { .batch_rows = batch_rows, .batch_streams = if (streams) (if (m.max_streams != 0) m.max_streams else 32) else 1, .node_probabilities = m.draft_probabilities or m.draft_confidence != null, + .granted_depth = m.granted_depth and m.draft_probabilities, .draft_streams = m.draft_streams, .head_trees = m.head_trees, .join_tail = m.join_tail, @@ -160,7 +164,12 @@ pub const Config = struct { .shared_costs = try alloc.extendCosts(gpa, timed.items, batch_rows), .lane_costs = try alloc.extendCosts(gpa, m.lane_costs, 64), .mtp_step_ms = m.mtp_step_ms, - .k = .{ .depth_rate = if (m.depth_rate > 0) m.depth_rate else (Constants{}).depth_rate }, + .k = .{ + .depth_rate = if (m.depth_rate > 0) m.depth_rate else (Constants{}).depth_rate, + .draft_slack = if (m.draft_slack > 0) m.draft_slack else (Constants{}).draft_slack, + .grant_slack = if (m.draft_slack > 0) m.draft_slack else (Constants{}).grant_slack, + .short_slack = if (m.draft_slack > 0) m.draft_slack else (Constants{}).short_slack, + }, .draft_step_ms = m.draft_step_ms, }; } diff --git a/zig/src/core/lanes/depth.zig b/zig/src/core/lanes/depth.zig index 8595a67aab..2f80da8edd 100644 --- a/zig/src/core/lanes/depth.zig +++ b/zig/src/core/lanes/depth.zig @@ -12,6 +12,8 @@ pub const Constants = struct { plain_margin: f64 = 1.05, // a drafted depth must beat plain rounds' tokens a ms by this much plain_wait_most: u64 = 128, // plain rounds between probes at most draft_slack: i64 = 2, // drafts a stream offers past what its last shared round granted it + grant_slack: i64 = 1, // granted depths: drafts past a grant that verified every draft the stream held + short_slack: i64 = 0, // granted depths: drafts past a grant that left some of them unverified }; /// A stream's acceptance by depth and probe counters (Python `_depth_state[stream]`). @@ -35,6 +37,7 @@ pub const Who = struct { finished: bool, forced: bool, granted: ?u32 = null, + grant_short: bool = false, // the last grant left some held drafts unverified (the width did not bind) stream: ?*anyopaque = null, sampled: bool = false, // keyed draws: host work a row that a greedy round does not have }; @@ -59,6 +62,7 @@ pub const Rule = struct { draft_step_ms: f64 = 0, // a shared round's draft past a stream's first (budgets price it) plain_guard: bool, node_probabilities: bool, + granted_depth: bool = false, // node_probabilities: a stream drafts its last grant plus draft_slack, lone ones too confidence_floor: ?f64 = null, batch_rows: u32, batched_heads: bool = false, // one batch drafts every stream of a shared round (its steps: the deepest stream's) @@ -192,9 +196,28 @@ pub const Rule = struct { return true; } + /// Drafts past a grant: none when it stopped short of the held drafts (the width did not bind), else grant_slack. + fn slack(r: *const Rule, who: Who) i64 { + return if (who.grant_short) r.k.short_slack else r.k.grant_slack; + } + + /// The last grant plus its slack (the deepest chain before any); after a grant of none, a probe of draft_slack. + fn grantDepth(r: *Rule, who: Who) !i64 { + const most = @min(@as(i64, r.most_drafts), @max(1, who.draft_room - 1)); + const granted = who.granted orelse return most; + _ = try r.rates(who); + const st = &who.state.*.?; + if (granted == 0) return if (!r.plain_guard or r.probePlain(st)) @min(most, r.k.draft_slack) else 0; + st.plain = 0; + st.wait = r.k.depth_probe_every; + return @min(most, @as(i64, granted) + r.slack(who)); + } + /// Python `_head_depth`: the next round's head drafts for a stream (0 when forced, 1 with a copy ahead). pub fn headDepth(r: *Rule, who: *const Who, budget: ?i64, probe: Probe) !i64 { - const d: i64 = if (who.finished) 0 else if (budget) |b| b else if (r.node_probabilities) + const d: i64 = if (who.finished) 0 else if (budget) |b| b else if (r.granted_depth) + try r.grantDepth(who.*) + else if (r.node_probabilities) @min(@as(i64, r.most_drafts), @max(1, who.draft_room - 1)) else try r.depth(who.*); @@ -203,6 +226,15 @@ pub const Rule = struct { return d; } + /// A shared hold pads every block to its deepest: depths cut so the drafting streams' blocks fit `rows` (one pass). + fn capPass(out: []i64, rows: u32) void { + var n: i64 = 0; + for (out) |d| n += @intFromBool(d > 0); + if (n == 0) return; + const most = @max(1, @divFloor(@as(i64, rows), n)); + for (out) |*d| d.* = @min(d.*, most); + } + /// Head drafts for the streams of a shared round, sized by allocation over their chain chances. pub fn budgets(r: *Rule, whos: []Who, probe: Probe, sampled: bool, out: []i64) !void { if (r.confidence_floor != null) { @@ -213,8 +245,9 @@ pub const Rule = struct { for (whos, out) |*w, *o| { const inner = try r.headDepth(w, null, probe); const granted: i64 = if (w.granted) |g| g else r.most_drafts; - o.* = try r.headDepth(w, @min(inner, granted + r.k.draft_slack), probe); + o.* = try r.headDepth(w, if (r.granted_depth) inner else @min(inner, granted + r.k.draft_slack), probe); } + if (r.granted_depth) capPass(out, r.batch_rows); return; } const most = try r.gpa.alloc(i64, whos.len); @@ -381,3 +414,28 @@ test "a shared round's drafts come out the same whichever depths its overhead wa } try std.testing.expectEqualSlices(i64, &got[0], &got[1]); } + +test "granted depths in a shared round fit one hold pass: every block is as deep as the deepest" { + const empty: Table = .{}; + var rule: Rule = .{ .gpa = std.testing.allocator, .prior = &.{0.8}, .most_drafts = 9, .family_costs = &empty, .shared_costs = &empty, .mtp_step_ms = 0.5, .plain_guard = false, .node_probabilities = true, .granted_depth = true, .batch_rows = 32 }; + defer rule.deinit(); + var states: [8]?State = @splat(null); + defer for (&states) |*st| if (st.*) |*x| x.deinit(std.testing.allocator); + var whos: [8]Who = undefined; + for (&whos, &states, 0..) |*w, *st, i| w.* = .{ .state = st, .draft_room = 100, .finished = false, .forced = false, .granted = @intCast(i) }; + const No = struct { + fn copy(_: *anyopaque, _: *const Who) bool { + return false; + } + }; + var unused: u8 = 0; + var out: [8]i64 = undefined; + try rule.budgets(&whos, .{ .ptr = &unused, .copy = No.copy }, false, &out); + // a grant of none probes two; grants 1..7 that took every held draft, plus one; cut to 32 rows over eight blocks + try std.testing.expectEqualSlices(i64, &.{ 2, 2, 3, 4, 4, 4, 4, 4 }, &out); + try rule.budgets(whos[0..2], .{ .ptr = &unused, .copy = No.copy }, false, out[0..2]); + try std.testing.expectEqualSlices(i64, &.{ 2, 2 }, out[0..2]); // two blocks fit whole + for (whos[0..2]) |*w| w.grant_short = true; + try rule.budgets(whos[0..2], .{ .ptr = &unused, .copy = No.copy }, false, out[0..2]); + try std.testing.expectEqualSlices(i64, &.{ 2, 1 }, out[0..2]); // a grant short of the held drafts: no slack +} diff --git a/zig/src/core/lanes/drafted.zig b/zig/src/core/lanes/drafted.zig index b7b9ca5bc6..cee1ec3155 100644 --- a/zig/src/core/lanes/drafted.zig +++ b/zig/src/core/lanes/drafted.zig @@ -105,6 +105,13 @@ pub const Drafted = struct { m.draft_streams = d.batched; m.draft_probabilities = false; m.head_trees = d.trees; + if (d.chances and x.drafter.vtable.chances != null) { + // a chain's drafts priced by the drafter's own chances; the next hold as deep as the grant and its slack + m.draft_probabilities = d.confidence_floor == null; + m.draft_confidence = d.confidence_floor; + m.granted_depth = d.confidence_floor == null; + m.draft_slack = d.slack; + } if (d.tree_nodes > 0) { // the drafter grows its own trees, best first, and prices their nodes m.drafts = @min(d.tree_nodes, if (target.exact_width > 1) target.exact_width - 1 else 0); diff --git a/zig/src/core/lanes/drafted_chances_test.zig b/zig/src/core/lanes/drafted_chances_test.zig new file mode 100644 index 0000000000..02c5eb4cb1 --- /dev/null +++ b/zig/src/core/lanes/drafted_chances_test.zig @@ -0,0 +1,266 @@ +//! A drafter that prices its chains (Facts.chances): verify rows go by its chances and replies stay one-token decode. +const std = @import("std"); +const Config = @import("config.zig").Config; +const Model = @import("config.zig").Model; +const Cost = @import("config.zig").Cost; +const Engine = @import("engine.zig").Engine; +const be = @import("backend.zig"); +const sm = @import("stream.zig"); +const fake = @import("fake.zig"); +const dr = @import("drafter.zig"); +const depth = @import("depth.zig"); +const Table = @import("table.zig").Table; +const Drafted = @import("drafted.zig").Drafted; +const dt = @import("drafted_test.zig"); + +const gpa = std.testing.allocator; +const most_streams = 8; + +/// drafted_test's drafter with a chance for each held draft: `sure` where its guess is right, `unsure` where wrong. +const Priced = struct { + head: dt.FakeDrafter, + sure: f64 = 0.7, + unsure: f64 = 0.7, + vtable: dr.Drafter.VTable = undefined, + inner_hold: *const fn (*anyopaque, []const dr.Hold) anyerror!void = undefined, + at: std.AutoHashMapUnmanaged(*sm.Stream, u64) = .empty, // each stream's last hold's position + asked: u32 = 0, + held: u64 = 0, // drafts its holds asked for + + fn of(ptr: *anyopaque) *Priced { + return @fieldParentPtr("head", @as(*dt.FakeDrafter, @ptrCast(@alignCast(ptr)))); + } + + fn hold(ptr: *anyopaque, items: []const dr.Hold) anyerror!void { + const p = of(ptr); + for (items) |it| { + try p.at.put(gpa, it.stream, it.position); + p.held += it.depth; + } + return p.inner_hold(ptr, items); + } + + /// The running product along the chain, as the DSpark confidence head's survivals are. + fn chances(ptr: *anyopaque, s: *sm.Stream, out: []f64) anyerror!bool { + const p = of(ptr); + const lane = p.head.lanes.getPtr(s) orelse return false; + if (out.len > lane.held.items.len) return error.TooFewDrafts; + const position = p.at.get(s) orelse return error.NoHold; + p.asked += 1; + var reach: f64 = 1; + for (out, 0..) |*o, j| { + const at = position + j; + reach *= if ((at *% 2654435761 + j) % 5 == 0) p.unsure else p.sure; // drafted_test's wrong guesses + o.* = reach; + } + return true; + } + + fn drafter(p: *Priced) dr.Drafter { + const base_ = p.head.drafter(); + p.vtable = base_.vtable.*; + p.inner_hold = base_.vtable.hold; + p.vtable.hold = hold; + p.vtable.chances = chances; + return .{ .ptr = &p.head, .vtable = &p.vtable }; + } + + fn deinit(p: *Priced) void { + p.at.deinit(gpa); + p.head.deinit(); + } +}; + +/// Every shared verify's window rows: whether one round gave its streams unequal rows (a ragged verify). +const Seen = struct { + var inner: *const fn (*anyopaque, []const be.Window, []be.Verified) anyerror!void = undefined; + var ragged: u32 = 0; + var rows: u64 = 0; + + fn verify(ptr: *anyopaque, windows: []const be.Window, out: []be.Verified) anyerror!void { + for (windows) |w| { + rows += w.rows(); + if (w.rows() != windows[0].rows()) { + ragged += 1; + break; + } + } + return inner(ptr, windows, out); + } +}; + +var costs: [16]Cost = undefined; + +fn base() Model { + for (&costs, 1..) |*c, w| c.* = .{ .width = @intCast(w), .ms = 5.0 + 0.8 * @as(f64, @floatFromInt(w)) }; + return .{ .exact_width = 16, .gpu_tokens = true, .window_costs = &costs, .hidden_rows = true, .batch_rows = 32, .max_streams = most_streams }; +} + +const Run = struct { tokens: [most_streams][64]u32 = undefined, lens: [most_streams]usize = @splat(0), drafted: u64 = 0, holds: u32 = 0, asked: u32 = 0, ragged: u32 = 0, rows: u64 = 0, held: u64 = 0 }; + +/// The drafter's Facts.slack for the next runs (0: the depth rule's own; 9: every hold the whole block). +var slack: u32 = 0; + +/// The drafter's chances: equal everywhere, or knowing its wrong guesses (a confidence head that ranks well). +const Head = enum { flat, knows }; + +/// Each prompt decoded together (one stream: alone); `floor` null: the chances price rows, else a fixed window. +fn run(prompts: []const []const u32, drafts: bool, floor: ?f64, head: Head) !Run { + var target: fake.Fake = .{ .gpa = gpa }; + defer target.deinit(); + var p: Priced = .{ .head = .{ .facts_ = .{ .depth = 9, .step_ms = 0.5, .batched = true, .chances = true, .confidence_floor = floor, .slack = slack } } }; + if (head == .knows) { + p.sure = 0.95; + p.unsure = 0.05; + } + defer p.deinit(); + var strict: dt.Strict = .{ .inner = target.backend() }; + defer strict.deinit(); + var seen = strict.backend(); + var table = seen.vtable.*; + Seen.inner = table.verify; + Seen.ragged = 0; + Seen.rows = 0; + table.verify = Seen.verify; + seen.vtable = &table; + var wrapped = try Drafted.init(gpa, seen, p.drafter()); + defer wrapped.deinit(); + var cfg = try Config.init(gpa, if (drafts) wrapped.facts(base()) else base(), 16, 15); + defer cfg.deinit(gpa); + var clock: fake.FixedClock = .{}; + var engine = Engine.init(gpa, &cfg, if (drafts) wrapped.backend() else target.backend(), clock.clock()); + defer engine.deinit(); + var streams: [most_streams]sm.Stream = undefined; + const n = prompts.len; + for (prompts, streams[0..n]) |prompt, *s| s.* = try sm.Stream.init(gpa, .{ .id = "s", .prompt = prompt, .max_new = 48, .eos = &.{96}, .drafts = drafts }); + defer for (streams[0..n]) |*s| s.deinit(gpa); + for (streams[0..n]) |*s| try engine.addStream(s); + while (engine.activeCount() > 0) try engine.step(); + var out: Run = .{ .holds = p.head.holds, .asked = p.asked, .ragged = Seen.ragged, .rows = Seen.rows, .held = p.held }; + for (streams[0..n], 0..) |*s, i| { + out.lens[i] = s.emitted().len; + @memcpy(out.tokens[i][0..out.lens[i]], s.emitted()); + out.drafted += s.drafted; + } + return out; +} + +fn expectSame(a: Run, b: Run, n: usize) !void { + for (0..n) |i| try std.testing.expectEqualSlices(u32, a.tokens[i][0..a.lens[i]], b.tokens[i][0..b.lens[i]]); +} + +/// `n` streams of unequal prompts, each a prefix of drafted_test's long prompt. +fn unequal(buf: *[most_streams][]const u32, n: usize) []const []const u32 { + for (buf[0..n], 0..) |*p, i| p.* = dt.long[0 .. 11 + i]; + return buf[0..n]; +} + +test "a drafter's chances price its chain's rows, and drafted replies equal one-token decode alone and shared" { + const sets = [_][]const []const u32{ &.{&dt.p1}, &.{ &dt.p1, &dt.long, &dt.p1 } }; + for (sets) |ps| { + const plain = try run(ps, false, null, .flat); + const priced = try run(ps, true, null, .flat); + try expectSame(plain, priced, ps.len); + try std.testing.expect(priced.asked > 0 and priced.drafted > 0); // the window allocator read the chances + const fixed = try run(ps, true, 0, .flat); + try expectSame(plain, fixed, ps.len); + try std.testing.expect(fixed.drafted > priced.drafted); // a fixed window verifies every draft the rows fit + } +} + +test "at 1, 4 and 8 streams the scheduled and the fixed verify equal one-token decode; shared rounds are ragged" { + var buf: [most_streams][]const u32 = undefined; + for ([_]usize{ 1, 4, 8 }) |n| { + const ps = unequal(&buf, n); + const plain = try run(ps, false, null, .flat); + const scheduled = try run(ps, true, null, .knows); + try expectSame(plain, scheduled, n); + const fixed = try run(ps, true, 0, .knows); + try expectSame(plain, fixed, n); + try std.testing.expect(scheduled.asked > 0 and scheduled.drafted > 0); + // a confident head's survivals cut each stream's chain where its guess goes wrong: fewer rows verified + try std.testing.expect(scheduled.rows < fixed.rows); + if (n > 1) try std.testing.expect(scheduled.ragged > 0); // streams' verify lengths differ within a round + } +} + +test "the drafter's chances reach the lane facts: rows by chances and holds by grants, or a fixed window" { + var p: Priced = .{ .head = .{ .facts_ = .{ .depth = 9, .step_ms = 0.5, .chances = true } } }; + defer p.deinit(); + var target: fake.Fake = .{ .gpa = gpa }; + defer target.deinit(); + var wrapped: Drafted = .{ .gpa = gpa, .target = target.backend(), .drafter = p.drafter() }; + defer wrapped.deinit(); + const m = wrapped.facts(base()); + try std.testing.expect(m.draft_probabilities and m.granted_depth and m.draft_confidence == null); + var cfg = try Config.init(gpa, m, 16, 15); + defer cfg.deinit(gpa); + try std.testing.expectEqual(@as(i64, 2), cfg.k.draft_slack); // the depth rule's own + try std.testing.expect(cfg.k.grant_slack == 1 and cfg.k.short_slack == 0); // a granted width's own + p.head.facts_.slack = 1; + var one = try Config.init(gpa, wrapped.facts(base()), 16, 15); + defer one.deinit(gpa); + try std.testing.expectEqual(@as(i64, 1), one.k.draft_slack); // a drafter's own slack past its grant + try std.testing.expect(one.k.grant_slack == 1 and one.k.short_slack == 1); // short or not + p.head.facts_.confidence_floor = 0; + const f = wrapped.facts(base()); + try std.testing.expect(!f.draft_probabilities and !f.granted_depth and f.draft_confidence.? == 0); + p.head.facts_.chances = false; + const none = wrapped.facts(base()); + try std.testing.expect(!none.draft_probabilities and none.draft_confidence == null); +} + +test "a granted depth: the deepest chain first, then the grant plus its slack, and after a grant of none a probe" { + const empty: Table = .{}; + var rule: depth.Rule = .{ .gpa = gpa, .prior = &.{0.8}, .most_drafts = 9, .family_costs = &empty, .shared_costs = &empty, .mtp_step_ms = 0.5, .plain_guard = true, .node_probabilities = true, .granted_depth = true, .batch_rows = 32 }; + defer rule.deinit(); + var st: ?depth.State = null; + defer if (st) |*s| s.deinit(gpa); + const No = struct { + fn copy(_: *anyopaque, _: *const depth.Who) bool { + return false; + } + }; + var unused: u8 = 0; + const probe: depth.Probe = .{ .ptr = &unused, .copy = No.copy }; + var who: depth.Who = .{ .state = &st, .draft_room = 100, .finished = false, .forced = false }; + try std.testing.expectEqual(@as(i64, 9), try rule.headDepth(&who, null, probe)); + who.granted = 3; + try std.testing.expectEqual(@as(i64, 4), try rule.headDepth(&who, null, probe)); // every held draft granted: one more + who.grant_short = true; + try std.testing.expectEqual(@as(i64, 3), try rule.headDepth(&who, null, probe)); // drafts left unverified: the grant + who.grant_short = false; + who.granted = 8; + try std.testing.expectEqual(@as(i64, 9), try rule.headDepth(&who, null, probe)); + rule.k.grant_slack = 2; // a drafter's slack sets both + rule.k.short_slack = 2; + who.grant_short = true; + who.granted = 3; + try std.testing.expectEqual(@as(i64, 5), try rule.headDepth(&who, null, probe)); + who.granted = 0; + var drafted: usize = 0; + for (0..8) |_| drafted += @intFromBool(try rule.headDepth(&who, null, probe) > 0); + try std.testing.expectEqual(@as(usize, 1), drafted); // one probe of draft_slack drafts in the first eight rounds +} + +test "holds as wide as the grants: replies equal one-token decode at 1, 4 and 8 streams, fewer drafts than whole blocks" { + var buf: [most_streams][]const u32 = undefined; + defer slack = 0; + for ([_]usize{ 1, 4, 8 }) |n| { + const ps = unequal(&buf, n); + const plain = try run(ps, false, null, .flat); + for ([_]Head{ .flat, .knows }) |head| { + slack = 0; + const narrow = try run(ps, true, null, head); + slack = 2; + const two = try run(ps, true, null, head); + slack = 9; + const whole = try run(ps, true, null, head); + try expectSame(plain, narrow, n); + try expectSame(plain, two, n); + try expectSame(plain, whole, n); + try std.testing.expect(narrow.drafted > 0); + try std.testing.expect(narrow.held < two.held and two.held < whole.held); + } + } +} diff --git a/zig/src/core/lanes/drafter.zig b/zig/src/core/lanes/drafter.zig index 80577e0edf..8bb6b7377f 100644 --- a/zig/src/core/lanes/drafter.zig +++ b/zig/src/core/lanes/drafter.zig @@ -18,6 +18,9 @@ pub const Facts = struct { window_costs: []const Cost = &.{}, // the target's window ms by rows (timed at load), for the depth rule lane_costs: []const Cost = &.{}, // the same for the lane planner (Model.lane_costs) tree_nodes: u32 = 0, // hold() without lanes grows its own tree of up to `depth` nodes, at most this many (tree, chances) + chances: bool = false, // chances() gives each held chain draft's chance: verify rows go by them (Model.granted_depth) + confidence_floor: ?f64 = null, // with chances: every round drafts the deepest chain and verifies drafts this likely + slack: u32 = 0, // with chances: drafts a hold offers past the stream's last grant (0: the depth rule's own) }; /// Target rows [start, start + features.rows) and the token after each; `rows`, when given, picks which to absorb. diff --git a/zig/src/core/lanes/engine.zig b/zig/src/core/lanes/engine.zig index a90dd3166f..3bf6c297da 100644 --- a/zig/src/core/lanes/engine.zig +++ b/zig/src/core/lanes/engine.zig @@ -71,6 +71,7 @@ pub const Engine = struct { .draft_step_ms = cfg.draft_step_ms, .plain_guard = cfg.plain_guard, .node_probabilities = cfg.node_probabilities, + .granted_depth = cfg.granted_depth, .confidence_floor = cfg.draft_confidence, .batch_rows = cfg.batch_rows, .batched_heads = cfg.draft_streams, diff --git a/zig/src/core/lanes/lanes.zig b/zig/src/core/lanes/lanes.zig index 280b16ef10..df733517d0 100644 --- a/zig/src/core/lanes/lanes.zig +++ b/zig/src/core/lanes/lanes.zig @@ -57,6 +57,7 @@ test { _ = @import("recorder_test.zig"); _ = @import("gpu_full_test.zig"); _ = @import("drafted_test.zig"); + _ = @import("drafted_chances_test.zig"); _ = @import("root_copy_test.zig"); _ = @import("drafted_grammar_test.zig"); _ = @import("windows_grammar_test.zig"); diff --git a/zig/src/core/lanes/stream.zig b/zig/src/core/lanes/stream.zig index 700d607aa2..f9b0423d65 100644 --- a/zig/src/core/lanes/stream.zig +++ b/zig/src/core/lanes/stream.zig @@ -232,6 +232,7 @@ pub const Stream = struct { served: ?u64 = null, proposal_depth: ?u32 = null, // null: legacy; otherwise the last routed request, zero until it succeeds granted: ?u32 = null, + grant_short: bool = false, // the last grant verified fewer drafts than the stream held copy_width: ?u32 = null, pub fn init(gpa: Allocator, spec: Spec) !Stream { @@ -300,6 +301,7 @@ pub const Stream = struct { s.inflight = null; s.served = null; s.granted = null; + s.grant_short = false; s.copy_width = null; s.proposal_depth = null; } diff --git a/zig/src/core/lanes/windows.zig b/zig/src/core/lanes/windows.zig index fd7ffec95d..32ec0e999d 100644 --- a/zig/src/core/lanes/windows.zig +++ b/zig/src/core/lanes/windows.zig @@ -226,7 +226,7 @@ pub fn probe(e: *Engine) depth.Probe { } pub fn who(s: *Stream) depth.Who { - return .{ .state = &s.depth, .draft_room = s.draftRoom(), .finished = s.finished, .forced = s.force.items.len > 0, .granted = s.granted, .stream = s, .sampled = s.sampling != null }; + return .{ .state = &s.depth, .draft_room = s.draftRoom(), .finished = s.finished, .forced = s.force.items.len > 0, .granted = s.granted, .grant_short = s.grant_short, .stream = s, .sampled = s.sampling != null }; } /// Whether any of a round's streams draws keyed samples (its rows carry their host work). @@ -283,7 +283,10 @@ pub fn allocate(e: *Engine, plans: []Plan) !void { try trail.event(e, &.{ f("ev", trail.str("lone-price")), f("stream", trail.str(plans[0].stream.id)), f("rows", trail.int(counts[0] + 1)), f("expected", ev.bits(expected)), f("ms", if (ms) |value| ev.bits(value) else .null), f("probe", .{ .bool = trial != null }) }); } for (plans, counts) |*p, c| { - if (p.kind == .head) p.stream.granted = c; + if (p.kind == .head) { + p.stream.granted = c; + if (e.cfg.granted_depth) p.stream.grant_short = c < p.count(); + } if (p.kind == .forced or p.count() == 0 or c == p.count()) continue; try cut.prefix(a, p, c); } From 631736f8d73b24f4d2209a4762555c2649434b88 Mon Sep 17 00:00:00 2001 From: BlivionIaG <15323295+BlivionIaG@users.noreply.github.com> Date: Mon, 26 Oct 2026 09:07:53 +0100 Subject: [PATCH 07/39] lanes: a drafter whose hold is a pass of its own: no hold beside a copy, and holds kept only while they pay The depth rule asks one head draft for a stream with a copy ahead, and a shared round's granted holds were priced only by their verify rows. A drafter whose hold runs as a pass of its own (Facts.copy_first) pays that pass for the one draft, and the copy round then drops it; Model.copy_first now asks it for none. A drafter may also give its hold pass's timed cost by blocks and depth (Facts.hold_costs, priced by hold_price.zig). With granted depths a shared round then keeps granted holds, most expected drafts a row first, only while each raises the next round's expected tokens a ms over its verify rows and the pass; a probe after a grant of none keeps its hold. A stream's last grant records the drafts it expected to keep (Stream.grant_gain). Both are off by default (copy_first false, hold_costs empty). Tests: with copy_first no hold is asked for a copy round and replies equal one-token decode at 1 and 3 streams; a hold's cost between and past its timed passes; granted holds kept while they pay, none for a dear pass, and a probe's hold kept. --- zig/src/core/lanes/config.zig | 4 + zig/src/core/lanes/depth.zig | 123 ++++++++++++++++++++- zig/src/core/lanes/drafted.zig | 2 + zig/src/core/lanes/drafted_copy_test.zig | 135 +++++++++++++++++++++++ zig/src/core/lanes/drafter.zig | 2 + zig/src/core/lanes/hold_price.zig | 58 ++++++++++ zig/src/core/lanes/lanes.zig | 2 + zig/src/core/lanes/stream.zig | 2 + zig/src/core/lanes/windows.zig | 11 +- 9 files changed, 334 insertions(+), 5 deletions(-) create mode 100644 zig/src/core/lanes/drafted_copy_test.zig create mode 100644 zig/src/core/lanes/hold_price.zig diff --git a/zig/src/core/lanes/config.zig b/zig/src/core/lanes/config.zig index 71c455637a..deb76d4be2 100644 --- a/zig/src/core/lanes/config.zig +++ b/zig/src/core/lanes/config.zig @@ -39,6 +39,8 @@ pub const Model = struct { granted_depth: bool = false, // with draft_probabilities: every stream drafts its last grant plus a slack draft_slack: u32 = 0, // drafts past a stream's last grant, short or not (0: the Constants' own) draft_streams: bool = false, // the head drafts every stream of a shared round in one batch + copy_first: bool = false, // a copy ahead takes the round: the head holds no drafts for it (a drafter's own pass) + hold_costs: []const @import("hold_price.zig").HoldCost = &.{}, // a drafter's own hold pass, timed at load head_trees: bool = false, // the head drafts trees of lanes (DraftRequest.lanes) and holds them join_tail: bool = false, // a prompt in pieces leaves its last token for the next shared round's window (Backend.join) piece_runs: bool = false, // a round's pieces go to the backend in one call (Backend.pieces: one forward over them) @@ -169,6 +171,8 @@ pub const Config = struct { .draft_slack = if (m.draft_slack > 0) m.draft_slack else (Constants{}).draft_slack, .grant_slack = if (m.draft_slack > 0) m.draft_slack else (Constants{}).grant_slack, .short_slack = if (m.draft_slack > 0) m.draft_slack else (Constants{}).short_slack, + .copy_first = m.copy_first and !m.context_copies, + .hold_costs = m.hold_costs, }, .draft_step_ms = m.draft_step_ms, }; diff --git a/zig/src/core/lanes/depth.zig b/zig/src/core/lanes/depth.zig index 2f80da8edd..c992e0de4b 100644 --- a/zig/src/core/lanes/depth.zig +++ b/zig/src/core/lanes/depth.zig @@ -14,8 +14,13 @@ pub const Constants = struct { draft_slack: i64 = 2, // drafts a stream offers past what its last shared round granted it grant_slack: i64 = 1, // granted depths: drafts past a grant that verified every draft the stream held short_slack: i64 = 0, // granted depths: drafts past a grant that left some of them unverified + copy_first: bool = false, // a copy ahead takes the next round whole: the head drafts none for it (else one) + hold_costs: []const HoldCost = &.{}, // a drafter's own hold pass by blocks and depth: granted holds must pay for it }; +pub const HoldCost = @import("hold_price.zig").HoldCost; +const holdMs = @import("hold_price.zig").holdMs; + /// A stream's acceptance by depth and probe counters (Python `_depth_state[stream]`). pub const State = struct { p: []f64, @@ -38,6 +43,7 @@ pub const Who = struct { forced: bool, granted: ?u32 = null, grant_short: bool = false, // the last grant left some held drafts unverified (the width did not bind) + gain: ?f64 = null, // the drafts its last grant expected to keep (null: none priced) stream: ?*anyopaque = null, sampled: bool = false, // keyed draws: host work a row that a greedy round does not have }; @@ -213,7 +219,7 @@ pub const Rule = struct { return @min(most, @as(i64, granted) + r.slack(who)); } - /// Python `_head_depth`: the next round's head drafts for a stream (0 when forced, 1 with a copy ahead). + /// Python `_head_depth`: the next round's head drafts for a stream (0 when forced, 1 or none with a copy ahead). pub fn headDepth(r: *Rule, who: *const Who, budget: ?i64, probe: Probe) !i64 { const d: i64 = if (who.finished) 0 else if (budget) |b| b else if (r.granted_depth) try r.grantDepth(who.*) @@ -222,7 +228,7 @@ pub const Rule = struct { else try r.depth(who.*); if (who.forced) return 0; - if (d != 0 and probe.copy(probe.ptr, who)) return 1; + if (d != 0 and probe.copy(probe.ptr, who)) return @intFromBool(!r.k.copy_first); return d; } @@ -235,6 +241,65 @@ pub const Rule = struct { for (out) |*d| d.* = @min(d.*, most); } + /// Granted holds kept, most expected drafts a row first, while they raise the next round's expected tokens a ms. + fn payHolds(r: *Rule, whos: []const Who, sampled: bool, out: []i64) !void { + const order = try r.gpa.alloc(usize, whos.len); + defer r.gpa.free(order); + var m: usize = 0; + var probes: usize = 0; // a probe after a grant of none keeps its hold: it measures the stream again + var deepest: i64 = 0; + for (whos, out, 0..) |w, d, i| { + if (d <= 0) continue; + if (w.granted == @as(?u32, 0)) { + probes += 1; + deepest = @max(deepest, d); + } else { + order[m] = i; + m += 1; + } + } + if (m == 0) return; + const Ctx = struct { + r: *Rule, + whos: []const Who, + fn less(c: @This(), x: usize, y: usize) bool { + const gx = c.r.gainOf(c.whos[x]) catch 0; + const gy = c.r.gainOf(c.whos[y]) catch 0; + return gx * rowsOf(c.whos[y]) > gy * rowsOf(c.whos[x]) or (gx * rowsOf(c.whos[y]) == gy * rowsOf(c.whos[x]) and x < y); + } + }; + std.mem.sort(usize, order[0..m], Ctx{ .r = r, .whos = whos }, Ctx.less); + const fixed = r.fixedMs(whos.len, sampled); + var rows: f64 = @floatFromInt(whos.len); + var tokens = rows; + const plain = r.shared_costs.get(@intCast(whos.len)) orelse return; + const probe_ms = if (probes > 0) holdMs(r.k.hold_costs, probes, deepest) else 0; + var best = tokens / (plain + fixed + probe_ms); + var keep: usize = 0; + for (order[0..m], 1..) |i, k| { + rows += rowsOf(whos[i]); + tokens += try r.gainOf(whos[i]); + deepest = @max(deepest, out[i]); + const verify = r.shared_costs.get(@intFromFloat(@ceil(rows))) orelse break; + const now = tokens / (verify + fixed + holdMs(r.k.hold_costs, k + probes, deepest)); + if (now > best) { + best = now; + keep = k; + } + } + for (order[keep..m]) |i| out[i] = 0; + } + + /// The drafts a granted stream's next round is expected to keep: its last grant's, or its first draft's rate. + fn gainOf(r: *Rule, w: Who) !f64 { + if (w.gain) |g| if (w.granted != null) return g; + return (try r.rates(w))[0]; + } + + fn rowsOf(w: Who) f64 { + return @floatFromInt(@max(1, w.granted orelse 1)); + } + /// Head drafts for the streams of a shared round, sized by allocation over their chain chances. pub fn budgets(r: *Rule, whos: []Who, probe: Probe, sampled: bool, out: []i64) !void { if (r.confidence_floor != null) { @@ -248,6 +313,7 @@ pub const Rule = struct { o.* = try r.headDepth(w, if (r.granted_depth) inner else @min(inner, granted + r.k.draft_slack), probe); } if (r.granted_depth) capPass(out, r.batch_rows); + if (r.granted_depth and r.k.hold_costs.len > 0) try r.payHolds(whos, sampled, out); return; } const most = try r.gpa.alloc(i64, whos.len); @@ -439,3 +505,56 @@ test "granted depths in a shared round fit one hold pass: every block is as deep try rule.budgets(whos[0..2], .{ .ptr = &unused, .copy = No.copy }, false, out[0..2]); try std.testing.expectEqualSlices(i64, &.{ 2, 1 }, out[0..2]); // a grant short of the held drafts: no slack } + +test "a copy ahead: one head draft beside it, or none when the head's drafts cost a pass of their own" { + const empty: Table = .{}; + const Yes = struct { + fn copy(_: *anyopaque, _: *const Who) bool { + return true; + } + }; + var unused: u8 = 0; + const probe: Probe = .{ .ptr = &unused, .copy = Yes.copy }; + var st: ?State = null; + defer if (st) |*x| x.deinit(std.testing.allocator); + const who: Who = .{ .state = &st, .draft_room = 100, .finished = false, .forced = false, .granted = 3 }; + for ([_]bool{ false, true }) |first| { + var rule: Rule = .{ .gpa = std.testing.allocator, .k = .{ .copy_first = first }, .prior = &.{0.8}, .most_drafts = 9, .family_costs = &empty, .shared_costs = &empty, .mtp_step_ms = 0.5, .plain_guard = false, .node_probabilities = true, .granted_depth = true, .batch_rows = 32 }; + defer rule.deinit(); + try std.testing.expectEqual(@as(i64, @intFromBool(!first)), try rule.headDepth(&who, null, probe)); + var out: [1]i64 = undefined; + var whos = [_]Who{who}; + try rule.budgets(&whos, probe, false, &out); + try std.testing.expectEqual(@as(i64, @intFromBool(!first)), out[0]); + } +} + +test "a granted hold stays only while its stream's expected drafts pay for the verify rows and the hold pass" { + const gpa = std.testing.allocator; + var shared: Table = .{}; + defer shared.deinit(gpa); + for (1..33) |w| try shared.put(gpa, @intCast(w), 10.0 + @as(f64, @floatFromInt(w))); + const empty: Table = .{}; + const cheap = [_]HoldCost{ .{ .streams = 1, .depth = 1, .ms = 1.0 }, .{ .streams = 8, .depth = 1, .ms = 4.5 } }; + var states: [4]?State = @splat(null); + defer for (&states) |*st| if (st.*) |*x| x.deinit(gpa); + const granted = [_]?u32{ 2, 2, 0, 1 }; + const gains = [_]?f64{ 1.8, 0.2, null, 0.9 }; + var whos: [4]Who = undefined; + for (&whos, &states, granted, gains) |*w, *st, g, x| w.* = .{ .state = st, .draft_room = 100, .finished = false, .forced = false, .granted = g, .gain = x }; + var rule: Rule = .{ .gpa = gpa, .k = .{ .hold_costs = &cheap }, .prior = &.{0.8}, .most_drafts = 9, .family_costs = &empty, .shared_costs = &shared, .mtp_step_ms = 0.5, .plain_guard = true, .node_probabilities = true, .granted_depth = true, .batch_rows = 32 }; + defer rule.deinit(); + // 8 ms overhead: plain 4 / 22; stream 0 then 3 raise the rate (5.8 / 25, 6.7 / 26.5), stream 1 would lower it + var out = [_]i64{ 4, 4, 0, 3 }; + try rule.payHolds(&whos, false, &out); + try std.testing.expectEqualSlices(i64, &.{ 4, 0, 0, 3 }, &out); + const dear = [_]HoldCost{.{ .streams = 1, .depth = 1, .ms = 20.0 }}; + rule.k.hold_costs = &dear; + out = .{ 4, 4, 0, 3 }; + try rule.payHolds(&whos, false, &out); + try std.testing.expectEqualSlices(i64, &.{ 0, 0, 0, 0 }, &out); // no gain pays a 20 ms pass + whos[2].granted = 0; + out = .{ 4, 4, 2, 3 }; + try rule.payHolds(&whos, false, &out); + try std.testing.expectEqualSlices(i64, &.{ 0, 0, 2, 0 }, &out); // a probe after a grant of none keeps its hold +} diff --git a/zig/src/core/lanes/drafted.zig b/zig/src/core/lanes/drafted.zig index cee1ec3155..6c9f7edc4b 100644 --- a/zig/src/core/lanes/drafted.zig +++ b/zig/src/core/lanes/drafted.zig @@ -105,6 +105,8 @@ pub const Drafted = struct { m.draft_streams = d.batched; m.draft_probabilities = false; m.head_trees = d.trees; + m.copy_first = d.copy_first; + m.hold_costs = d.hold_costs; if (d.chances and x.drafter.vtable.chances != null) { // a chain's drafts priced by the drafter's own chances; the next hold as deep as the grant and its slack m.draft_probabilities = d.confidence_floor == null; diff --git a/zig/src/core/lanes/drafted_copy_test.zig b/zig/src/core/lanes/drafted_copy_test.zig new file mode 100644 index 0000000000..329731f69c --- /dev/null +++ b/zig/src/core/lanes/drafted_copy_test.zig @@ -0,0 +1,135 @@ +//! A drafter whose hold costs a pass of its own: no hold for a stream whose next round is a copy. +const std = @import("std"); +const Config = @import("config.zig").Config; +const Engine = @import("engine.zig").Engine; +const be = @import("backend.zig"); +const sm = @import("stream.zig"); +const fake = @import("fake.zig"); +const dr = @import("drafter.zig"); +const Proposer = @import("proposer.zig").Proposer; +const Drafted = @import("drafted.zig").Drafted; +const dt = @import("drafted_test.zig"); + +const gpa = std.testing.allocator; + +/// Copies the fake target's own continuation (it lands whole) whenever the context's length is a multiple of three. +const Copier = struct { + out: [3]u32 = undefined, + match: i64 = 0, + + fn propose(ptr: *anyopaque, context: []const u32, most: i64) anyerror![]const u32 { + const c: *Copier = @ptrCast(@alignCast(ptr)); + c.match = 0; + if (context.len % 3 != 0 or most < 2) return &.{}; + var history: [128]u32 = undefined; + @memcpy(history[0..context.len], context); + const n: usize = @intCast(@min(3, most)); + for (c.out[0..n], 0..) |*t, k| { + t.* = fake.next(history[0 .. context.len + k], null, 0); + history[context.len + k] = t.*; + } + c.match = 100; + return c.out[0..n]; + } + + fn lastMatch(ptr: *anyopaque) i64 { + return @as(*Copier, @ptrCast(@alignCast(ptr))).match; + } + + const vtable: Proposer.VTable = .{ .propose = propose, .last_match = lastMatch }; +}; + +/// Each stream's last hold position, and the copy windows verified at a position a hold was asked for. +const Watch = struct { + var inner_hold: *const fn (*anyopaque, []const dr.Hold) anyerror!void = undefined; + var inner_verify: *const fn (*anyopaque, []const be.Window, []be.Verified) anyerror!void = undefined; + var at: [4]?u64 = @splat(null); + var streams: [4]?*sm.Stream = @splat(null); + var wasted: u32 = 0; + var copies: u32 = 0; + var blocks: u32 = 0; // the streams' blocks held, over every hold + + fn slot(s: *sm.Stream) usize { + for (&streams, 0..) |*x, i| if (x.* == null or x.* == s) { + x.* = s; + return i; + }; + unreachable; + } + + fn hold(ptr: *anyopaque, items: []const dr.Hold) anyerror!void { + blocks += @intCast(items.len); + // a joining stream's first hold comes before its first token is read: no copy can be foreseen for it + for (items) |it| at[slot(it.stream)] = if (it.stream.emitted().len > 0) it.position else null; + return inner_hold(ptr, items); + } + + fn verify(ptr: *anyopaque, windows: []const be.Window, out: []be.Verified) anyerror!void { + for (windows) |w| if (w.held == 0 and w.tokens.len > 0) { + copies += 1; + if (at[slot(w.stream)] == w.positions[0]) wasted += 1; + }; + return inner_verify(ptr, windows, out); + } +}; + +const Run = struct { tokens: [3][64]u32 = undefined, lens: [3]usize = @splat(0), holds: u32 = 0, wasted: u32 = 0, copies: u32 = 0 }; + +fn run(n: usize, drafts: bool, copy_first: bool) !Run { + Watch.at = @splat(null); + Watch.streams = @splat(null); + Watch.wasted = 0; + Watch.copies = 0; + Watch.blocks = 0; + var target: fake.Fake = .{ .gpa = gpa }; + defer target.deinit(); + var head: dt.FakeDrafter = .{ .facts_ = .{ .depth = 4, .step_ms = 0.5, .batched = true, .copy_first = copy_first } }; + defer head.deinit(); + var hooked = head.drafter(); + var table = hooked.vtable.*; + Watch.inner_hold = table.hold; + table.hold = Watch.hold; + hooked.vtable = &table; + var wrapped = try Drafted.init(gpa, target.backend(), hooked); + defer wrapped.deinit(); + // the round loop's windows, before the wrapper turns held drafts into host tokens + var seen = if (drafts) wrapped.backend() else target.backend(); + var verified = seen.vtable.*; + Watch.inner_verify = verified.verify; + verified.verify = Watch.verify; + seen.vtable = &verified; + const m: @import("config.zig").Model = .{ .exact_width = 16, .gpu_tokens = true, .hidden_rows = true, .batch_rows = 32, .max_streams = 4 }; + var cfg = try Config.init(gpa, if (drafts) wrapped.facts(m) else m, 16, 15); + defer cfg.deinit(gpa); + var clock: fake.FixedClock = .{}; + var engine = Engine.init(gpa, &cfg, seen, clock.clock()); + defer engine.deinit(); + var copiers: [3]Copier = @splat(.{}); + var streams: [3]sm.Stream = undefined; + for (streams[0..n], copiers[0..n], 0..) |*s, *c, i| s.* = try sm.Stream.init(gpa, .{ .id = "s", .prompt = dt.long[0 .. 12 + i], .max_new = 40, .eos = &.{96}, .drafts = drafts, .proposer = .{ .ptr = c, .vtable = &Copier.vtable } }); + defer for (streams[0..n]) |*s| s.deinit(gpa); + for (streams[0..n]) |*s| try engine.addStream(s); + while (engine.activeCount() > 0) try engine.step(); + var out: Run = .{ .holds = Watch.blocks, .wasted = Watch.wasted, .copies = Watch.copies }; + for (streams[0..n], 0..) |*s, i| { + out.lens[i] = s.emitted().len; + @memcpy(out.tokens[i][0..out.lens[i]], s.emitted()); + } + return out; +} + +test "a copy ahead takes the round: with copy_first no hold is asked for it, and replies equal one-token decode" { + for ([_]usize{ 1, 3 }) |n| { + const plain = try run(n, false, false); + const before = try run(n, true, false); + const after = try run(n, true, true); + for (0..n) |i| { + try std.testing.expectEqualSlices(u32, plain.tokens[i][0..plain.lens[i]], before.tokens[i][0..before.lens[i]]); + try std.testing.expectEqualSlices(u32, plain.tokens[i][0..plain.lens[i]], after.tokens[i][0..after.lens[i]]); + } + try std.testing.expect(before.copies > 0 and after.copies > 0); + try std.testing.expect(before.wasted > 0); // a hold of one draft beside each copy, dropped by the copy round + try std.testing.expectEqual(@as(u32, 0), after.wasted); + try std.testing.expect(after.holds < before.holds); + } +} diff --git a/zig/src/core/lanes/drafter.zig b/zig/src/core/lanes/drafter.zig index 8bb6b7377f..408a80832d 100644 --- a/zig/src/core/lanes/drafter.zig +++ b/zig/src/core/lanes/drafter.zig @@ -21,6 +21,8 @@ pub const Facts = struct { chances: bool = false, // chances() gives each held chain draft's chance: verify rows go by them (Model.granted_depth) confidence_floor: ?f64 = null, // with chances: every round drafts the deepest chain and verifies drafts this likely slack: u32 = 0, // with chances: drafts a hold offers past the stream's last grant (0: the depth rule's own) + copy_first: bool = false, // a hold costs a pass of its own: none is asked for a stream with a copy ahead + hold_costs: []const @import("hold_price.zig").HoldCost = &.{}, // that pass's ms by blocks and depth (timed at load) }; /// Target rows [start, start + features.rows) and the token after each; `rows`, when given, picks which to absorb. diff --git a/zig/src/core/lanes/hold_price.zig b/zig/src/core/lanes/hold_price.zig new file mode 100644 index 0000000000..e4c854410e --- /dev/null +++ b/zig/src/core/lanes/hold_price.zig @@ -0,0 +1,58 @@ +//! A drafter's hold pass priced from its timed passes: by the blocks it holds and their depth. +const std = @import("std"); + +/// A drafter's hold pass of `streams` blocks, each `depth` drafts deep, in ms (timed at load). +pub const HoldCost = struct { streams: u32, depth: u32, ms: f64 }; + +/// A hold pass's ms: linear in depth at each timed stream count, then in streams (past the widest, by its slope). +pub fn holdMs(costs: []const HoldCost, streams: usize, depth: i64) f64 { + var below: ?u32 = null; + var above: ?u32 = null; + for (costs) |c| { + if (c.streams <= streams and (below == null or c.streams > below.?)) below = c.streams; + if (c.streams >= streams and (above == null or c.streams < above.?)) above = c.streams; + } + const lo = below orelse above.?; + const hi = above orelse lo; + const at_lo = atDepth(costs, lo, depth); + if (hi == lo) { + if (above != null) return at_lo; + // past the widest timed count: the slope from the count below it + var under: ?u32 = null; + for (costs) |c| if (c.streams < lo and (under == null or c.streams > under.?)) { + under = c.streams; + }; + const u = under orelse return at_lo * @as(f64, @floatFromInt(streams)) / @as(f64, @floatFromInt(lo)); + const slope = (at_lo - atDepth(costs, u, depth)) / @as(f64, @floatFromInt(lo - u)); + return at_lo + slope * @as(f64, @floatFromInt(streams - lo)); + } + const t = @as(f64, @floatFromInt(streams - lo)) / @as(f64, @floatFromInt(hi - lo)); + return at_lo + t * (atDepth(costs, hi, depth) - at_lo); +} + +/// One stream count's hold ms at `depth`: linear between its two nearest timed depths (one alone: as timed). +fn atDepth(costs: []const HoldCost, streams: u32, depth: i64) f64 { + var a: ?HoldCost = null; + var b: ?HoldCost = null; + for (costs) |c| { + if (c.streams != streams) continue; + if (a == null or c.depth < a.?.depth) { + b = a; + a = c; + } else if (b == null or c.depth < b.?.depth) b = c; + } + const x = a.?; + const y = b orelse return x.ms; + const t = (@as(f64, @floatFromInt(depth)) - @as(f64, @floatFromInt(x.depth))) / @as(f64, @floatFromInt(y.depth - x.depth)); + return x.ms + t * (y.ms - x.ms); +} + +test "a hold's ms between and past the timed passes" { + const t = [_]HoldCost{ .{ .streams = 1, .depth = 1, .ms = 1.0 }, .{ .streams = 1, .depth = 4, .ms = 2.5 }, .{ .streams = 4, .depth = 1, .ms = 3.0 }, .{ .streams = 4, .depth = 4, .ms = 4.5 }, .{ .streams = 8, .depth = 1, .ms = 4.0 } }; + try std.testing.expectApproxEqAbs(@as(f64, 1.0), holdMs(&t, 1, 1), 1e-12); + try std.testing.expectApproxEqAbs(@as(f64, 2.0), holdMs(&t, 1, 3), 1e-12); // linear in depth + try std.testing.expectApproxEqAbs(@as(f64, 3.0), holdMs(&t, 1, 5), 1e-12); // and past the deepest + try std.testing.expectApproxEqAbs(@as(f64, 1.0 + 2.0 / 3.0), holdMs(&t, 2, 1), 1e-12); // linear in streams + try std.testing.expectApproxEqAbs(@as(f64, 4.0), holdMs(&t, 8, 3), 1e-12); // one depth timed: as timed + try std.testing.expectApproxEqAbs(@as(f64, 5.0), holdMs(&t, 12, 1), 1e-12); // past the widest: the slope below it +} diff --git a/zig/src/core/lanes/lanes.zig b/zig/src/core/lanes/lanes.zig index df733517d0..cc0271d636 100644 --- a/zig/src/core/lanes/lanes.zig +++ b/zig/src/core/lanes/lanes.zig @@ -18,6 +18,7 @@ pub const cost_rule = @import("cost_rule.zig"); pub const gpu_rule = @import("gpu_rule.zig"); pub const gpu_full = @import("gpu_full.zig"); pub const depth = @import("depth.zig"); +pub const hold_price = @import("hold_price.zig"); pub const mtp_policy = @import("mtp_policy.zig"); pub const trim = @import("trim.zig"); pub const config = @import("config.zig"); @@ -58,6 +59,7 @@ test { _ = @import("gpu_full_test.zig"); _ = @import("drafted_test.zig"); _ = @import("drafted_chances_test.zig"); + _ = @import("drafted_copy_test.zig"); _ = @import("root_copy_test.zig"); _ = @import("drafted_grammar_test.zig"); _ = @import("windows_grammar_test.zig"); diff --git a/zig/src/core/lanes/stream.zig b/zig/src/core/lanes/stream.zig index f9b0423d65..e79cefe996 100644 --- a/zig/src/core/lanes/stream.zig +++ b/zig/src/core/lanes/stream.zig @@ -233,6 +233,7 @@ pub const Stream = struct { proposal_depth: ?u32 = null, // null: legacy; otherwise the last routed request, zero until it succeeds granted: ?u32 = null, grant_short: bool = false, // the last grant verified fewer drafts than the stream held + grant_gain: ?f64 = null, // the drafts its last grant expected to keep (the chances it was priced by) copy_width: ?u32 = null, pub fn init(gpa: Allocator, spec: Spec) !Stream { @@ -302,6 +303,7 @@ pub const Stream = struct { s.served = null; s.granted = null; s.grant_short = false; + s.grant_gain = null; s.copy_width = null; s.proposal_depth = null; } diff --git a/zig/src/core/lanes/windows.zig b/zig/src/core/lanes/windows.zig index 32ec0e999d..7e46b2c107 100644 --- a/zig/src/core/lanes/windows.zig +++ b/zig/src/core/lanes/windows.zig @@ -226,7 +226,7 @@ pub fn probe(e: *Engine) depth.Probe { } pub fn who(s: *Stream) depth.Who { - return .{ .state = &s.depth, .draft_room = s.draftRoom(), .finished = s.finished, .forced = s.force.items.len > 0, .granted = s.granted, .grant_short = s.grant_short, .stream = s, .sampled = s.sampling != null }; + return .{ .state = &s.depth, .draft_room = s.draftRoom(), .finished = s.finished, .forced = s.force.items.len > 0, .granted = s.granted, .grant_short = s.grant_short, .gain = s.grant_gain, .stream = s, .sampled = s.sampling != null }; } /// Whether any of a round's streams draws keyed samples (its rows carry their host work). @@ -282,10 +282,15 @@ pub fn allocate(e: *Engine, plans: []Plan) !void { const ms = own.get(1 + @as(i64, counts[0])); try trail.event(e, &.{ f("ev", trail.str("lone-price")), f("stream", trail.str(plans[0].stream.id)), f("rows", trail.int(counts[0] + 1)), f("expected", ev.bits(expected)), f("ms", if (ms) |value| ev.bits(value) else .null), f("probe", .{ .bool = trial != null }) }); } - for (plans, counts) |*p, c| { + for (plans, counts, probs) |*p, c, pr| { if (p.kind == .head) { p.stream.granted = c; - if (e.cfg.granted_depth) p.stream.grant_short = c < p.count(); + if (e.cfg.granted_depth) { + p.stream.grant_short = c < p.count(); + var gain: f64 = 0; + for (pr[0..@min(c, pr.len)]) |chance| gain += chance; + p.stream.grant_gain = gain; + } } if (p.kind == .forced or p.count() == 0 or c == p.count()) continue; try cut.prefix(a, p, c); From 98dee05c1d665a1afdd88d690fa37bc16db71488 Mon Sep 17 00:00:00 2001 From: BlivionIaG <15323295+BlivionIaG@users.noreply.github.com> Date: Mon, 26 Oct 2026 09:37:14 +0100 Subject: [PATCH 08/39] lfm2: a DSpark confidence logit kernel, registered with the kernel contract LiquidAI's DSpark drafter carries a confidence head: one linear over a draft's final block row (the row the target's head reads) and the W1 row of the token before it, 2,048 + 256 inputs, to one logit. lfm2_dspark_confidence (lfm2_dspark.metal) computes that logit for every row of a hold pass: 256 threads a row, fp32 products and sums, the eight simdgroups' partials summed in threadgroup memory. Nothing calls it yet. It registers with the kernel contract (metal_lfm2_confidence.zig): the served pass shapes against a float64 reference, the accumulation gate against reduced-precision sums, and gate 5 against every Apple chip we serve and this GPU. --- zig/kernels/metal/lfm2/lfm2_dspark.metal | 27 +++++ zig/src/families/lfm2_moe/kernels.zig | 1 + zig/tests/contract/metal_lfm2_confidence.zig | 116 +++++++++++++++++++ 3 files changed, 144 insertions(+) create mode 100644 zig/tests/contract/metal_lfm2_confidence.zig diff --git a/zig/kernels/metal/lfm2/lfm2_dspark.metal b/zig/kernels/metal/lfm2/lfm2_dspark.metal index 050fd5e8cc..7f613def9b 100644 --- a/zig/kernels/metal/lfm2/lfm2_dspark.metal +++ b/zig/kernels/metal/lfm2/lfm2_dspark.metal @@ -105,3 +105,30 @@ constant constexpr uint MARKOV_SPAN = 512; const uint token = picks[size_t(i) * p.x]; ids[i] = token < p.z ? token : 0u; } + +// DSpark's confidence logit of draft i of stream s: proj . [x row s * n + i, W1[previous token]] in fp32, no bias. +struct lfm2_confidence_args { uint n, pass, vocab, rank, hidden; }; +[[kernel]] void lfm2_dspark_confidence(const device bfloat* x [[buffer(0)]], const device bfloat* w1 [[buffer(1)]], + const device uint* ids [[buffer(2)]], const device uint* picks [[buffer(3)]], + const device bfloat* proj [[buffer(4)]], + constant lfm2_confidence_args& p [[buffer(5)]], device float* out [[buffer(6)]], + uint2 tg [[threadgroup_position_in_grid]], uint t [[thread_index_in_threadgroup]], + uint sg [[simdgroup_index_in_threadgroup]], + uint lane [[thread_index_in_simdgroup]]) { + threadgroup float part[8]; + const uint i = tg.x, s = tg.y; + // the previous token: the block's pending id for the first draft, else the Markov step's pick before it + const uint id = i == 0 ? ids[size_t(s) * p.n] : picks[2 * (size_t(i - 1) * p.pass + s)]; + const device bfloat* row = x + (size_t(s) * p.n + i) * p.hidden; + float d = 0.0f; + for (uint k = t; k < p.hidden; k += 256) d = fma(float(row[k]), float(proj[k]), d); + if (id < p.vocab) + for (uint k = t; k < p.rank; k += 256) d = fma(float(w1[size_t(id) * p.rank + k]), float(proj[p.hidden + k]), d); + d = simd_sum(d); + if (lane == 0) part[sg] = d; + threadgroup_barrier(mem_flags::mem_threadgroup); + if (t != 0) return; + float sum = 0.0f; + for (uint g = 0; g < 8; ++g) sum += part[g]; + out[size_t(i) * p.pass + s] = sum; +} diff --git a/zig/src/families/lfm2_moe/kernels.zig b/zig/src/families/lfm2_moe/kernels.zig index 936ba33197..96997a1109 100644 --- a/zig/src/families/lfm2_moe/kernels.zig +++ b/zig/src/families/lfm2_moe/kernels.zig @@ -33,6 +33,7 @@ const names = [_][:0]const u8{ "lfm2_markov_scan", "lfm2_markov_pick", "lfm2_draft_ids", + "lfm2_dspark_confidence", }; pub const Kernels = struct { diff --git a/zig/tests/contract/metal_lfm2_confidence.zig b/zig/tests/contract/metal_lfm2_confidence.zig new file mode 100644 index 0000000000..f1543a183d --- /dev/null +++ b/zig/tests/contract/metal_lfm2_confidence.zig @@ -0,0 +1,116 @@ +//! LFM2's DSpark confidence logit (lfm2_dspark_confidence): a dot of a row and a W1 row with the head's 2,304 weights. +const std = @import("std"); +const mtl = @import("metal"); +const contract = @import("contract"); +const sources = @import("kernel_sources"); +const c = contract.spec; +const g = contract.gates; +const rows = contract.rows; +const rig_ = @import("rig.zig"); + +const hidden = 2048; +const rank = 256; +pub const name = "metal lfm2 dspark confidence"; + +/// rows: one stream's drafts (row i reads the block's row i and the W1 row of the token before draft i). +const shapes = [_]c.Shape{ .{ .rows = 1, .n = 1, .k = hidden + rank }, .{ .rows = 4, .n = 1, .k = hidden + rank }, .{ .rows = 9, .n = 1, .k = hidden + rank } }; + +/// A thread's chain over every 256th input, a 32-way simd_sum tree, then the eight simdgroups' partials in order. +fn depth(s: c.Shape) usize { + return (s.k + 255) / 256 + 5 + 8; +} + +pub fn spec() c.Spec { + return .{ .name = name, .accum = .f32, .depth = depth, .out = .f32, .served = &shapes }; +} + +pub const Probe = struct { + a: std.mem.Allocator, + rig: rig_.Rig, + pipe: mtl.Pipeline, + r: ?rows.Rows = null, + args: [6]rig_.Ref = undefined, // x, W1 rows, ids, picks, the head's weights, the shape + out: rig_.Ref = undefined, + + pub fn init(a: std.mem.Allocator, device: mtl.Device, queue: mtl.Queue) !Probe { + return .{ .a = a, .rig = .{ .a = a, .device = device, .queue = queue }, .pipe = try rig_.pipeline(device, sources.lfm2, "lfm2_dspark_confidence") }; + } + + pub fn deinit(p: *Probe) void { + if (p.r) |r| r.deinit(); + p.rig.deinit(); + p.pipe.deinit(); + } + + const vt = g.hooks(Probe); + pub fn kernel(p: *Probe) g.Kernel { + return .{ .ctx = p, .vt = &vt, .spec = spec() }; + } + + /// Row i's first 2,048 inputs as the block's row i, its last 256 as W1 row i: the token before draft i is i. + pub fn prepare(p: *Probe, s: c.Shape, f: g.Fill, seed: u64) !bool { + if (s.k != hidden + rank or s.rows == 0 or s.rows > 16) return false; + if (p.r) |r| r.deinit(); + p.r = null; + p.rig.reset(); + const r = try rows.Rows.init(p.a, .dot, s); + p.r = r; + r.fill(f, seed); + const x = try p.a.alloc(u16, s.rows * hidden); + defer p.a.free(x); + const w1 = try p.a.alloc(u16, s.rows * rank); + defer p.a.free(w1); + for (0..s.rows) |i| { + @memcpy(x[i * hidden ..][0..hidden], r.x[i * s.k ..][0..hidden]); + @memcpy(w1[i * rank ..][0..rank], r.x[i * s.k + hidden ..][0..rank]); + } + // the block's pending id is token 0; the pick before draft i (i > 0) is token i, a flag word after each + var picks: [32]u32 = @splat(0xdead); + for (1..s.rows) |i| picks[2 * (i - 1)] = @intCast(i); + const ids = [1]u32{0}; + const args = [5]u32{ @intCast(s.rows), 1, @intCast(s.rows), rank, hidden }; + p.args[0] = try p.rig.input(std.mem.sliceAsBytes(x)); + p.args[1] = try p.rig.input(std.mem.sliceAsBytes(w1)); + p.args[2] = try p.rig.input(std.mem.asBytes(&ids)); + p.args[3] = try p.rig.input(std.mem.asBytes(&picks)); + p.args[4] = try p.rig.input(std.mem.sliceAsBytes(r.w)); + p.args[5] = try p.rig.input(std.mem.asBytes(&args)); + p.out = try p.rig.output(s.rows * 4); + return true; + } + + pub fn run(p: *Probe) !void { + const n = p.r.?.shape.rows; + const pass = p.rig.begin(); + const e = pass.e; + e.setPipeline(p.pipe); + for (p.args, 0..) |ref, i| e.setBuffer(ref.buf, ref.off, i); + e.setBuffer(p.out.buf, p.out.off, 6); + e.dispatchGroups(mtl.Size.of(n, 1, 1), mtl.Size.of(256, 1, 1)); + try p.rig.finish(pass); + } + + pub fn count(p: *Probe) usize { + return p.r.?.outputs(); + } + + pub fn reference(p: *Probe, want: []f64, mag: []f64) !void { + return p.r.?.reference(want, mag); + } + + pub fn simulate(p: *Probe, acc: c.Accum, order: g.Order, out: []f64) !void { + return p.r.?.simulate(acc, order, out); + } +}; + +/// Gate 5 for the launch on every Apple chip we serve and this GPU: 256 threads, the eight simdgroups' partials. +pub fn limits(a: std.mem.Allocator, device: mtl.Device, only: ?[]const u8) !usize { + if (only) |s| if (std.mem.indexOf(u8, name, s) == null) return 0; + const d = contract.limits.MetalLaunch{ .name = name, .threads = 256, .tg_memory = 32, .offline = false }; + var notes: std.ArrayList(u8) = .empty; + defer notes.deinit(a); + var problems: usize = 0; + for (contract.limits.apple) |chip| problems += try contract.limits.metal(d, chip, ¬es, a); + std.debug.print("{s}", .{notes.items}); + return problems + try @import("metal_limits.zig").onThisGpu(a, device, .{ .decl = d, .function = "lfm2_dspark_confidence", .here = sources.lfm2, .pre_m5 = "" }, false); +} From 7d7e5fcc36bf238621040521d10e2999c07ed15f Mon Sep 17 00:00:00 2001 From: BlivionIaG <15323295+BlivionIaG@users.noreply.github.com> Date: Mon, 26 Oct 2026 09:37:43 +0100 Subject: [PATCH 09/39] lfm2: the DSpark confidence head schedules the verify (--draft-verify, --verify-costs) With the scheduled verify a hold also runs lfm2_dspark_confidence, and each lane keeps sigmoid(logit + bias) for its drafts. chances() gives their running product, the chance that a draft and every draft before it land, as Facts.chances: the lane core's window allocator then picks each stream's verify rows by those chances and the verify costs, and the next hold drafts as deep as the stream's grant. No second scheduler. While a hold still runs, its rows are priced by the lane's last settled confidences (draft j by the last hold's draft j, the deepest one repeated past it), so a verify does not wait on the GPU for them. A lane with none settled yet waits. Two serve flags on macOS, listed by capabilities --json and refused unless an lfm2_moe checkpoint is served with --drafter: - --draft-verify depth|fixed|scheduled: the core's depth rule, the whole block every round (the published fixed window), or the confidence schedule. scheduled is the default. - --verify-costs FILE: a table of verify forward ms by streams and rows a stream, written by tf-lfm2-verify-costs on the serving Mac, in place of the costs timed at load. Shared rounds keep the shared-round guard in every mode but fixed. Tests: test-lfm2-dspark reads a cost table, refuses malformed ones and checks the default mode; the server's parser tests take both flags and refuse an unknown mode. tf-lfm2-dspark checks the head against its fp32 reference. --- zig/build/metal_targets.zig | 1 + zig/src/core/engine_api.zig | 6 ++ zig/src/families/lfm2_moe/dspark.zig | 59 ++++++++++++++++- zig/src/families/lfm2_moe/dspark_block.zig | 28 ++++++++ zig/src/families/lfm2_moe/dspark_test.zig | 5 ++ zig/src/families/lfm2_moe/dspark_weights.zig | 12 ++++ zig/src/families/lfm2_moe/lfm2_moe.zig | 1 + zig/src/families/lfm2_moe/verify_costs.zig | 44 +++++++++++++ zig/src/native/lfm2_moe.zig | 30 ++++++++- zig/src/native/metal.zig | 1 + zig/src/server/cli.zig | 6 ++ zig/src/server/cli_test.zig | 7 ++ zig/src/server/main.zig | 2 +- zig/src/tensorfold.zig | 1 + zig/tests/lfm2_dspark.zig | 15 ++++- zig/tests/lfm2_verify_costs.zig | 67 ++++++++++++++++++++ 16 files changed, 277 insertions(+), 8 deletions(-) create mode 100644 zig/src/families/lfm2_moe/verify_costs.zig create mode 100644 zig/tests/lfm2_verify_costs.zig diff --git a/zig/build/metal_targets.zig b/zig/build/metal_targets.zig index ea81d7ea7a..4bc5ac5da3 100644 --- a/zig/build/metal_targets.zig +++ b/zig/build/metal_targets.zig @@ -293,6 +293,7 @@ pub fn metalTargets(b: *std.Build, target: std.Build.ResolvedTarget, optimize: s .{ .name = "tf-lfm2-bench", .path = "zig/tests/lfm2_bench.zig", .about = "LFM2-MoE prompt-chunk and one-row decode timing" }, .{ .name = "tf-lfm2-exact", .path = "zig/tests/lfm2_exact.zig", .about = "LFM2-MoE prompt chunks, windows and committed-state exactness" }, .{ .name = "tf-lfm2-dspark", .path = "zig/tests/lfm2_dspark.zig", .about = "LFM2's DSpark drafter on the GPU against an fp32 host reference of the released drafter" }, + .{ .name = "tf-lfm2-verify-costs", .path = "zig/tests/lfm2_verify_costs.zig", .about = "LFM2 verify forward ms by stream count and rows a stream, written as a cost table" }, .{ .name = "tf-lfm2-fp64", .path = "zig/tests/lfm2_fp64.zig", .about = "LFM2 projection classes and attention on the tensor units, the tiles and the twin, each against float64" }, .{ .name = "tensorfold", .path = "zig/src/main.zig", .about = "The native engine's command line" }, .{ .name = "tf-nemotron-fixtures", .path = "zig/tests/nemotron_fixtures.zig", .about = "Nemotron kernels against the Python engine's captured ops" }, diff --git a/zig/src/core/engine_api.zig b/zig/src/core/engine_api.zig index 93d830f364..49bbbbb2b4 100644 --- a/zig/src/core/engine_api.zig +++ b/zig/src/core/engine_api.zig @@ -274,6 +274,9 @@ pub const Family = struct { model_type: []const u8, formats: []const []const u8, /// The math an NVFP4 checkpoint runs: its own (FP4 rows into NVFP4 layers, FP8 into FP8 ones), or bf16 rows throughout. pub const Precision = enum { checkpoint, full }; +/// An external drafter's verify: the depth rule's rows, its whole block, or its confidence head's survivals. +pub const DraftVerify = enum { depth, fixed, scheduled }; + /// What a server asks of the engine it opens: the checkpoint, and the serve flags an engine reads. pub const Open = struct { pub fn fsrRefusal(o: Open, backend: []const u8) ?[]const u8 { @@ -301,6 +304,9 @@ pub const Open = struct { /// --drafter: a draft model's directory (Qwen3.8-27B's DFlash2; null: none), and --drafter-bits: 4, or 0 for bf16. drafter: ?[]const u8 = null, drafter_bits: u8 = 4, + /// --draft-verify and --verify-costs (a tf-lfm2-verify-costs table): the LFM2 DSpark drafter's verify rows. + draft_verify: ?DraftVerify = null, + verify_costs: ?[]const u8 = null, /// --record: each stream's committed rows and tapped features written under this folder, for drafter training. record: ?[]const u8 = null, speed_up: ?[]const u8 = null, diff --git a/zig/src/families/lfm2_moe/dspark.zig b/zig/src/families/lfm2_moe/dspark.zig index 850c1ce34c..0263d03f64 100644 --- a/zig/src/families/lfm2_moe/dspark.zig +++ b/zig/src/families/lfm2_moe/dspark.zig @@ -22,8 +22,13 @@ pub const absorb_rows = 2048; pub const stage_rows = 128; const kw = c.kv_heads * c.head_dim; +/// Verify rows by the core's depth rule, the whole block (fixed), or the confidence head's chances (scheduled). +pub const Verify = enum { depth, fixed, scheduled }; +/// The verify when none is asked: the confidence-scheduled one. +pub const default_verify: Verify = .scheduled; + /// `bits`: the width the bf16 linears are quantized to (8 keeps the fp32 reference's drafts; 4 does not). -pub const Options = struct { capacity: usize, bits: u8 = 8 }; +pub const Options = struct { capacity: usize, bits: u8 = 8, verify: Verify = default_verify }; /// One stream's context keys and values a layer ([kv head][cap][head dim], as the target's attention reads them). pub const Lane = struct { @@ -35,6 +40,8 @@ pub const Lane = struct { staged: u32 = 0, held: [16]u32 = @splat(0), held_len: u32 = 0, + conf: [16]f32 = @splat(0), // each held draft's chance given the drafts before it landed (the confidence head) + conf_len: u32 = 0, // the drafts `conf` covers: the last settled hold's depth slot: ?u32 = null, // its block's place in the last signalled hold, whose picks still hold its drafts }; @@ -62,6 +69,7 @@ pub const Scratch = struct { best: mtl.Buffer, // the Markov scan's best score and token of each threadgroup, by stream at: mtl.Buffer, picks: mtl.Buffer, + conf: mtl.Buffer, // the confidence logits, step by step, a pass's blocks each }; pub const Drafter = struct { @@ -147,6 +155,7 @@ pub const Drafter = struct { .best = try d.buffer(r * block.scan_groups * 4), .at = try d.buffer(r * block.scan_groups * 4), .picks = try d.buffer(d.config.block * r * 8), + .conf = try d.buffer(d.config.block * r * 4), }; } @@ -156,7 +165,12 @@ pub const Drafter = struct { } pub fn drafter(d: *Drafter) dr.Drafter { - return .{ .ptr = d, .vtable = &.{ .taps = tapsFn, .facts = factsFn, .open = openFn, .absorb = absorbFn, .hold = holdFn, .held = heldFn, .release = releaseFn } }; + return .{ .ptr = d, .vtable = &.{ .taps = tapsFn, .facts = factsFn, .open = openFn, .absorb = absorbFn, .hold = holdFn, .held = heldFn, .release = releaseFn, .chances = chancesFn } }; + } + + /// Whether holds compute the confidence head (the scheduled verify, on a checkpoint that has the head). + pub fn confident(d: *const Drafter) bool { + return d.options.verify == .scheduled and d.w.confidence != null; } fn cast(ptr: *anyopaque) *Drafter { @@ -173,7 +187,46 @@ pub const Drafter = struct { fn factsFn(ptr: *anyopaque) dr.Facts { const d = cast(ptr); - return .{ .depth = d.config.block, .step_ms = d.step_ms, .prior = prior[0..d.config.block], .batched = true, .shared_guard = true, .stream_bytes = d.laneBytes(d.options.capacity) }; + var f: dr.Facts = .{ .depth = d.config.block, .step_ms = d.step_ms, .prior = prior[0..d.config.block], .batched = true, .shared_guard = true, .stream_bytes = d.laneBytes(d.options.capacity) }; + switch (d.options.verify) { + .depth => {}, + .fixed => { // every round the whole block, no plain stretches + f.chances = true; + f.confidence_floor = 0; + f.shared_guard = false; + }, + .scheduled => f.chances = d.confident(), + } + return f; + } + + /// A held chain's chances: the confidence head's running product, or for a fixed window, ones that fall by depth. + fn chancesFn(ptr: *anyopaque, s: *Stream, out: []f64) anyerror!bool { + const d = cast(ptr); + const l = d.lanes.get(s) orelse return false; + if (d.options.verify == .fixed) { // no wait: equal chances at a depth share a capped round's rows evenly + for (out, 1..) |*o, j| o.* = 1 - 1e-3 * @as(f64, @floatFromInt(j)); + return true; + } + if (!d.confident()) return false; + if (d.pending != null and l.slot != null and l.conf_len > 0) { + // the hold still runs: the lane's last settled confidences price it, so the host does not wait on the GPU + if (out.len > d.depths[l.slot.?]) return error.DraftHeldFewer; + var reach: f64 = 1; + for (out, 0..) |*o, j| { + reach *= l.conf[@min(j, l.conf_len - 1)]; + o.* = reach; + } + return true; + } + try d.settle(); + if (out.len > l.held_len) return error.DraftHeldFewer; + var reach: f64 = 1; + for (out, l.conf[0..out.len]) |*o, p| { + reach *= p; + o.* = reach; + } + return true; } pub fn laneBytes(d: *const Drafter, capacity: usize) usize { diff --git a/zig/src/families/lfm2_moe/dspark_block.zig b/zig/src/families/lfm2_moe/dspark_block.zig index d3f6cbabb8..0b712bf2dd 100644 --- a/zig/src/families/lfm2_moe/dspark_block.zig +++ b/zig/src/families/lfm2_moe/dspark_block.zig @@ -16,6 +16,7 @@ const QkArgs = extern struct { len: u32, cap: u32, eps: f32, log2_theta: f32 }; const AttnArgs = extern struct { len: u32, cap: u32, scale: f32, subs: u32 }; const Pick = round.contract.Pick; const MarkovArgs = extern struct { stride: u32, rank: u32, vocab: u32, prev_stride: u32 }; +const ConfidenceArgs = extern struct { n: u32, pass: u32, vocab: u32, rank: u32, hidden: u32 }; /// Tokens a Markov scan threadgroup takes (the kernel's MARKOV_SPAN), and threadgroups a stream's row takes. pub const scan_span = 512; pub const scan_groups = (c.vocab + scan_span - 1) / scan_span; @@ -233,6 +234,7 @@ pub fn hold(d: *Drafter, blocks: []const Block, signal: ?Signal) !?mtl.CommandBu x.norm(s.h, s.r, d.w.norm, true, s.x, tail); try x.project(fwd.matrix(emb), .{ .buf = s.x }, .{ .buf = s.logits }, tail); try markov(x, blocks.len, n, steps); + if (d.confident()) confidence(x, blocks.len, n, steps); if (signal) |sg| { // committed unwaited, the command buffer returned retained e.end(); cb.signalEvent(sg.event, sg.value); @@ -250,9 +252,18 @@ pub fn hold(d: *Drafter, blocks: []const Block, signal: ?Signal) !?mtl.CommandBu } b.lane.held_len = b.depth; } + for (blocks, 0..) |b, bi| readConfidence(d, b.lane, b.depth, bi); return null; } +/// Lane `l`'s drafts' chances from slot `bi` of the last hold's logits: sigmoid(logit + bias); none: left as they are. +fn readConfidence(d: *Drafter, l: *Lane, depth: u32, bi: usize) void { + if (!d.confident()) return; + const logits = d.s.conf.slice(f32, d.config.block * ds.pass_rows); + for (l.conf[0..depth], 0..) |*p, i| p.* = 1 / (1 + @exp(-(logits[i * ds.pass_rows + bi] + d.w.confidence_bias))); + l.conf_len = depth; +} + /// A finished signalled hold's picks into the lanes' held chains (a pick past the vocabulary reads as token 0). pub fn readPicks(d: *Drafter, lanes: []const *Lane, depths: []const u32) void { const picks = d.s.picks.slice(Pick, d.config.block * ds.pass_rows); @@ -262,9 +273,26 @@ pub fn readPicks(d: *Drafter, lanes: []const *Lane, depths: []const u32) void { h.* = if (p.token < c.vocab) p.token else 0; } l.held_len = depth; + readConfidence(d, l, depth, bi); } } +/// Each stream's draft i confidence logit (the final norm's row i, W1 of the token before draft i), for readConfidence. +fn confidence(x: Enc, streams: usize, n: u32, depth: u32) void { + const d = x.d; + const e = x.e; + x.pipe("lfm2_dspark_confidence"); + e.setBuffer(d.s.x, 0, 0); + e.setBuffer(d.w.w1, 0, 1); + e.setBuffer(d.s.ids, 0, 2); + e.setBuffer(d.s.picks, 0, 3); + e.setBuffer(d.w.confidence.?, 0, 4); + const args = ConfidenceArgs{ .n = n, .pass = ds.pass_rows, .vocab = c.vocab, .rank = d.config.rank, .hidden = c.hidden }; + e.setBytes(std.mem.asBytes(&args), 5); + e.setBuffer(d.s.conf, 0, 6); + e.dispatchGroups(mtl.Size.of(depth, streams, 1), mtl.Size.of(256, 1, 1)); +} + /// Step i of each stream: its row i's logits plus W2 W1[previous draft] (the pending token first), then the argmax. fn markov(x: Enc, streams: usize, n: u32, depth: u32) !void { const d = x.d; diff --git a/zig/src/families/lfm2_moe/dspark_test.zig b/zig/src/families/lfm2_moe/dspark_test.zig index 9ee31d868e..8fd1eb32e1 100644 --- a/zig/src/families/lfm2_moe/dspark_test.zig +++ b/zig/src/families/lfm2_moe/dspark_test.zig @@ -206,3 +206,8 @@ test "lfm2 dspark: lfm2_qk on half-split rows and gains gives interleaved RoPE's } } } + +test "lfm2 dspark: the drafter verifies by its confidence head unless asked otherwise" { + const o: @import("dspark.zig").Options = .{ .capacity = 64 }; + try std.testing.expectEqual(@import("dspark.zig").Verify.scheduled, o.verify); +} diff --git a/zig/src/families/lfm2_moe/dspark_weights.zig b/zig/src/families/lfm2_moe/dspark_weights.zig index 81dc727e56..bfd6d078e1 100644 --- a/zig/src/families/lfm2_moe/dspark_weights.zig +++ b/zig/src/families/lfm2_moe/dspark_weights.zig @@ -35,6 +35,9 @@ pub const Weights = struct { /// The Markov head: W1 [vocab, rank] bf16 as stored (rows gathered by token), W2 [vocab, rank] as 8-bit codes. w1: mtl.Buffer, w2: tile.Matrix, + /// The confidence head: its [hidden + rank] bf16 weights and fp32 bias (null: the checkpoint has none). + confidence: ?mtl.Buffer = null, + confidence_bias: f32 = 0, owned: std.ArrayList(mtl.Buffer) = .empty, pub fn deinit(w: *Weights, gpa: std.mem.Allocator) void { @@ -140,5 +143,14 @@ pub fn load(gpa: std.mem.Allocator, device: mtl.Device, io: std.Io, dir: []const w.w1 = try l.buffer(c.vocab * cfg.rank * 2); @memcpy(w.w1.slice(u16, c.vocab * cfg.rank), w1); w.w2 = try l.matrix("markov_head.markov_w2.weight", c.vocab, cfg.rank, false); + w.confidence = null; + w.confidence_bias = 0; + if (check.tensors.contains("confidence_head.proj.weight")) { + const width = c.hidden + cfg.rank; // the final norm's row, then W1 of the token before the draft + const proj = (try l.tensor("confidence_head.proj.weight", &.{ 1, width })).host(u16); + w.confidence = try l.buffer(width * 2); + @memcpy(w.confidence.?.slice(u16, width), proj); + w.confidence_bias = quant.toF32((try l.tensor("confidence_head.proj.bias", &.{1})).host(u16)[0]); + } return w; } diff --git a/zig/src/families/lfm2_moe/lfm2_moe.zig b/zig/src/families/lfm2_moe/lfm2_moe.zig index 8f861e31b4..75b40e5938 100644 --- a/zig/src/families/lfm2_moe/lfm2_moe.zig +++ b/zig/src/families/lfm2_moe/lfm2_moe.zig @@ -11,3 +11,4 @@ pub const dspark = @import("dspark.zig"); pub const dspark_config = @import("dspark_config.zig"); pub const dspark_block = @import("dspark_block.zig"); pub const dspark_quant = @import("dspark_quant.zig"); +pub const verify_costs = @import("verify_costs.zig"); diff --git a/zig/src/families/lfm2_moe/verify_costs.zig b/zig/src/families/lfm2_moe/verify_costs.zig new file mode 100644 index 0000000000..f80e4cdcef --- /dev/null +++ b/zig/src/families/lfm2_moe/verify_costs.zig @@ -0,0 +1,44 @@ +//! A verify cost table from tf-lfm2-verify-costs: shared forward ms by total rows, for the lane core's shared costs. +const std = @import("std"); +const lanes = @import("lanes"); +const st = @import("state.zig"); + +const Cost = lanes.config.Cost; + +/// Lines `streams rows_each total_rows ms` ('#' comments): each total's ms at the fewest streams that reach it. +pub fn parse(gpa: std.mem.Allocator, text: []const u8) ![]Cost { + var best: [st.batch_rows + 1]?struct { streams: u32, ms: f64 } = @splat(null); + var lines = std.mem.tokenizeAny(u8, text, "\r\n"); + while (lines.next()) |line| { + const body = std.mem.trim(u8, line, " \t"); + if (body.len == 0 or body[0] == '#') continue; + var fields = std.mem.tokenizeAny(u8, body, " \t"); + var v: [4][]const u8 = undefined; + for (&v) |*x| x.* = fields.next() orelse return error.BadVerifyCosts; + if (fields.next() != null) return error.BadVerifyCosts; + const streams = try std.fmt.parseInt(u32, v[0], 10); + const each = try std.fmt.parseInt(u32, v[1], 10); + const total = try std.fmt.parseInt(u32, v[2], 10); + const ms = try std.fmt.parseFloat(f64, v[3]); + if (streams == 0 or each == 0 or total != streams * each or total > st.batch_rows or !(ms > 0)) return error.BadVerifyCosts; + if (best[total]) |b| if (b.streams <= streams) continue; + best[total] = .{ .streams = streams, .ms = ms }; + } + var out: std.ArrayList(Cost) = .empty; + errdefer out.deinit(gpa); + for (best, 0..) |b, total| if (b) |x| try out.append(gpa, .{ .width = @intCast(total), .ms = x.ms }); + if (out.items.len == 0) return error.BadVerifyCosts; + return out.toOwnedSlice(gpa); +} + +test "lfm2 dspark: a verify cost table gives each total its fewest streams' ms and refuses malformed lines" { + const gpa = std.testing.allocator; + const t = try parse(gpa, "# streams rows_each total_rows ms\n1 1 1 3.7\n2 1 2 4.9\n1 2 2 4.6\n\n4 8 32 40.5\n"); + defer gpa.free(t); + try std.testing.expectEqual(@as(usize, 3), t.len); + try std.testing.expectEqual(@as(f64, 4.6), t[1].ms); // one stream of two rows, not two of one + try std.testing.expectEqual(@as(u32, 32), t[2].width); + try std.testing.expectError(error.BadVerifyCosts, parse(gpa, "2 2 5 4.0\n")); + try std.testing.expectError(error.BadVerifyCosts, parse(gpa, "8 8 64 9.0\n")); + try std.testing.expectError(error.BadVerifyCosts, parse(gpa, "# nothing\n")); +} diff --git a/zig/src/native/lfm2_moe.zig b/zig/src/native/lfm2_moe.zig index a78d67d97b..c8183fbac2 100644 --- a/zig/src/native/lfm2_moe.zig +++ b/zig/src/native/lfm2_moe.zig @@ -87,7 +87,8 @@ pub fn open(a: std.mem.Allocator, gpa: std.mem.Allocator, io: std.Io, o: api.Ope release(h); return null; } - h.draft = q.dspark.Drafter.init(gpa, io, model, dir, .{ .capacity = window }) catch |err| { + const verify = verifyMode(o.draft_verify); + h.draft = q.dspark.Drafter.init(gpa, io, model, dir, .{ .capacity = window, .verify = verify }) catch |err| { problem.* = try std.fmt.allocPrint(a, "the native LFM2 drafter cannot load {s} ({s}): it takes LiquidAI/LFM2.5-24B-A2B-DSpark", .{ dir, @errorName(err) }); release(h); return null; @@ -109,7 +110,15 @@ pub fn open(a: std.mem.Allocator, gpa: std.mem.Allocator, io: std.Io, o: api.Ope h.wrap.?.held_on_target = true; } errdefer if (h.wrap) |*w| w.deinit(); - const facts = if (h.wrap) |*w| w.facts(h.metal.facts()) else h.metal.facts(); + var facts = if (h.wrap) |*w| w.facts(h.metal.facts()) else h.metal.facts(); + const profiled = if (h.draft != null) verifyCosts(a, io, o.verify_costs) catch |err| { + problem.* = try std.fmt.allocPrint(a, "--verify-costs {s} is not a tf-lfm2-verify-costs table ({s})", .{ o.verify_costs.?, @errorName(err) }); + h.wrap.?.deinit(); + h.draft.?.deinit(); + release(h); + return null; + } else null; + if (profiled) |t| facts.shared_costs = t; h.config = try lanes.Config.init(gpa, facts, q.state.window_rows, q.state.window_rows - 1); errdefer h.config.deinit(gpa); h.clock = .{ .io = io }; @@ -150,10 +159,25 @@ pub fn open(a: std.mem.Allocator, gpa: std.mem.Allocator, io: std.Io, o: api.Ope }; h.host.keepalive_target = .{ .ctx = &h.warm, .tick = mtl.keepalive.Target.tick }; try h.host.start(); - if (h.draft) |d| std.log.info("lfm2: DSpark drafts from --drafter, {d} a round at most, {d}-bit codes", .{ d.config.block, d.options.bits }); + if (h.draft) |d| std.log.info("lfm2: DSpark drafts from --drafter, {d} a round at most, {d}-bit codes, {t} verify", .{ d.config.block, d.options.bits, d.options.verify }); return .{ .engine = h.host.engine(), .close = Host.close, .ctx = h }; } +/// --verify-costs: a tf-lfm2-verify-costs table's shared forward ms by total rows (null: none named; `a`-owned). +fn verifyCosts(a: std.mem.Allocator, io: std.Io, path: ?[]const u8) !?[]const lanes.config.Cost { + const file = path orelse return null; + const text = try std.Io.Dir.cwd().readFileAlloc(io, file, a, .limited(1 << 20)); + const t = try q.verify_costs.parse(a, text); + std.log.info("lfm2: verify costs from {s}, {d} totals of rows", .{ file, t.len }); + return t; +} + +/// --draft-verify as the drafter's verify (null: the default). +fn verifyMode(v: ?api.DraftVerify) q.dspark.Verify { + const asked = v orelse return q.dspark.default_verify; + return std.meta.stringToEnum(q.dspark.Verify, @tagName(asked)).?; +} + /// The prompt cache's copies of a stream: the target's state at a mark (backend.zig), and the drafter's context there. const Snaps = struct { const S = q.backend.Snaps; diff --git a/zig/src/native/metal.zig b/zig/src/native/metal.zig index 104bbfc22b..eddee040ee 100644 --- a/zig/src/native/metal.zig +++ b/zig/src/native/metal.zig @@ -337,6 +337,7 @@ fn refusal(a: Allocator, o: api.Open, dense: bool) !?[]const u8 { const one_mac: ?[]const u8 = if (std.mem.eql(u8, t, "nemotron_h")) "Nemotron" else if (std.mem.eql(u8, t, "lfm2_moe")) "LFM2.5-24B-A2B" else if (isGemma4(t)) "Gemma 4" else if (std.mem.eql(u8, t, "step5_vl")) "Step-5" else if (std.mem.eql(u8, t, "spark2_5")) "Spark-X2.6" else if (!std.mem.eql(u8, t, "qwen3_5")) null else if (dense) "Qwen3.8-27B" else "Qwen3.5-2B"; if (o.speed_up != null) if (one_mac) |name| return try std.fmt.allocPrint(a, "--speed-up joins two Macs for Flash Next, GLM-5.3-Flash and GLM-5.3 only: serve {s} without it", .{name}); if (o.recall and std.mem.eql(u8, t, "qwen4_exp")) return recall_refusal; + if ((o.draft_verify != null or o.verify_costs != null) and (o.drafter == null or !std.mem.eql(u8, t, "lfm2_moe"))) return "--draft-verify and --verify-costs set the verify of LFM2.5-24B-A2B's DSpark drafter: serve it with --drafter"; if (spill.refusal(o)) |why| return why; if (o.learn == null) return null; // the 27B writes conversation states diff --git a/zig/src/server/cli.zig b/zig/src/server/cli.zig index cf5d600b9c..df7ea7937f 100644 --- a/zig/src/server/cli.zig +++ b/zig/src/server/cli.zig @@ -102,6 +102,8 @@ pub const flags = [_]Flag{ .{ .name = "--slide-graph", .native = true }, .{ .name = "--drafter", .native = true }, .{ .name = "--drafter-bits", .choices = &.{ "0", "4" }, .native = true }, + .{ .name = "--draft-verify", .choices = &.{ "depth", "fixed", "scheduled" }, .native = builtin.os.tag == .macos }, + .{ .name = "--verify-costs", .native = builtin.os.tag == .macos }, .{ .name = "--record", .native = true }, .{ .name = "--kv-cache", .choices = &.{ "bf16", "fp8" }, .native = true }, .{ .name = "--prompt-precision", .choices = &.{ "bf16", "int8" }, .native = true }, @@ -218,6 +220,8 @@ pub const Args = struct { compact_memory: ?[]const u8 = null, drafter: ?[]const u8 = null, drafter_bits: u8 = 4, + draft_verify: ?api.DraftVerify = null, // the LFM2 DSpark drafter's verify rows (null: the engine's default) + verify_costs: ?[]const u8 = null, // a tf-lfm2-verify-costs table pricing them (null: costs timed at load) record: ?[]const u8 = null, // drafter training rows written under this folder kv_cache: []const u8 = "bf16", // fp8: e4m3 attention caches with a scale a row, about twice the context prompt_precision: []const u8 = "bf16", // int8: prompt matmuls quantize their inputs to int8 a 64-input group @@ -548,6 +552,8 @@ pub fn request(a: Allocator, dir: []const u8, model_type: []const u8, args: Args .drafts = !args.no_drafts, .drafter = args.drafter, .drafter_bits = args.drafter_bits, + .draft_verify = args.draft_verify, + .verify_costs = args.verify_costs, .record = args.record, .kv8 = std.mem.eql(u8, args.kv_cache, "fp8"), .prompt_int8 = std.mem.eql(u8, args.prompt_precision, "int8"), diff --git a/zig/src/server/cli_test.zig b/zig/src/server/cli_test.zig index ca4455cf16..7065a8a49e 100644 --- a/zig/src/server/cli_test.zig +++ b/zig/src/server/cli_test.zig @@ -165,6 +165,13 @@ test "parse and capabilities share the table" { try std.testing.expectEqualStrings("x", (try parse(a, &.{ "m", "--drafter", "x" }, &u)).drafter.?); try std.testing.expectEqual(@as(u8, 0), (try parse(a, &.{ "m", "--drafter-bits", "0" }, &u)).drafter_bits); try std.testing.expectError(error.Usage, parse(a, &.{ "m", "--drafter-bits", "8" }, &u)); + if (@import("builtin").os.tag == .macos) { + const verify = try parse(a, &.{ "m", "--draft-verify", "scheduled", "--verify-costs", "/t" }, &u); + try std.testing.expectEqual(api.DraftVerify.scheduled, verify.draft_verify.?); + try std.testing.expectEqual(api.DraftVerify.scheduled, (try request(a, "d", "t", verify)).draft_verify.?); + try std.testing.expectEqualStrings("/t", (try request(a, "d", "t", verify)).verify_costs.?); + try std.testing.expectError(error.Usage, parse(a, &.{ "m", "--draft-verify", "wide" }, &u)); + } try std.testing.expectEqualStrings("/r", (try parse(a, &.{ "m", "--record", "/r" }, &u)).record.?); try std.testing.expectEqualStrings("fp8", (try parse(a, &.{ "m", "--kv-cache", "fp8" }, &u)).kv_cache); try std.testing.expectError(error.Usage, parse(a, &.{ "m", "--kv-cache", "int4" }, &u)); diff --git a/zig/src/server/main.zig b/zig/src/server/main.zig index 20e2fdb773..339ed441b9 100644 --- a/zig/src/server/main.zig +++ b/zig/src/server/main.zig @@ -26,7 +26,7 @@ const usage_line = "usage: tensorfold serve [-h] [--host HOST] [--port PORT] [-- " [--thinking | --no-thinking] [--reasoning-effort {low,medium,high,xhigh}]" ++ " [--thinking-budget THINKING_BUDGET] [--thinking-reserve THINKING_RESERVE] [--loop-guard] [--loop-guard-stop] [--no-drafts]" ++ " [--drafter DRAFTER]" ++ - " [--drafter-bits {0,4}] [--record RECORD] [--keep-warm SECONDS] [--idle-unload SECONDS]" ++ + " [--drafter-bits {0,4}] [--draft-verify {depth,fixed,scheduled}] [--verify-costs VERIFY_COSTS] [--record RECORD] [--keep-warm SECONDS] [--idle-unload SECONDS]" ++ " [--silence-floor-dbfs DBFS] [--silence-seconds SECONDS]" ++ " [--silence-onset-s SECONDS] [--silence-onset-voiced-s SECONDS]" ++ " [--silence-mid-s SECONDS] [--silence-mid-min-voiced-s SECONDS] [--silence-redraws N]" ++ diff --git a/zig/src/tensorfold.zig b/zig/src/tensorfold.zig index 99591ca724..6089264618 100644 --- a/zig/src/tensorfold.zig +++ b/zig/src/tensorfold.zig @@ -91,6 +91,7 @@ test { _ = @import("families/lfm2_moe/dspark_config.zig"); _ = @import("families/lfm2_moe/dspark_quant.zig"); _ = @import("families/lfm2_moe/dspark_test.zig"); + _ = @import("families/lfm2_moe/verify_costs.zig"); } pub const lane_projection = @import("core/lane_projection.zig"); diff --git a/zig/tests/lfm2_dspark.zig b/zig/tests/lfm2_dspark.zig index f95ce615e5..53157c06a1 100644 --- a/zig/tests/lfm2_dspark.zig +++ b/zig/tests/lfm2_dspark.zig @@ -125,7 +125,7 @@ pub fn main(init: std.process.Init) !void { defer pool.pop(); const m = try q.Model.load(gpa, init.io, args[1]); defer m.deinit(); - const d = try q.dspark.Drafter.init(gpa, init.io, m, args[2], .{ .capacity = length + 64, .bits = bits }); + const d = try q.dspark.Drafter.init(gpa, init.io, m, args[2], .{ .capacity = length + 64, .bits = bits, .verify = .scheduled }); defer d.deinit(); const cfg = d.config; // the compared block's rows (a hold of depth d runs a d-row block) @@ -156,6 +156,8 @@ pub fn main(init: std.process.Init) !void { _ = try q.dspark_block.hold(d, &.{.{ .lane = lane, .pending = ids[ctx], .anchor = ctx, .depth = block }}, null); var held: [16]u32 = undefined; @memcpy(held[0..block], lane.held[0..block]); + var conf: [16]f32 = undefined; + @memcpy(conf[0..block], lane.conf[0..block]); const gpu = held[0..block]; for ([_]usize{ 1, 4, 16 }) |rows| { var best: f64 = std.math.floatMax(f64); @@ -269,6 +271,9 @@ pub fn main(init: std.process.Init) !void { const w1 = try r.get("markov_head.markov_w1.weight"); const w2 = try r.get("markov_head.markov_w2.weight"); const bias = try a.alloc(f32, c.vocab); + const proj = try r.get("confidence_head.proj.weight"); + const proj_bias = (try r.get("confidence_head.proj.bias"))[0]; + var conf_worst: f32 = 0; var prev = ids[ctx]; var same: usize = 0; var worst: f32 = 0; @@ -294,8 +299,16 @@ pub fn main(init: std.process.Init) !void { same += @intFromBool(ok); std.debug.print("step {d}: reference {d} (margin {d:.3}, second {d}), GPU {d} {s}\n", .{ i, best, margin, second, gpu[i], if (ok) "equal" else "DIFFERENT" }); if (!ok) worst = @max(worst, margin); + // the confidence head: [final norm row i, W1 of the token before draft i] . proj + bias, through a sigmoid + var logit: f32 = proj_bias; + for (h[i * c.hidden ..][0..c.hidden], proj[0..c.hidden]) |x0, w0| logit += x0 * w0; + for (w1[@as(usize, prev) * cfg.rank ..][0..cfg.rank], proj[c.hidden..]) |x0, w0| logit += x0 * w0; + const want = 1 / (1 + @exp(-logit)); + conf_worst = @max(conf_worst, @abs(want - conf[i])); + std.debug.print("step {d}: confidence reference {d:.4}, GPU {d:.4}\n", .{ i, want, conf[i] }); prev = gpu[i]; // teacher-forced: step i + 1 follows the GPU's pick, so each step is compared alone } std.debug.print("the reference's largest margin over a different GPU pick: {d:.3}\n", .{worst}); + std.debug.print("the confidences' worst difference from the fp32 reference: {d:.5}\n", .{conf_worst}); std.debug.print("{d} of {d} drafts equal the fp32 reference's picks on the same chain ({d}-token context)\n", .{ same, n, ctx }); } diff --git a/zig/tests/lfm2_verify_costs.zig b/zig/tests/lfm2_verify_costs.zig new file mode 100644 index 0000000000..eef8ad7be3 --- /dev/null +++ b/zig/tests/lfm2_verify_costs.zig @@ -0,0 +1,67 @@ +//! LFM2 verify forwards timed by stream count and rows a stream: the round costs a DSpark verify budget weighs. +const std = @import("std"); +const mtl = @import("metal"); +const q = @import("tensorfold").lfm2_moe; +const st = q.state; +const c = q.config; + +fn now() i128 { + var ts: std.c.timespec = undefined; + _ = std.c.clock_gettime(.MONOTONIC, &ts); + return @as(i128, ts.sec) * std.time.ns_per_s + ts.nsec; +} + +/// Stream counts profiled; each runs every rows-a-stream width whose total fits a shared forward. +const counts = [_]usize{ 1, 2, 3, 4, 6, 8, 12, 16 }; +const reps = 9; + +pub fn main(init: std.process.Init) !void { + const gpa = init.gpa; + const args = try init.minimal.args.toSlice(init.arena.allocator()); + if (args.len < 3 or args.len > 4) return error.ExpectedModelOutputHistory; + const history: usize = if (args.len == 4) try std.fmt.parseInt(usize, args[3], 10) else 1024; + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const m = try q.Model.load(gpa, init.io, args[1]); + defer m.deinit(); + var scratch = try st.Scratch.init(gpa, m.device, &m.config.attention, @max(history, st.batch_rows)); + defer scratch.deinit(); + var caches: [16]st.Cache = undefined; + var made: usize = 0; + defer for (caches[0..made]) |*k| k.deinit(); + const ids = try gpa.alloc(u32, history); + defer gpa.free(ids); + for (caches[0..], 0..) |*k, i| { + k.* = try st.Cache.init(gpa, m.device, &m.config.attention, history + st.window_rows + 1); + made += 1; + for (ids, 0..) |*id, j| id.* = @intCast((j * 7919 + 13 + i * 104729) % 120000); + try q.forward.run(m, &scratch, &.{.{ .cache = k, .rows = history }}, ids, false, .last); + } + var out: std.Io.Writer.Allocating = .init(gpa); + defer out.deinit(); + const w = &out.writer; + try w.print("# LFM2 verify forward ms (median of {d}) by streams and rows a stream, {d}-token histories\n", .{ reps, history }); + try w.print("# streams rows_each total_rows ms\n", .{}); + var segments: [st.batch_rows]q.forward.Segment = undefined; + var tokens: [st.batch_rows]u32 = undefined; + for (&tokens, 0..) |*t, i| t.* = @intCast((i * 31 + 7) % 120000); + for (counts) |streams| { + var each: usize = 1; + while (each <= st.window_rows and streams * each <= st.batch_rows) : (each += 1) { + for (segments[0..streams], caches[0..streams]) |*s, *k| s.* = .{ .cache = k, .rows = each }; + var times: [reps + 2]f64 = undefined; + for (×) |*t| { + const t0 = now(); + try q.forward.run(m, &scratch, segments[0..streams], tokens[0 .. streams * each], true, .all); + t.* = @as(f64, @floatFromInt(now() - t0)) / 1e6; + for (caches[0..streams]) |*k| k.len = history; + } + const kept = times[2..]; // the first two calls warm a new width's pipelines + std.mem.sort(f64, kept, {}, std.sort.asc(f64)); + const ms = kept[reps / 2]; + try w.print("{d} {d} {d} {d:.3}\n", .{ streams, each, streams * each, ms }); + std.debug.print("{d} streams x {d} rows ({d}): {d:.3} ms\n", .{ streams, each, streams * each, ms }); + } + } + try std.Io.Dir.cwd().writeFile(init.io, .{ .sub_path = args[2], .data = out.written() }); +} From c811111ac1c5e1fba09ec292c3f97478f5f42c0c Mon Sep 17 00:00:00 2001 From: BlivionIaG <15323295+BlivionIaG@users.noreply.github.com> Date: Mon, 26 Oct 2026 09:37:47 +0100 Subject: [PATCH 10/39] lfm2: a short prompt's tapped rows wait in the DSpark stage like verified rows Each prompt chunk's rows were absorbed at once in a command buffer of their own, waited on before the first token, and the prompt's last row in a second one at the first draft request. The prefill pass is waited on before its rows are handed over, so a chunk that fits the lane's stage (128 rows) can be copied there on the host like a round's verified rows. The next hold, or a kept prompt state, absorbs the stage first, as before. Longer chunks are absorbed at once. --- zig/src/families/lfm2_moe/dspark.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zig/src/families/lfm2_moe/dspark.zig b/zig/src/families/lfm2_moe/dspark.zig index 0263d03f64..404c23516c 100644 --- a/zig/src/families/lfm2_moe/dspark.zig +++ b/zig/src/families/lfm2_moe/dspark.zig @@ -348,7 +348,7 @@ pub const Drafter = struct { }; } - /// Rows from `start` replace the context there: verified rows wait on the host for the next hold, others run now. + /// Rows from `start` replace the context there: rows that fit the stage wait on the host for the next hold. fn absorbFn(ptr: *anyopaque, items: []const dr.Absorb) anyerror!void { const d = cast(ptr); try d.settle(); @@ -371,7 +371,7 @@ pub const Drafter = struct { // rows from `start` replace what was staged past it; rows before the absorbed end go to the GPU now const keep: u32 = if (it.start >= l.end) @intCast(it.start - l.end) else 0; l.staged = @min(l.staged, keep); - if (!it.prompt and it.start >= l.end and l.staged + f.rows <= stage_rows) { + if (it.start >= l.end and l.staged + f.rows <= stage_rows) { const from = (src.contents() + f.offset)[0 .. f.rows * width]; @memcpy((l.stage.contents() + l.staged * width)[0 .. f.rows * width], from); l.staged += f.rows; From 7746c32ebcf05ab7733a34a45b7ec696f1b45f4b Mon Sep 17 00:00:00 2001 From: BlivionIaG <15323295+BlivionIaG@users.noreply.github.com> Date: Mon, 26 Oct 2026 09:38:01 +0100 Subject: [PATCH 11/39] lfm2: DSpark prices its own hold pass for the lane core A DSpark hold is a pass of its own: the drafter runs its five layers and the target's head for every drafting stream before the verify. The lane core can now weigh that pass (Facts.copy_first, Facts.hold_costs, Model.own_copy_rates); the drafter gives it what it needs. - copy_first: a stream with a copy ahead verifies the copy, and the drafter is asked no hold for it. - hold_costs: after its warm-up the drafter times holds of 1, 2, 4, 8 and 16 blocks at one draft and at four (two at 16 blocks, one 32-row pass) on empty lanes, best of three. Each settled one-pass hold then moves the timed passes at its block counts a tenth of the way toward its own GPU ms, so the price follows the served contexts. - own_copy_rates: with a drafter, a stream's copy windows are priced by its own copies. Test: a 4-block hold at twice its price raises the 4-block passes by a tenth and leaves the others; one between 4 and 8 blocks moves both; a hold without GPU time teaches nothing. --- zig/src/families/lfm2_moe/dspark.zig | 56 +++++++++++++++++++++-- zig/src/families/lfm2_moe/dspark_test.zig | 22 +++++++++ zig/src/native/lfm2_moe.zig | 2 + 3 files changed, 75 insertions(+), 5 deletions(-) diff --git a/zig/src/families/lfm2_moe/dspark.zig b/zig/src/families/lfm2_moe/dspark.zig index 404c23516c..eb3fa4d90f 100644 --- a/zig/src/families/lfm2_moe/dspark.zig +++ b/zig/src/families/lfm2_moe/dspark.zig @@ -20,6 +20,8 @@ pub const pass_rows = 32; pub const absorb_rows = 2048; /// Verified rows a lane keeps on the host until its next hold absorbs them (a round's rows for no GPU work). pub const stage_rows = 128; +/// The blocks a hold is timed with at load, each at one draft and at four (two at 16 blocks: one 32-row pass). +const hold_grid = [_]u32{ 1, 2, 4, 8, 16 }; const kw = c.kv_heads * c.head_dim; /// Verify rows by the core's depth rule, the whole block (fixed), or the confidence head's chances (scheduled). @@ -85,6 +87,8 @@ pub const Drafter = struct { /// One stream's hold wall ms at full depth, and that over the block (the depth rule's cost a draft). hold_ms: f64 = 0, step_ms: f64 = 0, + holds: [hold_grid.len * 2]lanes.hold_price.HoldCost = undefined, // hold passes timed at load, then by settled holds + holds_len: usize = 0, event: mtl.Event = undefined, signalled: u64 = 0, pending: ?mtl.CommandBuffer = null, @@ -187,7 +191,7 @@ pub const Drafter = struct { fn factsFn(ptr: *anyopaque) dr.Facts { const d = cast(ptr); - var f: dr.Facts = .{ .depth = d.config.block, .step_ms = d.step_ms, .prior = prior[0..d.config.block], .batched = true, .shared_guard = true, .stream_bytes = d.laneBytes(d.options.capacity) }; + var f: dr.Facts = .{ .depth = d.config.block, .step_ms = d.step_ms, .prior = prior[0..d.config.block], .batched = true, .shared_guard = true, .stream_bytes = d.laneBytes(d.options.capacity), .copy_first = true }; switch (d.options.verify) { .depth => {}, .fixed => { // every round the whole block, no plain stretches @@ -195,7 +199,10 @@ pub const Drafter = struct { f.confidence_floor = 0; f.shared_guard = false; }, - .scheduled => f.chances = d.confident(), + .scheduled => { + f.chances = d.confident(); + f.hold_costs = d.holds[0..d.holds_len]; + }, } return f; } @@ -461,9 +468,29 @@ pub const Drafter = struct { std.log.err("lfm2 dspark: a hold failed on the GPU: {s}", .{msg}); return error.DraftGpuFailed; } + d.priceHold((cb.gpuEnd() - cb.gpuStart()) * 1000); block.readPicks(d, d.pass[0..d.pass_len], d.depths[0..d.pass_len]); } + /// A settled hold's GPU ms against its price: the timed passes at its block counts move a tenth of the way to it. + pub fn priceHold(d: *Drafter, ms: f64) void { + const timed = d.holds[0..d.holds_len]; + if (timed.len == 0 or d.pass_len == 0 or !(ms > 0)) return; + const deepest = std.mem.max(u32, d.depths[0..d.pass_len]); + const priced = lanes.hold_price.holdMs(timed, d.pass_len, deepest); + if (!(priced > 0)) return; + const step = 1 + 0.1 * (ms / priced - 1); + var lo: u32 = 0; + var hi: u32 = std.math.maxInt(u32); + for (timed) |h| { + if (h.streams <= d.pass_len) lo = @max(lo, h.streams); + if (h.streams >= d.pass_len) hi = @min(hi, h.streams); + } + for (timed) |*h| if (h.streams >= lo and h.streams <= hi) { + h.ms *= step; + }; + } + /// The signalled hold settled and its lanes' picks forgotten (a lane may be dropped, or a hold may overwrite). fn unpass(d: *Drafter) !void { defer { @@ -490,10 +517,13 @@ pub const Drafter = struct { return heldFn(ptr, &.{s}, &.{out}); } - /// Holds on an empty context: the pipelines compiled, then one-draft and full-depth holds timed (best of three). + /// Holds on empty contexts: the pipelines compiled, then holds timed by blocks and depth (best of three). fn warm(d: *Drafter) !void { - const l = try d.newLane(64); - defer d.dropLane(l); + var made: [hold_grid[hold_grid.len - 1]]*Lane = undefined; + var n: usize = 0; + defer for (made[0..n]) |l| d.dropLane(l); + while (n < made.len) : (n += 1) made[n] = try d.newLane(64); + const l = made[0]; var best: [2]f64 = @splat(std.math.floatMax(f64)); for (0..4) |i| for ([_]u32{ 1, d.config.block }, &best) |depth, *b| { const t0 = std.Io.Clock.awake.now(d.io); @@ -504,5 +534,21 @@ pub const Drafter = struct { d.hold_ms = best[1]; d.step_ms = best[1] / @as(f64, @floatFromInt(d.config.block)); std.log.info("lfm2 dspark: a 1-draft hold takes {d:.2} ms, a {d}-draft hold {d:.2} ms", .{ best[0], d.config.block, best[1] }); + var blocks: [hold_grid[hold_grid.len - 1]]block.Block = undefined; + for (made, &blocks) |lane, *b| b.* = .{ .lane = lane, .pending = 1, .anchor = 0, .depth = 1 }; + d.holds_len = 0; + for (hold_grid) |streams| for ([_]u32{ 1, @min(4, pass_rows / streams) }, 0..) |depth, j| { + if (j == 1 and depth == 1) continue; + for (blocks[0..streams]) |*b| b.depth = depth; + var ms_best = std.math.floatMax(f64); + for (0..4) |i| { + const t0 = std.Io.Clock.awake.now(d.io); + _ = try block.hold(d, blocks[0..streams], null); + const ms = @as(f64, @floatFromInt(t0.durationTo(std.Io.Clock.awake.now(d.io)).toNanoseconds())) / 1e6; + if (i > 0) ms_best = @min(ms_best, ms); + } + d.holds[d.holds_len] = .{ .streams = streams, .depth = depth, .ms = ms_best }; + d.holds_len += 1; + }; } }; diff --git a/zig/src/families/lfm2_moe/dspark_test.zig b/zig/src/families/lfm2_moe/dspark_test.zig index 8fd1eb32e1..385866b291 100644 --- a/zig/src/families/lfm2_moe/dspark_test.zig +++ b/zig/src/families/lfm2_moe/dspark_test.zig @@ -4,6 +4,8 @@ const mtl = @import("metal"); const c = @import("config.zig"); const kernels = @import("kernels.zig"); const quant = @import("dspark_quant.zig"); +const ds = @import("dspark.zig"); +const lanes = @import("lanes"); const shared = mtl.ResourceOptions.shared; @@ -211,3 +213,23 @@ test "lfm2 dspark: the drafter verifies by its confidence head unless asked othe const o: @import("dspark.zig").Options = .{ .capacity = 64 }; try std.testing.expectEqual(@import("dspark.zig").Verify.scheduled, o.verify); } + +test "lfm2 dspark: a settled hold moves the timed passes at its block counts a tenth of the way to its GPU ms" { + var d: ds.Drafter = undefined; + const grid = [_]lanes.hold_price.HoldCost{ .{ .streams = 1, .depth = 1, .ms = 1 }, .{ .streams = 4, .depth = 1, .ms = 2 }, .{ .streams = 4, .depth = 4, .ms = 4 }, .{ .streams = 8, .depth = 1, .ms = 3 } }; + @memcpy(d.holds[0..grid.len], &grid); + d.holds_len = grid.len; + d.pass_len = 4; + d.depths[0..4].* = .{ 1, 1, 1, 1 }; + d.priceHold(4); // priced 2 ms: the 4-block passes rise by a tenth of the doubling + try std.testing.expectApproxEqAbs(@as(f64, 2.2), d.holds[1].ms, 1e-12); + try std.testing.expectApproxEqAbs(@as(f64, 4.4), d.holds[2].ms, 1e-12); + try std.testing.expectEqual(@as(f64, 1), d.holds[0].ms); + try std.testing.expectEqual(@as(f64, 3), d.holds[3].ms); + d.pass_len = 6; // between 4 and 8 blocks: both move + d.depths[0..6].* = .{ 1, 1, 1, 1, 1, 1 }; + d.priceHold(0); + try std.testing.expectEqual(@as(f64, 2.2), d.holds[1].ms); // no GPU time: nothing learned + d.priceHold(2.6 * 2); + try std.testing.expect(d.holds[1].ms > 2.2 and d.holds[3].ms > 3 and d.holds[0].ms == 1); +} diff --git a/zig/src/native/lfm2_moe.zig b/zig/src/native/lfm2_moe.zig index c8183fbac2..767cbbe975 100644 --- a/zig/src/native/lfm2_moe.zig +++ b/zig/src/native/lfm2_moe.zig @@ -119,6 +119,8 @@ pub fn open(a: std.mem.Allocator, gpa: std.mem.Allocator, io: std.Io, o: api.Ope return null; } else null; if (profiled) |t| facts.shared_costs = t; + // a drafting stream prices its copy windows by its own copies: on distinct prompts they keep breaking + if (h.wrap != null) facts.own_copy_rates = true; h.config = try lanes.Config.init(gpa, facts, q.state.window_rows, q.state.window_rows - 1); errdefer h.config.deinit(gpa); h.clock = .{ .io = io }; From 82a14bc1dd30458e978d20cd66a3c008a88f1ade Mon Sep 17 00:00:00 2001 From: BlivionIaG <15323295+BlivionIaG@users.noreply.github.com> Date: Mon, 26 Oct 2026 09:38:12 +0100 Subject: [PATCH 12/39] lfm2: tf-lfm2-dspark times shared holds; tf-lfm2-verify-costs reads real rows tf-lfm2-dspark times a shared round's hold over 2 to 16 streams, by the wall clock and on the GPU, and absorbs of 128 rows and of a whole absorb pass, so a hold's price can be checked against a verify's. tf-lfm2-verify-costs reads token id files after HISTORY: each of the 16 streams takes its history and then its verify rows from its own sixteenth of the files' ids (the history shrinks to fit), so a shared round's rows route the experts their own text routes. Without id files the rows are synthetic as before; the table's header names their kind. --- zig/tests/lfm2_dspark.zig | 42 ++++++++++++++++++++++++++++++++- zig/tests/lfm2_verify_costs.zig | 30 +++++++++++++++++++---- 2 files changed, 66 insertions(+), 6 deletions(-) diff --git a/zig/tests/lfm2_dspark.zig b/zig/tests/lfm2_dspark.zig index 53157c06a1..5ee8140de2 100644 --- a/zig/tests/lfm2_dspark.zig +++ b/zig/tests/lfm2_dspark.zig @@ -109,6 +109,16 @@ fn compare(label: []const u8, gpu: []const u16, row: []const f32, bias: []const std.debug.print("{s}: cosine {d:.5}, worst difference {d:.3}\n", .{ label, dot / @sqrt(gg * rr), worst }); } +/// A hold's GPU ms: the blocks held in a signalled command buffer, waited on and read for its GPU times. +fn gpuHold(d: *q.dspark.Drafter, blocks: []const q.dspark_block.Block) !f64 { + d.signalled += 1; + const cb = (try q.dspark_block.hold(d, blocks, .{ .event = d.event, .value = d.signalled })).?; + defer cb.release(); + cb.wait(); + if (cb.failure() != null) return error.GpuFailed; + return (cb.gpuEnd() - cb.gpuStart()) * 1000; +} + fn absorb(d: *q.dspark.Drafter, piece: q.dspark_block.Piece) !void { const run = q.dspark_block.begin(d); try q.dspark_block.absorbPass(d, run.e, &.{piece}); @@ -159,7 +169,8 @@ pub fn main(init: std.process.Init) !void { var conf: [16]f32 = undefined; @memcpy(conf[0..block], lane.conf[0..block]); const gpu = held[0..block]; - for ([_]usize{ 1, 4, 16 }) |rows| { + for ([_]usize{ 1, 4, 16, 128, q.dspark.absorb_rows }) |rows| { + if (rows > ctx) continue; var best: f64 = std.math.floatMax(f64); for (0..5) |_| { const t0 = std.Io.Clock.awake.now(init.io); @@ -178,6 +189,35 @@ pub fn main(init: std.process.Init) !void { } std.debug.print("hold at depth {d}: {d:.2} ms (best of 5)\n", .{ depth, best }); } + // a shared round's hold: S streams' blocks of d rows in one pass (S x d at most pass_rows), each on this context + var lanes_: [16]*q.dspark.Lane = undefined; + lanes_[0] = lane; + var made: usize = 1; + defer for (lanes_[1..made]) |l| d.dropLane(l); + while (made < lanes_.len) : (made += 1) { + const l = try d.newLane(length + 32); + lanes_[made] = l; + var from: usize = 0; + while (from < ctx) : (from += q.dspark.absorb_rows) { + const rows = @min(q.dspark.absorb_rows, ctx - from); + try absorb(d, .{ .lane = l, .src = .{ .buf = taps.buf, .off = from * width * 2 }, .rows = rows, .p0 = from }); + } + l.end = ctx; + } + for ([_]usize{ 2, 4, 8, 16 }) |streams| for ([_]u32{ 1, 2, 3, 4, 8 }) |depth| { + if (streams * depth > q.dspark.pass_rows) continue; + var blocks: [16]q.dspark_block.Block = undefined; + for (blocks[0..streams], lanes_[0..streams]) |*b, l| b.* = .{ .lane = l, .pending = ids[ctx], .anchor = ctx, .depth = depth }; + var best: f64 = std.math.floatMax(f64); + var gpu_best: f64 = std.math.floatMax(f64); + for (0..5) |_| { + const t0 = std.Io.Clock.awake.now(init.io); + _ = try q.dspark_block.hold(d, blocks[0..streams], null); + best = @min(best, @as(f64, @floatFromInt(t0.durationTo(std.Io.Clock.awake.now(init.io)).toNanoseconds())) / 1e6); + gpu_best = @min(gpu_best, try gpuHold(d, blocks[0..streams])); + } + std.debug.print("hold of {d} streams at depth {d}: {d:.2} ms, GPU {d:.2} ms (best of 5)\n", .{ streams, depth, best, gpu_best }); + }; _ = try q.dspark_block.hold(d, &.{.{ .lane = lane, .pending = ids[ctx], .anchor = ctx, .depth = block }}, null); // the host reference, fp32 throughout var r: Ref = .{ .gpa = gpa, .check = ck.Checkpoint.init(gpa), .cfg = cfg }; diff --git a/zig/tests/lfm2_verify_costs.zig b/zig/tests/lfm2_verify_costs.zig index eef8ad7be3..f3100d633e 100644 --- a/zig/tests/lfm2_verify_costs.zig +++ b/zig/tests/lfm2_verify_costs.zig @@ -15,11 +15,27 @@ fn now() i128 { const counts = [_]usize{ 1, 2, 3, 4, 6, 8, 12, 16 }; const reps = 9; +fn readIds(gpa: std.mem.Allocator, io: std.Io, path: []const u8, list: *std.ArrayList(u32)) !void { + const bytes = try std.Io.Dir.cwd().readFileAlloc(io, path, gpa, .limited(1 << 22)); + defer gpa.free(bytes); + var it = std.mem.tokenizeAny(u8, bytes, " \n\r\t,"); + while (it.next()) |t| try list.append(gpa, try std.fmt.parseInt(u32, t, 10)); +} + pub fn main(init: std.process.Init) !void { const gpa = init.gpa; const args = try init.minimal.args.toSlice(init.arena.allocator()); - if (args.len < 3 or args.len > 4) return error.ExpectedModelOutputHistory; - const history: usize = if (args.len == 4) try std.fmt.parseInt(usize, args[3], 10) else 1024; + if (args.len < 3) return error.ExpectedModelOutputHistory; + var history: usize = if (args.len >= 4) try std.fmt.parseInt(usize, args[3], 10) else 1024; + // Token id files: every file's ids end to end; stream i reads its history and then its rows from its own sixteenth. + var real: std.ArrayList(u32) = .empty; + defer real.deinit(gpa); + for (args[@min(args.len, 4)..]) |path| try readIds(gpa, init.io, path, &real); + const part = real.items.len / 16; + if (real.items.len > 0) { + if (part < st.window_rows + 64) return error.IdFilesTooShort; + history = @min(history, part - st.window_rows); + } const pool = mtl.objc.Pool.push(); defer pool.pop(); const m = try q.Model.load(gpa, init.io, args[1]); @@ -34,13 +50,13 @@ pub fn main(init: std.process.Init) !void { for (caches[0..], 0..) |*k, i| { k.* = try st.Cache.init(gpa, m.device, &m.config.attention, history + st.window_rows + 1); made += 1; - for (ids, 0..) |*id, j| id.* = @intCast((j * 7919 + 13 + i * 104729) % 120000); + for (ids, 0..) |*id, j| id.* = if (part > 0) real.items[i * part + j] else @intCast((j * 7919 + 13 + i * 104729) % 120000); try q.forward.run(m, &scratch, &.{.{ .cache = k, .rows = history }}, ids, false, .last); } var out: std.Io.Writer.Allocating = .init(gpa); defer out.deinit(); const w = &out.writer; - try w.print("# LFM2 verify forward ms (median of {d}) by streams and rows a stream, {d}-token histories\n", .{ reps, history }); + try w.print("# LFM2 verify forward ms (median of {d}) by streams and rows a stream, {d}-token histories, {s} rows\n", .{ reps, history, if (part > 0) "real" else "synthetic" }); try w.print("# streams rows_each total_rows ms\n", .{}); var segments: [st.batch_rows]q.forward.Segment = undefined; var tokens: [st.batch_rows]u32 = undefined; @@ -48,7 +64,11 @@ pub fn main(init: std.process.Init) !void { for (counts) |streams| { var each: usize = 1; while (each <= st.window_rows and streams * each <= st.batch_rows) : (each += 1) { - for (segments[0..streams], caches[0..streams]) |*s, *k| s.* = .{ .cache = k, .rows = each }; + for (segments[0..streams], caches[0..streams], 0..) |*s, *k, i| { + s.* = .{ .cache = k, .rows = each }; + // real rows: the tokens after the stream's history, as a verify of drafts that all land + if (part > 0) @memcpy(tokens[i * each ..][0..each], real.items[i * part + history ..][0..each]); + } var times: [reps + 2]f64 = undefined; for (×) |*t| { const t0 = now(); From eba0950dfccb74862877d87f7fc5b09930b74916 Mon Sep 17 00:00:00 2001 From: BlivionIaG <15323295+BlivionIaG@users.noreply.github.com> Date: Mon, 26 Oct 2026 10:06:31 +0100 Subject: [PATCH 13/39] docs: the LFM2 recipe's DSpark verify schedule and its flags The recipe names the confidence head, the three --draft-verify modes (scheduled by default), how the lane core prices a hold pass and a copy, the lone-stream guard, and --verify-costs with tf-lfm2-verify-costs. --- docs/recipes/lfm2.5-24b-a2b.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/docs/recipes/lfm2.5-24b-a2b.md b/docs/recipes/lfm2.5-24b-a2b.md index f4272b59a5..64e26c258b 100644 --- a/docs/recipes/lfm2.5-24b-a2b.md +++ b/docs/recipes/lfm2.5-24b-a2b.md @@ -82,3 +82,29 @@ drafted rounds measure faster than plain ones at that stream count. against an fp32 host reference of the released checkpoint (the same taps and the target's embedding and head), prints each draft against the reference's pick with the margin between the reference's two best tokens, and times the drafter's passes; a near tie can pick the other token. + +### Verify schedule + +The drafter also carries a confidence head: a linear map of a draft's final block row and `W1[previous token]` +(2,304 inputs) to one logit, through a sigmoid (`lfm2_dspark_confidence`). It does not change the drafts. +`--draft-verify` picks how they are verified: + +- `scheduled` (the default): a chain's chance at draft j is the product of its first j confidences. Each round the + lane core's window allocator takes the likeliest drafts over all streams while the expected tokens a ms rise, so + each stream gets its own verify length, and the next hold drafts as deep as the stream's grant. A hold still + running on the GPU is priced by its lane's last confidences, so the host does not wait for it. +- `depth`: the lane core's depth rule, from each stream's measured acceptance by depth. +- `fixed`: every round verifies the whole block, as LiquidAI's published benchmarks did; a shared round still fits + 32 rows. + +With `scheduled` a stream with a copy ahead verifies the copy and asks no hold, and a drafting stream prices its copy +windows by its own copies. The lane core also weighs the hold pass itself: the drafter times holds of 1 to 16 +blocks at load and corrects those prices toward each settled hold's GPU time. A lone stream holds drafts only while +its drafted rounds measure faster than plain ones (`zig/src/core/lanes/lone_guard.zig`); shared rounds keep the +shared-round guard. + +`--verify-costs FILE` replaces the costs timed at load with a table from `tf-lfm2-verify-costs MODEL FILE [HISTORY +[IDS...]]` (lines `streams rows_each total_rows ms`), measured on the serving Mac. With token id files each stream +reads its history and verify rows from its own part of the ids, so its rows route its own text's experts. Both flags +are refused unless an `lfm2_moe` checkpoint is served with `--drafter`. `tf-lfm2-dspark` also prints the confidence +head against its fp32 reference and times shared holds of 2 to 16 streams. From c40129e1611566c7f7c395d190a0f24012cf54ae Mon Sep 17 00:00:00 2001 From: BlivionIaG <15323295+BlivionIaG@users.noreply.github.com> Date: Mon, 26 Oct 2026 09:40:54 +0100 Subject: [PATCH 14/39] lfm2: the DSpark drafter loads one matrix at a time and keeps a lane in one buffer Memory of the drafter, no change to its drafts: - The loader keeps the matrices in the safetensors file (Checkpoint.addFileSplit) and reads each into one reused stage, quantizes it and drops it. Load no longer holds the 655 MB bf16 file beside the 8-bit codes, only the largest matrix (W1 and W2). The codes are the same bytes. - A lane is one Metal buffer of laneBytes: its layers' keys and values, then its staged rows. Opening a stream allocated eleven buffers. Lane.kv gives a layer's offset; the rows and their layout are unchanged. - The absorb norms its context rows in place, so the rotated query heads go to the freed buffer and x, which only a hold reads, takes 32 rows instead of 2,048. - The waited passes end in one helper that commits, waits and reports a failed command buffer. - At load the drafter logs its weights, scratch and a stream's bytes. tf-lfm2-dspark counts host allocations: twenty rounds of hold, chances, held drafts and a staged absorb through the drafter's interface, after four warm ones, must take no allocation or resize of the drafter's allocator and one unwaited hold pass each. --- zig/src/families/lfm2_moe/dspark.zig | 62 +++++++++----------- zig/src/families/lfm2_moe/dspark_block.zig | 41 ++++++------- zig/src/families/lfm2_moe/dspark_weights.zig | 21 +++++-- zig/tests/lfm2_dspark.zig | 42 +++++++++++++ 4 files changed, 104 insertions(+), 62 deletions(-) diff --git a/zig/src/families/lfm2_moe/dspark.zig b/zig/src/families/lfm2_moe/dspark.zig index eb3fa4d90f..76d8369521 100644 --- a/zig/src/families/lfm2_moe/dspark.zig +++ b/zig/src/families/lfm2_moe/dspark.zig @@ -34,17 +34,21 @@ pub const Options = struct { capacity: usize, bits: u8 = 8, verify: Verify = def /// One stream's context keys and values a layer ([kv head][cap][head dim], as the target's attention reads them). pub const Lane = struct { - keys: [8]mtl.Buffer, - values: [8]mtl.Buffer, + mem: mtl.Buffer, // each layer's keys then values, then the staged rows (shared storage) cap: usize, // positions each holds: the context and one block past it + stage_at: usize, // the staged rows' byte offset in `mem` end: u64 = 0, // context rows absorbed: positions [0, end) - stage: mtl.Buffer, // tapped rows [end, end + staged), not absorbed yet (shared storage) - staged: u32 = 0, + staged: u32 = 0, // tapped rows [end, end + staged), not absorbed yet held: [16]u32 = @splat(0), held_len: u32 = 0, conf: [16]f32 = @splat(0), // each held draft's chance given the drafts before it landed (the confidence head) conf_len: u32 = 0, // the drafts `conf` covers: the last settled hold's depth slot: ?u32 = null, // its block's place in the last signalled hold, whose picks still hold its drafts + + /// Layer `layer`'s keys, or its values when `values`, in `mem`. + pub fn kv(l: *const Lane, layer: usize, values: bool) block.Ref { + return .{ .buf = l.mem, .off = (2 * layer + @intFromBool(values)) * l.cap * kw * 2 }; + } }; /// The passes' working buffers, one pass at a time on the target's queue. @@ -66,8 +70,8 @@ pub const Scratch = struct { act: mtl.Buffer, logits: mtl.Buffer, sums: mtl.Buffer, - fcx: mtl.Buffer, - hctx: mtl.Buffer, + fcx: mtl.Buffer, // an absorb pass's context rows: fc's output, normed in place + dump: mtl.Buffer, // the query heads lfm2_qk rotates from context rows, which no one reads best: mtl.Buffer, // the Markov scan's best score and token of each threadgroup, by stream at: mtl.Buffer, picks: mtl.Buffer, @@ -139,7 +143,7 @@ pub const Drafter = struct { d.s = .{ .ids = try d.buffer(r * 4), .h = try d.buffer(r * c.hidden * 2), - .x = try d.buffer(wide * c.hidden * 2), + .x = try d.buffer(r * c.hidden * 2), .r = try d.buffer(r * c.hidden * 2), .y = try d.buffer(r * c.hidden * 2), .q = try d.buffer(r * c.hidden * 2), @@ -155,7 +159,7 @@ pub const Drafter = struct { .logits = try d.buffer(r * c.vocab * 2), .sums = try d.buffer(sums * 4), .fcx = try d.buffer(absorb_rows * c.hidden * 2), - .hctx = try d.buffer(absorb_rows * c.hidden * 2), + .dump = try d.buffer(absorb_rows * c.hidden * 2), .best = try d.buffer(r * block.scan_groups * 4), .at = try d.buffer(r * block.scan_groups * 4), .picks = try d.buffer(d.config.block * r * 8), @@ -236,8 +240,9 @@ pub const Drafter = struct { return true; } + /// A stream's lane at `capacity` positions: its context keys and values a layer, and its staged rows. pub fn laneBytes(d: *const Drafter, capacity: usize) usize { - return 2 * d.config.layers * (capacity + d.config.block) * kw * 2 + stage_rows * d.tapBytes(); + return @intCast(d.keptBytes(capacity + d.config.block) + stage_rows * d.tapBytes()); } /// A tapped row's bytes: every tap's hidden row, bf16. @@ -248,7 +253,7 @@ pub const Drafter = struct { /// The lane's staged rows as an absorb piece (null: none staged). pub fn stagedPiece(l: *Lane) ?block.Piece { if (l.staged == 0) return null; - return .{ .lane = l, .src = .{ .buf = l.stage }, .rows = l.staged, .p0 = l.end }; + return .{ .lane = l, .src = .{ .buf = l.mem, .off = l.stage_at }, .rows = l.staged, .p0 = l.end }; } /// The lane's staged rows absorbed now, in a command buffer of their own. @@ -272,33 +277,17 @@ pub const Drafter = struct { try d.lanes.put(d.gpa, s, l); } + /// A lane in one buffer of laneBytes(capacity). pub fn newLane(d: *Drafter, capacity: usize) !*Lane { const l = try d.gpa.create(Lane); errdefer d.gpa.destroy(l); - l.* = .{ .keys = undefined, .values = undefined, .stage = try d.target.device.buffer(stage_rows * d.tapBytes(), shared), .cap = capacity + d.config.block }; - errdefer l.stage.deinit(); - var made: usize = 0; - errdefer for (0..made) |i| { - l.keys[i].deinit(); - l.values[i].deinit(); - }; - for (0..d.config.layers) |i| { - l.keys[i] = try d.target.device.buffer(l.cap * kw * 2, shared); - l.values[i] = d.target.device.buffer(l.cap * kw * 2, shared) catch |e| { - l.keys[i].deinit(); - return e; - }; - made += 1; - } + const at: usize = @intCast(d.keptBytes(capacity + d.config.block)); + l.* = .{ .mem = try d.target.device.buffer(d.laneBytes(capacity), shared), .cap = capacity + d.config.block, .stage_at = at }; return l; } pub fn dropLane(d: *Drafter, l: *Lane) void { - l.stage.deinit(); - for (0..d.config.layers) |i| { - l.keys[i].deinit(); - l.values[i].deinit(); - } + l.mem.deinit(); d.gpa.destroy(l); } @@ -348,8 +337,8 @@ pub const Drafter = struct { fn copyLane(d: *const Drafter, l: *const Lane, at: usize, bytes: []u8, out: bool) void { const n = at * c.head_dim * 2; var off: usize = 0; - for (0..d.config.layers) |i| for ([_]mtl.Buffer{ l.keys[i], l.values[i] }) |buf| for (0..c.kv_heads) |h| { - const live = (buf.contents() + h * l.cap * c.head_dim * 2)[0..n]; + for (0..d.config.layers) |i| for ([_]block.Ref{ l.kv(i, false), l.kv(i, true) }) |r| for (0..c.kv_heads) |h| { + const live = (r.buf.contents() + r.off + h * l.cap * c.head_dim * 2)[0..n]; if (out) @memcpy(bytes[off..][0..n], live) else @memcpy(live, bytes[off..][0..n]); off += n; }; @@ -380,7 +369,7 @@ pub const Drafter = struct { l.staged = @min(l.staged, keep); if (it.start >= l.end and l.staged + f.rows <= stage_rows) { const from = (src.contents() + f.offset)[0 .. f.rows * width]; - @memcpy((l.stage.contents() + l.staged * width)[0 .. f.rows * width], from); + @memcpy((l.mem.contents() + l.stage_at + l.staged * width)[0 .. f.rows * width], from); l.staged += f.rows; continue; } @@ -517,6 +506,12 @@ pub const Drafter = struct { return heldFn(ptr, &.{s}, &.{out}); } + fn mibOf(bufs: []const mtl.Buffer) f64 { + var n: usize = 0; + for (bufs) |b| n += b.length(); + return @as(f64, @floatFromInt(n)) / (1024.0 * 1024.0); + } + /// Holds on empty contexts: the pipelines compiled, then holds timed by blocks and depth (best of three). fn warm(d: *Drafter) !void { var made: [hold_grid[hold_grid.len - 1]]*Lane = undefined; @@ -534,6 +529,7 @@ pub const Drafter = struct { d.hold_ms = best[1]; d.step_ms = best[1] / @as(f64, @floatFromInt(d.config.block)); std.log.info("lfm2 dspark: a 1-draft hold takes {d:.2} ms, a {d}-draft hold {d:.2} ms", .{ best[0], d.config.block, best[1] }); + std.log.info("lfm2 dspark: weights {d:.1} MiB, scratch {d:.1} MiB, a stream {d} KiB a position and {d} KiB staged", .{ mibOf(d.w.buffers()), mibOf(d.owned.items), d.keptBytes(1) / 1024, stage_rows * d.tapBytes() / 1024 }); var blocks: [hold_grid[hold_grid.len - 1]]block.Block = undefined; for (made, &blocks) |lane, *b| b.* = .{ .lane = lane, .pending = 1, .anchor = 0, .depth = 1 }; d.holds_len = 0; diff --git a/zig/src/families/lfm2_moe/dspark_block.zig b/zig/src/families/lfm2_moe/dspark_block.zig index 0b712bf2dd..64e9dd38ce 100644 --- a/zig/src/families/lfm2_moe/dspark_block.zig +++ b/zig/src/families/lfm2_moe/dspark_block.zig @@ -73,8 +73,8 @@ const Enc = struct { const args = QkArgs{ .len = @intCast(len), .cap = @intCast(lane.cap), .eps = c.eps, .log2_theta = std.math.log2(x.d.config.theta) }; x.e.setBytes(std.mem.asBytes(&args), 5); x.e.setBuffer(queries.buf, queries.off, 6); - x.e.setBuffer(lane.keys[layer], 0, 7); - x.e.setBuffer(lane.values[layer], 0, 8); + x.e.setBuffer(lane.mem, lane.kv(layer, false).off, 7); + x.e.setBuffer(lane.mem, lane.kv(layer, true).off, 8); x.e.dispatchGroups(mtl.Size.of(c.query_heads + c.kv_heads, rows, 1), mtl.Size.of(32, 1, 1)); } @@ -85,8 +85,8 @@ const Enc = struct { const used = (keys + st.chunk_keys - 1) / st.chunk_keys * st.subs; x.pipe("lfm2_attn_block"); x.e.setBuffer(s.queries, row * c.hidden * 2, 0); - x.e.setBuffer(lane.keys[layer], 0, 1); - x.e.setBuffer(lane.values[layer], 0, 2); + x.e.setBuffer(lane.mem, lane.kv(layer, false).off, 1); + x.e.setBuffer(lane.mem, lane.kv(layer, true).off, 2); x.e.setBytes(std.mem.asBytes(&args), 3); x.e.setBuffer(s.pm, 0, 4); x.e.setBuffer(s.pl, 0, 5); @@ -103,18 +103,6 @@ const Enc = struct { } }; -fn failed(msg: [*:0]const u8) anyerror { - std.log.err("lfm2 dspark: command buffer failed: {s}", .{msg}); - return error.GpuFailed; -} - -fn finish(cb: mtl.CommandBuffer, e: mtl.ComputeEncoder) !void { - e.end(); - cb.commit(); - cb.wait(); - if (cb.failure()) |msg| return failed(msg); -} - /// One stream's tapped rows in an absorb pass: `rows` rows at `src`, the context from position `p0`. pub const Piece = struct { lane: *Lane, src: Ref, rows: usize, p0: u64 }; @@ -128,16 +116,15 @@ pub fn absorbPass(d: *Drafter, e: mtl.ComputeEncoder, pieces: []const Piece) !vo rows += pc.rows; } if (rows == 0 or rows > ds.absorb_rows) return error.DraftContextGap; - x.norm(s.fcx, s.fcx, d.w.hidden_norm, false, s.hctx, rows); + x.norm(s.fcx, s.fcx, d.w.hidden_norm, false, s.fcx, rows); // in place: a thread reads its values first for (0..d.config.layers) |i| { const ly = &d.w.layers[i]; - try x.project(ly.k, .{ .buf = s.hctx }, .{ .buf = s.k }, rows); - try x.project(ly.v, .{ .buf = s.hctx }, .{ .buf = s.v }, rows); + try x.project(ly.k, .{ .buf = s.fcx }, .{ .buf = s.k }, rows); + try x.project(ly.v, .{ .buf = s.fcx }, .{ .buf = s.v }, rows); var at: usize = 0; for (pieces) |pc| { - // the kernel rotates query heads too: they read the context rows and land in x, which no one reads - const q = Ref{ .buf = s.hctx, .off = at * c.hidden * 2 }; - x.qk(q, .{ .buf = s.k, .off = at * kw * 2 }, .{ .buf = s.v, .off = at * kw * 2 }, i, pc.lane, pc.p0, .{ .buf = s.x }, pc.rows); + const q = Ref{ .buf = s.fcx, .off = at * c.hidden * 2 }; + x.qk(q, .{ .buf = s.k, .off = at * kw * 2 }, .{ .buf = s.v, .off = at * kw * 2 }, i, pc.lane, pc.p0, .{ .buf = s.dump }, pc.rows); at += pc.rows; } } @@ -150,7 +137,13 @@ pub fn begin(d: *Drafter) struct { cb: mtl.CommandBuffer, e: mtl.ComputeEncoder } pub fn end(cb: mtl.CommandBuffer, e: mtl.ComputeEncoder) !void { - return finish(cb, e); + e.end(); + cb.commit(); + cb.wait(); + if (cb.failure()) |msg| { + std.log.err("lfm2 dspark: command buffer failed: {s}", .{msg}); + return error.GpuFailed; + } } /// A GPU event a hold signals instead of being waited on, and the value it sets. @@ -242,7 +235,7 @@ pub fn hold(d: *Drafter, blocks: []const Block, signal: ?Signal) !?mtl.CommandBu cb.commit(); return kept; } - try finish(cb, e); + try end(cb, e); const picks = s.picks.slice(Pick, d.config.block * ds.pass_rows); for (blocks, 0..) |b, bi| { for (b.lane.held[0..steps], 0..) |*h, i| { diff --git a/zig/src/families/lfm2_moe/dspark_weights.zig b/zig/src/families/lfm2_moe/dspark_weights.zig index bfd6d078e1..f0e081cb25 100644 --- a/zig/src/families/lfm2_moe/dspark_weights.zig +++ b/zig/src/families/lfm2_moe/dspark_weights.zig @@ -55,6 +55,7 @@ const Loader = struct { gpa: std.mem.Allocator, device: mtl.Device, check: *const ckpt.Checkpoint, + stage: *ckpt.Stage, // a matrix read from the file, one at a time, into one reused buffer owned: *std.ArrayList(mtl.Buffer), bits: u8, @@ -65,8 +66,11 @@ const Loader = struct { return b; } + /// A vector as loaded, or a matrix read into the stage (valid until the next matrix), of `shape` in bf16. fn tensor(l: Loader, name: []const u8, shape: []const usize) !ckpt.Tensor { - const t = try l.check.get(name); + l.stage.reset(); + if (shape.len == 2) try l.stage.reserve(l.check, &.{name}); + const t = if (shape.len == 2) try l.stage.read(l.check, name) else try l.check.get(name); if (t.dtype != .bf16 or t.rank != shape.len or !std.mem.eql(usize, t.shape[0..t.rank], shape)) { std.log.err("{s}: unexpected DSpark tensor dtype or shape", .{name}); return error.BadDraftWeight; @@ -108,18 +112,25 @@ fn part(buf: []u8, layer: usize, name: []const u8) ![]const u8 { return std.fmt.bufPrint(buf, "layers.{d}.{s}.weight", .{ layer, name }); } -/// The drafter's checkpoint read and quantized at `bits` (4 or 8); the checkpoint's own buffers are released after. +/// Matrices stay in the file until the loader reads them; vectors load with the header. +fn matrices(_: []const u8, rank: usize) bool { + return rank == 2; +} + +/// The drafter's checkpoint read and quantized at `bits` (4 or 8), a matrix at a time; its buffers are released after. pub fn load(gpa: std.mem.Allocator, device: mtl.Device, io: std.Io, dir: []const u8, cfg: *const dc.Config, bits: u8) !Weights { if (bits != 4 and bits != 8) return error.UnsupportedDraftBits; var check = ckpt.Checkpoint.init(gpa); defer check.deinit(); + var stage: ckpt.Stage = .{ .device = device }; + defer stage.deinit(); const files = try shards.shardFiles(gpa, io, dir); defer shards.freeShardFiles(gpa, files); - for (files) |path| try check.addFile(device, path, ""); + for (files) |path| try check.addFileSplit(device, path, "", null, matrices); var w: Weights = undefined; w.owned = .empty; errdefer w.deinit(gpa); - const l: Loader = .{ .gpa = gpa, .device = device, .check = &check, .owned = &w.owned, .bits = bits }; + const l: Loader = .{ .gpa = gpa, .device = device, .check = &check, .stage = &stage, .owned = &w.owned, .bits = bits }; const qw = c.query_heads * c.head_dim; const kw = c.kv_heads * c.head_dim; w.fc = try l.matrix("fc.weight", c.hidden, cfg.tap_count * c.hidden, false); @@ -145,7 +156,7 @@ pub fn load(gpa: std.mem.Allocator, device: mtl.Device, io: std.Io, dir: []const w.w2 = try l.matrix("markov_head.markov_w2.weight", c.vocab, cfg.rank, false); w.confidence = null; w.confidence_bias = 0; - if (check.tensors.contains("confidence_head.proj.weight")) { + if (check.has("confidence_head.proj.weight")) { const width = c.hidden + cfg.rank; // the final norm's row, then W1 of the token before the draft const proj = (try l.tensor("confidence_head.proj.weight", &.{ 1, width })).host(u16); w.confidence = try l.buffer(width * 2); diff --git a/zig/tests/lfm2_dspark.zig b/zig/tests/lfm2_dspark.zig index 5ee8140de2..5a9885b98e 100644 --- a/zig/tests/lfm2_dspark.zig +++ b/zig/tests/lfm2_dspark.zig @@ -125,6 +125,47 @@ fn absorb(d: *q.dspark.Drafter, piece: q.dspark_block.Piece) !void { try q.dspark_block.end(run.cb, run.e); } +/// Served rounds through the drafter's interface: after four warm rounds, no host allocation and one hold pass a round. +fn steady(gpa: std.mem.Allocator, d: *q.dspark.Drafter, taps: mtl.Buffer, ids: []const u32, ctx: usize) !void { + const lanes = tf.lanes; + var s = try lanes.Stream.init(gpa, .{ .id = "steady", .prompt = ids[0 .. ctx + 1], .max_new = 160 }); + defer s.deinit(gpa); + const dr = d.drafter(); + try dr.open(&s); + defer dr.release(&s); + const width: u32 = @intCast(d.tapBytes()); + const feats = struct { + fn at(buf: mtl.Buffer, row: usize, rows: usize, w: u32) lanes.backend.Features { + return .{ .buffer = @intFromPtr(buf.id), .offset = row * w, .rows = @intCast(rows), .row_bytes = w, .space = .device, .dtype = .bf16 }; + } + }; + try dr.absorb(&.{.{ .stream = &s, .features = feats.at(taps, 0, ctx, width), .start = 0, .follow = &.{} }}); + var counted = std.testing.FailingAllocator.init(gpa, .{}); + var anchor: u64 = ctx; + const rounds = 24; + const warm = 4; + var holds: u64 = 0; + for (0..rounds) |r| { + if (r == warm) { + d.gpa = counted.allocator(); + holds = d.signalled; + } + try dr.hold(&.{.{ .stream = &s, .pending = ids[@intCast(anchor % ctx)], .position = anchor + 1, .depth = 3 }}); + var chances: [3]f64 = undefined; + _ = try dr.chances(&s, &chances); + var drafts: [3]u32 = undefined; + try dr.held(&.{&s}, &.{&drafts}); + const rows = r % 3 + 1; + const at: usize = @intCast(anchor % (ctx - 3)); + try dr.absorb(&.{.{ .stream = &s, .features = feats.at(taps, at, rows, width), .start = anchor, .follow = &.{}, .prompt = false }}); + anchor += rows; + } + d.gpa = gpa; + const n = rounds - warm; + std.debug.print("steady rounds: {d} rounds of hold, chances, held and a verified absorb: {d} host allocations, {d} resizes, {d} unwaited holds\n", .{ n, counted.allocations, counted.resize_index, d.signalled - holds }); + if (counted.allocations != 0 or counted.resize_index != 0 or d.signalled - holds != n) return error.SteadyRoundsAllocate; +} + pub fn main(init: std.process.Init) !void { const gpa = init.gpa; const args = try init.minimal.args.toSlice(init.arena.allocator()); @@ -218,6 +259,7 @@ pub fn main(init: std.process.Init) !void { } std.debug.print("hold of {d} streams at depth {d}: {d:.2} ms, GPU {d:.2} ms (best of 5)\n", .{ streams, depth, best, gpu_best }); }; + try steady(gpa, d, taps.buf, ids, ctx); _ = try q.dspark_block.hold(d, &.{.{ .lane = lane, .pending = ids[ctx], .anchor = ctx, .depth = block }}, null); // the host reference, fp32 throughout var r: Ref = .{ .gpa = gpa, .check = ck.Checkpoint.init(gpa), .cfg = cfg }; From 4671e17768b982972d3c30c2d30143a8ea68d549 Mon Sep 17 00:00:00 2001 From: BlivionIaG <15323295+BlivionIaG@users.noreply.github.com> Date: Mon, 26 Oct 2026 10:06:40 +0100 Subject: [PATCH 15/39] docs: the LFM2 recipe's DSpark loader and lane memory The recipe says how the drafter loads and keeps its lanes, what it logs at load and that tf-lfm2-dspark counts host allocations over served rounds. --- docs/recipes/lfm2.5-24b-a2b.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/recipes/lfm2.5-24b-a2b.md b/docs/recipes/lfm2.5-24b-a2b.md index 64e26c258b..48b2666927 100644 --- a/docs/recipes/lfm2.5-24b-a2b.md +++ b/docs/recipes/lfm2.5-24b-a2b.md @@ -108,3 +108,8 @@ shared-round guard. reads its history and verify rows from its own part of the ids, so its rows route its own text's experts. Both flags are refused unless an `lfm2_moe` checkpoint is served with `--drafter`. `tf-lfm2-dspark` also prints the confidence head against its fp32 reference and times shared holds of 2 to 16 streams. + +The loader reads the checkpoint one matrix at a time and keeps only the 8-bit codes; a stream's drafter lane is one +Metal buffer (its layers' keys and values, then its staged rows), and its bytes join the streams the load limit +admits. At load the drafter logs its weights, scratch and a stream's bytes. `tf-lfm2-dspark` also counts host +allocations over twenty served drafter rounds, which must be none. From 7b997000da5c3cfe58a3a7c77ed05e8fe546fd3c Mon Sep 17 00:00:00 2001 From: BlivionIaG <15323295+BlivionIaG@users.noreply.github.com> Date: Mon, 26 Oct 2026 09:43:46 +0100 Subject: [PATCH 16/39] core: tile_qmm's twin takes a caller's row count for its window kernel encodeTwin picks the window kernel from window_rows rows on, for matrices of window_n outputs or more, and the scalar twin below. A caller that measured its own cut (the DSpark drafter's hold projections) now passes it to encodeTwinFrom; encodeTwin keeps its choice and its callers are unchanged. The kernels are unchanged. --- zig/src/core/tile_qmm.zig | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/zig/src/core/tile_qmm.zig b/zig/src/core/tile_qmm.zig index 030bd097e1..dbc0da8b68 100644 --- a/zig/src/core/tile_qmm.zig +++ b/zig/src/core/tile_qmm.zig @@ -167,10 +167,26 @@ pub fn encodeTwin( y: mtl.Buffer, y_off: usize, rows: usize, +) !void { + // the window kernel only for matrices of window_n outputs or more + return encodeTwinFrom(e, p, m, x, x_off, y, y_off, rows, if (m.n >= window_n) window_rows else twin_rows + 1); +} + +/// encodeTwin with the window kernel from `from` rows on (the scalar twin below them), for a caller's measured cut. +pub fn encodeTwinFrom( + e: mtl.ComputeEncoder, + p: *const Pipelines, + m: Matrix, + x: mtl.Buffer, + x_off: usize, + y: mtl.Buffer, + y_off: usize, + rows: usize, + from: usize, ) !void { const i = try width(m); if (rows == 0 or rows > twin_rows) return error.NoRows; - const window = rows >= window_rows and m.n >= window_n; + const window = rows >= from; e.setPipeline(if (window) p.rows[i] else if (rows > 1) p.twin_few[i] else p.twin[i]); bind(e, m, .{ .k = @intCast(m.k), .n = @intCast(m.n), .r = @intCast(rows) }, x, x_off, y, y_off); if (window) return e.dispatchGroups(.{ .width = m.n / window_columns }, .{ .width = 64 }); From 3d735f17c4630dcf852e38fe458b3b8e66ed9daf Mon Sep 17 00:00:00 2001 From: BlivionIaG <15323295+BlivionIaG@users.noreply.github.com> Date: Mon, 26 Oct 2026 09:43:59 +0100 Subject: [PATCH 17/39] lfm2: a DSpark hold runs in one concurrent encoder A hold ran every dispatch in a serial encoder, its blocks' attention one block after the other. A hold now encodes on a concurrent encoder and puts a barrier only where a step reads an earlier step's writes: - q, k and v, and gate and up, are projections of one input: one sums pass, then their products free to overlap (projectAll). The absorb's k and v do the same. - The attention partials hold a whole hold pass's query rows, so the blocks of a shared hold attend side by side in one wave, then merge in one. - A projection picks its kernel by rows and output width: the scalar twin up to 8 rows, the window kernel past them where the GPU has no tensor units, and the tensor units from 9 rows (the target's head, 128,000 outputs, from 5) where it has them; outputs of the hidden width or fewer stay on the twin to its rows. The kernels are the target's own and core's, unchanged; only their order and the rows each launch takes change. Drafts are checked by tf-lfm2-dspark against the fp32 reference, and every draft is still verified by the target. --- zig/src/families/lfm2_moe/dspark.zig | 7 +- zig/src/families/lfm2_moe/dspark_block.zig | 177 +++++++++++++++------ 2 files changed, 136 insertions(+), 48 deletions(-) diff --git a/zig/src/families/lfm2_moe/dspark.zig b/zig/src/families/lfm2_moe/dspark.zig index 76d8369521..1b782c496f 100644 --- a/zig/src/families/lfm2_moe/dspark.zig +++ b/zig/src/families/lfm2_moe/dspark.zig @@ -138,7 +138,7 @@ pub const Drafter = struct { const r = pass_rows; const wide = @max(pass_rows, absorb_rows); const i = d.config.intermediate; - const partials = @max(st.split_rows, d.config.block) * st.subs * c.query_heads * st.chunks; + const partials = d.partRows() * st.subs * c.query_heads * st.chunks; const sums = @max(absorb_rows * (d.config.tap_count * c.hidden / 64), r * (i / 64)); d.s = .{ .ids = try d.buffer(r * 4), @@ -167,6 +167,11 @@ pub const Drafter = struct { }; } + /// Query rows the attention partials hold at once: a whole hold pass's, so its blocks attend side by side. + pub fn partRows(d: *const Drafter) usize { + return @max(pass_rows, d.config.block); + } + /// The drafter's buffers that every round reads (weights and scratch), for the server's residency set. pub fn buffers(d: *const Drafter, gpa: std.mem.Allocator) ![]mtl.Buffer { return std.mem.concat(gpa, mtl.Buffer, &.{ d.w.buffers(), d.owned.items }); diff --git a/zig/src/families/lfm2_moe/dspark_block.zig b/zig/src/families/lfm2_moe/dspark_block.zig index 64e9dd38ce..707e4df5ba 100644 --- a/zig/src/families/lfm2_moe/dspark_block.zig +++ b/zig/src/families/lfm2_moe/dspark_block.zig @@ -14,6 +14,12 @@ const Lane = ds.Lane; const kw = c.kv_heads * c.head_dim; const QkArgs = extern struct { len: u32, cap: u32, eps: f32, log2_theta: f32 }; const AttnArgs = extern struct { len: u32, cap: u32, scale: f32, subs: u32 }; +/// Partials a query row takes in the scratch (the kernels' QH x CHUNKS x SUBS). +const part_row = c.query_heads * st.chunks * st.subs; + +fn attnArgs(lane: *const Lane, keys: u64) AttnArgs { + return .{ .len = @intCast(keys - 1), .cap = @intCast(lane.cap), .scale = 1.0 / @sqrt(@as(f32, c.head_dim)), .subs = st.subs }; +} const Pick = round.contract.Pick; const MarkovArgs = extern struct { stride: u32, rank: u32, vocab: u32, prev_stride: u32 }; const ConfidenceArgs = extern struct { n: u32, pass: u32, vocab: u32, rank: u32, hidden: u32 }; @@ -21,6 +27,17 @@ const ConfidenceArgs = extern struct { n: u32, pass: u32, vocab: u32, rank: u32, pub const scan_span = 512; pub const scan_groups = (c.vocab + scan_span - 1) / scan_span; +/// Rows a drafter projection takes on the scalar twin at most (faster there than the window or tensor units). +const scalar_rows = 8; +/// Rows from which the target's head (128,000 outputs) runs on the tensor units, where the GPU has them. +const head_rows = 5; + +/// Rows from which `w` runs on the tensor units: outputs of the hidden width or fewer stay on the twin to its rows. +fn naxFrom(w: tile.Matrix) usize { + if (w.n <= c.hidden) return tile.twin_rows + 1; + return if (w.n == c.vocab) head_rows else scalar_rows + 1; +} + /// A buffer and byte offset. pub const Ref = struct { buf: mtl.Buffer, off: usize = 0 }; @@ -30,23 +47,46 @@ pub const Block = struct { lane: *Lane, pending: u32, anchor: u64, depth: u32 }; const Enc = struct { d: *Drafter, e: mtl.ComputeEncoder, + conc: bool = false, // a concurrent encoder: each step waits for the writes before it at `sync` + + /// Later dispatches see the writes before this point (a no-op on a serial encoder). + fn sync(x: Enc) void { + if (x.conc) x.e.barrier(); + } fn pipe(x: Enc, comptime key: []const u8) void { x.e.setPipeline(x.d.target.kernels.get(key)); } - /// y[rows] = x[rows] W^T: the twin up to its rows, past them the M5's tensor-unit kernels, else the tiles. + /// y[rows] = x[rows] W^T: the M5's tensor units from naxFrom rows, else the twin (scalar to scalar_rows) or tiles. fn project(x: Enc, w: tile.Matrix, in: Ref, out: Ref, rows: usize) !void { + return x.projectAll(&.{w}, in, &.{out}, rows); + } + + /// y_i[rows] = x[rows] W_i^T on one input and shape: one sums pass, then products free to overlap. + fn projectAll(x: Enc, ws: []const tile.Matrix, in: Ref, outs: []const Ref, rows: usize) !void { const t = &x.d.target.kernels.tiles; - if (rows > tile.twin_rows) if (x.d.target.kernels.naxPipes(w.bits)) |p| { - const sums = Ref{ .buf = x.d.s.sums }; - am.rowSums(x.e, p, c.group, in, sums, @intCast(rows), @intCast(w.k)); - const q = .{ .w = Ref{ .buf = w.w, .off = w.w_off }, .s = Ref{ .buf = w.scales, .off = w.s_off }, .b = Ref{ .buf = w.biases, .off = w.b_off }, .n = @as(u32, @intCast(w.n)), .k = @as(u32, @intCast(w.k)) }; - return am.dense(x.e, p, in, sums, q, out, @intCast(rows)); + const sums = Ref{ .buf = x.d.s.sums }; + const k = ws[0].k; + const nax = x.d.target.kernels.naxPipes(ws[0].bits); + if (rows >= naxFrom(ws[0])) if (nax) |p| { + am.rowSums(x.e, p, c.group, in, sums, @intCast(rows), @intCast(k)); + x.sync(); + for (ws, outs) |w, out| { + const q = .{ .w = Ref{ .buf = w.w, .off = w.w_off }, .s = Ref{ .buf = w.scales, .off = w.s_off }, .b = Ref{ .buf = w.biases, .off = w.b_off }, .n = @as(u32, @intCast(w.n)), .k = @as(u32, @intCast(w.k)) }; + am.dense(x.e, p, in, sums, q, out, @intCast(rows)); + } + return; }; - if (rows <= tile.twin_rows and x.d.target.kernels.twin) return tile.encodeTwin(x.e, t, w, in.buf, in.off, out.buf, out.off, rows); - tile.encodeSums(x.e, t, w.k, in.buf, in.off, x.d.s.sums, rows); - try tile.encode(x.e, t, w, in.buf, in.off, x.d.s.sums, 0, out.buf, out.off, rows); + if (rows <= tile.twin_rows and x.d.target.kernels.twin) { + // beside the tensor units the scalar twin takes every row count it has; elsewhere the window from 9 rows + const window: usize = if (nax != null) tile.twin_rows + 1 else scalar_rows + 1; + for (ws, outs) |w, out| try tile.encodeTwinFrom(x.e, t, w, in.buf, in.off, out.buf, out.off, rows, window); + return; + } + tile.encodeSums(x.e, t, k, in.buf, in.off, x.d.s.sums, rows); + x.sync(); + for (ws, outs) |w, out| try tile.encode(x.e, t, w, in.buf, in.off, x.d.s.sums, 0, out.buf, out.off, rows); } /// h += r when `add`, then out = norm(h) * gain: LFM2's own norm (the drafter's width and eps are the target's). @@ -78,25 +118,33 @@ const Enc = struct { x.e.dispatchGroups(mtl.Size.of(c.query_heads + c.kv_heads, rows, 1), mtl.Size.of(32, 1, 1)); } - /// Query rows [row, row + rows) against the lane's keys [0, keys), all of them visible to every row. - fn attend(x: Enc, layer: usize, lane: *const Lane, row: usize, rows: usize, keys: u64) void { + /// The chunks' partials of query rows [row, row + rows) against the lane's keys, at partial row `part`. + fn partials(x: Enc, layer: usize, lane: *const Lane, row: usize, rows: usize, keys: u64, part: usize) void { const s = &x.d.s; - const args = AttnArgs{ .len = @intCast(keys - 1), .cap = @intCast(lane.cap), .scale = 1.0 / @sqrt(@as(f32, c.head_dim)), .subs = st.subs }; + const args = attnArgs(lane, keys); const used = (keys + st.chunk_keys - 1) / st.chunk_keys * st.subs; + const at = part * part_row; x.pipe("lfm2_attn_block"); x.e.setBuffer(s.queries, row * c.hidden * 2, 0); x.e.setBuffer(lane.mem, lane.kv(layer, false).off, 1); x.e.setBuffer(lane.mem, lane.kv(layer, true).off, 2); x.e.setBytes(std.mem.asBytes(&args), 3); - x.e.setBuffer(s.pm, 0, 4); - x.e.setBuffer(s.pl, 0, 5); - x.e.setBuffer(s.po, 0, 6); + x.e.setBuffer(s.pm, at * 4, 4); + x.e.setBuffer(s.pl, at * 4, 5); + x.e.setBuffer(s.po, at * c.head_dim * 4, 6); x.e.setValue(@as(u32, @intCast(rows)), 7); x.e.dispatchGroups(mtl.Size.of((rows + 7) / 8, c.kv_heads, @intCast(used)), mtl.Size.of(128, 1, 1)); + } + + /// The partials from partial row `part` merged into the attention's output rows [row, row + rows). + fn merge(x: Enc, lane: *const Lane, row: usize, rows: usize, keys: u64, part: usize) void { + const s = &x.d.s; + const args = attnArgs(lane, keys); + const at = part * part_row; x.pipe("lfm2_attn_block_merge"); - x.e.setBuffer(s.pm, 0, 0); - x.e.setBuffer(s.pl, 0, 1); - x.e.setBuffer(s.po, 0, 2); + x.e.setBuffer(s.pm, at * 4, 0); + x.e.setBuffer(s.pl, at * 4, 1); + x.e.setBuffer(s.po, at * c.head_dim * 4, 2); x.e.setBytes(std.mem.asBytes(&args), 3); x.e.setBuffer(s.y, row * c.hidden * 2, 4); x.e.dispatchGroups(mtl.Size.of(c.query_heads, rows, 1), mtl.Size.of(32, 1, 1)); @@ -108,25 +156,32 @@ pub const Piece = struct { lane: *Lane, src: Ref, rows: usize, p0: u64 }; /// Each piece's rows into its lane's keys and values (fc, hidden_norm, k, v, k_norm, RoPE) in passes on `e`. pub fn absorbPass(d: *Drafter, e: mtl.ComputeEncoder, pieces: []const Piece) !void { + return absorbOn(.{ .d = d, .e = e }, pieces); +} + +fn absorbOn(x: Enc, pieces: []const Piece) !void { + const d = x.d; const s = &d.s; - const x = Enc{ .d = d, .e = e }; var rows: usize = 0; for (pieces) |pc| { try x.project(d.w.fc, pc.src, .{ .buf = s.fcx, .off = rows * c.hidden * 2 }, pc.rows); + x.sync(); // the pieces' projections share the sums scratch rows += pc.rows; } if (rows == 0 or rows > ds.absorb_rows) return error.DraftContextGap; x.norm(s.fcx, s.fcx, d.w.hidden_norm, false, s.fcx, rows); // in place: a thread reads its values first + x.sync(); for (0..d.config.layers) |i| { const ly = &d.w.layers[i]; - try x.project(ly.k, .{ .buf = s.fcx }, .{ .buf = s.k }, rows); - try x.project(ly.v, .{ .buf = s.fcx }, .{ .buf = s.v }, rows); + try x.projectAll(&.{ ly.k, ly.v }, .{ .buf = s.fcx }, &.{ .{ .buf = s.k }, .{ .buf = s.v } }, rows); + x.sync(); var at: usize = 0; for (pieces) |pc| { const q = Ref{ .buf = s.fcx, .off = at * c.hidden * 2 }; x.qk(q, .{ .buf = s.k, .off = at * kw * 2 }, .{ .buf = s.v, .off = at * kw * 2 }, i, pc.lane, pc.p0, .{ .buf = s.dump }, pc.rows); at += pc.rows; } + x.sync(); } } @@ -146,6 +201,30 @@ pub fn end(cb: mtl.CommandBuffer, e: mtl.ComputeEncoder) !void { } } +/// Layer `layer`'s q, k, v and every block's attention (a lone block's last layer: `out` rows), on a concurrent `x`. +fn attendBlocks(x: Enc, layer: usize, blocks: []const Block, n: u32, rows: usize, out: usize) !void { + const s = &x.d.s; + const ly = &x.d.w.layers[layer]; + // the three projections, every block's keys, then waves of as many blocks as the partials hold at once + try x.projectAll(&.{ ly.q, ly.k, ly.v }, .{ .buf = s.x }, &.{ .{ .buf = s.q }, .{ .buf = s.k }, .{ .buf = s.v } }, rows); + x.sync(); + for (blocks, 0..) |b, bi| { + const at = bi * n; + x.qk(.{ .buf = s.q, .off = at * c.hidden * 2 }, .{ .buf = s.k, .off = at * kw * 2 }, .{ .buf = s.v, .off = at * kw * 2 }, layer, b.lane, b.anchor, .{ .buf = s.queries, .off = at * c.hidden * 2 }, n); + } + x.sync(); + const each = if (blocks.len == 1) out else n; + const wave = @max(1, x.d.partRows() / n); + var first: usize = 0; + while (first < blocks.len) : (first += wave) { + const last = @min(blocks.len, first + wave); + for (blocks[first..last], first..) |b, bi| x.partials(layer, b.lane, bi * n, each, b.anchor + n, (bi - first) * n); + x.sync(); + for (blocks[first..last], first..) |b, bi| x.merge(b.lane, bi * n, each, b.anchor + n, (bi - first) * n); + x.sync(); + } +} + /// A GPU event a hold signals instead of being waited on, and the value it sets. pub const Signal = struct { event: mtl.Event, value: u64 }; @@ -169,15 +248,15 @@ pub fn hold(d: *Drafter, blocks: []const Block, signal: ?Signal) !?mtl.CommandBu // past the last layer's attention a lone block carries only the rows its chain reads; several keep them all const tail = if (blocks.len == 1) steps else rows; const cb = d.target.queue.commandBuffer(); - const x = Enc{ .d = d, .e = cb.compute(.serial) }; - const e = x.e; + // one concurrent encoder: independent dispatches overlap, each dependent step waits at a barrier + const x = Enc{ .d = d, .e = cb.compute(.concurrent), .conc = true }; // the blocks' staged context rows first, in the same command buffer var staged: [ds.pass_rows]Piece = undefined; var count: usize = 0; var held: usize = 0; for (blocks) |b| if (ds.Drafter.stagedPiece(b.lane)) |pc| { if (held + pc.rows > ds.absorb_rows) { - try absorbPass(d, e, staged[0..count]); + try absorbOn(x, staged[0..count]); count = 0; held = 0; } @@ -185,49 +264,51 @@ pub fn hold(d: *Drafter, blocks: []const Block, signal: ?Signal) !?mtl.CommandBu count += 1; held += pc.rows; }; - if (count > 0) try absorbPass(d, e, staged[0..count]); + if (count > 0) try absorbOn(x, staged[0..count]); for (blocks) |b| { b.lane.end += b.lane.staged; b.lane.staged = 0; } const t = d.target; const emb = t.weights.embedding; - e.setPipeline(t.kernels.embed(emb.bits)); - e.setBuffer(s.ids, 0, 0); - e.setBuffer(emb.w, emb.w_off, 1); - e.setBuffer(emb.scales, emb.s_off, 2); - e.setBuffer(emb.biases, emb.b_off, 3); - e.setBuffer(s.h, 0, 4); + x.e.setPipeline(t.kernels.embed(emb.bits)); + x.e.setBuffer(s.ids, 0, 0); + x.e.setBuffer(emb.w, emb.w_off, 1); + x.e.setBuffer(emb.scales, emb.s_off, 2); + x.e.setBuffer(emb.biases, emb.b_off, 3); + x.e.setBuffer(s.h, 0, 4); const units = c.hidden / @as(usize, if (emb.bits == 6) 4 else 8 / emb.bits); - e.dispatchThreads(mtl.Size.of(units, rows, 1), mtl.Size.of(256, 1, 1)); + x.e.dispatchThreads(mtl.Size.of(units, rows, 1), mtl.Size.of(256, 1, 1)); + x.sync(); for (0..d.config.layers) |i| { const ly = &d.w.layers[i]; const out = if (i + 1 == d.config.layers) tail else rows; x.norm(s.h, s.r, ly.input_norm, i > 0, s.x, rows); - try x.project(ly.q, .{ .buf = s.x }, .{ .buf = s.q }, rows); - try x.project(ly.k, .{ .buf = s.x }, .{ .buf = s.k }, rows); - try x.project(ly.v, .{ .buf = s.x }, .{ .buf = s.v }, rows); - for (blocks, 0..) |b, bi| { - const at = bi * n; - x.qk(.{ .buf = s.q, .off = at * c.hidden * 2 }, .{ .buf = s.k, .off = at * kw * 2 }, .{ .buf = s.v, .off = at * kw * 2 }, i, b.lane, b.anchor, .{ .buf = s.queries, .off = at * c.hidden * 2 }, n); - x.attend(i, b.lane, at, if (blocks.len == 1) out else n, b.anchor + n); - } + x.sync(); + try attendBlocks(x, i, blocks, n, rows, out); try x.project(ly.o, .{ .buf = s.y }, .{ .buf = s.r }, out); + x.sync(); x.norm(s.h, s.r, ly.post_norm, true, s.x, out); - try x.project(ly.gate, .{ .buf = s.x }, .{ .buf = s.gate }, out); - try x.project(ly.up, .{ .buf = s.x }, .{ .buf = s.up }, out); + x.sync(); + try x.projectAll(&.{ ly.gate, ly.up }, .{ .buf = s.x }, &.{ .{ .buf = s.gate }, .{ .buf = s.up } }, out); + x.sync(); x.pipe("lfm2_swiglu"); - e.setBuffer(s.gate, 0, 0); - e.setBuffer(s.up, 0, 1); - e.setBuffer(s.act, 0, 2); - e.setValue(@as(u32, @intCast(out * d.config.intermediate)), 3); - e.dispatchThreads(mtl.Size.of(out * d.config.intermediate / 8, 1, 1), mtl.Size.of(256, 1, 1)); + x.e.setBuffer(s.gate, 0, 0); + x.e.setBuffer(s.up, 0, 1); + x.e.setBuffer(s.act, 0, 2); + x.e.setValue(@as(u32, @intCast(out * d.config.intermediate)), 3); + x.e.dispatchThreads(mtl.Size.of(out * d.config.intermediate / 8, 1, 1), mtl.Size.of(256, 1, 1)); + x.sync(); try x.project(ly.down, .{ .buf = s.act }, .{ .buf = s.r }, out); + x.sync(); } x.norm(s.h, s.r, d.w.norm, true, s.x, tail); + x.sync(); try x.project(fwd.matrix(emb), .{ .buf = s.x }, .{ .buf = s.logits }, tail); + x.sync(); try markov(x, blocks.len, n, steps); if (d.confident()) confidence(x, blocks.len, n, steps); + const e = x.e; if (signal) |sg| { // committed unwaited, the command buffer returned retained e.end(); cb.signalEvent(sg.event, sg.value); @@ -305,11 +386,13 @@ fn markov(x: Enc, streams: usize, n: u32, depth: u32) !void { e.setBuffer(d.w.w2.scales, d.w.w2.s_off, 7); e.setBuffer(d.w.w2.biases, d.w.w2.b_off, 8); e.dispatchGroups(mtl.Size.of(scan_groups, streams, 1), mtl.Size.of(256, 1, 1)); + x.sync(); x.pipe("lfm2_markov_pick"); e.setBuffer(s.best, 0, 0); e.setBuffer(s.at, 0, 1); e.setValue(@as(u32, scan_groups), 2); e.setBuffer(s.picks, i * ds.pass_rows * @sizeOf(Pick), 3); e.dispatchGroups(mtl.Size.of(streams, 1, 1), mtl.Size.of(256, 1, 1)); + x.sync(); } } From b859d8e6a2240208fc687c77b93801dd98e8aa6d Mon Sep 17 00:00:00 2001 From: BlivionIaG <15323295+BlivionIaG@users.noreply.github.com> Date: Mon, 26 Oct 2026 09:43:59 +0100 Subject: [PATCH 18/39] lfm2: a full DSpark stage absorbs every lane's staged rows in its pass In plain rounds each lane stages its tapped rows and absorbs them in a waited pass when its 128-row stage fills. With one prompt on every stream the stages fill in the same round; with a different prompt a stream they fill on different rounds, so 16 streams waited on 16 passes of up to 128 rows each stage's worth of rounds. A pass that a full stage starts now takes every other lane's staged rows too (up to 2,048 rows, 32 pieces a pass), so the lanes' stages fill together after the first one. --- zig/src/families/lfm2_moe/dspark.zig | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/zig/src/families/lfm2_moe/dspark.zig b/zig/src/families/lfm2_moe/dspark.zig index 1b782c496f..2d898022a2 100644 --- a/zig/src/families/lfm2_moe/dspark.zig +++ b/zig/src/families/lfm2_moe/dspark.zig @@ -407,6 +407,21 @@ pub const Drafter = struct { l.staged = 0; } const r = run orelse return; + // the other lanes' staged rows join the pass: one waited pass a stage's worth of rounds, not one a stream + var others = d.lanes.valueIterator(); + while (others.next()) |lp| { + const pc = stagedPiece(lp.*) orelse continue; + if (count == pieces.len or rows + pc.rows > absorb_rows) { + try block.absorbPass(d, r.e, pieces[0..count]); + count = 0; + rows = 0; + } + pieces[count] = pc; + count += 1; + rows += pc.rows; + lp.*.end += lp.*.staged; + lp.*.staged = 0; + } if (count > 0) try block.absorbPass(d, r.e, pieces[0..count]); try block.end(r.cb, r.e); } From 2b7a9a26985eb0a581d7079100902fd80420414a Mon Sep 17 00:00:00 2001 From: BlivionIaG <15323295+BlivionIaG@users.noreply.github.com> Date: Mon, 26 Oct 2026 10:06:48 +0100 Subject: [PATCH 19/39] docs: the LFM2 recipe's cheaper DSpark hold The recipe says how a hold encodes its passes and picks a kernel for each projection, and how a full stage absorbs every lane's rows. --- docs/recipes/lfm2.5-24b-a2b.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/recipes/lfm2.5-24b-a2b.md b/docs/recipes/lfm2.5-24b-a2b.md index 48b2666927..824788cffd 100644 --- a/docs/recipes/lfm2.5-24b-a2b.md +++ b/docs/recipes/lfm2.5-24b-a2b.md @@ -113,3 +113,9 @@ The loader reads the checkpoint one matrix at a time and keeps only the 8-bit co Metal buffer (its layers' keys and values, then its staged rows), and its bytes join the streams the load limit admits. At load the drafter logs its weights, scratch and a stream's bytes. `tf-lfm2-dspark` also counts host allocations over twenty served drafter rounds, which must be none. + +A hold runs on one concurrent encoder with a barrier only where a step reads an earlier step's writes: q, k and v, +and gate and up, share one sums pass, and the blocks of a shared hold attend side by side. A projection takes the +scalar twin up to 8 rows, the window kernel past them where the GPU has no tensor units, and the tensor units from +9 rows (the head from 5) where it has them. When one lane's stage of 128 verified rows fills, its absorb pass takes +every lane's staged rows, so the lanes' stages fill together. From ca93db1e1acdbd91b1d5282411582532d4351fd0 Mon Sep 17 00:00:00 2001 From: BlivionIaG <15323295+BlivionIaG@users.noreply.github.com> Date: Mon, 26 Oct 2026 18:44:14 +0100 Subject: [PATCH 20/39] lfm2 dspark: a held pick that is nonfinite or past the vocabulary is refused, and hidden_act must be silu The unwaited hold read its picks with nonfinite ignored and a bad id as 0; the GPU copy into a verify window did the same. Both now refuse as the synchronous path does (DraftNonfinite): readPicks errors, and lfm2_draft_ids sets a flag the verify reads. Planted NaN, Inf and invalid-id tests cover the host and the kernel. The config refuses a hidden_act other than silu by name. --- zig/kernels/metal/lfm2/lfm2_dspark.metal | 10 ++-- zig/src/families/lfm2_moe/dspark.zig | 11 ++++- zig/src/families/lfm2_moe/dspark_block.zig | 23 +++++++--- zig/src/families/lfm2_moe/dspark_config.zig | 4 ++ zig/src/families/lfm2_moe/dspark_test.zig | 51 +++++++++++++++++++++ zig/src/families/lfm2_moe/forward.zig | 3 ++ zig/src/families/lfm2_moe/state.zig | 2 + zig/tests/contract/metal_lfm2_dspark.zig | 14 +++++- 8 files changed, 105 insertions(+), 13 deletions(-) diff --git a/zig/kernels/metal/lfm2/lfm2_dspark.metal b/zig/kernels/metal/lfm2/lfm2_dspark.metal index 7f613def9b..b4de152986 100644 --- a/zig/kernels/metal/lfm2/lfm2_dspark.metal +++ b/zig/kernels/metal/lfm2/lfm2_dspark.metal @@ -98,12 +98,16 @@ constant constexpr uint MARKOV_SPAN = 512; } } -// A verify window's held drafts from the drafter's picks: picks[i * stride] into ids[i], a token past the vocabulary 0. +// A verify window's held drafts from the drafter's picks: picks[i * stride] into ids[i]. A pick flagged nonfinite +// (the word after it) or past the vocabulary becomes token 0 and sets bad, which the host reads after the verify. [[kernel]] void lfm2_draft_ids(const device uint* picks [[buffer(0)]], constant uint4& p [[buffer(1)]], - device uint* ids [[buffer(2)]], uint i [[thread_position_in_grid]]) { + device uint* ids [[buffer(2)]], device atomic_uint* bad [[buffer(3)]], + uint i [[thread_position_in_grid]]) { if (i >= p.y) return; const uint token = picks[size_t(i) * p.x]; - ids[i] = token < p.z ? token : 0u; + const bool ok = token < p.z && picks[size_t(i) * p.x + 1] == 0u; + ids[i] = ok ? token : 0u; + if (!ok) atomic_store_explicit(bad, 1u, memory_order_relaxed); } // DSpark's confidence logit of draft i of stream s: proj . [x row s * n + i, W1[previous token]] in fp32, no bias. diff --git a/zig/src/families/lfm2_moe/dspark.zig b/zig/src/families/lfm2_moe/dspark.zig index 2d898022a2..e759edcb7a 100644 --- a/zig/src/families/lfm2_moe/dspark.zig +++ b/zig/src/families/lfm2_moe/dspark.zig @@ -30,7 +30,13 @@ pub const Verify = enum { depth, fixed, scheduled }; pub const default_verify: Verify = .scheduled; /// `bits`: the width the bf16 linears are quantized to (8 keeps the fp32 reference's drafts; 4 does not). -pub const Options = struct { capacity: usize, bits: u8 = 8, verify: Verify = default_verify }; +/// `verify_asked`: --draft-verify was given, so a scheduled verify without the confidence head is refused, not depth. +pub const Options = struct { capacity: usize, bits: u8 = 8, verify: Verify = default_verify, verify_asked: bool = false }; + +/// An asked scheduled verify needs the checkpoint's confidence head; the default one falls back to the depth rule. +pub fn admitVerify(o: Options, has_head: bool) !void { + if (o.verify == .scheduled and o.verify_asked and !has_head) return error.ScheduledVerifyNeedsConfidenceHead; +} /// One stream's context keys and values a layer ([kv head][cap][head dim], as the target's attention reads them). pub const Lane = struct { @@ -110,6 +116,7 @@ pub const Drafter = struct { errdefer d.w.deinit(gpa); errdefer for (d.owned.items) |b| b.deinit(); errdefer d.owned.deinit(gpa); + try admitVerify(options, d.w.confidence != null); try d.scratch(); try d.warm(); return d; @@ -478,7 +485,7 @@ pub const Drafter = struct { return error.DraftGpuFailed; } d.priceHold((cb.gpuEnd() - cb.gpuStart()) * 1000); - block.readPicks(d, d.pass[0..d.pass_len], d.depths[0..d.pass_len]); + try block.readPicks(d, d.pass[0..d.pass_len], d.depths[0..d.pass_len]); } /// A settled hold's GPU ms against its price: the timed passes at its block counts move a tenth of the way to it. diff --git a/zig/src/families/lfm2_moe/dspark_block.zig b/zig/src/families/lfm2_moe/dspark_block.zig index 707e4df5ba..a89e2fc885 100644 --- a/zig/src/families/lfm2_moe/dspark_block.zig +++ b/zig/src/families/lfm2_moe/dspark_block.zig @@ -338,19 +338,30 @@ fn readConfidence(d: *Drafter, l: *Lane, depth: u32, bi: usize) void { l.conf_len = depth; } -/// A finished signalled hold's picks into the lanes' held chains (a pick past the vocabulary reads as token 0). -pub fn readPicks(d: *Drafter, lanes: []const *Lane, depths: []const u32) void { +/// A finished signalled hold's picks into the lanes' held chains; a nonfinite or invalid pick is refused as the +/// synchronous path refuses it, and no lane keeps a chain then. +pub fn readPicks(d: *Drafter, lanes: []const *Lane, depths: []const u32) !void { const picks = d.s.picks.slice(Pick, d.config.block * ds.pass_rows); + var chains: [ds.pass_rows][]u32 = undefined; + for (lanes, chains[0..lanes.len]) |l, *ch| ch.* = &l.held; + try copyPicks(picks, chains[0..lanes.len], depths); for (lanes, depths, 0..) |l, depth, bi| { - for (l.held[0..depth], 0..) |*h, i| { - const p = picks[i * ds.pass_rows + bi]; - h.* = if (p.token < c.vocab) p.token else 0; - } l.held_len = depth; readConfidence(d, l, depth, bi); } } +/// Step i of block bi is picks[i * ds.pass_rows + bi]: each block's first `depth` picks into its chain. +pub fn copyPicks(picks: []const Pick, chains: []const []u32, depths: []const u32) !void { + for (chains, depths, 0..) |chain, depth, bi| { + for (chain[0..depth], 0..) |*h, i| { + const p = picks[i * ds.pass_rows + bi]; + if (p.nonfinite != 0 or p.token >= c.vocab) return error.DraftNonfinite; + h.* = p.token; + } + } +} + /// Each stream's draft i confidence logit (the final norm's row i, W1 of the token before draft i), for readConfidence. fn confidence(x: Enc, streams: usize, n: u32, depth: u32) void { const d = x.d; diff --git a/zig/src/families/lfm2_moe/dspark_config.zig b/zig/src/families/lfm2_moe/dspark_config.zig index 7b3b628a02..79e518aa48 100644 --- a/zig/src/families/lfm2_moe/dspark_config.zig +++ b/zig/src/families/lfm2_moe/dspark_config.zig @@ -55,6 +55,9 @@ pub const Config = struct { } out.tap_count = @intCast(taps.array.items.len); try kinds(o, out.layers); + // the block's MLP is the fixed SiLU gate + const act = o.get("hidden_act") orelse return error.BadDraftConfig; + if (act != .string or !std.mem.eql(u8, act.string, "silu")) return error.UnsupportedDraftActivation; // the vendored defaults decide the block: non-causal attention, interleaved RoPE, a plain Markov head if (o.get("is_causal") != null) return error.UnsupportedDraftMask; const neox = o.get("rope_is_neox_style") orelse return error.BadDraftConfig; @@ -126,6 +129,7 @@ test "lfm2 dspark: other drafters and shapes are refused with their reason" { try std.testing.expectError(error.DraftForAnotherTarget, edited("\"num_target_layers\":40", "\"num_target_layers\":24")); try std.testing.expectError(error.DraftForAnotherTarget, edited("26,34]", "26,44]")); try std.testing.expectError(error.UnsupportedDraftShape, edited("\"head_dim\":64", "\"head_dim\":128")); + try std.testing.expectError(error.UnsupportedDraftActivation, edited("\"silu\"", "\"gelu\"")); try std.testing.expectError(error.UnsupportedDraftRope, edited("\"rope_is_neox_style\":false", "\"rope_is_neox_style\":true")); try std.testing.expectError(error.UnsupportedMarkovHead, edited("\"vanilla\"", "\"gated\"")); try std.testing.expectError(error.UnsupportedDraftMask, edited("\"full_attention\"]", "\"sliding_attention\"]")); diff --git a/zig/src/families/lfm2_moe/dspark_test.zig b/zig/src/families/lfm2_moe/dspark_test.zig index 385866b291..4d96731210 100644 --- a/zig/src/families/lfm2_moe/dspark_test.zig +++ b/zig/src/families/lfm2_moe/dspark_test.zig @@ -4,6 +4,8 @@ const mtl = @import("metal"); const c = @import("config.zig"); const kernels = @import("kernels.zig"); const quant = @import("dspark_quant.zig"); +const block = @import("dspark_block.zig"); +const Pick = @import("tree_round_gpu").contract.Pick; const ds = @import("dspark.zig"); const lanes = @import("lanes"); @@ -233,3 +235,52 @@ test "lfm2 dspark: a settled hold moves the timed passes at its block counts a t d.priceHold(2.6 * 2); try std.testing.expect(d.holds[1].ms > 2.2 and d.holds[3].ms > 3 and d.holds[0].ms == 1); } + +/// lfm2_draft_ids on one pick (token, nonfinite flag) at a stride of 4 words: the id it wrote and the bad flag. +fn draftIds(r: *Rig, pick: Pick) ![2]u32 { + const picks = try r.buffer(u32, 4); + defer picks.deinit(); + const ids = try r.buffer(u32, 1); + defer ids.deinit(); + const bad = try r.buffer(u32, 1); + defer bad.deinit(); + picks.slice(u32, 4)[0] = pick.token; + picks.slice(u32, 4)[1] = pick.nonfinite; + try r.run("lfm2_draft_ids", &.{ picks, ids, bad }, [4]u32{ 4, 1, c.vocab, 0 }, 1, .{ 1, 1 }); + return .{ ids.slice(u32, 1)[0], bad.slice(u32, 1)[0] }; +} + +test "lfm2 dspark: held ids carry a flagged or invalid pick as token 0 and the bad flag, a good pick untouched" { + var r = try Rig.init(); + defer r.deinit(); + try std.testing.expectEqual([2]u32{ 5, 0 }, try draftIds(&r, .{ .token = 5 })); + try std.testing.expectEqual([2]u32{ c.vocab - 1, 0 }, try draftIds(&r, .{ .token = c.vocab - 1 })); + for ([_]u32{ 1, 2, 4, 7 }) |kind| try std.testing.expectEqual([2]u32{ 0, 1 }, try draftIds(&r, .{ .token = 9, .nonfinite = kind })); + try std.testing.expectEqual([2]u32{ 0, 1 }, try draftIds(&r, .{ .token = std.math.maxInt(u32) })); + try std.testing.expectEqual([2]u32{ 0, 1 }, try draftIds(&r, .{ .token = c.vocab })); +} + +test "lfm2 dspark: a settled hold refuses a NaN, infinity or invalid pick as the synchronous path does" { + var picks: [4 * ds.pass_rows]Pick = @splat(.{ .token = 3 }); + var a: [4]u32 = undefined; + var b: [4]u32 = undefined; + const chains = [_][]u32{ &a, &b }; + try block.copyPicks(&picks, &chains, &.{ 2, 2 }); + try std.testing.expectEqualSlices(u32, &.{ 3, 3 }, a[0..2]); + const bad = [_]Pick{ .{ .token = 3, .nonfinite = 1 }, .{ .token = 3, .nonfinite = 2 }, .{ .token = 3, .nonfinite = 4 }, .{ .token = std.math.maxInt(u32) }, .{ .token = c.vocab } }; + for (bad) |p| { + picks[ds.pass_rows + 1] = p; // the second block's second step + try std.testing.expectError(error.DraftNonfinite, block.copyPicks(&picks, &chains, &.{ 2, 2 })); + picks[ds.pass_rows + 1] = .{ .token = 3 }; + } + picks[3 * ds.pass_rows] = .{ .nonfinite = 1 }; // a step past the block's depth is not read + try block.copyPicks(&picks, &chains, &.{ 2, 2 }); +} + +test "lfm2 dspark: an asked scheduled verify without the confidence head is refused by name, the default falls back" { + const asked: ds.Options = .{ .capacity = 64, .verify_asked = true }; + try std.testing.expectError(error.ScheduledVerifyNeedsConfidenceHead, ds.admitVerify(asked, false)); + try ds.admitVerify(asked, true); + try ds.admitVerify(.{ .capacity = 64 }, false); + try ds.admitVerify(.{ .capacity = 64, .verify = .depth, .verify_asked = true }, false); +} diff --git a/zig/src/families/lfm2_moe/forward.zig b/zig/src/families/lfm2_moe/forward.zig index 10040c0d77..51408e2fd4 100644 --- a/zig/src/families/lfm2_moe/forward.zig +++ b/zig/src/families/lfm2_moe/forward.zig @@ -146,11 +146,13 @@ pub fn runHeld(m: *Model, s: *st.Scratch, segments: []const Segment, tokens: []c const e = cb.compute(.serial); const plan = if (segs.batched(record, segments.len)) segs.prepare(e, s, segments) else null; const enc = Encoder{ .e = e, .m = m, .s = s, .prompt = prompt, .record = record, .plan = plan }; + if (held.len > 0) @memset(s.bad.contents()[0..4], 0); for (held) |h| { enc.pipe("lfm2_draft_ids"); e.setBuffer(h.src, h.off, 0); e.setValue([4]u32{ h.stride, h.count, c.vocab, 0 }, 1); e.setBuffer(s.ids, h.row * 4, 2); + e.setBuffer(s.bad, 0, 3); enc.threads(.{ h.count, 1, 1 }, .{ 32, 1, 1 }); } const embedding = m.weights.embedding; @@ -198,6 +200,7 @@ pub fn runHeld(m: *Model, s: *st.Scratch, segments: []const Segment, tokens: []c std.debug.print("LFM2 forward: {s}\n", .{msg}); return error.GpuFailed; } + if (held.len > 0 and std.mem.readInt(u32, s.bad.contents()[0..4], .little) != 0) return error.DraftNonfinite; var base: usize = 0; for (segments) |seg| { const logit_row = if (head == .all) base + seg.rows - 1 else 0; diff --git a/zig/src/families/lfm2_moe/state.zig b/zig/src/families/lfm2_moe/state.zig index 1bc749ae95..883f670fbe 100644 --- a/zig/src/families/lfm2_moe/state.zig +++ b/zig/src/families/lfm2_moe/state.zig @@ -119,6 +119,7 @@ pub const Scratch = struct { memory: Storage, rows: usize, ids: mtl.Buffer, + bad: mtl.Buffer, // set by lfm2_draft_ids when a held pick is flagged nonfinite or past the vocabulary h: mtl.Buffer, x: mtl.Buffer, r: mtl.Buffer, @@ -170,6 +171,7 @@ pub const Scratch = struct { const mem = &out.memory; const slots = rows * c.topk; out.ids = try mem.alloc(rows * 4); + out.bad = try mem.alloc(4); inline for (.{ "h", "x", "r", "y", "q" }) |name| @field(out, name) = try mem.alloc(rows * c.hidden * 2); out.queries = try mem.alloc((rows + attn_rows) * c.hidden * 2); // a prompt attention tile reads attn_rows from its first out.bcx = try mem.alloc(rows * 3 * c.hidden * 2); diff --git a/zig/tests/contract/metal_lfm2_dspark.zig b/zig/tests/contract/metal_lfm2_dspark.zig index 63d7bcf590..f119d00c66 100644 --- a/zig/tests/contract/metal_lfm2_dspark.zig +++ b/zig/tests/contract/metal_lfm2_dspark.zig @@ -63,6 +63,7 @@ pub const Probe = struct { ids: []u32 = &.{}, args: [8]rig_.Ref = undefined, y: rig_.Ref = undefined, + bad: rig_.Ref = undefined, // draft_ids: the flag word it sets for a flagged or invalid pick pub fn init(a: std.mem.Allocator, device: mtl.Device, queue: mtl.Queue, w: Which) !Probe { const kernels: [2][:0]const u8 = switch (w) { @@ -109,10 +110,13 @@ pub const Probe = struct { 1 => vocab - 1, else => vocab + @as(u32, @intCast(i % 5)), }; + // the flag word after each pick: zero, but planted nonzero on the last of every four streams + for (0..s.rows) |r| p.ids[r * stride + 1] = if (r % 4 == 3) 1 else 0; p.args[0] = try p.rig.input(std.mem.sliceAsBytes(p.ids)); const at = [4]u32{ stride, @intCast(s.rows), vocab, 0 }; p.args[1] = try p.rig.input(std.mem.asBytes(&at)); p.y = try p.rig.output(p.count()); + p.bad = try p.rig.output(4); return true; } if (s.n != vocab or s.k != rank or s.rows == 0) return false; @@ -165,6 +169,7 @@ pub const Probe = struct { e.setBuffer(p.args[0].buf, p.args[0].off, 0); e.setBuffer(p.args[1].buf, p.args[1].off, 1); e.setBuffer(p.y.buf, p.y.off, 2); + e.setBuffer(p.bad.buf, p.bad.off, 3); e.dispatchThreads(mtl.Size.of(p.rows, 1, 1), mtl.Size.of(32, 1, 1)); return p.rig.finish(pass); } @@ -194,11 +199,16 @@ pub const Probe = struct { /// Each stream's pick against float64 scores with the kernel's bf16 roundings: the best, but for near ties. pub fn check(p: *Probe, _: g.Gate) !?[]const u8 { const out = p.rig.bytes(p.y, p.count()); - if (p.which == .draft_ids) { // exact: each stream's first pick, or 0 for a token past the vocabulary + if (p.which == .draft_ids) { // exact: each stream's first pick, or 0 for a token past the vocabulary or a flag + var any = false; for (0..p.rows) |r| { const token = p.ids[r * stride]; - if (std.mem.readInt(u32, out[4 * r ..][0..4], .little) != (if (token < vocab) token else 0)) return "an id differs from its pick"; + const ok = token < vocab and p.ids[r * stride + 1] == 0; + any = any or !ok; + if (std.mem.readInt(u32, out[4 * r ..][0..4], .little) != (if (ok) token else 0)) return "an id differs from its pick"; } + const raised = std.mem.readInt(u32, p.rig.bytes(p.bad, 4)[0..4], .little); + if ((raised != 0) != any) return "the bad flag does not follow the flagged and invalid picks"; return null; } for (0..p.rows) |s| { From 25a5d3b3dafe5e7fcb44c129d8a5c6d818f13253 Mon Sep 17 00:00:00 2001 From: BlivionIaG <15323295+BlivionIaG@users.noreply.github.com> Date: Mon, 26 Oct 2026 18:44:43 +0100 Subject: [PATCH 21/39] tf-lfm2-dspark: the reference reads the target's embedding width from the checkpoint and joins its workers on every path --- zig/tests/lfm2_dspark.zig | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/zig/tests/lfm2_dspark.zig b/zig/tests/lfm2_dspark.zig index 5a9885b98e..b1ab9b12fc 100644 --- a/zig/tests/lfm2_dspark.zig +++ b/zig/tests/lfm2_dspark.zig @@ -49,12 +49,12 @@ fn matmul(gpa: std.mem.Allocator, w: []const f32, x: []const f32, y: []f32, n: u const hs = try gpa.alloc(std.Thread, threads); defer gpa.free(hs); var made: usize = 0; + defer for (hs[0..made]) |h| h.join(); // a failed spawn still joins the workers already running var at: usize = 0; while (at < n) : (at += per) { hs[made] = try std.Thread.spawn(.{}, Job.run, .{ w, x, y, n, k, rows, at, @min(n, at + per) }); made += 1; } - for (hs[0..made]) |h| h.join(); } fn rms(x: []f32, gain: []const f32) void { @@ -76,19 +76,21 @@ fn rope(x: []f32, pos: u64, theta: f32) void { }; } -/// The target's embedding row `id`, dequantized from its 6-bit codes (MLX's scale * q + bias). +/// The target's embedding row `id`, dequantized from its codes at the checkpoint's own width (scale * q + bias). fn embedRow(m: *q.Model, id: u32, out: []f32) void { const e = m.weights.embedding; - const codes = (e.w.contents() + e.w_off)[@as(usize, id) * c.hidden * 6 / 8 ..][0 .. c.hidden * 6 / 8]; + const bits: usize = e.bits; + const row_bytes = c.hidden * bits / 8; + const codes = (e.w.contents() + e.w_off)[@as(usize, id) * row_bytes ..][0..row_bytes]; const sc = @as([*]const u16, @ptrCast(@alignCast(e.scales.contents() + e.s_off)))[@as(usize, id) * c.hidden / 64 ..][0 .. c.hidden / 64]; const bi = @as([*]const u16, @ptrCast(@alignCast(e.biases.contents() + e.b_off)))[@as(usize, id) * c.hidden / 64 ..][0 .. c.hidden / 64]; - for (0..c.hidden / 4) |u| { - const w = codes[u * 3 ..][0..3]; - const qs = [4]u32{ w[0] & 0x3f, (w[0] >> 6) | ((@as(u32, w[1]) & 0x0f) << 2), (w[1] >> 4) | ((@as(u32, w[2]) & 0x03) << 4), w[2] >> 2 }; - for (qs, 0..) |qv, j| { - const g = (u * 4 + j) / 64; - out[u * 4 + j] = quant.toF32(sc[g]) * @as(f32, @floatFromInt(qv)) + quant.toF32(bi[g]); - } + for (0..c.hidden) |j| { + // codes are a little-endian bit stream: code j takes bits [j * bits, (j + 1) * bits) + const at = j * bits; + var word: u32 = codes[at / 8]; + if (at / 8 + 1 < row_bytes) word |= @as(u32, codes[at / 8 + 1]) << 8; + const qv = (word >> @intCast(at % 8)) & ((@as(u32, 1) << @intCast(bits)) - 1); + out[j] = quant.toF32(sc[j / 64]) * @as(f32, @floatFromInt(qv)) + quant.toF32(bi[j / 64]); } } From 1eb0476e7356c57fab32212531bd25ea5e1e2060 Mon Sep 17 00:00:00 2001 From: BlivionIaG <15323295+BlivionIaG@users.noreply.github.com> Date: Mon, 26 Oct 2026 18:48:29 +0100 Subject: [PATCH 22/39] lfm2 dspark: one-line comments for the held-pick refusal --- zig/kernels/metal/lfm2/lfm2_dspark.metal | 3 +-- zig/src/families/lfm2_moe/dspark_block.zig | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/zig/kernels/metal/lfm2/lfm2_dspark.metal b/zig/kernels/metal/lfm2/lfm2_dspark.metal index b4de152986..551a695467 100644 --- a/zig/kernels/metal/lfm2/lfm2_dspark.metal +++ b/zig/kernels/metal/lfm2/lfm2_dspark.metal @@ -98,8 +98,7 @@ constant constexpr uint MARKOV_SPAN = 512; } } -// A verify window's held drafts from the drafter's picks: picks[i * stride] into ids[i]. A pick flagged nonfinite -// (the word after it) or past the vocabulary becomes token 0 and sets bad, which the host reads after the verify. +// Held drafts from picks[i * stride] into ids[i]; a nonfinite or past-vocabulary pick is token 0 and sets bad. [[kernel]] void lfm2_draft_ids(const device uint* picks [[buffer(0)]], constant uint4& p [[buffer(1)]], device uint* ids [[buffer(2)]], device atomic_uint* bad [[buffer(3)]], uint i [[thread_position_in_grid]]) { diff --git a/zig/src/families/lfm2_moe/dspark_block.zig b/zig/src/families/lfm2_moe/dspark_block.zig index a89e2fc885..06fbade4a2 100644 --- a/zig/src/families/lfm2_moe/dspark_block.zig +++ b/zig/src/families/lfm2_moe/dspark_block.zig @@ -338,8 +338,7 @@ fn readConfidence(d: *Drafter, l: *Lane, depth: u32, bi: usize) void { l.conf_len = depth; } -/// A finished signalled hold's picks into the lanes' held chains; a nonfinite or invalid pick is refused as the -/// synchronous path refuses it, and no lane keeps a chain then. +/// A finished hold's picks into the lanes' held chains; a nonfinite or invalid pick is refused, as when synchronous. pub fn readPicks(d: *Drafter, lanes: []const *Lane, depths: []const u32) !void { const picks = d.s.picks.slice(Pick, d.config.block * ds.pass_rows); var chains: [ds.pass_rows][]u32 = undefined; From cf9fad0e97e62a87c26dff9852da67e92f0dc194 Mon Sep 17 00:00:00 2001 From: BlivionIaG <15323295+BlivionIaG@users.noreply.github.com> Date: Mon, 26 Oct 2026 18:49:03 +0100 Subject: [PATCH 23/39] lfm2 dspark: one-line comment for Options --- zig/src/families/lfm2_moe/dspark.zig | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/zig/src/families/lfm2_moe/dspark.zig b/zig/src/families/lfm2_moe/dspark.zig index e759edcb7a..cec88c5330 100644 --- a/zig/src/families/lfm2_moe/dspark.zig +++ b/zig/src/families/lfm2_moe/dspark.zig @@ -29,8 +29,7 @@ pub const Verify = enum { depth, fixed, scheduled }; /// The verify when none is asked: the confidence-scheduled one. pub const default_verify: Verify = .scheduled; -/// `bits`: the width the bf16 linears are quantized to (8 keeps the fp32 reference's drafts; 4 does not). -/// `verify_asked`: --draft-verify was given, so a scheduled verify without the confidence head is refused, not depth. +/// `bits`: the quantized width (8 keeps the fp32 reference's drafts; 4 does not); `verify_asked`: --draft-verify given. pub const Options = struct { capacity: usize, bits: u8 = 8, verify: Verify = default_verify, verify_asked: bool = false }; /// An asked scheduled verify needs the checkpoint's confidence head; the default one falls back to the depth rule. From d4abce91ac26c7fbd60820713a3c7041f71112ac Mon Sep 17 00:00:00 2001 From: BlivionIaG <15323295+BlivionIaG@users.noreply.github.com> Date: Mon, 26 Oct 2026 18:52:03 +0100 Subject: [PATCH 24/39] kernel contract: the draft ids probe zeroes its bad flag each run --- zig/tests/contract/metal_lfm2_dspark.zig | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/zig/tests/contract/metal_lfm2_dspark.zig b/zig/tests/contract/metal_lfm2_dspark.zig index f119d00c66..bd1def8d1b 100644 --- a/zig/tests/contract/metal_lfm2_dspark.zig +++ b/zig/tests/contract/metal_lfm2_dspark.zig @@ -63,7 +63,7 @@ pub const Probe = struct { ids: []u32 = &.{}, args: [8]rig_.Ref = undefined, y: rig_.Ref = undefined, - bad: rig_.Ref = undefined, // draft_ids: the flag word it sets for a flagged or invalid pick + bad: rig_.Ref = undefined, // draft_ids: the flag word it sets for a flagged or invalid pick, zeroed each run pub fn init(a: std.mem.Allocator, device: mtl.Device, queue: mtl.Queue, w: Which) !Probe { const kernels: [2][:0]const u8 = switch (w) { @@ -116,7 +116,6 @@ pub const Probe = struct { const at = [4]u32{ stride, @intCast(s.rows), vocab, 0 }; p.args[1] = try p.rig.input(std.mem.asBytes(&at)); p.y = try p.rig.output(p.count()); - p.bad = try p.rig.output(4); return true; } if (s.n != vocab or s.k != rank or s.rows == 0) return false; @@ -169,6 +168,8 @@ pub const Probe = struct { e.setBuffer(p.args[0].buf, p.args[0].off, 0); e.setBuffer(p.args[1].buf, p.args[1].off, 1); e.setBuffer(p.y.buf, p.y.off, 2); + p.bad = try p.rig.scratch(4); + @memset(p.rig.bytes(p.bad, 4), 0); e.setBuffer(p.bad.buf, p.bad.off, 3); e.dispatchThreads(mtl.Size.of(p.rows, 1, 1), mtl.Size.of(32, 1, 1)); return p.rig.finish(pass); From 0a9a87c6b90108f325f3adf198ab1e5cc3919cc5 Mon Sep 17 00:00:00 2001 From: BlivionIaG <15323295+BlivionIaG@users.noreply.github.com> Date: Mon, 26 Oct 2026 18:46:01 +0100 Subject: [PATCH 25/39] lfm2 dspark: run the released bf16 matrices as stored (--drafter-bits 0, the default), no quantizing at load The drafter's linears and Markov W2 stay bf16: new rows and tile kernels for the linears and a bf16 scan. An explicit --drafter-bits other than 0 is refused by name. 8-bit codes stay behind Options.bits for tools. --- zig/kernels/metal/lfm2/lfm2_dspark.metal | 131 ++++++++++++++++-- zig/src/core/engine_api.zig | 2 + zig/src/families/lfm2_moe/dspark.zig | 4 +- .../families/lfm2_moe/dspark_bf16_test.zig | 121 ++++++++++++++++ zig/src/families/lfm2_moe/dspark_block.zig | 50 +++++-- zig/src/families/lfm2_moe/dspark_test.zig | 10 +- zig/src/families/lfm2_moe/dspark_weights.zig | 43 +++--- zig/src/families/lfm2_moe/kernels.zig | 3 + zig/src/native/lfm2_moe.zig | 8 +- zig/src/server/cli.zig | 5 +- zig/src/server/cli_test.zig | 2 +- zig/src/tensorfold.zig | 1 + zig/tests/lfm2_dspark.zig | 2 +- 13 files changed, 332 insertions(+), 50 deletions(-) create mode 100644 zig/src/families/lfm2_moe/dspark_bf16_test.zig diff --git a/zig/kernels/metal/lfm2/lfm2_dspark.metal b/zig/kernels/metal/lfm2/lfm2_dspark.metal index 551a695467..82866af620 100644 --- a/zig/kernels/metal/lfm2/lfm2_dspark.metal +++ b/zig/kernels/metal/lfm2/lfm2_dspark.metal @@ -1,18 +1,22 @@ // DSpark's Markov step, compiled after lfm2.metal's text (its includes and constants). -// DSpark's Markov scan: a threadgroup's best of 512 tokens of a row, bf16(base + bf16(W2 . W1[prev])), W2 8-bit codes. +// DSpark's Markov scan, W2 as 8-bit codes (BF16 false) or bf16 as released (true): a threadgroup's best of 512 tokens. struct lfm2_markov_args { uint stride, rank, vocab, prev_stride; }; constant constexpr uint MARKOV_SPAN = 512; -[[kernel]] void lfm2_markov_scan(const device bfloat* logits [[buffer(0)]], const device bfloat* w1 [[buffer(1)]], - const device uint8_t* w2 [[buffer(2)]], const device uint* prev [[buffer(3)]], - constant lfm2_markov_args& p [[buffer(4)]], device float* best [[buffer(5)]], - device uint* at [[buffer(6)]], const device bfloat* w2s [[buffer(7)]], - const device bfloat* w2b [[buffer(8)]], uint2 tg [[threadgroup_position_in_grid]], - uint2 grid [[threadgroups_per_grid]], uint sg [[simdgroup_index_in_threadgroup]], - uint lane [[thread_index_in_simdgroup]]) { - threadgroup float e[256]; - threadgroup float sv[8]; - threadgroup uint si[8]; + +// Eight bf16 in a uint4 as floats (a bf16 is the high half of its float). +inline void lfm2_unpack8(uint4 v, thread float* f) { + for (int j = 0; j < 4; ++j) { + f[2 * j] = as_type(v[j] << 16); + f[2 * j + 1] = as_type(v[j] & 0xffff0000u); + } +} +template +inline void lfm2_markov_scan_t(const device bfloat* logits, const device bfloat* w1, const device uint8_t* w2, + const device uint* prev, constant lfm2_markov_args& p, device float* best, + device uint* at, const device bfloat* w2s, const device bfloat* w2b, uint2 tg, + uint2 grid, uint sg, uint lane, threadgroup float* e, threadgroup float* sv, + threadgroup uint* si) { const uint s = tg.y, t = sg * 32 + lane, id = prev[size_t(s) * p.prev_stride]; for (uint i = t; i < p.rank; i += 256) e[i] = id < p.vocab ? float(w1[size_t(id) * p.rank + i]) : 0.0f; threadgroup_barrier(mem_flags::mem_threadgroup); @@ -24,6 +28,12 @@ constant constexpr uint MARKOV_SPAN = 512; for (uint v = first + sg; v < last; v += 8) { float d = 0.0f; for (uint i = 8 * lane; i < p.rank; i += 256) { + if (BF16) { + float f[8]; + lfm2_unpack8(*(const device uint4*)((const device bfloat*)w2 + size_t(v) * p.rank + i), f); + for (int j = 0; j < 8; ++j) d = fma(f[j], e[i + j], d); + continue; + } const size_t g = (size_t(v) * p.rank + i) / 64; const float sc = w2s[g], bi = w2b[g]; const uchar4 lo = *(const device uchar4*)(w2 + size_t(v) * p.rank + i); @@ -58,6 +68,32 @@ constant constexpr uint MARKOV_SPAN = 512; at[size_t(s) * grid.x + tg.x] = bi; } +[[kernel]] void lfm2_markov_scan(const device bfloat* logits [[buffer(0)]], const device bfloat* w1 [[buffer(1)]], + const device uint8_t* w2 [[buffer(2)]], const device uint* prev [[buffer(3)]], + constant lfm2_markov_args& p [[buffer(4)]], device float* best [[buffer(5)]], + device uint* at [[buffer(6)]], const device bfloat* w2s [[buffer(7)]], + const device bfloat* w2b [[buffer(8)]], uint2 tg [[threadgroup_position_in_grid]], + uint2 grid [[threadgroups_per_grid]], uint sg [[simdgroup_index_in_threadgroup]], + uint lane [[thread_index_in_simdgroup]]) { + threadgroup float e[256]; + threadgroup float sv[8]; + threadgroup uint si[8]; + lfm2_markov_scan_t(logits, w1, w2, prev, p, best, at, w2s, w2b, tg, grid, sg, lane, e, sv, si); +} + +// The same scan over the released bf16 W2 [vocab, rank] (buffers 7 and 8 unused). +[[kernel]] void lfm2_markov_scan_bf16(const device bfloat* logits [[buffer(0)]], const device bfloat* w1 [[buffer(1)]], + const device uint8_t* w2 [[buffer(2)]], const device uint* prev [[buffer(3)]], + constant lfm2_markov_args& p [[buffer(4)]], device float* best [[buffer(5)]], + device uint* at [[buffer(6)]], uint2 tg [[threadgroup_position_in_grid]], + uint2 grid [[threadgroups_per_grid]], uint sg [[simdgroup_index_in_threadgroup]], + uint lane [[thread_index_in_simdgroup]]) { + threadgroup float e[256]; + threadgroup float sv[8]; + threadgroup uint si[8]; + lfm2_markov_scan_t(logits, w1, w2, prev, p, best, at, w1, w1, tg, grid, sg, lane, e, sv, si); +} + // DSpark's Markov step pick: the scan's best over a stream's threadgroups (ties to the lowest token), as a Pick. [[kernel]] void lfm2_markov_pick(const device float* best [[buffer(0)]], const device uint* at [[buffer(1)]], constant uint& groups [[buffer(2)]], device uint* picks [[buffer(3)]], @@ -135,3 +171,76 @@ struct lfm2_confidence_args { uint n, pass, vocab, rank, hidden; }; for (uint g = 0; g < 8; ++g) sum += part[g]; out[size_t(i) * p.pass + s] = sum; } + +// DSpark's bf16 linears (the released matrices as stored): y [rows, n] = x [rows, k] W [n, k]^T, fp32 sums, bf16 out. +struct lfm2_bf16_args { uint rows, n, k; }; +constant constexpr uint BF16_ROWS = 16; + +// Few rows: a simdgroup a column of W over 16 rows, so W is read once a 16 rows. k a multiple of 8. +[[kernel]] void lfm2_bf16_rows(const device bfloat* x [[buffer(0)]], const device bfloat* w [[buffer(1)]], + constant lfm2_bf16_args& p [[buffer(2)]], device bfloat* y [[buffer(3)]], + uint2 tg [[threadgroup_position_in_grid]], uint sg [[simdgroup_index_in_threadgroup]], + uint lane [[thread_index_in_simdgroup]]) { + const uint col = tg.x * 8 + sg, r0 = tg.y * BF16_ROWS; + if (col >= p.n) return; + const uint rows = min(BF16_ROWS, p.rows - r0); + float acc[BF16_ROWS]; + for (uint r = 0; r < BF16_ROWS; ++r) acc[r] = 0.0f; + const device bfloat* wr = w + size_t(col) * p.k; + const device bfloat* xr = x + size_t(r0) * p.k; + for (uint i = 8 * lane; i < p.k; i += 256) { + float wv[8]; + lfm2_unpack8(*(const device uint4*)(wr + i), wv); + for (uint r = 0; r < BF16_ROWS; ++r) { + if (r >= rows) break; + float xv[8]; + lfm2_unpack8(*(const device uint4*)(xr + size_t(r) * p.k + i), xv); + for (int j = 0; j < 8; ++j) acc[r] = fma(wv[j], xv[j], acc[r]); + } + } + for (uint r = 0; r < BF16_ROWS; ++r) { + const float sum = simd_sum(acc[r]); + if (lane == 0 && r < rows) y[size_t(r0 + r) * p.n + col] = static_cast(sum); + } +} + +// Many rows: 64x64 tiles of y, four simdgroups of 32x32, k in steps of 32. k a multiple of 32. +[[kernel]] void lfm2_bf16_mma(const device bfloat* x [[buffer(0)]], const device bfloat* w [[buffer(1)]], + constant lfm2_bf16_args& p [[buffer(2)]], device bfloat* y [[buffer(3)]], + uint2 tg [[threadgroup_position_in_grid]], uint sg [[simdgroup_index_in_threadgroup]], + uint t [[thread_index_in_threadgroup]]) { + threadgroup bfloat xs[64 * 32]; + threadgroup bfloat ws[64 * 32]; + threadgroup float out[4][32 * 32]; + const uint c0 = tg.x * 64, r0 = tg.y * 64; + simdgroup_matrix acc[4][4]; + for (uint i = 0; i < 4; ++i) + for (uint j = 0; j < 4; ++j) acc[i][j] = simdgroup_matrix(0.0f); + const uint sr = (sg / 2) * 32, sc = (sg % 2) * 32; + for (uint k0 = 0; k0 < p.k; k0 += 32) { + for (uint i = t; i < 64 * 4; i += 128) { // 64 rows of 32 bf16 as 4 chunks of 8 + const uint r = i / 4, q = (i % 4) * 8; + for (uint j = 0; j < 8; ++j) { + xs[r * 32 + q + j] = r0 + r < p.rows ? x[size_t(r0 + r) * p.k + k0 + q + j] : bfloat(0.0f); + ws[r * 32 + q + j] = c0 + r < p.n ? w[size_t(c0 + r) * p.k + k0 + q + j] : bfloat(0.0f); + } + } + threadgroup_barrier(mem_flags::mem_threadgroup); + for (uint kk = 0; kk < 32; kk += 8) { + simdgroup_matrix a[4], b[4]; + for (uint i = 0; i < 4; ++i) simdgroup_load(a[i], xs + (sr + 8 * i) * 32 + kk, 32); + for (uint j = 0; j < 4; ++j) simdgroup_load(b[j], ws + (sc + 8 * j) * 32 + kk, 32, ulong2(0, 0), true); + for (uint i = 0; i < 4; ++i) + for (uint j = 0; j < 4; ++j) simdgroup_multiply_accumulate(acc[i][j], a[i], b[j], acc[i][j]); + } + threadgroup_barrier(mem_flags::mem_threadgroup); + } + for (uint i = 0; i < 4; ++i) + for (uint j = 0; j < 4; ++j) simdgroup_store(acc[i][j], out[sg] + 8 * i * 32 + 8 * j, 32); + simdgroup_barrier(mem_flags::mem_threadgroup); + for (uint i = t % 32; i < 32 * 32; i += 32) { + const uint r = i / 32, c = i % 32; + const uint row = r0 + sr + r, col = c0 + sc + c; + if (row < p.rows && col < p.n) y[size_t(row) * p.n + col] = static_cast(out[sg][i]); + } +} diff --git a/zig/src/core/engine_api.zig b/zig/src/core/engine_api.zig index 49bbbbb2b4..afb4b320c7 100644 --- a/zig/src/core/engine_api.zig +++ b/zig/src/core/engine_api.zig @@ -307,6 +307,8 @@ pub const Open = struct { /// --draft-verify and --verify-costs (a tf-lfm2-verify-costs table): the LFM2 DSpark drafter's verify rows. draft_verify: ?DraftVerify = null, verify_costs: ?[]const u8 = null, + /// Whether --drafter-bits was given: LFM2's drafter runs bf16 unless it is refused a width it has no mode for. + drafter_bits_given: bool = false, /// --record: each stream's committed rows and tapped features written under this folder, for drafter training. record: ?[]const u8 = null, speed_up: ?[]const u8 = null, diff --git a/zig/src/families/lfm2_moe/dspark.zig b/zig/src/families/lfm2_moe/dspark.zig index cec88c5330..2f801a3e89 100644 --- a/zig/src/families/lfm2_moe/dspark.zig +++ b/zig/src/families/lfm2_moe/dspark.zig @@ -29,8 +29,8 @@ pub const Verify = enum { depth, fixed, scheduled }; /// The verify when none is asked: the confidence-scheduled one. pub const default_verify: Verify = .scheduled; -/// `bits`: the quantized width (8 keeps the fp32 reference's drafts; 4 does not); `verify_asked`: --draft-verify given. -pub const Options = struct { capacity: usize, bits: u8 = 8, verify: Verify = default_verify, verify_asked: bool = false }; +/// `bits`: 0 runs the released bf16 matrices as stored, 8 quantizes to 8-bit codes; `verify_asked`: --draft-verify given. +pub const Options = struct { capacity: usize, bits: u8 = 0, verify: Verify = default_verify, verify_asked: bool = false }; /// An asked scheduled verify needs the checkpoint's confidence head; the default one falls back to the depth rule. pub fn admitVerify(o: Options, has_head: bool) !void { diff --git a/zig/src/families/lfm2_moe/dspark_bf16_test.zig b/zig/src/families/lfm2_moe/dspark_bf16_test.zig new file mode 100644 index 0000000000..293549bde1 --- /dev/null +++ b/zig/src/families/lfm2_moe/dspark_bf16_test.zig @@ -0,0 +1,121 @@ +//! The DSpark drafter's bf16 kernels on the GPU against host references: the linears and the Markov scan over bf16 W2. +const std = @import("std"); +const mtl = @import("metal"); +const quant = @import("dspark_quant.zig"); +const base = @import("dspark_test.zig"); + +const Args = extern struct { rows: u32, n: u32, k: u32 }; + +/// y = x W^T through `key` on random bf16 data, each output within a bf16 step of the fp64 product. +fn linear(comptime key: []const u8, rows: usize, n: usize, k: usize, groups: [2]usize, threads: usize) !void { + var r = try base.Rig.init(); + defer r.deinit(); + const x = try r.buffer(u16, rows * k); + defer x.deinit(); + const w = try r.buffer(u16, n * k); + defer w.deinit(); + const y = try r.buffer(u16, rows * n); + defer y.deinit(); + base.noise(31, x.slice(u16, rows * k), 2); + base.noise(32, w.slice(u16, n * k), 0.5); + const cb = r.queue.commandBuffer(); + const e = cb.compute(.serial); + e.setPipeline(r.k.get(key)); + e.setBuffer(x, 0, 0); + e.setBuffer(w, 0, 1); + e.setValue(Args{ .rows = @intCast(rows), .n = @intCast(n), .k = @intCast(k) }, 2); + e.setBuffer(y, 0, 3); + e.dispatchGroups(mtl.Size.of(groups[0], groups[1], 1), mtl.Size.of(threads, 1, 1)); + e.end(); + cb.commit(); + cb.wait(); + if (cb.failure()) |_| return error.GpuFailed; + const xs = x.slice(u16, rows * k); + const ws = w.slice(u16, n * k); + const ys = y.slice(u16, rows * n); + for (0..rows) |i| for (0..n) |j| { + var dot: f64 = 0; + for (0..k) |t| dot += @as(f64, quant.toF32(xs[i * k + t])) * @as(f64, quant.toF32(ws[j * k + t])); + const got: f64 = quant.toF32(ys[i * n + j]); + try std.testing.expect(@abs(got - dot) <= @abs(dot) * 0x1p-7 + 1e-3); + }; +} + +test "lfm2 dspark: the bf16 rows kernel is x W^T for one row, a full 16 and a ragged 19 (two row groups)" { + for ([_]usize{ 1, 16, 19 }) |rows| try linear("lfm2_bf16_rows", rows, 100, 288, .{ 13, (rows + 15) / 16 }, 256); +} + +test "lfm2 dspark: the bf16 tile kernel is x W^T for a ragged 70 rows and 100 columns, k a multiple of 32" { + try linear("lfm2_bf16_mma", 70, 100, 288, .{ 2, 2 }, 128); +} + +test "lfm2 dspark: the bf16 tile kernel at the drafter's widths" { + try linear("lfm2_bf16_mma", 33, 512, 2048, .{ 8, 1 }, 128); +} + +test "lfm2 dspark: a Markov step over bf16 W2 picks the argmax of bf16(base + bf16(W2 . W1[prev])), a NaN row flagged" { + var r = try base.Rig.init(); + defer r.deinit(); + const rank = 256; + const vocab = 1300; + const streams = 4; + const groups = (vocab + 511) / 512; + const w1 = try r.buffer(u16, vocab * rank); + defer w1.deinit(); + base.noise(11, w1.slice(u16, vocab * rank), 1); + const w2 = try r.buffer(u16, vocab * rank); + defer w2.deinit(); + base.noise(12, w2.slice(u16, vocab * rank), 0.5); + const ids = try r.buffer(u32, 2 * streams); + defer ids.deinit(); + @memcpy(ids.slice(u32, 2 * streams), &[_]u32{ 17, 9, vocab - 1, 9, vocab + 5, 0, 3, 3 }); + const logits = try r.buffer(u16, streams * 2 * vocab); + defer logits.deinit(); + base.noise(14, logits.slice(u16, streams * 2 * vocab), 4); + logits.slice(u16, streams * 2 * vocab)[3 * 2 * vocab + 700] = 0x7fc0; + const best = try r.buffer(f32, streams * groups); + defer best.deinit(); + const at = try r.buffer(u32, streams * groups); + defer at.deinit(); + const picks = try r.buffer(u32, 2 * streams); + defer picks.deinit(); + const cb = r.queue.commandBuffer(); + const e = cb.compute(.serial); + e.setPipeline(r.k.get("lfm2_markov_scan_bf16")); + for ([_]mtl.Buffer{ logits, w1, w2, ids }, 0..) |b, i| e.setBuffer(b, 0, i); + e.setValue([4]u32{ 2 * vocab, rank, vocab, 2 }, 4); + for ([_]mtl.Buffer{ best, at }, 5..) |b, i| e.setBuffer(b, 0, i); + e.dispatchGroups(mtl.Size.of(groups, streams, 1), mtl.Size.of(256, 1, 1)); + e.setPipeline(r.k.get("lfm2_markov_pick")); + e.setBuffer(best, 0, 0); + e.setBuffer(at, 0, 1); + e.setValue(@as(u32, groups), 2); + e.setBuffer(picks, 0, 3); + e.dispatchGroups(mtl.Size.of(streams, 1, 1), mtl.Size.of(256, 1, 1)); + e.end(); + cb.commit(); + cb.wait(); + const got = picks.slice(u32, 2 * streams); + const logit = logits.slice(u16, streams * 2 * vocab); + for (0..streams) |s| { + if (s == 3) { + try std.testing.expectEqual(@as(u32, 1), got[2 * s + 1]); + continue; + } + try std.testing.expectEqual(@as(u32, 0), got[2 * s + 1]); + const id = ids.slice(u32, 2 * streams)[2 * s]; + var top: f64 = -std.math.inf(f64); + var mine: f64 = 0; + for (0..vocab) |v| { + var dot: f64 = 0; + for (0..rank) |i| { + const e1: f64 = if (id < vocab) quant.toF32(w1.slice(u16, vocab * rank)[id * rank + i]) else 0; + dot += @as(f64, quant.toF32(w2.slice(u16, vocab * rank)[v * rank + i])) * e1; + } + const score: f64 = quant.toF32(quant.toBf16(quant.toF32(logit[s * 2 * vocab + v]) + quant.toF32(quant.toBf16(@floatCast(dot))))); + top = @max(top, score); + if (v == got[2 * s]) mine = score; + } + try std.testing.expect(got[2 * s] < vocab and mine >= top - @abs(top) * 0x1p-7); + } +} diff --git a/zig/src/families/lfm2_moe/dspark_block.zig b/zig/src/families/lfm2_moe/dspark_block.zig index 06fbade4a2..f27a380771 100644 --- a/zig/src/families/lfm2_moe/dspark_block.zig +++ b/zig/src/families/lfm2_moe/dspark_block.zig @@ -8,6 +8,7 @@ const tile = @import("../../core/tile_qmm.zig"); const am = @import("../../core/affine_mm.zig"); const round = @import("tree_round_gpu"); const ds = @import("dspark.zig"); +const dw = @import("dspark_weights.zig"); const Drafter = ds.Drafter; const Lane = ds.Lane; @@ -21,6 +22,7 @@ fn attnArgs(lane: *const Lane, keys: u64) AttnArgs { return .{ .len = @intCast(keys - 1), .cap = @intCast(lane.cap), .scale = 1.0 / @sqrt(@as(f32, c.head_dim)), .subs = st.subs }; } const Pick = round.contract.Pick; +const Bf16Args = extern struct { rows: u32, n: u32, k: u32 }; const MarkovArgs = extern struct { stride: u32, rank: u32, vocab: u32, prev_stride: u32 }; const ConfidenceArgs = extern struct { n: u32, pass: u32, vocab: u32, rank: u32, hidden: u32 }; /// Tokens a Markov scan threadgroup takes (the kernel's MARKOV_SPAN), and threadgroups a stream's row takes. @@ -59,12 +61,19 @@ const Enc = struct { } /// y[rows] = x[rows] W^T: the M5's tensor units from naxFrom rows, else the twin (scalar to scalar_rows) or tiles. - fn project(x: Enc, w: tile.Matrix, in: Ref, out: Ref, rows: usize) !void { + fn project(x: Enc, w: dw.Linear, in: Ref, out: Ref, rows: usize) !void { return x.projectAll(&.{w}, in, &.{out}, rows); } - /// y_i[rows] = x[rows] W_i^T on one input and shape: one sums pass, then products free to overlap. - fn projectAll(x: Enc, ws: []const tile.Matrix, in: Ref, outs: []const Ref, rows: usize) !void { + /// y_i[rows] = x[rows] W_i^T on one input and shape: bf16 matrices straight, else one sums pass, products free to overlap. + fn projectAll(x: Enc, lins: []const dw.Linear, in: Ref, outs: []const Ref, rows: usize) !void { + if (lins[0] == .bf16) { + for (lins, outs) |lin, out| x.bf16(lin.bf16, in, out, rows); + return; + } + var buf: [4]tile.Matrix = undefined; + for (lins, 0..) |lin, i| buf[i] = lin.codes; + const ws = buf[0..lins.len]; const t = &x.d.target.kernels.tiles; const sums = Ref{ .buf = x.d.s.sums }; const k = ws[0].k; @@ -89,6 +98,20 @@ const Enc = struct { for (ws, outs) |w, out| try tile.encode(x.e, t, w, in.buf, in.off, x.d.s.sums, 0, out.buf, out.off, rows); } + /// y[rows] = x[rows] W^T over the released bf16 matrix: 16 rows a pass of W up to 32 rows, else 64x64 tiles. + fn bf16(x: Enc, m: dw.Bf16, in: Ref, out: Ref, rows: usize) void { + const args = Bf16Args{ .rows = @intCast(rows), .n = @intCast(m.n), .k = @intCast(m.k) }; + const few = rows <= 32; + if (few) x.pipe("lfm2_bf16_rows") else x.pipe("lfm2_bf16_mma"); + x.e.setBuffer(in.buf, in.off, 0); + x.e.setBuffer(m.w, 0, 1); + x.e.setBytes(std.mem.asBytes(&args), 2); + x.e.setBuffer(out.buf, out.off, 3); + if (few) { + x.e.dispatchGroups(mtl.Size.of((m.n + 7) / 8, (rows + 15) / 16, 1), mtl.Size.of(256, 1, 1)); + } else x.e.dispatchGroups(mtl.Size.of((m.n + 63) / 64, (rows + 63) / 64, 1), mtl.Size.of(128, 1, 1)); + } + /// h += r when `add`, then out = norm(h) * gain: LFM2's own norm (the drafter's width and eps are the target's). fn norm(x: Enc, h: mtl.Buffer, r: mtl.Buffer, gain: mtl.Buffer, add: bool, out: mtl.Buffer, rows: usize) void { x.pipe("lfm2_norm"); @@ -304,7 +327,7 @@ pub fn hold(d: *Drafter, blocks: []const Block, signal: ?Signal) !?mtl.CommandBu } x.norm(s.h, s.r, d.w.norm, true, s.x, tail); x.sync(); - try x.project(fwd.matrix(emb), .{ .buf = s.x }, .{ .buf = s.logits }, tail); + try x.project(.{ .codes = fwd.matrix(emb) }, .{ .buf = s.x }, .{ .buf = s.logits }, tail); x.sync(); try markov(x, blocks.len, n, steps); if (d.confident()) confidence(x, blocks.len, n, steps); @@ -384,17 +407,28 @@ fn markov(x: Enc, streams: usize, n: u32, depth: u32) !void { const e = x.e; for (0..depth) |i| { // step i's previous token: the pending one (each block's first id), then step i - 1's pick - x.pipe("lfm2_markov_scan"); + switch (d.w.w2) { + .codes => x.pipe("lfm2_markov_scan"), + .bf16 => x.pipe("lfm2_markov_scan_bf16"), + } e.setBuffer(s.logits, i * c.vocab * 2, 0); e.setBuffer(d.w.w1, 0, 1); - e.setBuffer(d.w.w2.w, d.w.w2.w_off, 2); + e.setBuffer(switch (d.w.w2) { + .codes => |m| m.w, + .bf16 => |m| m.w, + }, switch (d.w.w2) { + .codes => |m| m.w_off, + .bf16 => 0, + }, 2); if (i == 0) e.setBuffer(s.ids, 0, 3) else e.setBuffer(s.picks, (i - 1) * ds.pass_rows * @sizeOf(Pick), 3); const args = MarkovArgs{ .stride = n * c.vocab, .rank = d.config.rank, .vocab = c.vocab, .prev_stride = if (i == 0) n else @sizeOf(Pick) / 4 }; e.setBytes(std.mem.asBytes(&args), 4); e.setBuffer(s.best, 0, 5); e.setBuffer(s.at, 0, 6); - e.setBuffer(d.w.w2.scales, d.w.w2.s_off, 7); - e.setBuffer(d.w.w2.biases, d.w.w2.b_off, 8); + if (d.w.w2 == .codes) { + e.setBuffer(d.w.w2.codes.scales, d.w.w2.codes.s_off, 7); + e.setBuffer(d.w.w2.codes.biases, d.w.w2.codes.b_off, 8); + } e.dispatchGroups(mtl.Size.of(scan_groups, streams, 1), mtl.Size.of(256, 1, 1)); x.sync(); x.pipe("lfm2_markov_pick"); diff --git a/zig/src/families/lfm2_moe/dspark_test.zig b/zig/src/families/lfm2_moe/dspark_test.zig index 4d96731210..55c341161c 100644 --- a/zig/src/families/lfm2_moe/dspark_test.zig +++ b/zig/src/families/lfm2_moe/dspark_test.zig @@ -11,12 +11,12 @@ const lanes = @import("lanes"); const shared = mtl.ResourceOptions.shared; -const Rig = struct { +pub const Rig = struct { device: mtl.Device, queue: mtl.Queue, k: kernels.Kernels, - fn init() !Rig { + pub fn init() !Rig { const device = mtl.Device.init() catch return error.SkipZigTest; errdefer device.deinit(); const queue = try device.queue(); @@ -24,13 +24,13 @@ const Rig = struct { return .{ .device = device, .queue = queue, .k = try kernels.load(device) }; } - fn deinit(r: *Rig) void { + pub fn deinit(r: *Rig) void { r.k.deinit(); r.queue.deinit(); r.device.deinit(); } - fn buffer(r: *Rig, comptime T: type, n: usize) !mtl.Buffer { + pub fn buffer(r: *Rig, comptime T: type, n: usize) !mtl.Buffer { const b = try r.device.buffer(n * @sizeOf(T), shared); @memset(b.slice(u8, n * @sizeOf(T)), 0); return b; @@ -50,7 +50,7 @@ const Rig = struct { } }; -fn noise(seed: u64, out: []u16, scale: f32) void { +pub fn noise(seed: u64, out: []u16, scale: f32) void { var prng = std.Random.DefaultPrng.init(seed); for (out) |*v| v.* = quant.toBf16((prng.random().float(f32) * 2 - 1) * scale); } diff --git a/zig/src/families/lfm2_moe/dspark_weights.zig b/zig/src/families/lfm2_moe/dspark_weights.zig index f0e081cb25..c97a647b8b 100644 --- a/zig/src/families/lfm2_moe/dspark_weights.zig +++ b/zig/src/families/lfm2_moe/dspark_weights.zig @@ -1,4 +1,4 @@ -//! The DSpark drafter's weights: bf16 linears quantized to MLX affine codes at load, norms and W1 kept as stored. +//! The DSpark drafter's weights: the released bf16 linears as stored, or on request quantized to 8-bit affine codes. const std = @import("std"); const mtl = @import("metal"); const c = @import("config.zig"); @@ -10,17 +10,23 @@ const tile = @import("../../core/tile_qmm.zig"); const shared = mtl.ResourceOptions.shared | mtl.ResourceOptions.untracked; +/// An [n, k] bf16 matrix as released (rows only reordered for half-split RoPE where the checkpoint is read). +pub const Bf16 = struct { w: mtl.Buffer, n: usize, k: usize }; + +/// A linear: bf16 as stored (bits 0), or MLX affine codes at the loader's width. +pub const Linear = union(enum) { bf16: Bf16, codes: tile.Matrix }; + /// A bf16 vector in its own buffer (norm gains; RoPE-paired ones in the half-split order the kernels rotate). pub const Vector = mtl.Buffer; pub const Layer = struct { - q: tile.Matrix, - k: tile.Matrix, - v: tile.Matrix, - o: tile.Matrix, - gate: tile.Matrix, - up: tile.Matrix, - down: tile.Matrix, + q: Linear, + k: Linear, + v: Linear, + o: Linear, + gate: Linear, + up: Linear, + down: Linear, input_norm: Vector, post_norm: Vector, q_norm: Vector, @@ -28,13 +34,13 @@ pub const Layer = struct { }; pub const Weights = struct { - fc: tile.Matrix, + fc: Linear, hidden_norm: Vector, norm: Vector, layers: [8]Layer, - /// The Markov head: W1 [vocab, rank] bf16 as stored (rows gathered by token), W2 [vocab, rank] as 8-bit codes. + /// The Markov head: W1 [vocab, rank] bf16 as stored (rows gathered by token), W2 [vocab, rank] as the linears. w1: mtl.Buffer, - w2: tile.Matrix, + w2: Linear, /// The confidence head: its [hidden + rank] bf16 weights and fp32 bias (null: the checkpoint has none). confidence: ?mtl.Buffer = null, confidence_bias: f32 = 0, @@ -87,8 +93,8 @@ const Loader = struct { return b; } - /// An [n, k] bf16 matrix as affine codes at the loader's width; rows reordered for half-split RoPE when `paired`. - fn matrix(l: Loader, name: []const u8, n: usize, k: usize, paired: bool) !tile.Matrix { + /// An [n, k] bf16 matrix as it is or as affine codes; rows reordered for half-split RoPE when `paired`. + fn matrix(l: Loader, name: []const u8, n: usize, k: usize, paired: bool) !Linear { const t = try l.tensor(name, &.{ n, k }); var rows = t.host(u16); var moved: ?[]u16 = null; @@ -99,12 +105,17 @@ const Loader = struct { for (0..n) |r| @memcpy(m[r * k ..][0..k], rows[quant.halfSplit(r, c.head_dim) * k ..][0..k]); rows = m; } + if (l.bits == 0) { + const b = try l.buffer(n * k * 2); + @memcpy(b.slice(u16, n * k), rows); + return .{ .bf16 = .{ .w = b, .n = n, .k = k } }; + } const codes = try l.buffer(n * k * l.bits / 8); const scales = try l.buffer(n * k / 64 * 2); const biases = try l.buffer(n * k / 64 * 2); const job: quant.Job = .{ .src = rows, .k = k, .bits = l.bits, .codes = codes.slice(u8, n * k * l.bits / 8), .scales = scales.slice(u16, n * k / 64), .biases = biases.slice(u16, n * k / 64) }; try job.all(l.gpa, n); - return .{ .w = codes, .scales = scales, .biases = biases, .n = n, .k = k, .bits = l.bits }; + return .{ .codes = .{ .w = codes, .scales = scales, .biases = biases, .n = n, .k = k, .bits = l.bits } }; } }; @@ -117,9 +128,9 @@ fn matrices(_: []const u8, rank: usize) bool { return rank == 2; } -/// The drafter's checkpoint read and quantized at `bits` (4 or 8), a matrix at a time; its buffers are released after. +/// The drafter's checkpoint read as released (`bits` 0) or quantized to 8-bit codes, a matrix at a time; buffers freed after. pub fn load(gpa: std.mem.Allocator, device: mtl.Device, io: std.Io, dir: []const u8, cfg: *const dc.Config, bits: u8) !Weights { - if (bits != 4 and bits != 8) return error.UnsupportedDraftBits; + if (bits != 0 and bits != 8) return error.UnsupportedDraftBits; var check = ckpt.Checkpoint.init(gpa); defer check.deinit(); var stage: ckpt.Stage = .{ .device = device }; diff --git a/zig/src/families/lfm2_moe/kernels.zig b/zig/src/families/lfm2_moe/kernels.zig index 96997a1109..c83c9a0b4a 100644 --- a/zig/src/families/lfm2_moe/kernels.zig +++ b/zig/src/families/lfm2_moe/kernels.zig @@ -31,6 +31,9 @@ const names = [_][:0]const u8{ "lfm2_attn_merge_segs", "lfm2_tap", "lfm2_markov_scan", + "lfm2_markov_scan_bf16", + "lfm2_bf16_rows", + "lfm2_bf16_mma", "lfm2_markov_pick", "lfm2_draft_ids", "lfm2_dspark_confidence", diff --git a/zig/src/native/lfm2_moe.zig b/zig/src/native/lfm2_moe.zig index 767cbbe975..d36e4a9805 100644 --- a/zig/src/native/lfm2_moe.zig +++ b/zig/src/native/lfm2_moe.zig @@ -82,13 +82,13 @@ pub fn open(a: std.mem.Allocator, gpa: std.mem.Allocator, io: std.Io, o: api.Ope h.draft = null; h.wrap = null; if (if (o.drafts) o.drafter else null) |dir| { - if (o.drafter_bits == 0) { - problem.* = "--drafter-bits 0: the LFM2 DSpark drafter runs its bf16 linears as 8-bit codes, quantized at load"; + if (o.drafter_bits_given and o.drafter_bits != 0) { + problem.* = "--drafter-bits: the LFM2 DSpark drafter runs the released bf16 matrices; it takes 0 and no other width"; release(h); return null; } const verify = verifyMode(o.draft_verify); - h.draft = q.dspark.Drafter.init(gpa, io, model, dir, .{ .capacity = window, .verify = verify }) catch |err| { + h.draft = q.dspark.Drafter.init(gpa, io, model, dir, .{ .capacity = window, .bits = 0, .verify = verify }) catch |err| { problem.* = try std.fmt.allocPrint(a, "the native LFM2 drafter cannot load {s} ({s}): it takes LiquidAI/LFM2.5-24B-A2B-DSpark", .{ dir, @errorName(err) }); release(h); return null; @@ -161,7 +161,7 @@ pub fn open(a: std.mem.Allocator, gpa: std.mem.Allocator, io: std.Io, o: api.Ope }; h.host.keepalive_target = .{ .ctx = &h.warm, .tick = mtl.keepalive.Target.tick }; try h.host.start(); - if (h.draft) |d| std.log.info("lfm2: DSpark drafts from --drafter, {d} a round at most, {d}-bit codes, {t} verify", .{ d.config.block, d.options.bits, d.options.verify }); + if (h.draft) |d| std.log.info("lfm2: DSpark drafts from --drafter, {d} a round at most, bf16 matrices as released, {t} verify", .{ d.config.block, d.options.verify }); return .{ .engine = h.host.engine(), .close = Host.close, .ctx = h }; } diff --git a/zig/src/server/cli.zig b/zig/src/server/cli.zig index df7ea7937f..03ad685b96 100644 --- a/zig/src/server/cli.zig +++ b/zig/src/server/cli.zig @@ -219,7 +219,7 @@ pub const Args = struct { compact_keep: ?u32 = null, compact_memory: ?[]const u8 = null, drafter: ?[]const u8 = null, - drafter_bits: u8 = 4, + drafter_bits: ?u8 = null, // null: not given (a family without its own default takes 4) draft_verify: ?api.DraftVerify = null, // the LFM2 DSpark drafter's verify rows (null: the engine's default) verify_costs: ?[]const u8 = null, // a tf-lfm2-verify-costs table pricing them (null: costs timed at load) record: ?[]const u8 = null, // drafter training rows written under this folder @@ -551,7 +551,8 @@ pub fn request(a: Allocator, dir: []const u8, model_type: []const u8, args: Args .lanes_fixed = parallelFixed(args.parallel), .drafts = !args.no_drafts, .drafter = args.drafter, - .drafter_bits = args.drafter_bits, + .drafter_bits = args.drafter_bits orelse 4, + .drafter_bits_given = args.drafter_bits != null, .draft_verify = args.draft_verify, .verify_costs = args.verify_costs, .record = args.record, diff --git a/zig/src/server/cli_test.zig b/zig/src/server/cli_test.zig index 7065a8a49e..c4b9a4011d 100644 --- a/zig/src/server/cli_test.zig +++ b/zig/src/server/cli_test.zig @@ -163,7 +163,7 @@ test "parse and capabilities share the table" { try std.testing.expectEqual(@as(usize, 2), args.api_key.len); try std.testing.expect(!args.thinking); try std.testing.expectEqualStrings("x", (try parse(a, &.{ "m", "--drafter", "x" }, &u)).drafter.?); - try std.testing.expectEqual(@as(u8, 0), (try parse(a, &.{ "m", "--drafter-bits", "0" }, &u)).drafter_bits); + try std.testing.expectEqual(@as(?u8, 0), (try parse(a, &.{ "m", "--drafter-bits", "0" }, &u)).drafter_bits); try std.testing.expectError(error.Usage, parse(a, &.{ "m", "--drafter-bits", "8" }, &u)); if (@import("builtin").os.tag == .macos) { const verify = try parse(a, &.{ "m", "--draft-verify", "scheduled", "--verify-costs", "/t" }, &u); diff --git a/zig/src/tensorfold.zig b/zig/src/tensorfold.zig index 6089264618..0191cabc8e 100644 --- a/zig/src/tensorfold.zig +++ b/zig/src/tensorfold.zig @@ -92,6 +92,7 @@ test { _ = @import("families/lfm2_moe/dspark_quant.zig"); _ = @import("families/lfm2_moe/dspark_test.zig"); _ = @import("families/lfm2_moe/verify_costs.zig"); + _ = @import("families/lfm2_moe/dspark_bf16_test.zig"); } pub const lane_projection = @import("core/lane_projection.zig"); diff --git a/zig/tests/lfm2_dspark.zig b/zig/tests/lfm2_dspark.zig index b1ab9b12fc..a1814189c3 100644 --- a/zig/tests/lfm2_dspark.zig +++ b/zig/tests/lfm2_dspark.zig @@ -172,7 +172,7 @@ pub fn main(init: std.process.Init) !void { const gpa = init.gpa; const args = try init.minimal.args.toSlice(init.arena.allocator()); if (args.len < 4 or args.len > 6) return error.ExpectedModelDrafterPromptLengthRowsBits; - const bits: u8 = if (args.len == 6) try std.fmt.parseInt(u8, args[5], 10) else 8; + const bits: u8 = if (args.len == 6) try std.fmt.parseInt(u8, args[5], 10) else 0; const length = try std.fmt.parseInt(usize, args[3], 10); const pool = mtl.objc.Pool.push(); defer pool.pop(); From cf5997dbbf70d3727a906e26a547487a1c8e51dd Mon Sep 17 00:00:00 2001 From: BlivionIaG <15323295+BlivionIaG@users.noreply.github.com> Date: Mon, 26 Oct 2026 19:18:44 +0100 Subject: [PATCH 26/39] lfm2 dspark: Options' comment fits 120 columns --- zig/src/families/lfm2_moe/dspark.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zig/src/families/lfm2_moe/dspark.zig b/zig/src/families/lfm2_moe/dspark.zig index 2f801a3e89..40766e9fe1 100644 --- a/zig/src/families/lfm2_moe/dspark.zig +++ b/zig/src/families/lfm2_moe/dspark.zig @@ -29,7 +29,7 @@ pub const Verify = enum { depth, fixed, scheduled }; /// The verify when none is asked: the confidence-scheduled one. pub const default_verify: Verify = .scheduled; -/// `bits`: 0 runs the released bf16 matrices as stored, 8 quantizes to 8-bit codes; `verify_asked`: --draft-verify given. +/// `bits`: 0 runs the bf16 matrices as stored, 8 quantizes them at load; `verify_asked`: --draft-verify given. pub const Options = struct { capacity: usize, bits: u8 = 0, verify: Verify = default_verify, verify_asked: bool = false }; /// An asked scheduled verify needs the checkpoint's confidence head; the default one falls back to the depth rule. From 4767a9f410319475dbf442096a32673f056645f1 Mon Sep 17 00:00:00 2001 From: BlivionIaG <15323295+BlivionIaG@users.noreply.github.com> Date: Mon, 26 Oct 2026 19:21:22 +0100 Subject: [PATCH 27/39] lfm2 dspark: the weights loader's comment fits 120 columns --- zig/src/families/lfm2_moe/dspark_weights.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zig/src/families/lfm2_moe/dspark_weights.zig b/zig/src/families/lfm2_moe/dspark_weights.zig index c97a647b8b..9d87dd6eed 100644 --- a/zig/src/families/lfm2_moe/dspark_weights.zig +++ b/zig/src/families/lfm2_moe/dspark_weights.zig @@ -128,7 +128,7 @@ fn matrices(_: []const u8, rank: usize) bool { return rank == 2; } -/// The drafter's checkpoint read as released (`bits` 0) or quantized to 8-bit codes, a matrix at a time; buffers freed after. +/// The checkpoint read as released (`bits` 0) or quantized to 8-bit codes, a matrix at a time; buffers freed after. pub fn load(gpa: std.mem.Allocator, device: mtl.Device, io: std.Io, dir: []const u8, cfg: *const dc.Config, bits: u8) !Weights { if (bits != 0 and bits != 8) return error.UnsupportedDraftBits; var check = ckpt.Checkpoint.init(gpa); From e28f27e09fab65e70c4ac55892489067fb05c4cc Mon Sep 17 00:00:00 2001 From: BlivionIaG <15323295+BlivionIaG@users.noreply.github.com> Date: Mon, 26 Oct 2026 19:23:49 +0100 Subject: [PATCH 28/39] lfm2 dspark: the block's linear comment fits 120 columns --- zig/src/families/lfm2_moe/dspark_block.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zig/src/families/lfm2_moe/dspark_block.zig b/zig/src/families/lfm2_moe/dspark_block.zig index f27a380771..0dd20d8ef5 100644 --- a/zig/src/families/lfm2_moe/dspark_block.zig +++ b/zig/src/families/lfm2_moe/dspark_block.zig @@ -65,7 +65,7 @@ const Enc = struct { return x.projectAll(&.{w}, in, &.{out}, rows); } - /// y_i[rows] = x[rows] W_i^T on one input and shape: bf16 matrices straight, else one sums pass, products free to overlap. + /// y_i[rows] = x[rows] W_i^T on one input and shape: bf16 matrices straight, else one sums pass, products overlap. fn projectAll(x: Enc, lins: []const dw.Linear, in: Ref, outs: []const Ref, rows: usize) !void { if (lins[0] == .bf16) { for (lins, outs) |lin, out| x.bf16(lin.bf16, in, out, rows); From a6c1d325d392eddab6db7c76ac4005306296a1f6 Mon Sep 17 00:00:00 2001 From: BlivionIaG <15323295+BlivionIaG@users.noreply.github.com> Date: Mon, 26 Oct 2026 19:09:04 +0100 Subject: [PATCH 29/39] lfm2 dspark: the bf16 tile kernel loads its tiles 16 bytes at a time --- zig/kernels/metal/lfm2/lfm2_dspark.metal | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/zig/kernels/metal/lfm2/lfm2_dspark.metal b/zig/kernels/metal/lfm2/lfm2_dspark.metal index 82866af620..a8685313d2 100644 --- a/zig/kernels/metal/lfm2/lfm2_dspark.metal +++ b/zig/kernels/metal/lfm2/lfm2_dspark.metal @@ -218,12 +218,12 @@ constant constexpr uint BF16_ROWS = 16; for (uint j = 0; j < 4; ++j) acc[i][j] = simdgroup_matrix(0.0f); const uint sr = (sg / 2) * 32, sc = (sg % 2) * 32; for (uint k0 = 0; k0 < p.k; k0 += 32) { - for (uint i = t; i < 64 * 4; i += 128) { // 64 rows of 32 bf16 as 4 chunks of 8 + for (uint i = t; i < 64 * 4; i += 128) { // 64 rows of 32 bf16 as 4 chunks of 8, 16 bytes a load const uint r = i / 4, q = (i % 4) * 8; - for (uint j = 0; j < 8; ++j) { - xs[r * 32 + q + j] = r0 + r < p.rows ? x[size_t(r0 + r) * p.k + k0 + q + j] : bfloat(0.0f); - ws[r * 32 + q + j] = c0 + r < p.n ? w[size_t(c0 + r) * p.k + k0 + q + j] : bfloat(0.0f); - } + const uint4 xv = r0 + r < p.rows ? *(const device uint4*)(x + size_t(r0 + r) * p.k + k0 + q) : uint4(0); + const uint4 wv = c0 + r < p.n ? *(const device uint4*)(w + size_t(c0 + r) * p.k + k0 + q) : uint4(0); + *(threadgroup uint4*)(xs + r * 32 + q) = xv; + *(threadgroup uint4*)(ws + r * 32 + q) = wv; } threadgroup_barrier(mem_flags::mem_threadgroup); for (uint kk = 0; kk < 32; kk += 8) { From d50c4eabfadb9ac05370909fd9cec2de2c623a12 Mon Sep 17 00:00:00 2001 From: BlivionIaG <15323295+BlivionIaG@users.noreply.github.com> Date: Mon, 26 Oct 2026 19:20:32 +0100 Subject: [PATCH 30/39] lfm2 dspark: --drafter-bits 8 serves the 8-bit codes, named in --help, the README, the recipe and the startup log --drafter-bits takes 0, 4 and 8. LFM2's drafter runs bf16 as released when it is not given or 0, 8-bit codes quantized at load on 8, and refuses 4. The other families refuse 8 by name. --- README.md | 2 +- docs/recipes/lfm2.5-24b-a2b.md | 6 ++++-- zig/src/core/engine_api.zig | 4 ++-- zig/src/native/cuda_options.zig | 4 ++++ zig/src/native/lfm2_moe.zig | 13 ++++++++----- zig/src/server/cli.zig | 2 +- zig/src/server/cli_test.zig | 4 +++- zig/src/server/main.zig | 2 +- 8 files changed, 24 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 0847d13abd..dd8da91501 100644 --- a/README.md +++ b/README.md @@ -290,7 +290,7 @@ The binary's `capabilities --json` response lists its supported flags and platfo | `--loop-guard-stop` | As `--loop-guard`, but end the reply where the guard fires: the reasoning so far, no answer or tool calls, `finish_reason` `content_filter` and a `reasoning_loop` error ([API](docs/api.md)). | | `--no-drafts` | Produce the plain reference through the same engine. | | `--drafter DRAFTER` | An external drafter, as a directory or a repo id in the Hugging Face cache, for models that take one: Qwen3.8-27B uses DFlash2 and Gemma 4 31B the drafter named in its recipe. Without one they decode plain. | -| `--drafter-bits 0\|4` | Drafter weight precision where the model offers a choice: 4 bits (the default), or 0 to keep bf16. Qwen3.8-27B on CUDA takes 4 only. | +| `--drafter-bits 0\|4\|8` | Drafter weight precision where the model offers a choice: 4 bits (the default), or 0 to keep bf16. Qwen3.8-27B on CUDA takes 4 only. LFM2.5-24B-A2B's DSpark drafter runs bf16 as released (0, the default when not given) or 8-bit codes quantized at load (8); it has no 4-bit mode. | | `--record DIR` | Write each stream's committed tokens, positions and tapped rows under DIR, for drafter training. CUDA Nemotron only; a single stream runs through the lane rounds, and recording stops below 25 GiB free disk. | | `--keep-warm SECONDS` | Keep the Metal GPU active while idle for this long after a request, default 900; zero disables it. | | `--cfg-gamma GAMMA` | Metal Drama speech: semantic guidance strength, finite 1–5; 0/default disables it. Only reference-conditioned requests are guided. Enabled replies report `X-Cfg-Gamma`; codec heads stay conditional. | diff --git a/docs/recipes/lfm2.5-24b-a2b.md b/docs/recipes/lfm2.5-24b-a2b.md index 824788cffd..bfd79f85a0 100644 --- a/docs/recipes/lfm2.5-24b-a2b.md +++ b/docs/recipes/lfm2.5-24b-a2b.md @@ -69,8 +69,10 @@ The drafter has no embedding and no head of its own: drafter was trained on. - Markov head: row i's logits plus `W2 W1[previous token]` (rank 256); draft i is their argmax. -Its bf16 linears are quantized at load to 8-bit MLX affine codes in groups of 64 and run on the family's kernels -and `core/tile_qmm`; the q and k rows are reordered so that the target's half-split RoPE kernel gives the +The drafter runs the released bf16 matrices as they are (`--drafter-bits 0`, the default). `--drafter-bits 8` instead +quantizes them at load to 8-bit MLX affine codes in groups of 64, about 5% faster on one stream and 361 MiB of weights +against 625 MiB; the server logs which mode runs, and 4 bits is refused. Both run on the family's kernels +(bf16 on `lfm2_bf16_rows`, `lfm2_bf16_mma` and `lfm2_markov_scan_bf16`, 8-bit on `core/tile_qmm`); the q and k rows are reordered so that the target's half-split RoPE kernel gives the checkpoint's interleaved RoPE. Its own kernels are `lfm2_tap`, `lfm2_attn_block` and `lfm2_attn_block_merge` (non-causal block attention, the target's attention kernels with a template flag), `lfm2_markov_scan` and `lfm2_markov_pick` (a Markov step) and `lfm2_draft_ids` (a hold's drafts into the next verify's rows on the GPU, so diff --git a/zig/src/core/engine_api.zig b/zig/src/core/engine_api.zig index afb4b320c7..caae72e6a0 100644 --- a/zig/src/core/engine_api.zig +++ b/zig/src/core/engine_api.zig @@ -301,13 +301,13 @@ pub const Open = struct { /// --parallel named a number: an engine whose memory fits fewer streams refuses instead of serving fewer. lanes_fixed: bool = false, drafts: bool = true, - /// --drafter: a draft model's directory (Qwen3.8-27B's DFlash2; null: none), and --drafter-bits: 4, or 0 for bf16. + /// --drafter: a draft model's directory (null: none); --drafter-bits: 4, 0 for bf16, or 8 (LFM2 only). drafter: ?[]const u8 = null, drafter_bits: u8 = 4, /// --draft-verify and --verify-costs (a tf-lfm2-verify-costs table): the LFM2 DSpark drafter's verify rows. draft_verify: ?DraftVerify = null, verify_costs: ?[]const u8 = null, - /// Whether --drafter-bits was given: LFM2's drafter runs bf16 unless it is refused a width it has no mode for. + /// Whether --drafter-bits was given: LFM2's drafter runs bf16 as released unless it is asked for 8-bit codes. drafter_bits_given: bool = false, /// --record: each stream's committed rows and tapped features written under this folder, for drafter training. record: ?[]const u8 = null, diff --git a/zig/src/native/cuda_options.zig b/zig/src/native/cuda_options.zig index d6a6eefe28..0c07152bdb 100644 --- a/zig/src/native/cuda_options.zig +++ b/zig/src/native/cuda_options.zig @@ -36,6 +36,10 @@ pub fn set(comptime F: type, o: api.Open, opts: *F.Options, problem: *[]const u8 return false; }; } + if (o.drafter_bits_given and o.drafter_bits == 8) { + problem.* = "--drafter-bits 8: only LFM2.5-24B-A2B's Metal DSpark drafter takes it; this family takes 0 (bf16) or 4"; + return false; + } if (comptime @hasField(F.Options, "drafter")) opts.drafter = o.drafter; if (comptime @hasField(F.Options, "drafter_bits")) opts.drafter_bits = o.drafter_bits; if (@hasField(F.Options, "kv8")) opts.kv8 = o.kv8; diff --git a/zig/src/native/lfm2_moe.zig b/zig/src/native/lfm2_moe.zig index d36e4a9805..d1c180391a 100644 --- a/zig/src/native/lfm2_moe.zig +++ b/zig/src/native/lfm2_moe.zig @@ -82,13 +82,13 @@ pub fn open(a: std.mem.Allocator, gpa: std.mem.Allocator, io: std.Io, o: api.Ope h.draft = null; h.wrap = null; if (if (o.drafts) o.drafter else null) |dir| { - if (o.drafter_bits_given and o.drafter_bits != 0) { - problem.* = "--drafter-bits: the LFM2 DSpark drafter runs the released bf16 matrices; it takes 0 and no other width"; + const bits: u8 = if (o.drafter_bits_given) o.drafter_bits else 0; + if (bits != 0 and bits != 8) { + problem.* = "--drafter-bits: the LFM2 DSpark drafter takes 0 (bf16 as released, the default) or 8 (8-bit codes quantized at load)"; release(h); return null; } - const verify = verifyMode(o.draft_verify); - h.draft = q.dspark.Drafter.init(gpa, io, model, dir, .{ .capacity = window, .bits = 0, .verify = verify }) catch |err| { + h.draft = q.dspark.Drafter.init(gpa, io, model, dir, .{ .capacity = window, .bits = bits }) catch |err| { problem.* = try std.fmt.allocPrint(a, "the native LFM2 drafter cannot load {s} ({s}): it takes LiquidAI/LFM2.5-24B-A2B-DSpark", .{ dir, @errorName(err) }); release(h); return null; @@ -161,7 +161,10 @@ pub fn open(a: std.mem.Allocator, gpa: std.mem.Allocator, io: std.Io, o: api.Ope }; h.host.keepalive_target = .{ .ctx = &h.warm, .tick = mtl.keepalive.Target.tick }; try h.host.start(); - if (h.draft) |d| std.log.info("lfm2: DSpark drafts from --drafter, {d} a round at most, bf16 matrices as released, {t} verify", .{ d.config.block, d.options.verify }); + if (h.draft) |d| { + const mode = if (d.options.bits == 8) "8-bit codes quantized at load (--drafter-bits 8), not the released bf16" else "bf16 matrices as released"; + std.log.info("lfm2: DSpark drafts from --drafter, {d} a round at most, {s}", .{ d.config.block, mode }); + } return .{ .engine = h.host.engine(), .close = Host.close, .ctx = h }; } diff --git a/zig/src/server/cli.zig b/zig/src/server/cli.zig index 03ad685b96..6e3d16654e 100644 --- a/zig/src/server/cli.zig +++ b/zig/src/server/cli.zig @@ -101,7 +101,7 @@ pub const flags = [_]Flag{ .{ .name = "--slide", .kind = .store_true, .native = true }, .{ .name = "--slide-graph", .native = true }, .{ .name = "--drafter", .native = true }, - .{ .name = "--drafter-bits", .choices = &.{ "0", "4" }, .native = true }, + .{ .name = "--drafter-bits", .choices = &.{ "0", "4", "8" }, .native = true }, .{ .name = "--draft-verify", .choices = &.{ "depth", "fixed", "scheduled" }, .native = builtin.os.tag == .macos }, .{ .name = "--verify-costs", .native = builtin.os.tag == .macos }, .{ .name = "--record", .native = true }, diff --git a/zig/src/server/cli_test.zig b/zig/src/server/cli_test.zig index c4b9a4011d..b6cfc59f71 100644 --- a/zig/src/server/cli_test.zig +++ b/zig/src/server/cli_test.zig @@ -164,7 +164,9 @@ test "parse and capabilities share the table" { try std.testing.expect(!args.thinking); try std.testing.expectEqualStrings("x", (try parse(a, &.{ "m", "--drafter", "x" }, &u)).drafter.?); try std.testing.expectEqual(@as(?u8, 0), (try parse(a, &.{ "m", "--drafter-bits", "0" }, &u)).drafter_bits); - try std.testing.expectError(error.Usage, parse(a, &.{ "m", "--drafter-bits", "8" }, &u)); + try std.testing.expectEqual(@as(?u8, 8), (try parse(a, &.{ "m", "--drafter-bits", "8" }, &u)).drafter_bits); + try std.testing.expectEqual(@as(?u8, null), (try parse(a, &.{ "m", "--drafter", "x" }, &u)).drafter_bits); + try std.testing.expectError(error.Usage, parse(a, &.{ "m", "--drafter-bits", "3" }, &u)); if (@import("builtin").os.tag == .macos) { const verify = try parse(a, &.{ "m", "--draft-verify", "scheduled", "--verify-costs", "/t" }, &u); try std.testing.expectEqual(api.DraftVerify.scheduled, verify.draft_verify.?); diff --git a/zig/src/server/main.zig b/zig/src/server/main.zig index 339ed441b9..c4d1742897 100644 --- a/zig/src/server/main.zig +++ b/zig/src/server/main.zig @@ -26,7 +26,7 @@ const usage_line = "usage: tensorfold serve [-h] [--host HOST] [--port PORT] [-- " [--thinking | --no-thinking] [--reasoning-effort {low,medium,high,xhigh}]" ++ " [--thinking-budget THINKING_BUDGET] [--thinking-reserve THINKING_RESERVE] [--loop-guard] [--loop-guard-stop] [--no-drafts]" ++ " [--drafter DRAFTER]" ++ - " [--drafter-bits {0,4}] [--draft-verify {depth,fixed,scheduled}] [--verify-costs VERIFY_COSTS] [--record RECORD] [--keep-warm SECONDS] [--idle-unload SECONDS]" ++ + " [--drafter-bits {0,4,8}] [--draft-verify {depth,fixed,scheduled}] [--verify-costs VERIFY_COSTS] [--record RECORD] [--keep-warm SECONDS] [--idle-unload SECONDS]" ++ " [--silence-floor-dbfs DBFS] [--silence-seconds SECONDS]" ++ " [--silence-onset-s SECONDS] [--silence-onset-voiced-s SECONDS]" ++ " [--silence-mid-s SECONDS] [--silence-mid-min-voiced-s SECONDS] [--silence-redraws N]" ++ From 1027db08300878496ac43ab8dd04ec89c3a63b98 Mon Sep 17 00:00:00 2001 From: BlivionIaG <15323295+BlivionIaG@users.noreply.github.com> Date: Mon, 26 Oct 2026 19:33:11 +0100 Subject: [PATCH 31/39] kernel contract: the drafter's bf16 linears and bf16 Markov scan carry gates 1-4, recorded M1/M2 limits and six planted controls The few-rows and 64x64 tile kernels and the scan over a bf16 W2 pass gates 1 to 4. Six controls edit the source: a lane skipping half of k, a group's last row, an untransposed W tile, a lost last column, a NaN score not flagged and every stream reading the first stream's previous token. Each goes red at its gate. The tile kernel compiles for 832 threads on M1 and M2. --- docs/kernel-contract.md | 6 +- zig/src/core/contract/lfm2_records.zig | 11 ++ zig/tests/contract/metal_lfm2_bf16.zig | 133 +++++++++++++++++++++ zig/tests/contract/metal_lfm2_controls.zig | 41 ++++++- zig/tests/contract/metal_lfm2_dspark.zig | 26 ++-- 5 files changed, 205 insertions(+), 12 deletions(-) create mode 100644 zig/tests/contract/metal_lfm2_bf16.zig diff --git a/docs/kernel-contract.md b/docs/kernel-contract.md index af412e6887..31e9054632 100644 --- a/docs/kernel-contract.md +++ b/docs/kernel-contract.md @@ -135,8 +135,10 @@ tile_qmm's pipelines and GLM-5.3-Flash's hc_expand_10. LFM2.5-24B-A2B's shared rounds: the greedy pick (`lfm2_argmax`) and the conv and attention kernels that run every stream's rows in one dispatch, each bit-equal to the per-stream kernels it replaces. Four controls edit the source: a tie to the highest id, a nonfinite logit not flagged, and every stream reading or writing the first stream's state or cache. -Their launches are in `lfm2_records.zig` with the M1 and M2 limits from `--offline`; `lfm2_attn_segs` reserves its 128 -threads in its source, as Metal cannot build it for M1 or M2 at another count. +The DSpark drafter's tap copy, Markov scan and pick, draft ids and block attention are there too, with six more controls. +Its bf16 linears (few rows, 64x64 tiles) and the Markov scan over a bf16 W2 have four and two more. +Their launches are in `lfm2_records.zig` with the M1 and M2 limits from `--offline`; `lfm2_attn_segs` and +`lfm2_attn_block` reserve their 128 threads in their source, as Metal cannot build them for M1 or M2 at another count. Metal also checks Nemotron's routed experts on the engine's route tables, one row to 128: `tf_xup_w3` and `tf_xdown_w3` against float64 at tf_load16's bf16 x sums (up's relu2 as an interval over its two bf16 roundings), and the rows2, rows4 and six geometry pipelines of each projection bit-equal to them (two controls: member rows read from one row, and diff --git a/zig/src/core/contract/lfm2_records.zig b/zig/src/core/contract/lfm2_records.zig index 355559b1ff..2e7587fa3d 100644 --- a/zig/src/core/contract/lfm2_records.zig +++ b/zig/src/core/contract/lfm2_records.zig @@ -1,12 +1,23 @@ //! LFM2-MoE's shared-round and greedy-pick launches (lfm2.metal, lfm2_segments.metal), as the encoders launch them. const std = @import("std"); const M = @import("limits.zig").MetalLaunch; +// The drafter kernels carry M1 and M2 limits too, recorded from the offline compile; no run covers those chips. pub const records = [_]M{ .{ .name = "metal lfm2 argmax lfm2_argmax", .threads = 1024, .tg_memory = 384, .m1 = 1024, .m2 = 1024 }, .{ .name = "metal lfm2 stream conv lfm2_conv_segs", .threads = 256, .tg_memory = 0, .m1 = 1024, .m2 = 1024 }, .{ .name = "metal lfm2 stream attention qk lfm2_qk_segs", .threads = 32, .tg_memory = 0, .m1 = 832, .m2 = 832 }, .{ .name = "metal lfm2 stream attention lfm2_attn_segs", .threads = 128, .tg_memory = 18688, .reserved = 128 }, .{ .name = "metal lfm2 stream attention merge lfm2_attn_merge_segs", .threads = 32, .tg_memory = 0, .m1 = 1024, .m2 = 1024 }, + .{ .name = "metal lfm2 tap lfm2_tap", .threads = 256, .tg_memory = 0, .m1 = 1024, .m2 = 1024 }, + .{ .name = "metal lfm2 markov scan lfm2_markov_scan", .threads = 256, .tg_memory = 1088, .m1 = 1024, .m2 = 1024 }, + .{ .name = "metal lfm2 markov scan bf16 lfm2_markov_scan_bf16", .threads = 256, .tg_memory = 1088, .m1 = 1024, .m2 = 1024 }, + .{ .name = "metal lfm2 bf16 rows lfm2_bf16_rows", .threads = 256, .tg_memory = 0, .m1 = 1024, .m2 = 1024 }, + .{ .name = "metal lfm2 bf16 tile lfm2_bf16_mma", .threads = 128, .tg_memory = 24576, .m1 = 832, .m2 = 832 }, + .{ .name = "metal lfm2 markov pick lfm2_markov_pick", .threads = 256, .tg_memory = 3072, .m1 = 1024, .m2 = 1024 }, + .{ .name = "metal lfm2 draft ids lfm2_draft_ids", .threads = 32, .tg_memory = 0, .m1 = 1024, .m2 = 1024 }, + .{ .name = "metal lfm2 block attention lfm2_attn_block", .threads = 128, .tg_memory = 18688, .reserved = 128 }, + .{ .name = "metal lfm2 block attention merge lfm2_attn_block_merge", .threads = 32, .tg_memory = 0, .m1 = 1024, .m2 = 1024 }, + .{ .name = "metal lfm2 dspark confidence lfm2_dspark_confidence", .threads = 256, .tg_memory = 32, .m1 = 1024, .m2 = 1024 }, }; test "the LFM2 launches fit every Apple family" { diff --git a/zig/tests/contract/metal_lfm2_bf16.zig b/zig/tests/contract/metal_lfm2_bf16.zig new file mode 100644 index 0000000000..0a191a48fa --- /dev/null +++ b/zig/tests/contract/metal_lfm2_bf16.zig @@ -0,0 +1,133 @@ +//! LFM2's DSpark bf16 linears under gates 1-4: x W^T with bf16 inputs and fp32 sums, few rows or 64x64 tiles. +const std = @import("std"); +const mtl = @import("metal"); +const contract = @import("contract"); +const rig_ = @import("rig.zig"); +const controls = @import("metal_lfm2_controls.zig"); + +const c = contract.spec; +const g = contract.gates; +const Dense = contract.dense.Dense; + +pub const Which = enum { rows, tile }; +const names = [_][]const u8{ "metal lfm2 bf16 rows", "metal lfm2 bf16 tile" }; + +/// The drafter's matrices: q and o (2,048 from 2,048), k and v (512), gate and up (6,144), down (2,048 from 6,144). +const rows_served = [_]c.Shape{ + .{ .rows = 1, .n = 2048, .k = 2048 }, .{ .rows = 9, .n = 512, .k = 2048 }, .{ .rows = 16, .n = 6144, .k = 2048 }, + .{ .rows = 32, .n = 2048, .k = 6144 }, .{ .rows = 4, .n = 2048, .k = 10240 }, +}; +/// A cold prompt's absorb: the taps' projection (2,048 from 10,240) and k and v, up to 2,048 rows a pass. +const tile_served = [_]c.Shape{ + .{ .rows = 33, .n = 2048, .k = 2048 }, .{ .rows = 128, .n = 512, .k = 2048 }, + .{ .rows = 2048, .n = 2048, .k = 10240 }, .{ .rows = 70, .n = 6144, .k = 2048 }, +}; + +/// A lane's k / 32 fp32 fmas in turn, then the simdgroup's five-level sum. +fn rowsDepth(s: c.Shape) usize { + return s.k / 32 + 8; +} + +/// One 8-wide MMA step a k step of 8, accumulating in fp32. +fn tileDepth(s: c.Shape) usize { + return s.k / 8 + 8; +} + +pub fn spec(w: Which) c.Spec { + return switch (w) { + // rows in groups of 16, a column a simdgroup (any n), k in 256-wide lane steps of 8 + .rows => .{ .name = names[0], .accum = .f32, .depth = rowsDepth, .out = .bf16, .tiles = .{ .{ .size = 16 }, .{ .size = 8 }, .{ .size = 256, .step = 8, .min = 8 } }, .served = &rows_served }, + .tile => .{ .name = names[1], .accum = .f32, .depth = tileDepth, .out = .bf16, .tiles = .{ .{ .size = 64 }, .{ .size = 64 }, .{ .size = 32, .step = 32, .min = 32 } }, .served = &tile_served }, + }; +} + +const Args = extern struct { rows: u32, n: u32, k: u32 }; + +pub const Probe = struct { + a: std.mem.Allocator, + rig: rig_.Rig, + which: Which, + mutation: controls.Mutation, + pipe: mtl.Pipeline, + d: ?Dense = null, + x: rig_.Ref = undefined, + w: rig_.Ref = undefined, + out: rig_.Ref = undefined, + + pub fn init(a: std.mem.Allocator, device: mtl.Device, queue: mtl.Queue, w: Which, mutation: controls.Mutation) !Probe { + const text = try controls.source(a, mutation); + defer if (mutation != .none) a.free(text); + const key: []const u8 = if (w == .rows) "lfm2_bf16_rows" else "lfm2_bf16_mma"; + return .{ .a = a, .rig = .{ .a = a, .device = device, .queue = queue }, .which = w, .mutation = mutation, .pipe = try rig_.pipeline(device, text, key) }; + } + + pub fn deinit(p: *Probe) void { + if (p.d) |d| d.deinit(); + p.rig.deinit(); + p.pipe.deinit(); + } + + const vt = g.hooks(Probe); + pub fn kernel(p: *Probe) g.Kernel { + var s = spec(p.which); + if (p.mutation != .none) s.name = p.mutation.name(); + return .{ .spec = s, .ctx = p, .vt = &vt, .control = p.mutation.gate() }; + } + + pub fn prepare(p: *Probe, shape: c.Shape, fill: g.Fill, seed: u64) !bool { + if (shape.rows == 0 or shape.n == 0 or shape.k == 0) return false; + if (p.which == .rows and (shape.k % 8 != 0 or shape.rows > 32)) return false; + if (p.which == .tile and shape.k % 32 != 0) return false; + if (p.d) |d| d.deinit(); + p.d = null; + p.rig.reset(); + const d = try Dense.init(p.a, shape, 1); + p.d = d; + d.fillX(fill, seed, 2); + // W as bf16 bits in [-1, 1), or gate 4's mirrored integers whose halves cancel (partials under 2^24) + const bits = try p.a.alloc(u16, shape.n * shape.k); + defer p.a.free(bits); + var prng = std.Random.DefaultPrng.init(seed ^ 0x6d); + const rng = prng.random(); + const k = shape.k; + for (bits) |*b| b.* = c.toBf16(if (fill == .stress) 0 else rng.float(f32) * 2 - 1); + if (fill == .stress) for (0..shape.n) |j| for (0..k / 2) |i| { + const v: f32 = @floatFromInt(1 + rng.uintLessThan(u32, 7)); + bits[j * k + i] = c.toBf16(v); + bits[j * k + k / 2 + i] = c.toBf16(-v); + }; + for (d.w, bits) |*wv, b| wv.* = c.fromBf16(b); + p.x = try p.rig.input(std.mem.sliceAsBytes(d.x)); + p.w = try p.rig.input(std.mem.sliceAsBytes(bits)); + p.out = try p.rig.output(d.outputs() * c.size(spec(p.which).out)); + return true; + } + + pub fn run(p: *Probe) !void { + const d = p.d.?; + const pass = p.rig.begin(); + const e = pass.e; + e.setPipeline(p.pipe); + e.setBuffer(p.x.buf, p.x.off, 0); + e.setBuffer(p.w.buf, p.w.off, 1); + const args = Args{ .rows = @intCast(d.shape.rows), .n = @intCast(d.shape.n), .k = @intCast(d.shape.k) }; + e.setBytes(std.mem.asBytes(&args), 2); + e.setBuffer(p.out.buf, p.out.off, 3); + if (p.which == .rows) { + e.dispatchGroups(mtl.Size.of((d.shape.n + 7) / 8, (d.shape.rows + 15) / 16, 1), mtl.Size.of(256, 1, 1)); + } else e.dispatchGroups(mtl.Size.of((d.shape.n + 63) / 64, (d.shape.rows + 63) / 64, 1), mtl.Size.of(128, 1, 1)); + try p.rig.finish(pass); + } + + pub fn count(p: *Probe) usize { + return p.d.?.outputs(); + } + + pub fn reference(p: *Probe, want: []f64, mag: []f64) !void { + return p.d.?.reference(want, mag); + } + + pub fn simulate(p: *Probe, acc: c.Accum, order: g.Order, out: []f64) !void { + return p.d.?.simulate(acc, order, out); + } +}; diff --git a/zig/tests/contract/metal_lfm2_controls.zig b/zig/tests/contract/metal_lfm2_controls.zig index 655a97d4fc..bd8a890529 100644 --- a/zig/tests/contract/metal_lfm2_controls.zig +++ b/zig/tests/contract/metal_lfm2_controls.zig @@ -9,6 +9,19 @@ pub const Mutation = enum { argmax_no_flag, conv_first_state, attention_first_cache, + tap_packed_rows, + tap_clobbers_column, + markov_nan_dropped, + markov_first_prev, + draft_ids_unzeroed, + block_attention_causal, + confidence_no_w1, + markov_bf16_nan_dropped, + markov_bf16_first_prev, + bf16_rows_short_k, + bf16_rows_last_row, + bf16_tile_untransposed, + bf16_tile_last_column, pub fn name(m: Mutation) []const u8 { return switch (m) { @@ -17,6 +30,19 @@ pub const Mutation = enum { .argmax_no_flag => "control: metal lfm2 argmax, a nonfinite logit is not flagged", .conv_first_state => "control: metal lfm2 stream conv, every stream reads the first stream's state", .attention_first_cache => "control: metal lfm2 stream attention, every stream writes the first stream's cache", + .tap_packed_rows => "control: metal lfm2 tap copy, rows packed at the hidden size, not the taps' row", + .tap_clobbers_column => "control: metal lfm2 tap copy, a column of another tap is overwritten", + .markov_nan_dropped => "control: metal lfm2 markov step, a NaN score is not flagged", + .markov_first_prev => "control: metal lfm2 markov step, every stream reads the first stream's previous token", + .draft_ids_unzeroed => "control: metal lfm2 draft ids, a pick past the vocabulary keeps its token", + .confidence_no_w1 => "control: metal lfm2 dspark confidence, the previous token's W1 row is not added", + .block_attention_causal => "control: metal lfm2 block attention, a row misses the last key", + .markov_bf16_nan_dropped => "control: metal lfm2 markov step bf16, a NaN score is not flagged", + .markov_bf16_first_prev => "control: metal lfm2 markov step bf16, every stream reads the first stream's previous token", + .bf16_rows_short_k => "control: metal lfm2 bf16 rows, a lane skips every second step of k", + .bf16_rows_last_row => "control: metal lfm2 bf16 rows, a group's last row is not computed", + .bf16_tile_untransposed => "control: metal lfm2 bf16 tile, W's tile is read untransposed", + .bf16_tile_last_column => "control: metal lfm2 bf16 tile, the last column is not written", }; } @@ -24,7 +50,7 @@ pub const Mutation = enum { pub fn gate(m: Mutation) ?contract.gates.Gate { return switch (m) { .none => null, - .argmax_no_flag => .nonfinite, + .argmax_no_flag, .markov_nan_dropped, .markov_bf16_nan_dropped => .nonfinite, else => .shapes, }; } @@ -38,6 +64,19 @@ pub const Mutation = enum { .argmax_no_flag => .{ .from = "if (!isfinite(f)) bad = 1;", .to = "if (false) bad = 1;" }, .conv_first_state => .{ .from = "lfm2_conv_row(bcx + size_t(s.base) * 3 * D, s.conv,", .to = "lfm2_conv_row(bcx + size_t(s.base) * 3 * D, segs[0].conv," }, .attention_first_cache => .{ .from = "queries + b * QH * HD, s.keys, s.values,", .to = "queries + b * QH * HD, segs[0].keys, segs[0].values," }, + .tap_packed_rows => .{ .from = "(taps + size_t(i.y) * at.x + at.y))[i.x]", .to = "(taps + size_t(i.y) * D + at.y))[i.x]" }, + .markov_nan_dropped => .{ .from = "nan = nan || isnan(sc);", .to = "nan = false;" }, + .markov_first_prev => .{ .from = "id = prev[size_t(s) * p.prev_stride];", .to = "id = prev[0];" }, + .draft_ids_unzeroed => .{ .from = "ids[i] = ok ? token : 0u;", .to = "ids[i] = token;" }, + .confidence_no_w1 => .{ .from = "if (id < p.vocab)\n for (uint k = t; k < p.rank", .to = "if (false)\n for (uint k = t; k < p.rank" }, + .block_attention_causal => .{ .from = "count[b] = FULL ? p.len + 1 :", .to = "count[b] = FULL ? p.len :" }, + .markov_bf16_nan_dropped => .{ .from = "nan = nan || isnan(sc);", .to = "nan = false;" }, + .markov_bf16_first_prev => .{ .from = "id = prev[size_t(s) * p.prev_stride];", .to = "id = prev[0];" }, + .bf16_rows_short_k => .{ .from = "for (uint i = 8 * lane; i < p.k; i += 256) {", .to = "for (uint i = 8 * lane; i < p.k; i += 512) {" }, + .bf16_rows_last_row => .{ .from = "if (r >= rows) break;", .to = "if (r + 1 >= rows) break;" }, + .bf16_tile_untransposed => .{ .from = "32, ulong2(0, 0), true);", .to = "32, ulong2(0, 0), false);" }, + .bf16_tile_last_column => .{ .from = "if (row < p.rows && col < p.n) y[", .to = "if (row < p.rows && col + 1 < p.n) y[" }, + .tap_clobbers_column => .{ .from = "[i.x] = from[i.x];\n}\n", .to = "[i.x] = from[i.x];\n *(device bfloat*)(taps + size_t(i.y) * at.x) = 0;\n}\n" }, }; } }; diff --git a/zig/tests/contract/metal_lfm2_dspark.zig b/zig/tests/contract/metal_lfm2_dspark.zig index bd1def8d1b..8979bea1d5 100644 --- a/zig/tests/contract/metal_lfm2_dspark.zig +++ b/zig/tests/contract/metal_lfm2_dspark.zig @@ -7,13 +7,13 @@ const c = contract.spec; const g = contract.gates; const rig_ = @import("rig.zig"); -pub const Which = enum { markov_step, draft_ids }; +pub const Which = enum { markov_step, draft_ids, markov_step_bf16 }; const rank = 256; const vocab = 128000; const span = 512; // tokens a scan threadgroup takes (the kernel's MARKOV_SPAN) const groups = (vocab + span - 1) / span; const table = 1000; // W1's rows in the check: ids are inside it or past the vocabulary (a zero row) -const names = [_][]const u8{ "metal lfm2 markov step", "metal lfm2 draft ids" }; +const names = [_][]const u8{ "metal lfm2 markov step", "metal lfm2 draft ids", "metal lfm2 markov step bf16" }; const stride = 64; // words between a stream's picks: a hold pass's 32 blocks, a token and a flag each const ids_shapes = [_]c.Shape{ .{ .rows = 1, .n = 1, .k = 1 }, .{ .rows = 4, .n = 1, .k = 1 }, .{ .rows = 15, .n = 1, .k = 1 } }; @@ -30,7 +30,7 @@ fn depth(_: c.Shape) usize { pub fn spec(w: Which) c.Spec { return switch (w) { .draft_ids => .{ .name = names[1], .accum = .none, .depth = none, .out = .raw, .served = &ids_shapes, .sums = .{ .f16 = 0, .bf16 = 0, .why = "a copy moves tokens: no sums" } }, - .markov_step => .{ .name = names[0], .accum = .none, .depth = depth, .out = .raw, .served = &step_shapes, .nan = .{ .stated = "a row with a NaN score is flagged nonfinite" }, .sums = .{ .f16 = 0, .bf16 = 0, .why = "picks are tokens: the check bounds each pick's score by gamma(256), not sum bits" } }, + .markov_step, .markov_step_bf16 => .{ .name = if (w == .markov_step) names[0] else names[2], .accum = .none, .depth = depth, .out = .raw, .served = &step_shapes, .nan = .{ .stated = "a row with a NaN score is flagged nonfinite" }, .sums = .{ .f16 = 0, .bf16 = 0, .why = "picks are tokens: the check bounds each pick's score by gamma(256), not sum bits" } }, }; } @@ -60,6 +60,7 @@ pub const Probe = struct { codes: []u8 = &.{}, scales: []u16 = &.{}, biases: []u16 = &.{}, + w2: []u16 = &.{}, // the bf16 step's W2 ids: []u32 = &.{}, args: [8]rig_.Ref = undefined, y: rig_.Ref = undefined, @@ -68,6 +69,7 @@ pub const Probe = struct { pub fn init(a: std.mem.Allocator, device: mtl.Device, queue: mtl.Queue, w: Which) !Probe { const kernels: [2][:0]const u8 = switch (w) { .markov_step => .{ "lfm2_markov_scan", "lfm2_markov_pick" }, + .markov_step_bf16 => .{ "lfm2_markov_scan_bf16", "lfm2_markov_pick" }, .draft_ids => .{ "lfm2_draft_ids", "lfm2_draft_ids" }, }; const first = try rig_.pipeline(device, sources.lfm2, kernels[0]); @@ -77,7 +79,7 @@ pub const Probe = struct { } fn drop(p: *Probe) void { - inline for (.{ "src", "w1", "codes", "scales", "biases", "ids" }) |f| { + inline for (.{ "src", "w1", "codes", "scales", "biases", "w2", "ids" }) |f| { p.a.free(@field(p, f)); @field(p, f) = &.{}; } @@ -132,6 +134,10 @@ pub const Probe = struct { sc.* = c.toBf16(@as(f32, @floatFromInt(i % 7 + 1)) / 4096); bi.* = c.toBf16(@as(f32, @floatFromInt(i % 5)) / 64 - 0.03); } + if (p.which == .markov_step_bf16) { + p.w2 = try p.a.alloc(u16, vocab * rank); + fill(p.w2, seed +% 9, .random); + } // the previous token every second word (the picks' layout): inside W1, its last row, or past the vocabulary p.ids = try p.a.alloc(u32, 2 * s.rows); for (p.ids, 0..) |*id, i| id.* = if (i % 2 == 1) 0xdead else switch ((i / 2) % 3) { @@ -141,7 +147,7 @@ pub const Probe = struct { }; p.args[0] = try p.rig.input(std.mem.sliceAsBytes(p.src)); p.args[1] = try p.rig.input(std.mem.sliceAsBytes(p.w1)); - p.args[2] = try p.rig.input(p.codes); + p.args[2] = if (p.which == .markov_step_bf16) try p.rig.input(std.mem.sliceAsBytes(p.w2)) else try p.rig.input(p.codes); p.args[3] = try p.rig.input(std.mem.sliceAsBytes(p.ids)); const args = [4]u32{ 2 * vocab, rank, vocab, 2 }; p.args[4] = try p.rig.input(std.mem.asBytes(&args)); @@ -155,7 +161,7 @@ pub const Probe = struct { pub fn count(p: *Probe) usize { return switch (p.which) { - .markov_step => p.rows * 8, + .markov_step, .markov_step_bf16 => p.rows * 8, .draft_ids => p.rows * 4, }; } @@ -180,8 +186,10 @@ pub const Probe = struct { for (p.args[0..5], 0..) |r, i| e.setBuffer(r.buf, r.off, i); e.setBuffer(best.buf, best.off, 5); e.setBuffer(at.buf, at.off, 6); - e.setBuffer(p.args[5].buf, p.args[5].off, 7); - e.setBuffer(p.args[6].buf, p.args[6].off, 8); + if (p.which == .markov_step) { + e.setBuffer(p.args[5].buf, p.args[5].off, 7); + e.setBuffer(p.args[6].buf, p.args[6].off, 8); + } e.dispatchGroups(mtl.Size.of(groups, p.rows, 1), mtl.Size.of(256, 1, 1)); e.setPipeline(p.pipes[1]); e.setBuffer(best.buf, best.off, 0); @@ -225,7 +233,7 @@ pub const Probe = struct { var size: f64 = 0; if (id < table) for (0..rank) |i| { const k = (v * rank + i) / 64; - const w = @as(f64, c.fromBf16(p.scales[k])) * @as(f64, @floatFromInt(p.codes[v * rank + i])) + c.fromBf16(p.biases[k]); + const w: f64 = if (p.which == .markov_step_bf16) c.fromBf16(p.w2[v * rank + i]) else @as(f64, c.fromBf16(p.scales[k])) * @as(f64, @floatFromInt(p.codes[v * rank + i])) + c.fromBf16(p.biases[k]); dot += w * c.fromBf16(p.w1[id * rank + i]); size += @abs(w * c.fromBf16(p.w1[id * rank + i])); }; From 487f6a5b2813acffe24e33be744c000270aeaa2a Mon Sep 17 00:00:00 2001 From: BlivionIaG <15323295+BlivionIaG@users.noreply.github.com> Date: Mon, 26 Oct 2026 20:55:29 +0100 Subject: [PATCH 32/39] lfm2: the drafter's verify is not a choice: the confidence head's chances, else the depth rule --draft-verify and --verify-costs are gone with the fixed mode, the cost-table loader and tf-lfm2-verify-costs. A checkpoint with the confidence head verifies by its chances. One without it verifies by the depth rule. --- docs/recipes/lfm2.5-24b-a2b.md | 21 ++---- zig/build/metal_targets.zig | 1 - zig/src/core/engine_api.zig | 6 -- zig/src/families/lfm2_moe/dspark.zig | 37 ++------- zig/src/families/lfm2_moe/dspark_test.zig | 14 ---- zig/src/families/lfm2_moe/lfm2_moe.zig | 1 - zig/src/families/lfm2_moe/verify_costs.zig | 44 ----------- zig/src/native/lfm2_moe.zig | 23 ------ zig/src/native/metal.zig | 1 - zig/src/server/cli.zig | 6 -- zig/src/server/cli_test.zig | 7 -- zig/src/server/main.zig | 2 +- zig/src/tensorfold.zig | 1 - zig/tests/lfm2_dspark.zig | 2 +- zig/tests/lfm2_verify_costs.zig | 87 ---------------------- 15 files changed, 15 insertions(+), 238 deletions(-) delete mode 100644 zig/src/families/lfm2_moe/verify_costs.zig delete mode 100644 zig/tests/lfm2_verify_costs.zig diff --git a/docs/recipes/lfm2.5-24b-a2b.md b/docs/recipes/lfm2.5-24b-a2b.md index bfd79f85a0..e5a86b8761 100644 --- a/docs/recipes/lfm2.5-24b-a2b.md +++ b/docs/recipes/lfm2.5-24b-a2b.md @@ -89,26 +89,19 @@ the drafter's passes; a near tie can pick the other token. The drafter also carries a confidence head: a linear map of a draft's final block row and `W1[previous token]` (2,304 inputs) to one logit, through a sigmoid (`lfm2_dspark_confidence`). It does not change the drafts. -`--draft-verify` picks how they are verified: +The verify rows follow it: a chain's chance at draft j is the product of its first j confidences. Each round the +lane core's window allocator takes the likeliest drafts over all streams while the expected tokens a ms rise, so +each stream gets its own verify length, and the next hold drafts as deep as the stream's grant. A hold still +running on the GPU is priced by its lane's last confidences, so the host does not wait for it. A checkpoint +without the head verifies by the core's depth rule, from each stream's measured acceptance by depth. -- `scheduled` (the default): a chain's chance at draft j is the product of its first j confidences. Each round the - lane core's window allocator takes the likeliest drafts over all streams while the expected tokens a ms rise, so - each stream gets its own verify length, and the next hold drafts as deep as the stream's grant. A hold still - running on the GPU is priced by its lane's last confidences, so the host does not wait for it. -- `depth`: the lane core's depth rule, from each stream's measured acceptance by depth. -- `fixed`: every round verifies the whole block, as LiquidAI's published benchmarks did; a shared round still fits - 32 rows. - -With `scheduled` a stream with a copy ahead verifies the copy and asks no hold, and a drafting stream prices its copy +A stream with a copy ahead verifies the copy and asks no hold, and a drafting stream prices its copy windows by its own copies. The lane core also weighs the hold pass itself: the drafter times holds of 1 to 16 blocks at load and corrects those prices toward each settled hold's GPU time. A lone stream holds drafts only while its drafted rounds measure faster than plain ones (`zig/src/core/lanes/lone_guard.zig`); shared rounds keep the shared-round guard. -`--verify-costs FILE` replaces the costs timed at load with a table from `tf-lfm2-verify-costs MODEL FILE [HISTORY -[IDS...]]` (lines `streams rows_each total_rows ms`), measured on the serving Mac. With token id files each stream -reads its history and verify rows from its own part of the ids, so its rows route its own text's experts. Both flags -are refused unless an `lfm2_moe` checkpoint is served with `--drafter`. `tf-lfm2-dspark` also prints the confidence +`tf-lfm2-dspark` also prints the confidence head against its fp32 reference and times shared holds of 2 to 16 streams. The loader reads the checkpoint one matrix at a time and keeps only the 8-bit codes; a stream's drafter lane is one diff --git a/zig/build/metal_targets.zig b/zig/build/metal_targets.zig index 4bc5ac5da3..ea81d7ea7a 100644 --- a/zig/build/metal_targets.zig +++ b/zig/build/metal_targets.zig @@ -293,7 +293,6 @@ pub fn metalTargets(b: *std.Build, target: std.Build.ResolvedTarget, optimize: s .{ .name = "tf-lfm2-bench", .path = "zig/tests/lfm2_bench.zig", .about = "LFM2-MoE prompt-chunk and one-row decode timing" }, .{ .name = "tf-lfm2-exact", .path = "zig/tests/lfm2_exact.zig", .about = "LFM2-MoE prompt chunks, windows and committed-state exactness" }, .{ .name = "tf-lfm2-dspark", .path = "zig/tests/lfm2_dspark.zig", .about = "LFM2's DSpark drafter on the GPU against an fp32 host reference of the released drafter" }, - .{ .name = "tf-lfm2-verify-costs", .path = "zig/tests/lfm2_verify_costs.zig", .about = "LFM2 verify forward ms by stream count and rows a stream, written as a cost table" }, .{ .name = "tf-lfm2-fp64", .path = "zig/tests/lfm2_fp64.zig", .about = "LFM2 projection classes and attention on the tensor units, the tiles and the twin, each against float64" }, .{ .name = "tensorfold", .path = "zig/src/main.zig", .about = "The native engine's command line" }, .{ .name = "tf-nemotron-fixtures", .path = "zig/tests/nemotron_fixtures.zig", .about = "Nemotron kernels against the Python engine's captured ops" }, diff --git a/zig/src/core/engine_api.zig b/zig/src/core/engine_api.zig index caae72e6a0..6bffc34646 100644 --- a/zig/src/core/engine_api.zig +++ b/zig/src/core/engine_api.zig @@ -274,9 +274,6 @@ pub const Family = struct { model_type: []const u8, formats: []const []const u8, /// The math an NVFP4 checkpoint runs: its own (FP4 rows into NVFP4 layers, FP8 into FP8 ones), or bf16 rows throughout. pub const Precision = enum { checkpoint, full }; -/// An external drafter's verify: the depth rule's rows, its whole block, or its confidence head's survivals. -pub const DraftVerify = enum { depth, fixed, scheduled }; - /// What a server asks of the engine it opens: the checkpoint, and the serve flags an engine reads. pub const Open = struct { pub fn fsrRefusal(o: Open, backend: []const u8) ?[]const u8 { @@ -304,9 +301,6 @@ pub const Open = struct { /// --drafter: a draft model's directory (null: none); --drafter-bits: 4, 0 for bf16, or 8 (LFM2 only). drafter: ?[]const u8 = null, drafter_bits: u8 = 4, - /// --draft-verify and --verify-costs (a tf-lfm2-verify-costs table): the LFM2 DSpark drafter's verify rows. - draft_verify: ?DraftVerify = null, - verify_costs: ?[]const u8 = null, /// Whether --drafter-bits was given: LFM2's drafter runs bf16 as released unless it is asked for 8-bit codes. drafter_bits_given: bool = false, /// --record: each stream's committed rows and tapped features written under this folder, for drafter training. diff --git a/zig/src/families/lfm2_moe/dspark.zig b/zig/src/families/lfm2_moe/dspark.zig index 40766e9fe1..f0b7e112cc 100644 --- a/zig/src/families/lfm2_moe/dspark.zig +++ b/zig/src/families/lfm2_moe/dspark.zig @@ -24,18 +24,8 @@ pub const stage_rows = 128; const hold_grid = [_]u32{ 1, 2, 4, 8, 16 }; const kw = c.kv_heads * c.head_dim; -/// Verify rows by the core's depth rule, the whole block (fixed), or the confidence head's chances (scheduled). -pub const Verify = enum { depth, fixed, scheduled }; -/// The verify when none is asked: the confidence-scheduled one. -pub const default_verify: Verify = .scheduled; - -/// `bits`: 0 runs the bf16 matrices as stored, 8 quantizes them at load; `verify_asked`: --draft-verify given. -pub const Options = struct { capacity: usize, bits: u8 = 0, verify: Verify = default_verify, verify_asked: bool = false }; - -/// An asked scheduled verify needs the checkpoint's confidence head; the default one falls back to the depth rule. -pub fn admitVerify(o: Options, has_head: bool) !void { - if (o.verify == .scheduled and o.verify_asked and !has_head) return error.ScheduledVerifyNeedsConfidenceHead; -} +/// `bits`: 0 runs the bf16 matrices as stored, 8 quantizes them at load. +pub const Options = struct { capacity: usize, bits: u8 = 0 }; /// One stream's context keys and values a layer ([kv head][cap][head dim], as the target's attention reads them). pub const Lane = struct { @@ -115,7 +105,6 @@ pub const Drafter = struct { errdefer d.w.deinit(gpa); errdefer for (d.owned.items) |b| b.deinit(); errdefer d.owned.deinit(gpa); - try admitVerify(options, d.w.confidence != null); try d.scratch(); try d.warm(); return d; @@ -187,9 +176,9 @@ pub const Drafter = struct { return .{ .ptr = d, .vtable = &.{ .taps = tapsFn, .facts = factsFn, .open = openFn, .absorb = absorbFn, .hold = holdFn, .held = heldFn, .release = releaseFn, .chances = chancesFn } }; } - /// Whether holds compute the confidence head (the scheduled verify, on a checkpoint that has the head). + /// Whether holds compute the confidence head: the verify rows follow its chances, else the core's depth rule. pub fn confident(d: *const Drafter) bool { - return d.options.verify == .scheduled and d.w.confidence != null; + return d.w.confidence != null; } fn cast(ptr: *anyopaque) *Drafter { @@ -207,18 +196,8 @@ pub const Drafter = struct { fn factsFn(ptr: *anyopaque) dr.Facts { const d = cast(ptr); var f: dr.Facts = .{ .depth = d.config.block, .step_ms = d.step_ms, .prior = prior[0..d.config.block], .batched = true, .shared_guard = true, .stream_bytes = d.laneBytes(d.options.capacity), .copy_first = true }; - switch (d.options.verify) { - .depth => {}, - .fixed => { // every round the whole block, no plain stretches - f.chances = true; - f.confidence_floor = 0; - f.shared_guard = false; - }, - .scheduled => { - f.chances = d.confident(); - f.hold_costs = d.holds[0..d.holds_len]; - }, - } + f.chances = d.confident(); + f.hold_costs = d.holds[0..d.holds_len]; return f; } @@ -226,10 +205,6 @@ pub const Drafter = struct { fn chancesFn(ptr: *anyopaque, s: *Stream, out: []f64) anyerror!bool { const d = cast(ptr); const l = d.lanes.get(s) orelse return false; - if (d.options.verify == .fixed) { // no wait: equal chances at a depth share a capped round's rows evenly - for (out, 1..) |*o, j| o.* = 1 - 1e-3 * @as(f64, @floatFromInt(j)); - return true; - } if (!d.confident()) return false; if (d.pending != null and l.slot != null and l.conf_len > 0) { // the hold still runs: the lane's last settled confidences price it, so the host does not wait on the GPU diff --git a/zig/src/families/lfm2_moe/dspark_test.zig b/zig/src/families/lfm2_moe/dspark_test.zig index 55c341161c..043a590353 100644 --- a/zig/src/families/lfm2_moe/dspark_test.zig +++ b/zig/src/families/lfm2_moe/dspark_test.zig @@ -210,12 +210,6 @@ test "lfm2 dspark: lfm2_qk on half-split rows and gains gives interleaved RoPE's } } } - -test "lfm2 dspark: the drafter verifies by its confidence head unless asked otherwise" { - const o: @import("dspark.zig").Options = .{ .capacity = 64 }; - try std.testing.expectEqual(@import("dspark.zig").Verify.scheduled, o.verify); -} - test "lfm2 dspark: a settled hold moves the timed passes at its block counts a tenth of the way to its GPU ms" { var d: ds.Drafter = undefined; const grid = [_]lanes.hold_price.HoldCost{ .{ .streams = 1, .depth = 1, .ms = 1 }, .{ .streams = 4, .depth = 1, .ms = 2 }, .{ .streams = 4, .depth = 4, .ms = 4 }, .{ .streams = 8, .depth = 1, .ms = 3 } }; @@ -276,11 +270,3 @@ test "lfm2 dspark: a settled hold refuses a NaN, infinity or invalid pick as the picks[3 * ds.pass_rows] = .{ .nonfinite = 1 }; // a step past the block's depth is not read try block.copyPicks(&picks, &chains, &.{ 2, 2 }); } - -test "lfm2 dspark: an asked scheduled verify without the confidence head is refused by name, the default falls back" { - const asked: ds.Options = .{ .capacity = 64, .verify_asked = true }; - try std.testing.expectError(error.ScheduledVerifyNeedsConfidenceHead, ds.admitVerify(asked, false)); - try ds.admitVerify(asked, true); - try ds.admitVerify(.{ .capacity = 64 }, false); - try ds.admitVerify(.{ .capacity = 64, .verify = .depth, .verify_asked = true }, false); -} diff --git a/zig/src/families/lfm2_moe/lfm2_moe.zig b/zig/src/families/lfm2_moe/lfm2_moe.zig index 75b40e5938..8f861e31b4 100644 --- a/zig/src/families/lfm2_moe/lfm2_moe.zig +++ b/zig/src/families/lfm2_moe/lfm2_moe.zig @@ -11,4 +11,3 @@ pub const dspark = @import("dspark.zig"); pub const dspark_config = @import("dspark_config.zig"); pub const dspark_block = @import("dspark_block.zig"); pub const dspark_quant = @import("dspark_quant.zig"); -pub const verify_costs = @import("verify_costs.zig"); diff --git a/zig/src/families/lfm2_moe/verify_costs.zig b/zig/src/families/lfm2_moe/verify_costs.zig deleted file mode 100644 index f80e4cdcef..0000000000 --- a/zig/src/families/lfm2_moe/verify_costs.zig +++ /dev/null @@ -1,44 +0,0 @@ -//! A verify cost table from tf-lfm2-verify-costs: shared forward ms by total rows, for the lane core's shared costs. -const std = @import("std"); -const lanes = @import("lanes"); -const st = @import("state.zig"); - -const Cost = lanes.config.Cost; - -/// Lines `streams rows_each total_rows ms` ('#' comments): each total's ms at the fewest streams that reach it. -pub fn parse(gpa: std.mem.Allocator, text: []const u8) ![]Cost { - var best: [st.batch_rows + 1]?struct { streams: u32, ms: f64 } = @splat(null); - var lines = std.mem.tokenizeAny(u8, text, "\r\n"); - while (lines.next()) |line| { - const body = std.mem.trim(u8, line, " \t"); - if (body.len == 0 or body[0] == '#') continue; - var fields = std.mem.tokenizeAny(u8, body, " \t"); - var v: [4][]const u8 = undefined; - for (&v) |*x| x.* = fields.next() orelse return error.BadVerifyCosts; - if (fields.next() != null) return error.BadVerifyCosts; - const streams = try std.fmt.parseInt(u32, v[0], 10); - const each = try std.fmt.parseInt(u32, v[1], 10); - const total = try std.fmt.parseInt(u32, v[2], 10); - const ms = try std.fmt.parseFloat(f64, v[3]); - if (streams == 0 or each == 0 or total != streams * each or total > st.batch_rows or !(ms > 0)) return error.BadVerifyCosts; - if (best[total]) |b| if (b.streams <= streams) continue; - best[total] = .{ .streams = streams, .ms = ms }; - } - var out: std.ArrayList(Cost) = .empty; - errdefer out.deinit(gpa); - for (best, 0..) |b, total| if (b) |x| try out.append(gpa, .{ .width = @intCast(total), .ms = x.ms }); - if (out.items.len == 0) return error.BadVerifyCosts; - return out.toOwnedSlice(gpa); -} - -test "lfm2 dspark: a verify cost table gives each total its fewest streams' ms and refuses malformed lines" { - const gpa = std.testing.allocator; - const t = try parse(gpa, "# streams rows_each total_rows ms\n1 1 1 3.7\n2 1 2 4.9\n1 2 2 4.6\n\n4 8 32 40.5\n"); - defer gpa.free(t); - try std.testing.expectEqual(@as(usize, 3), t.len); - try std.testing.expectEqual(@as(f64, 4.6), t[1].ms); // one stream of two rows, not two of one - try std.testing.expectEqual(@as(u32, 32), t[2].width); - try std.testing.expectError(error.BadVerifyCosts, parse(gpa, "2 2 5 4.0\n")); - try std.testing.expectError(error.BadVerifyCosts, parse(gpa, "8 8 64 9.0\n")); - try std.testing.expectError(error.BadVerifyCosts, parse(gpa, "# nothing\n")); -} diff --git a/zig/src/native/lfm2_moe.zig b/zig/src/native/lfm2_moe.zig index d1c180391a..d5114d2a0c 100644 --- a/zig/src/native/lfm2_moe.zig +++ b/zig/src/native/lfm2_moe.zig @@ -111,14 +111,6 @@ pub fn open(a: std.mem.Allocator, gpa: std.mem.Allocator, io: std.Io, o: api.Ope } errdefer if (h.wrap) |*w| w.deinit(); var facts = if (h.wrap) |*w| w.facts(h.metal.facts()) else h.metal.facts(); - const profiled = if (h.draft != null) verifyCosts(a, io, o.verify_costs) catch |err| { - problem.* = try std.fmt.allocPrint(a, "--verify-costs {s} is not a tf-lfm2-verify-costs table ({s})", .{ o.verify_costs.?, @errorName(err) }); - h.wrap.?.deinit(); - h.draft.?.deinit(); - release(h); - return null; - } else null; - if (profiled) |t| facts.shared_costs = t; // a drafting stream prices its copy windows by its own copies: on distinct prompts they keep breaking if (h.wrap != null) facts.own_copy_rates = true; h.config = try lanes.Config.init(gpa, facts, q.state.window_rows, q.state.window_rows - 1); @@ -168,21 +160,6 @@ pub fn open(a: std.mem.Allocator, gpa: std.mem.Allocator, io: std.Io, o: api.Ope return .{ .engine = h.host.engine(), .close = Host.close, .ctx = h }; } -/// --verify-costs: a tf-lfm2-verify-costs table's shared forward ms by total rows (null: none named; `a`-owned). -fn verifyCosts(a: std.mem.Allocator, io: std.Io, path: ?[]const u8) !?[]const lanes.config.Cost { - const file = path orelse return null; - const text = try std.Io.Dir.cwd().readFileAlloc(io, file, a, .limited(1 << 20)); - const t = try q.verify_costs.parse(a, text); - std.log.info("lfm2: verify costs from {s}, {d} totals of rows", .{ file, t.len }); - return t; -} - -/// --draft-verify as the drafter's verify (null: the default). -fn verifyMode(v: ?api.DraftVerify) q.dspark.Verify { - const asked = v orelse return q.dspark.default_verify; - return std.meta.stringToEnum(q.dspark.Verify, @tagName(asked)).?; -} - /// The prompt cache's copies of a stream: the target's state at a mark (backend.zig), and the drafter's context there. const Snaps = struct { const S = q.backend.Snaps; diff --git a/zig/src/native/metal.zig b/zig/src/native/metal.zig index eddee040ee..104bbfc22b 100644 --- a/zig/src/native/metal.zig +++ b/zig/src/native/metal.zig @@ -337,7 +337,6 @@ fn refusal(a: Allocator, o: api.Open, dense: bool) !?[]const u8 { const one_mac: ?[]const u8 = if (std.mem.eql(u8, t, "nemotron_h")) "Nemotron" else if (std.mem.eql(u8, t, "lfm2_moe")) "LFM2.5-24B-A2B" else if (isGemma4(t)) "Gemma 4" else if (std.mem.eql(u8, t, "step5_vl")) "Step-5" else if (std.mem.eql(u8, t, "spark2_5")) "Spark-X2.6" else if (!std.mem.eql(u8, t, "qwen3_5")) null else if (dense) "Qwen3.8-27B" else "Qwen3.5-2B"; if (o.speed_up != null) if (one_mac) |name| return try std.fmt.allocPrint(a, "--speed-up joins two Macs for Flash Next, GLM-5.3-Flash and GLM-5.3 only: serve {s} without it", .{name}); if (o.recall and std.mem.eql(u8, t, "qwen4_exp")) return recall_refusal; - if ((o.draft_verify != null or o.verify_costs != null) and (o.drafter == null or !std.mem.eql(u8, t, "lfm2_moe"))) return "--draft-verify and --verify-costs set the verify of LFM2.5-24B-A2B's DSpark drafter: serve it with --drafter"; if (spill.refusal(o)) |why| return why; if (o.learn == null) return null; // the 27B writes conversation states diff --git a/zig/src/server/cli.zig b/zig/src/server/cli.zig index 6e3d16654e..010160f588 100644 --- a/zig/src/server/cli.zig +++ b/zig/src/server/cli.zig @@ -102,8 +102,6 @@ pub const flags = [_]Flag{ .{ .name = "--slide-graph", .native = true }, .{ .name = "--drafter", .native = true }, .{ .name = "--drafter-bits", .choices = &.{ "0", "4", "8" }, .native = true }, - .{ .name = "--draft-verify", .choices = &.{ "depth", "fixed", "scheduled" }, .native = builtin.os.tag == .macos }, - .{ .name = "--verify-costs", .native = builtin.os.tag == .macos }, .{ .name = "--record", .native = true }, .{ .name = "--kv-cache", .choices = &.{ "bf16", "fp8" }, .native = true }, .{ .name = "--prompt-precision", .choices = &.{ "bf16", "int8" }, .native = true }, @@ -220,8 +218,6 @@ pub const Args = struct { compact_memory: ?[]const u8 = null, drafter: ?[]const u8 = null, drafter_bits: ?u8 = null, // null: not given (a family without its own default takes 4) - draft_verify: ?api.DraftVerify = null, // the LFM2 DSpark drafter's verify rows (null: the engine's default) - verify_costs: ?[]const u8 = null, // a tf-lfm2-verify-costs table pricing them (null: costs timed at load) record: ?[]const u8 = null, // drafter training rows written under this folder kv_cache: []const u8 = "bf16", // fp8: e4m3 attention caches with a scale a row, about twice the context prompt_precision: []const u8 = "bf16", // int8: prompt matmuls quantize their inputs to int8 a 64-input group @@ -553,8 +549,6 @@ pub fn request(a: Allocator, dir: []const u8, model_type: []const u8, args: Args .drafter = args.drafter, .drafter_bits = args.drafter_bits orelse 4, .drafter_bits_given = args.drafter_bits != null, - .draft_verify = args.draft_verify, - .verify_costs = args.verify_costs, .record = args.record, .kv8 = std.mem.eql(u8, args.kv_cache, "fp8"), .prompt_int8 = std.mem.eql(u8, args.prompt_precision, "int8"), diff --git a/zig/src/server/cli_test.zig b/zig/src/server/cli_test.zig index b6cfc59f71..123e4dfe10 100644 --- a/zig/src/server/cli_test.zig +++ b/zig/src/server/cli_test.zig @@ -167,13 +167,6 @@ test "parse and capabilities share the table" { try std.testing.expectEqual(@as(?u8, 8), (try parse(a, &.{ "m", "--drafter-bits", "8" }, &u)).drafter_bits); try std.testing.expectEqual(@as(?u8, null), (try parse(a, &.{ "m", "--drafter", "x" }, &u)).drafter_bits); try std.testing.expectError(error.Usage, parse(a, &.{ "m", "--drafter-bits", "3" }, &u)); - if (@import("builtin").os.tag == .macos) { - const verify = try parse(a, &.{ "m", "--draft-verify", "scheduled", "--verify-costs", "/t" }, &u); - try std.testing.expectEqual(api.DraftVerify.scheduled, verify.draft_verify.?); - try std.testing.expectEqual(api.DraftVerify.scheduled, (try request(a, "d", "t", verify)).draft_verify.?); - try std.testing.expectEqualStrings("/t", (try request(a, "d", "t", verify)).verify_costs.?); - try std.testing.expectError(error.Usage, parse(a, &.{ "m", "--draft-verify", "wide" }, &u)); - } try std.testing.expectEqualStrings("/r", (try parse(a, &.{ "m", "--record", "/r" }, &u)).record.?); try std.testing.expectEqualStrings("fp8", (try parse(a, &.{ "m", "--kv-cache", "fp8" }, &u)).kv_cache); try std.testing.expectError(error.Usage, parse(a, &.{ "m", "--kv-cache", "int4" }, &u)); diff --git a/zig/src/server/main.zig b/zig/src/server/main.zig index c4d1742897..2506a58854 100644 --- a/zig/src/server/main.zig +++ b/zig/src/server/main.zig @@ -26,7 +26,7 @@ const usage_line = "usage: tensorfold serve [-h] [--host HOST] [--port PORT] [-- " [--thinking | --no-thinking] [--reasoning-effort {low,medium,high,xhigh}]" ++ " [--thinking-budget THINKING_BUDGET] [--thinking-reserve THINKING_RESERVE] [--loop-guard] [--loop-guard-stop] [--no-drafts]" ++ " [--drafter DRAFTER]" ++ - " [--drafter-bits {0,4,8}] [--draft-verify {depth,fixed,scheduled}] [--verify-costs VERIFY_COSTS] [--record RECORD] [--keep-warm SECONDS] [--idle-unload SECONDS]" ++ + " [--drafter-bits {0,4,8}] [--record RECORD] [--keep-warm SECONDS] [--idle-unload SECONDS]" ++ " [--silence-floor-dbfs DBFS] [--silence-seconds SECONDS]" ++ " [--silence-onset-s SECONDS] [--silence-onset-voiced-s SECONDS]" ++ " [--silence-mid-s SECONDS] [--silence-mid-min-voiced-s SECONDS] [--silence-redraws N]" ++ diff --git a/zig/src/tensorfold.zig b/zig/src/tensorfold.zig index 0191cabc8e..cd9d82bc08 100644 --- a/zig/src/tensorfold.zig +++ b/zig/src/tensorfold.zig @@ -91,7 +91,6 @@ test { _ = @import("families/lfm2_moe/dspark_config.zig"); _ = @import("families/lfm2_moe/dspark_quant.zig"); _ = @import("families/lfm2_moe/dspark_test.zig"); - _ = @import("families/lfm2_moe/verify_costs.zig"); _ = @import("families/lfm2_moe/dspark_bf16_test.zig"); } diff --git a/zig/tests/lfm2_dspark.zig b/zig/tests/lfm2_dspark.zig index a1814189c3..4fb630fb13 100644 --- a/zig/tests/lfm2_dspark.zig +++ b/zig/tests/lfm2_dspark.zig @@ -178,7 +178,7 @@ pub fn main(init: std.process.Init) !void { defer pool.pop(); const m = try q.Model.load(gpa, init.io, args[1]); defer m.deinit(); - const d = try q.dspark.Drafter.init(gpa, init.io, m, args[2], .{ .capacity = length + 64, .bits = bits, .verify = .scheduled }); + const d = try q.dspark.Drafter.init(gpa, init.io, m, args[2], .{ .capacity = length + 64, .bits = bits }); defer d.deinit(); const cfg = d.config; // the compared block's rows (a hold of depth d runs a d-row block) diff --git a/zig/tests/lfm2_verify_costs.zig b/zig/tests/lfm2_verify_costs.zig deleted file mode 100644 index f3100d633e..0000000000 --- a/zig/tests/lfm2_verify_costs.zig +++ /dev/null @@ -1,87 +0,0 @@ -//! LFM2 verify forwards timed by stream count and rows a stream: the round costs a DSpark verify budget weighs. -const std = @import("std"); -const mtl = @import("metal"); -const q = @import("tensorfold").lfm2_moe; -const st = q.state; -const c = q.config; - -fn now() i128 { - var ts: std.c.timespec = undefined; - _ = std.c.clock_gettime(.MONOTONIC, &ts); - return @as(i128, ts.sec) * std.time.ns_per_s + ts.nsec; -} - -/// Stream counts profiled; each runs every rows-a-stream width whose total fits a shared forward. -const counts = [_]usize{ 1, 2, 3, 4, 6, 8, 12, 16 }; -const reps = 9; - -fn readIds(gpa: std.mem.Allocator, io: std.Io, path: []const u8, list: *std.ArrayList(u32)) !void { - const bytes = try std.Io.Dir.cwd().readFileAlloc(io, path, gpa, .limited(1 << 22)); - defer gpa.free(bytes); - var it = std.mem.tokenizeAny(u8, bytes, " \n\r\t,"); - while (it.next()) |t| try list.append(gpa, try std.fmt.parseInt(u32, t, 10)); -} - -pub fn main(init: std.process.Init) !void { - const gpa = init.gpa; - const args = try init.minimal.args.toSlice(init.arena.allocator()); - if (args.len < 3) return error.ExpectedModelOutputHistory; - var history: usize = if (args.len >= 4) try std.fmt.parseInt(usize, args[3], 10) else 1024; - // Token id files: every file's ids end to end; stream i reads its history and then its rows from its own sixteenth. - var real: std.ArrayList(u32) = .empty; - defer real.deinit(gpa); - for (args[@min(args.len, 4)..]) |path| try readIds(gpa, init.io, path, &real); - const part = real.items.len / 16; - if (real.items.len > 0) { - if (part < st.window_rows + 64) return error.IdFilesTooShort; - history = @min(history, part - st.window_rows); - } - const pool = mtl.objc.Pool.push(); - defer pool.pop(); - const m = try q.Model.load(gpa, init.io, args[1]); - defer m.deinit(); - var scratch = try st.Scratch.init(gpa, m.device, &m.config.attention, @max(history, st.batch_rows)); - defer scratch.deinit(); - var caches: [16]st.Cache = undefined; - var made: usize = 0; - defer for (caches[0..made]) |*k| k.deinit(); - const ids = try gpa.alloc(u32, history); - defer gpa.free(ids); - for (caches[0..], 0..) |*k, i| { - k.* = try st.Cache.init(gpa, m.device, &m.config.attention, history + st.window_rows + 1); - made += 1; - for (ids, 0..) |*id, j| id.* = if (part > 0) real.items[i * part + j] else @intCast((j * 7919 + 13 + i * 104729) % 120000); - try q.forward.run(m, &scratch, &.{.{ .cache = k, .rows = history }}, ids, false, .last); - } - var out: std.Io.Writer.Allocating = .init(gpa); - defer out.deinit(); - const w = &out.writer; - try w.print("# LFM2 verify forward ms (median of {d}) by streams and rows a stream, {d}-token histories, {s} rows\n", .{ reps, history, if (part > 0) "real" else "synthetic" }); - try w.print("# streams rows_each total_rows ms\n", .{}); - var segments: [st.batch_rows]q.forward.Segment = undefined; - var tokens: [st.batch_rows]u32 = undefined; - for (&tokens, 0..) |*t, i| t.* = @intCast((i * 31 + 7) % 120000); - for (counts) |streams| { - var each: usize = 1; - while (each <= st.window_rows and streams * each <= st.batch_rows) : (each += 1) { - for (segments[0..streams], caches[0..streams], 0..) |*s, *k, i| { - s.* = .{ .cache = k, .rows = each }; - // real rows: the tokens after the stream's history, as a verify of drafts that all land - if (part > 0) @memcpy(tokens[i * each ..][0..each], real.items[i * part + history ..][0..each]); - } - var times: [reps + 2]f64 = undefined; - for (×) |*t| { - const t0 = now(); - try q.forward.run(m, &scratch, segments[0..streams], tokens[0 .. streams * each], true, .all); - t.* = @as(f64, @floatFromInt(now() - t0)) / 1e6; - for (caches[0..streams]) |*k| k.len = history; - } - const kept = times[2..]; // the first two calls warm a new width's pipelines - std.mem.sort(f64, kept, {}, std.sort.asc(f64)); - const ms = kept[reps / 2]; - try w.print("{d} {d} {d} {d:.3}\n", .{ streams, each, streams * each, ms }); - std.debug.print("{d} streams x {d} rows ({d}): {d:.3} ms\n", .{ streams, each, streams * each, ms }); - } - } - try std.Io.Dir.cwd().writeFile(init.io, .{ .sub_path = args[2], .data = out.written() }); -} From 14565fa73451591bb30a1b68f243df2042d49c49 Mon Sep 17 00:00:00 2001 From: BlivionIaG <15323295+BlivionIaG@users.noreply.github.com> Date: Wed, 28 Oct 2026 09:48:03 +0100 Subject: [PATCH 33/39] lfm2: address DSpark stack review and retain main contracts --- docs/kernel-contract.md | 2 +- docs/recipes/lfm2.5-24b-a2b.md | 10 +++--- zig/kernels/metal/lfm2/lfm2.metal | 1 - zig/kernels/metal/lfm2/lfm2_dspark.metal | 12 +++++++ zig/src/core/lanes/depth.zig | 15 ++++++-- zig/src/core/lanes/hold_price.zig | 2 +- zig/src/families/lfm2_moe/dspark.zig | 36 +++++++++++++++++--- zig/src/families/lfm2_moe/dspark_config.zig | 6 ++-- zig/src/families/lfm2_moe/dspark_quant.zig | 2 +- zig/src/native/lfm2_moe.zig | 5 +++ zig/tests/contract/metal_lfm2_attn.zig | 30 ++++++---------- zig/tests/contract/metal_lfm2_confidence.zig | 24 +++++-------- zig/tests/contract/metal_lfm2_dspark.zig | 32 ++++++----------- zig/tests/contract/metal_lfm2_tap.zig | 24 +++++-------- zig/tests/contract/metal_main.zig | 17 ++++++++- 15 files changed, 128 insertions(+), 90 deletions(-) diff --git a/docs/kernel-contract.md b/docs/kernel-contract.md index 31e9054632..6449dad3cc 100644 --- a/docs/kernel-contract.md +++ b/docs/kernel-contract.md @@ -135,7 +135,7 @@ tile_qmm's pipelines and GLM-5.3-Flash's hc_expand_10. LFM2.5-24B-A2B's shared rounds: the greedy pick (`lfm2_argmax`) and the conv and attention kernels that run every stream's rows in one dispatch, each bit-equal to the per-stream kernels it replaces. Four controls edit the source: a tie to the highest id, a nonfinite logit not flagged, and every stream reading or writing the first stream's state or cache. -The DSpark drafter's tap copy, Markov scan and pick, draft ids and block attention are there too, with six more controls. +The DSpark drafter's tap copy, Markov scan and pick, draft ids, block attention and confidence logit are there too, with seven more controls. Its bf16 linears (few rows, 64x64 tiles) and the Markov scan over a bf16 W2 have four and two more. Their launches are in `lfm2_records.zig` with the M1 and M2 limits from `--offline`; `lfm2_attn_segs` and `lfm2_attn_block` reserve their 128 threads in their source, as Metal cannot build them for M1 or M2 at another count. diff --git a/docs/recipes/lfm2.5-24b-a2b.md b/docs/recipes/lfm2.5-24b-a2b.md index e5a86b8761..9f13a18e37 100644 --- a/docs/recipes/lfm2.5-24b-a2b.md +++ b/docs/recipes/lfm2.5-24b-a2b.md @@ -77,8 +77,7 @@ checkpoint's interleaved RoPE. Its own kernels are `lfm2_tap`, `lfm2_attn_block` (non-causal block attention, the target's attention kernels with a template flag), `lfm2_markov_scan` and `lfm2_markov_pick` (a Markov step) and `lfm2_draft_ids` (a hold's drafts into the next verify's rows on the GPU, so the host does not wait for a hold that fits one pass). Each kept prompt state keeps the drafter's context too, so a -drafting stream resumes prompts. Shared rounds take the lane core's shared-round guard: they draft only while -drafted rounds measure faster than plain ones at that stream count. +drafting stream resumes prompts. The lane core prices drafts across the admitted stream counts. `tf-kernel-contract --only lfm2` checks these kernels. `tf-lfm2-dspark MODEL DRAFTER TOKENS [ROWS]` runs the drafter against an fp32 host reference of the released checkpoint (the same taps and the target's embedding and head), @@ -97,14 +96,13 @@ without the head verifies by the core's depth rule, from each stream's measured A stream with a copy ahead verifies the copy and asks no hold, and a drafting stream prices its copy windows by its own copies. The lane core also weighs the hold pass itself: the drafter times holds of 1 to 16 -blocks at load and corrects those prices toward each settled hold's GPU time. A lone stream holds drafts only while -its drafted rounds measure faster than plain ones (`zig/src/core/lanes/lone_guard.zig`); shared rounds keep the -shared-round guard. +blocks at load and corrects those prices toward each settled hold's GPU time. `tf-lfm2-dspark` also prints the confidence head against its fp32 reference and times shared holds of 2 to 16 streams. -The loader reads the checkpoint one matrix at a time and keeps only the 8-bit codes; a stream's drafter lane is one +The loader reads the checkpoint one matrix at a time and keeps the released bf16 matrices by default, or the +8-bit codes with `--drafter-bits 8`; a stream's drafter lane is one Metal buffer (its layers' keys and values, then its staged rows), and its bytes join the streams the load limit admits. At load the drafter logs its weights, scratch and a stream's bytes. `tf-lfm2-dspark` also counts host allocations over twenty served drafter rounds, which must be none. diff --git a/zig/kernels/metal/lfm2/lfm2.metal b/zig/kernels/metal/lfm2/lfm2.metal index 76c223805d..2da8578970 100644 --- a/zig/kernels/metal/lfm2/lfm2.metal +++ b/zig/kernels/metal/lfm2/lfm2.metal @@ -353,7 +353,6 @@ template } template [[host_name("lfm2_attn_mma")]] [[kernel]] decltype(lfm2_attn_mma<1>) lfm2_attn_mma<1>; template [[host_name("lfm2_attn_mma_r2")]] [[kernel]] decltype(lfm2_attn_mma<2>) lfm2_attn_mma<2>; -template [[host_name("lfm2_attn_block")]] [[kernel]] decltype(lfm2_attn_mma<1, true>) lfm2_attn_mma<1, true>; // The chunks merged in chunk order for (head, row): Y [rows, QH*HD] bf16. template diff --git a/zig/kernels/metal/lfm2/lfm2_dspark.metal b/zig/kernels/metal/lfm2/lfm2_dspark.metal index a8685313d2..7440805007 100644 --- a/zig/kernels/metal/lfm2/lfm2_dspark.metal +++ b/zig/kernels/metal/lfm2/lfm2_dspark.metal @@ -244,3 +244,15 @@ constant constexpr uint BF16_ROWS = 16; if (row < p.rows && col < p.n) y[size_t(row) * p.n + col] = static_cast(out[sg][i]); } } + +// A drafter block's attention: lfm2_attn_mma where every row sees all keys, its 128 threads reserved for M1 and M2. +[[max_total_threads_per_threadgroup(128)]] [[kernel]] void lfm2_attn_block( + const device bfloat* queries [[buffer(0)]], const device bfloat* keys [[buffer(1)]], + const device bfloat* values [[buffer(2)]], constant lfm2_attn_args& p [[buffer(3)]], device float* pm [[buffer(4)]], + device float* pl [[buffer(5)]], device float* po [[buffer(6)]], constant uint& rows [[buffer(7)]], + uint3 tg [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]], + uint sg [[simdgroup_index_in_threadgroup]], uint lane [[thread_index_in_simdgroup]]) { + threadgroup uint ks[2][HD * (KT / 2)]; + threadgroup bfloat vs[2][KB * KP]; + lfm2_attn_rows<1, true>(queries, keys, values, p, pm, pl, po, rows, tg, tid, sg, lane, ks, vs); +} diff --git a/zig/src/core/lanes/depth.zig b/zig/src/core/lanes/depth.zig index c992e0de4b..6620dcb69c 100644 --- a/zig/src/core/lanes/depth.zig +++ b/zig/src/core/lanes/depth.zig @@ -78,7 +78,10 @@ pub const Rule = struct { shared_rounds: u64 = 0, greedy_overhead: std.ArrayList(Overhead) = .empty, // overhead from rounds without a sampled stream + hold_order: []usize = &.{}, // reused hold ordering, at most batch_rows entries + pub fn deinit(r: *Rule) void { + r.gpa.free(r.hold_order); r.round_ms.deinit(r.gpa); r.overhead.deinit(r.gpa); r.greedy_ms.deinit(r.gpa); @@ -243,8 +246,9 @@ pub const Rule = struct { /// Granted holds kept, most expected drafts a row first, while they raise the next round's expected tokens a ms. fn payHolds(r: *Rule, whos: []const Who, sampled: bool, out: []i64) !void { - const order = try r.gpa.alloc(usize, whos.len); - defer r.gpa.free(order); + if (whos.len > r.batch_rows) return error.BadHoldStreamCount; + if (r.hold_order.len == 0) r.hold_order = try r.gpa.alloc(usize, r.batch_rows); + const order = r.hold_order[0..whos.len]; var m: usize = 0; var probes: usize = 0; // a probe after a grant of none keeps its hold: it measures the stream again var deepest: i64 = 0; @@ -548,6 +552,13 @@ test "a granted hold stays only while its stream's expected drafts pay for the v var out = [_]i64{ 4, 4, 0, 3 }; try rule.payHolds(&whos, false, &out); try std.testing.expectEqualSlices(i64, &.{ 4, 0, 0, 3 }, &out); + var failing = std.testing.FailingAllocator.init(gpa, .{ .fail_index = 0 }); + rule.gpa = failing.allocator(); + out = .{ 4, 4, 0, 3 }; + try rule.payHolds(&whos, false, &out); + try std.testing.expect(!failing.has_induced_failure); + try std.testing.expectEqualSlices(i64, &.{ 4, 0, 0, 3 }, &out); + rule.gpa = gpa; const dear = [_]HoldCost{.{ .streams = 1, .depth = 1, .ms = 20.0 }}; rule.k.hold_costs = &dear; out = .{ 4, 4, 0, 3 }; diff --git a/zig/src/core/lanes/hold_price.zig b/zig/src/core/lanes/hold_price.zig index e4c854410e..807531a6cd 100644 --- a/zig/src/core/lanes/hold_price.zig +++ b/zig/src/core/lanes/hold_price.zig @@ -30,7 +30,7 @@ pub fn holdMs(costs: []const HoldCost, streams: usize, depth: i64) f64 { return at_lo + t * (atDepth(costs, hi, depth) - at_lo); } -/// One stream count's hold ms at `depth`: linear between its two nearest timed depths (one alone: as timed). +/// One stream count's hold ms at `depth`: linear through its two smallest timed depths (one alone: as timed). fn atDepth(costs: []const HoldCost, streams: u32, depth: i64) f64 { var a: ?HoldCost = null; var b: ?HoldCost = null; diff --git a/zig/src/families/lfm2_moe/dspark.zig b/zig/src/families/lfm2_moe/dspark.zig index f0b7e112cc..6993f95fcc 100644 --- a/zig/src/families/lfm2_moe/dspark.zig +++ b/zig/src/families/lfm2_moe/dspark.zig @@ -103,11 +103,17 @@ pub const Drafter = struct { errdefer d.event.deinit(); d.w = try dw.load(gpa, target.device, io, dir, &d.config, options.bits); errdefer d.w.deinit(gpa); - errdefer for (d.owned.items) |b| b.deinit(); - errdefer d.owned.deinit(gpa); + try d.prepare(); + return d; + } + + fn prepare(d: *Drafter) !void { + errdefer { + for (d.owned.items) |b| b.deinit(); + d.owned.deinit(d.gpa); + } try d.scratch(); try d.warm(); - return d; } pub fn deinit(d: *Drafter) void { @@ -195,7 +201,7 @@ pub const Drafter = struct { fn factsFn(ptr: *anyopaque) dr.Facts { const d = cast(ptr); - var f: dr.Facts = .{ .depth = d.config.block, .step_ms = d.step_ms, .prior = prior[0..d.config.block], .batched = true, .shared_guard = true, .stream_bytes = d.laneBytes(d.options.capacity), .copy_first = true }; + var f: dr.Facts = .{ .depth = d.config.block, .step_ms = d.step_ms, .prior = prior[0..d.config.block], .batched = true, .stream_bytes = d.laneBytes(d.options.capacity), .copy_first = true }; f.chances = d.confident(); f.hold_costs = d.holds[0..d.holds_len]; return f; @@ -549,3 +555,25 @@ pub const Drafter = struct { }; } }; + +test "lfm2 dspark: scratch and warm allocation failures release buffers before their list" { + const device = mtl.Device.init() catch return error.SkipZigTest; + defer device.deinit(); + var target: Model = undefined; + target.device = device; + for ([_]usize{ 1, 3 }) |fail_index| { + var failing = std.testing.FailingAllocator.init(std.testing.allocator, .{ .fail_index = fail_index }); + var d: Drafter = .{ + .gpa = failing.allocator(), + .io = undefined, + .target = &target, + .config = try dc.Config.parse(std.testing.allocator, dc.stock), + .w = undefined, + .options = .{ .capacity = 64 }, + .s = undefined, + }; + try std.testing.expectError(error.OutOfMemory, d.prepare()); + try std.testing.expect(failing.has_induced_failure); + try std.testing.expectEqual(failing.allocated_bytes, failing.freed_bytes); + } +} diff --git a/zig/src/families/lfm2_moe/dspark_config.zig b/zig/src/families/lfm2_moe/dspark_config.zig index 79e518aa48..68acc6d30d 100644 --- a/zig/src/families/lfm2_moe/dspark_config.zig +++ b/zig/src/families/lfm2_moe/dspark_config.zig @@ -66,8 +66,8 @@ pub const Config = struct { if (head != .string or !std.mem.eql(u8, head.string, "vanilla")) return error.UnsupportedMarkovHead; if (o.get("sample_from_anchor")) |v| if (v != .bool or !v.bool) return error.UnsupportedDraftAnchor; if (out.layers == 0 or out.layers > 8 or out.block < 2 or out.block > 16) return error.UnsupportedDraftShape; - if (out.intermediate == 0 or out.intermediate % 64 != 0 or out.rank == 0 or out.rank % 64 != 0) return error.UnsupportedDraftShape; - if (out.mask >= c.vocab or !(out.theta > 0)) return error.BadDraftConfig; + if (out.intermediate == 0 or out.intermediate % 64 != 0 or out.rank == 0 or out.rank > 256 or out.rank % 64 != 0) return error.UnsupportedDraftShape; + if (out.mask >= c.vocab or !std.math.isFinite(out.theta) or !(out.theta > 0)) return error.BadDraftConfig; return out; } }; @@ -136,5 +136,7 @@ test "lfm2 dspark: other drafters and shapes are refused with their reason" { try std.testing.expectError(error.UnsupportedDraftMask, edited("\"block_size\"", "\"is_causal\":true,\"block_size\"")); try std.testing.expectError(error.UnsupportedDraftAnchor, edited("\"block_size\"", "\"sample_from_anchor\":false,\"block_size\"")); try std.testing.expectError(error.UnsupportedDraftShape, edited("\"block_size\":9", "\"block_size\":17")); + try std.testing.expectError(error.UnsupportedDraftShape, edited("\"markov_rank\":256", "\"markov_rank\":512")); + try std.testing.expectError(error.BadDraftConfig, edited("10000000.0", "1e100")); try std.testing.expectError(error.BadDraftConfig, edited("\"mask_token_id\":125017", "\"mask_token_id\":128000")); } diff --git a/zig/src/families/lfm2_moe/dspark_quant.zig b/zig/src/families/lfm2_moe/dspark_quant.zig index a6a5baccd6..a34b52dd20 100644 --- a/zig/src/families/lfm2_moe/dspark_quant.zig +++ b/zig/src/families/lfm2_moe/dspark_quant.zig @@ -85,7 +85,7 @@ test "lfm2 dspark: 4- and 8-bit codes keep the larger edge exact, as MLX quantiz var codes: [64]u8 = undefined; var s: u16 = undefined; var b: u16 = undefined; - group(&x, bits, codes[0 .. 64 * bits / 8], &s, &b); + group(&x, bits, codes[0 .. 64 * @as(usize, bits) / 8], &s, &b); try std.testing.expect(toF32(s) < 0 and toF32(b) == 5.875); // the max is the edge: codes descend from it const last: u8 = if (bits == 8) codes[63] else codes[31] >> 4; try std.testing.expectEqual(@as(u8, 0), last); diff --git a/zig/src/native/lfm2_moe.zig b/zig/src/native/lfm2_moe.zig index d5114d2a0c..3734c41a3c 100644 --- a/zig/src/native/lfm2_moe.zig +++ b/zig/src/native/lfm2_moe.zig @@ -82,6 +82,11 @@ pub fn open(a: std.mem.Allocator, gpa: std.mem.Allocator, io: std.Io, o: api.Ope h.draft = null; h.wrap = null; if (if (o.drafts) o.drafter else null) |dir| { + if (h.metal.width == 1) { + problem.* = "--drafter: LFM2 wider lanes failed the local exactness check; drafts require verified wider lanes"; + release(h); + return null; + } const bits: u8 = if (o.drafter_bits_given) o.drafter_bits else 0; if (bits != 0 and bits != 8) { problem.* = "--drafter-bits: the LFM2 DSpark drafter takes 0 (bf16 as released, the default) or 8 (8-bit codes quantized at load)"; diff --git a/zig/tests/contract/metal_lfm2_attn.zig b/zig/tests/contract/metal_lfm2_attn.zig index 799b337c59..60fa6939ae 100644 --- a/zig/tests/contract/metal_lfm2_attn.zig +++ b/zig/tests/contract/metal_lfm2_attn.zig @@ -6,6 +6,7 @@ const sources = @import("kernel_sources"); const c = contract.spec; const g = contract.gates; const rig_ = @import("rig.zig"); +const controls = @import("metal_lfm2_controls.zig"); const qh = 32; const kvh = 8; @@ -34,6 +35,7 @@ fn value(i: usize, seed: u64, scale: f32) u16 { pub const Probe = struct { a: std.mem.Allocator, rig: rig_.Rig, + mutation: controls.Mutation, pipes: [2]mtl.Pipeline, shape: c.Shape = undefined, cap: usize = 0, @@ -44,10 +46,12 @@ pub const Probe = struct { args: [5]rig_.Ref = undefined, y: rig_.Ref = undefined, - pub fn init(a: std.mem.Allocator, device: mtl.Device, queue: mtl.Queue) !Probe { - const first = try rig_.pipeline(device, sources.lfm2, "lfm2_attn_block"); + pub fn init(a: std.mem.Allocator, device: mtl.Device, queue: mtl.Queue, mutation: controls.Mutation) !Probe { + const text = try controls.source(a, mutation); + defer if (mutation != .none) a.free(text); + const first = try rig_.pipeline(device, text, "lfm2_attn_block"); errdefer first.deinit(); - return .{ .a = a, .rig = .{ .a = a, .device = device, .queue = queue }, .pipes = .{ first, try rig_.pipeline(device, sources.lfm2, "lfm2_attn_block_merge") } }; + return .{ .a = a, .rig = .{ .a = a, .device = device, .queue = queue }, .pipes = .{ first, try rig_.pipeline(device, text, "lfm2_attn_block_merge") }, .mutation = mutation }; } fn drop(p: *Probe) void { @@ -66,7 +70,9 @@ pub const Probe = struct { const vt = g.hooks(Probe); pub fn kernel(p: *Probe) g.Kernel { - return .{ .ctx = p, .vt = &vt, .spec = spec() }; + var s = spec(); + if (p.mutation != .none) s.name = p.mutation.name(); + return .{ .ctx = p, .vt = &vt, .spec = s, .control = p.mutation.gate() }; } pub fn prepare(p: *Probe, s: c.Shape, f: g.Fill, seed: u64) !bool { @@ -164,19 +170,3 @@ pub const Probe = struct { return null; } }; - -/// Gate 5 for the block attention's two launches on every Apple chip we serve and this GPU. -pub fn limits(a: std.mem.Allocator, device: mtl.Device, only: ?[]const u8) !usize { - var problems: usize = 0; - const launches = [_]struct { []const u8, []const u8, u32, u32 }{ .{ "metal lfm2 block attention", "lfm2_attn_block", 128, 18688 }, .{ "metal lfm2 block attention merge", "lfm2_attn_block_merge", 32, 0 } }; - for (launches) |l| { - if (only) |s| if (std.mem.indexOf(u8, l[0], s) == null) continue; - const d = contract.limits.MetalLaunch{ .name = l[0], .threads = l[2], .tg_memory = l[3], .offline = false }; - var notes: std.ArrayList(u8) = .empty; - defer notes.deinit(a); - for (contract.limits.apple) |chip| problems += try contract.limits.metal(d, chip, ¬es, a); - std.debug.print("{s}", .{notes.items}); - problems += try @import("metal_limits.zig").onThisGpu(a, device, .{ .decl = d, .function = l[1], .here = sources.lfm2, .pre_m5 = "" }, false); - } - return problems; -} diff --git a/zig/tests/contract/metal_lfm2_confidence.zig b/zig/tests/contract/metal_lfm2_confidence.zig index f1543a183d..ebf74ea5f2 100644 --- a/zig/tests/contract/metal_lfm2_confidence.zig +++ b/zig/tests/contract/metal_lfm2_confidence.zig @@ -7,6 +7,7 @@ const c = contract.spec; const g = contract.gates; const rows = contract.rows; const rig_ = @import("rig.zig"); +const controls = @import("metal_lfm2_controls.zig"); const hidden = 2048; const rank = 256; @@ -28,12 +29,15 @@ pub const Probe = struct { a: std.mem.Allocator, rig: rig_.Rig, pipe: mtl.Pipeline, + mutation: controls.Mutation, r: ?rows.Rows = null, args: [6]rig_.Ref = undefined, // x, W1 rows, ids, picks, the head's weights, the shape out: rig_.Ref = undefined, - pub fn init(a: std.mem.Allocator, device: mtl.Device, queue: mtl.Queue) !Probe { - return .{ .a = a, .rig = .{ .a = a, .device = device, .queue = queue }, .pipe = try rig_.pipeline(device, sources.lfm2, "lfm2_dspark_confidence") }; + pub fn init(a: std.mem.Allocator, device: mtl.Device, queue: mtl.Queue, mutation: controls.Mutation) !Probe { + const text = try controls.source(a, mutation); + defer if (mutation != .none) a.free(text); + return .{ .a = a, .rig = .{ .a = a, .device = device, .queue = queue }, .pipe = try rig_.pipeline(device, text, "lfm2_dspark_confidence"), .mutation = mutation }; } pub fn deinit(p: *Probe) void { @@ -44,7 +48,9 @@ pub const Probe = struct { const vt = g.hooks(Probe); pub fn kernel(p: *Probe) g.Kernel { - return .{ .ctx = p, .vt = &vt, .spec = spec() }; + var s = spec(); + if (p.mutation != .none) s.name = p.mutation.name(); + return .{ .ctx = p, .vt = &vt, .spec = s, .control = p.mutation.gate() }; } /// Row i's first 2,048 inputs as the block's row i, its last 256 as W1 row i: the token before draft i is i. @@ -102,15 +108,3 @@ pub const Probe = struct { return p.r.?.simulate(acc, order, out); } }; - -/// Gate 5 for the launch on every Apple chip we serve and this GPU: 256 threads, the eight simdgroups' partials. -pub fn limits(a: std.mem.Allocator, device: mtl.Device, only: ?[]const u8) !usize { - if (only) |s| if (std.mem.indexOf(u8, name, s) == null) return 0; - const d = contract.limits.MetalLaunch{ .name = name, .threads = 256, .tg_memory = 32, .offline = false }; - var notes: std.ArrayList(u8) = .empty; - defer notes.deinit(a); - var problems: usize = 0; - for (contract.limits.apple) |chip| problems += try contract.limits.metal(d, chip, ¬es, a); - std.debug.print("{s}", .{notes.items}); - return problems + try @import("metal_limits.zig").onThisGpu(a, device, .{ .decl = d, .function = "lfm2_dspark_confidence", .here = sources.lfm2, .pre_m5 = "" }, false); -} diff --git a/zig/tests/contract/metal_lfm2_dspark.zig b/zig/tests/contract/metal_lfm2_dspark.zig index 8979bea1d5..04b7fc489a 100644 --- a/zig/tests/contract/metal_lfm2_dspark.zig +++ b/zig/tests/contract/metal_lfm2_dspark.zig @@ -6,6 +6,7 @@ const sources = @import("kernel_sources"); const c = contract.spec; const g = contract.gates; const rig_ = @import("rig.zig"); +const controls = @import("metal_lfm2_controls.zig"); pub const Which = enum { markov_step, draft_ids, markov_step_bf16 }; const rank = 256; @@ -53,6 +54,7 @@ pub const Probe = struct { a: std.mem.Allocator, rig: rig_.Rig, which: Which, + mutation: controls.Mutation, pipes: [2]mtl.Pipeline, rows: usize = 0, src: []u16 = &.{}, // the base logits rows @@ -66,16 +68,18 @@ pub const Probe = struct { y: rig_.Ref = undefined, bad: rig_.Ref = undefined, // draft_ids: the flag word it sets for a flagged or invalid pick, zeroed each run - pub fn init(a: std.mem.Allocator, device: mtl.Device, queue: mtl.Queue, w: Which) !Probe { + pub fn init(a: std.mem.Allocator, device: mtl.Device, queue: mtl.Queue, w: Which, mutation: controls.Mutation) !Probe { + const text = try controls.source(a, mutation); + defer if (mutation != .none) a.free(text); const kernels: [2][:0]const u8 = switch (w) { .markov_step => .{ "lfm2_markov_scan", "lfm2_markov_pick" }, .markov_step_bf16 => .{ "lfm2_markov_scan_bf16", "lfm2_markov_pick" }, .draft_ids => .{ "lfm2_draft_ids", "lfm2_draft_ids" }, }; - const first = try rig_.pipeline(device, sources.lfm2, kernels[0]); + const first = try rig_.pipeline(device, text, kernels[0]); errdefer first.deinit(); - const second = try rig_.pipeline(device, sources.lfm2, kernels[1]); - return .{ .a = a, .rig = .{ .a = a, .device = device, .queue = queue }, .which = w, .pipes = .{ first, second } }; + const second = try rig_.pipeline(device, text, kernels[1]); + return .{ .a = a, .rig = .{ .a = a, .device = device, .queue = queue }, .which = w, .mutation = mutation, .pipes = .{ first, second } }; } fn drop(p: *Probe) void { @@ -94,7 +98,9 @@ pub const Probe = struct { const vt = g.hooks(Probe); pub fn kernel(p: *Probe) g.Kernel { - return .{ .ctx = p, .vt = &vt, .spec = spec(p.which) }; + var s = spec(p.which); + if (p.mutation != .none) s.name = p.mutation.name(); + return .{ .ctx = p, .vt = &vt, .spec = s, .control = p.mutation.gate() }; } fn fill(out: []u16, seed: u64, f: g.Fill) void { @@ -256,19 +262,3 @@ pub const Probe = struct { return null; } }; - -/// Gate 5 for the three launches on every Apple chip and this GPU: ids 32 threads, others 256, the scan 1,088 bytes. -pub fn limits(a: std.mem.Allocator, device: mtl.Device, only: ?[]const u8) !usize { - var problems: usize = 0; - const launches = [_]struct { []const u8, []const u8, u32, u32 }{ .{ "metal lfm2 markov scan", "lfm2_markov_scan", 1088, 256 }, .{ "metal lfm2 markov pick", "lfm2_markov_pick", 3072, 256 }, .{ "metal lfm2 draft ids", "lfm2_draft_ids", 0, 32 } }; - for (launches) |l| { - if (only) |s| if (std.mem.indexOf(u8, l[0], s) == null) continue; - const d = contract.limits.MetalLaunch{ .name = l[0], .threads = l[3], .tg_memory = l[2], .offline = false }; - var notes: std.ArrayList(u8) = .empty; - defer notes.deinit(a); - for (contract.limits.apple) |chip| problems += try contract.limits.metal(d, chip, ¬es, a); - std.debug.print("{s}", .{notes.items}); - problems += try @import("metal_limits.zig").onThisGpu(a, device, .{ .decl = d, .function = l[1], .here = sources.lfm2, .pre_m5 = "" }, false); - } - return problems; -} diff --git a/zig/tests/contract/metal_lfm2_tap.zig b/zig/tests/contract/metal_lfm2_tap.zig index 871ea8a8a2..021baffc9e 100644 --- a/zig/tests/contract/metal_lfm2_tap.zig +++ b/zig/tests/contract/metal_lfm2_tap.zig @@ -6,6 +6,7 @@ const sources = @import("kernel_sources"); const c = contract.spec; const g = contract.gates; const rig_ = @import("rig.zig"); +const controls = @import("metal_lfm2_controls.zig"); const hidden = 2048; const taps = 5; @@ -41,14 +42,17 @@ pub const Probe = struct { a: std.mem.Allocator, rig: rig_.Rig, pipe: mtl.Pipeline, + mutation: controls.Mutation, rows: usize = 0, src: []u16 = &.{}, // H rows prior: []u16 = &.{}, // the taps rows as the call finds them (the other taps' columns must stay) args: [2]rig_.Ref = undefined, y: rig_.Ref = undefined, - pub fn init(a: std.mem.Allocator, device: mtl.Device, queue: mtl.Queue) !Probe { - return .{ .a = a, .rig = .{ .a = a, .device = device, .queue = queue }, .pipe = try rig_.pipeline(device, sources.lfm2, "lfm2_tap") }; + pub fn init(a: std.mem.Allocator, device: mtl.Device, queue: mtl.Queue, mutation: controls.Mutation) !Probe { + const text = try controls.source(a, mutation); + defer if (mutation != .none) a.free(text); + return .{ .a = a, .rig = .{ .a = a, .device = device, .queue = queue }, .pipe = try rig_.pipeline(device, text, "lfm2_tap"), .mutation = mutation }; } fn drop(p: *Probe) void { @@ -67,7 +71,9 @@ pub const Probe = struct { const vt = g.hooks(Probe); pub fn kernel(p: *Probe) g.Kernel { - return .{ .ctx = p, .vt = &vt, .spec = spec() }; + var s = spec(); + if (p.mutation != .none) s.name = p.mutation.name(); + return .{ .ctx = p, .vt = &vt, .spec = s, .control = p.mutation.gate() }; } fn fill(out: []u16, seed: u64, f: g.Fill) void { @@ -129,15 +135,3 @@ pub const Probe = struct { return null; } }; - -/// Gate 5 for the tap's launch of 256 threads and no threadgroup memory: every Apple chip we serve, then this GPU. -pub fn limits(a: std.mem.Allocator, device: mtl.Device, only: ?[]const u8) !usize { - if (only) |s| if (std.mem.indexOf(u8, name, s) == null) return 0; - const d = contract.limits.MetalLaunch{ .name = name, .threads = 256, .tg_memory = 0, .offline = false }; - var notes: std.ArrayList(u8) = .empty; - defer notes.deinit(a); - var problems: usize = 0; - for (contract.limits.apple) |chip| problems += try contract.limits.metal(d, chip, ¬es, a); - std.debug.print("{s}", .{notes.items}); - return problems + try @import("metal_limits.zig").onThisGpu(a, device, .{ .decl = d, .function = "lfm2_tap", .here = sources.lfm2, .pre_m5 = "" }, false); -} diff --git a/zig/tests/contract/metal_main.zig b/zig/tests/contract/metal_main.zig index 4007a7566a..52a0d205c3 100644 --- a/zig/tests/contract/metal_main.zig +++ b/zig/tests/contract/metal_main.zig @@ -20,17 +20,18 @@ const glm_rows_k = @import("metal_glm_rows.zig"); const flash_pool = @import("metal_flashnext_pool.zig"); const flash = @import("metal_flashnext.zig"); +const fsr = @import("metal_fsr.zig"); const gemma = @import("metal_gemma4.zig"); const gemma_embed = @import("metal_gemma4_embed.zig"); const gemma_mixed = @import("metal_gemma4_mixed.zig"); const lfm2_argmax = @import("metal_lfm2_argmax.zig"); const lfm2_streams = @import("metal_lfm2_streams.zig"); +const lfm2_controls = @import("metal_lfm2_controls.zig"); const lfm2_tap = @import("metal_lfm2_tap.zig"); const lfm2_dspark = @import("metal_lfm2_dspark.zig"); const lfm2_bf16 = @import("metal_lfm2_bf16.zig"); const lfm2_attn = @import("metal_lfm2_attn.zig"); const lfm2_conf = @import("metal_lfm2_confidence.zig"); -const lfm2_controls = @import("metal_lfm2_controls.zig"); const g = contract.gates; const usage = "usage: tf-kernel-contract [--only TEXT] [--no-gates] [--no-limits] [--offline DIR] [--seed N] [--repro] [--rotary-long] [--rotary-accuracy FILE]"; @@ -325,6 +326,15 @@ pub fn main(init: std.process.Init) !u8 { try kernels.append(a, flash_list[flash_made - 1].kernel()); } }; + var fsr_probes: [4]fsr.Probe = undefined; + var fsr_made: usize = 0; + defer for (fsr_probes[0..fsr_made]) |*probe| probe.deinit(); + for ([_]usize{ 4, 6 }) |bits| for ([_]bool{ false, true }) |control| { + if (!wanted(o, fsr.Probe.name(bits, control))) continue; + fsr_probes[fsr_made] = .{ .a = a, .device = device, .queue = queue, .bits = bits, .control = control }; + fsr_made += 1; + try kernels.append(a, fsr_probes[fsr_made - 1].kernel()); + }; var d3norm: ?drama3_k.Norm = null; defer if (d3norm) |*n| n.deinit(); if (wanted(o, drama3_k.spec().name)) { @@ -416,6 +426,9 @@ pub fn main(init: std.process.Init) !u8 { var glm_windows: glm_rows_k.Suite = .{ .a = a, .device = device, .queue = queue, .kernels = &kernels }; defer glm_windows.deinit(); try glm_windows.collect(o.only); + var nemotron_experts: @import("metal_nemotron_experts.zig").Suite = .{ .a = a, .device = device, .queue = queue, .kernels = &kernels }; + defer nemotron_experts.deinit(); + try nemotron_experts.collect(o.only); const weight_only = @import("metal_weight_only.zig"); const weight_source = try weight_only.text(a); defer a.free(weight_source); @@ -454,6 +467,8 @@ pub fn main(init: std.process.Init) !u8 { defer native_ple.destroy(); const native_gdn = try @import("metal_qwen4_gdn.zig").Suite.create(a, device, queue, &kernels, o.only); defer native_gdn.destroy(); + const moss_image = try @import("metal_moss_image.zig").Suite.create(a, device, queue, &kernels, o.only); + defer moss_image.destroy(); var t: g.Tally = .{}; const t0 = std.Io.Clock.awake.now(init.io); if (o.gates) { From 017919fbd1b7bdb54429b0622c6e985844ae544a Mon Sep 17 00:00:00 2001 From: BlivionIaG <15323295+BlivionIaG@users.noreply.github.com> Date: Wed, 28 Oct 2026 09:57:03 +0100 Subject: [PATCH 34/39] docs: record DSpark review checks and load-time peaks --- docs/receipts/lfm2-dspark-review/README.md | 80 + .../lfm2-dspark-review/equality-and-load.json | 4517 +++++++++++++++++ 2 files changed, 4597 insertions(+) create mode 100644 docs/receipts/lfm2-dspark-review/README.md create mode 100644 docs/receipts/lfm2-dspark-review/equality-and-load.json diff --git a/docs/receipts/lfm2-dspark-review/README.md b/docs/receipts/lfm2-dspark-review/README.md new file mode 100644 index 0000000000..9db7cad86d --- /dev/null +++ b/docs/receipts/lfm2-dspark-review/README.md @@ -0,0 +1,80 @@ +# DSpark review fixes on current main + +Source `14565fa73`, rebased onto main `d0c9dc9fb`, tested on 2026-10-28. +Apple M4 Pro, 20 GPU cores, 48 GiB RAM, macOS 26.5.1, Zig 0.17.0, +Apple Metal compiler 32023.883. Builds target `apple_m1`. + +The shared and lone guards from the rejected PRs are absent. The lane core retains +confidence/grant pricing, hold-cost pricing and `copy_first`. Hold ordering reuses +one bounded allocation instead of allocating on every round. The initialization +cleanup releases Metal buffers before their owning list; injected scratch and warm +allocation failures exercise that path. Config admission refuses rank above 256 and +nonfinite f32 RoPE theta. The quantizer test widens its size calculation before +multiplication, and `--drafter` refuses a target restricted to one-row verification. + +## Checks + +- `zig build test test-kernel-contract test-golden fuzz-cortheon native dev-tools -Dcpu=apple_m1 --summary all`: + 749/749 steps succeeded; 8,414 tests passed, 46 skipped, 0 failed. +- `zig build -Dcpu=apple_m1 --summary all`: 239/239 steps succeeded. +- `zig build test-lfm2 test-lfm2-dspark -Dcpu=apple_m1 --summary all`: + 111/111 tests passed. The final scratch/warm failure test also passed in a subsequent + focused run: 51/51 DSpark tests. +- `zig build test-lfm2-dspark -Dcpu=apple_m1 -Doptimize=debug --summary all`: + 51/51 passed, including both injected allocation failures. +- Debug `zig test` on `dspark_config.zig`: 4/4 passed; on `dspark_quant.zig`: 2/2 passed. + The original quantizer test from `2e5966f68` aborted with integer overflow in Debug. +- `zig fmt --check zig` and `git diff --check`: passed. +- Lean: 428 findings on main, 428 on this source, 0 new. +- `tf-kernel-contract --only lfm2`: 11 kernels passed gates 1-4, 0 failures, + 0 known findings; all 17 planted controls red, 0 missed; 0 limit problems. +- `tf-kernel-contract --only tile_qmm`: 6 kernels passed gates 1-4, 0 failures, + 0 known findings; 0 limit problems. + +## Served equality + +Target `LiquidAI/LFM2.5-24B-A2B-MLX-4bit` at +`667408d494bde191c2cb9dd4f4549827816b9275`, with the local released DSpark weights. +The drafter safetensors SHA-256 is recorded in the accompanying JSON. + +For both default bf16 and explicit `--drafter-bits 8`, serve with `--parallel 16 +--context 4096 --prompt-cache-gib 0 --keep-warm 0`. Run: + +```sh +python3 tools/bench_concurrent.py http://127.0.0.1:18194 review \ + --tokens 64 --reps 1 --temperatures 0 --levels 1,4,8,16 --alone --serial +python3 tools/bench_concurrent.py http://127.0.0.1:18194 review \ + --tokens 64 --reps 1 --temperatures 0 --levels 1,4,8,16 --alone --serial \ + --prompts tools/bench_prompts.json +``` + +Each precision mode passed all 12 cells: eight same-prompt cells (code and chat) +and four distinct-prompt cells. Concurrent replies equal solo replies and drafted +solo replies equal `"draft": false` token hashes. Zero unequal replies and zero +failed requests. [Equality evidence](equality-and-load.json) retains per-request +concurrent, solo and plain token hashes. + +These are short correctness checks, with one repetition, while builds were also +running. Their timings do not establish the decode performance gate. + +## Peak during load + +One model process at a time under a model lock. Both servers used the same target +and bf16 drafter, `--parallel 1 --context 4096 --prompt-cache-gib 0 --keep-warm 0`. +Read `/memory` after readiness and before sending any request. Its lifetime peak +agreed with `/usr/bin/time -l` at process exit. + +- Exact #1392 head `1bb6e5db2`: 16,036,097,352 bytes, 14.935 GiB. +- Corrected stack source `14565fa73`: 15,527,930,776 bytes, 14.462 GiB. + +The measured reduction during load is 508,166,576 bytes (484.6 MiB). This compares +the original pre-allocation head with the corrected stack rebased onto current +main; it is not an isolated comparison of #1393's commit on the same base. + +## Remaining qualification + +No new long-context, cancellation/replacement stress, resumed-serving, sampled +serving or alternating speed qualification is claimed here. The maintainer's +requested current-main performance cells and other-family A/B remain to be run. +M1, M2, M3, M5, CUDA and HIP were not run for this revision. The focused Metal +contract checks above are not the full cross-family `kernel-contract` gate. diff --git a/docs/receipts/lfm2-dspark-review/equality-and-load.json b/docs/receipts/lfm2-dspark-review/equality-and-load.json new file mode 100644 index 0000000000..f66ec3b0d4 --- /dev/null +++ b/docs/receipts/lfm2-dspark-review/equality-and-load.json @@ -0,0 +1,4517 @@ +{ + "source_commit": "14565fa73", + "base_commit": "d0c9dc9fb", + "target_revision": "667408d494bde191c2cb9dd4f4549827816b9275", + "drafter_weights_sha256": "7d06a78556b0b1f386134508f09b00a3a7e0e5e0ec2cc73119229f4ef7c9b299", + "tokens": 64, + "seed": 1234, + "repetitions": 1, + "ignore_eos": true, + "prompt_cache_gib": 0, + "load_comparison": { + "before-alloc-bf16-load": { + "parallel": 1, + "context": 4096, + "mode": "bf16", + "memory": { + "source": "darwin_rusage_info_v4", + "physical_footprint_bytes": 15464950680, + "lifetime_peak_physical_footprint_bytes": 16036097352 + } + }, + "candidate-bf16-load": { + "parallel": 1, + "context": 4096, + "mode": "bf16", + "memory": { + "source": "darwin_rusage_info_v4", + "physical_footprint_bytes": 15504747440, + "lifetime_peak_physical_footprint_bytes": 15527930776 + } + } + }, + "cells": [ + { + "mode": "bf16", + "pool": "same", + "prompt": "code", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "alone": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "token_hash_comparisons": [ + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + } + ] + }, + { + "mode": "bf16", + "pool": "same", + "prompt": "code", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "alone": { + "equal": 4, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "token_hash_comparisons": [ + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + } + ] + }, + { + "mode": "bf16", + "pool": "same", + "prompt": "code", + "temperature": 0.0, + "streams": 8, + "failed": 0, + "alone": { + "equal": 8, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "token_hash_comparisons": [ + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + } + ] + }, + { + "mode": "bf16", + "pool": "same", + "prompt": "code", + "temperature": 0.0, + "streams": 16, + "failed": 0, + "alone": { + "equal": 16, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "token_hash_comparisons": [ + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + } + ] + }, + { + "mode": "bf16", + "pool": "same", + "prompt": "chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "alone": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "token_hash_comparisons": [ + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + } + ] + }, + { + "mode": "bf16", + "pool": "same", + "prompt": "chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "alone": { + "equal": 4, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "token_hash_comparisons": [ + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + } + ] + }, + { + "mode": "bf16", + "pool": "same", + "prompt": "chat", + "temperature": 0.0, + "streams": 8, + "failed": 0, + "alone": { + "equal": 8, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "token_hash_comparisons": [ + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + } + ] + }, + { + "mode": "bf16", + "pool": "same", + "prompt": "chat", + "temperature": 0.0, + "streams": 16, + "failed": 0, + "alone": { + "equal": 16, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "token_hash_comparisons": [ + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + } + ] + }, + { + "mode": "bf16", + "pool": "distinct", + "prompt": "distinct", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "alone": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 16, + "unequal": 0, + "failed": 0 + }, + "token_hash_comparisons": [ + { + "concurrent": { + "tokens": 64, + "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0", + "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0", + "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0", + "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7", + "complete": true, + "error": null + } + } + ] + }, + { + "mode": "bf16", + "pool": "distinct", + "prompt": "distinct", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "alone": { + "equal": 4, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 16, + "unequal": 0, + "failed": 0 + }, + "token_hash_comparisons": [ + { + "concurrent": { + "tokens": 64, + "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0", + "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0", + "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0", + "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "d99ee3b3ea17e26e8c5bbdadd3aa09e747b018ae41cf6eabe0b24a55eecfce4f", + "output_sha256": "ce549c26dbcd1281bf48df56c9e67359c959db7d0b1dec8bd78a0736364660e2", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "d99ee3b3ea17e26e8c5bbdadd3aa09e747b018ae41cf6eabe0b24a55eecfce4f", + "output_sha256": "ce549c26dbcd1281bf48df56c9e67359c959db7d0b1dec8bd78a0736364660e2", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "d99ee3b3ea17e26e8c5bbdadd3aa09e747b018ae41cf6eabe0b24a55eecfce4f", + "output_sha256": "ce549c26dbcd1281bf48df56c9e67359c959db7d0b1dec8bd78a0736364660e2", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "62bdf45358eadd229505c80c2e1d91b089d5eee70eb3644df4ec7347c496ad89", + "output_sha256": "502fedcdf2669b346ef9c7a01dc4004453e243072f995334caa465e57be9c309", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "62bdf45358eadd229505c80c2e1d91b089d5eee70eb3644df4ec7347c496ad89", + "output_sha256": "502fedcdf2669b346ef9c7a01dc4004453e243072f995334caa465e57be9c309", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "62bdf45358eadd229505c80c2e1d91b089d5eee70eb3644df4ec7347c496ad89", + "output_sha256": "502fedcdf2669b346ef9c7a01dc4004453e243072f995334caa465e57be9c309", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "d10f504ba0ea3e42aabf8d121c59354a0f6c4159a313f96f50cf28c2f109ec10", + "output_sha256": "c5f9d75515c1bdd5eef1de9bf1d825efead755ffb6c40d9fa211e9b73639a2e3", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "d10f504ba0ea3e42aabf8d121c59354a0f6c4159a313f96f50cf28c2f109ec10", + "output_sha256": "c5f9d75515c1bdd5eef1de9bf1d825efead755ffb6c40d9fa211e9b73639a2e3", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "d10f504ba0ea3e42aabf8d121c59354a0f6c4159a313f96f50cf28c2f109ec10", + "output_sha256": "c5f9d75515c1bdd5eef1de9bf1d825efead755ffb6c40d9fa211e9b73639a2e3", + "complete": true, + "error": null + } + } + ] + }, + { + "mode": "bf16", + "pool": "distinct", + "prompt": "distinct", + "temperature": 0.0, + "streams": 8, + "failed": 0, + "alone": { + "equal": 8, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 16, + "unequal": 0, + "failed": 0 + }, + "token_hash_comparisons": [ + { + "concurrent": { + "tokens": 64, + "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0", + "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0", + "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0", + "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "d99ee3b3ea17e26e8c5bbdadd3aa09e747b018ae41cf6eabe0b24a55eecfce4f", + "output_sha256": "ce549c26dbcd1281bf48df56c9e67359c959db7d0b1dec8bd78a0736364660e2", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "d99ee3b3ea17e26e8c5bbdadd3aa09e747b018ae41cf6eabe0b24a55eecfce4f", + "output_sha256": "ce549c26dbcd1281bf48df56c9e67359c959db7d0b1dec8bd78a0736364660e2", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "d99ee3b3ea17e26e8c5bbdadd3aa09e747b018ae41cf6eabe0b24a55eecfce4f", + "output_sha256": "ce549c26dbcd1281bf48df56c9e67359c959db7d0b1dec8bd78a0736364660e2", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "62bdf45358eadd229505c80c2e1d91b089d5eee70eb3644df4ec7347c496ad89", + "output_sha256": "502fedcdf2669b346ef9c7a01dc4004453e243072f995334caa465e57be9c309", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "62bdf45358eadd229505c80c2e1d91b089d5eee70eb3644df4ec7347c496ad89", + "output_sha256": "502fedcdf2669b346ef9c7a01dc4004453e243072f995334caa465e57be9c309", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "62bdf45358eadd229505c80c2e1d91b089d5eee70eb3644df4ec7347c496ad89", + "output_sha256": "502fedcdf2669b346ef9c7a01dc4004453e243072f995334caa465e57be9c309", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "d10f504ba0ea3e42aabf8d121c59354a0f6c4159a313f96f50cf28c2f109ec10", + "output_sha256": "c5f9d75515c1bdd5eef1de9bf1d825efead755ffb6c40d9fa211e9b73639a2e3", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "d10f504ba0ea3e42aabf8d121c59354a0f6c4159a313f96f50cf28c2f109ec10", + "output_sha256": "c5f9d75515c1bdd5eef1de9bf1d825efead755ffb6c40d9fa211e9b73639a2e3", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "d10f504ba0ea3e42aabf8d121c59354a0f6c4159a313f96f50cf28c2f109ec10", + "output_sha256": "c5f9d75515c1bdd5eef1de9bf1d825efead755ffb6c40d9fa211e9b73639a2e3", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "d80d7a1d94635a9a5ce9305bf22133261181963691b70407d3212b0557e0d458", + "output_sha256": "f04d026e2ba2a0f5b7d7043b15ba913e60871b429828345b00314bd3151438ee", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "d80d7a1d94635a9a5ce9305bf22133261181963691b70407d3212b0557e0d458", + "output_sha256": "f04d026e2ba2a0f5b7d7043b15ba913e60871b429828345b00314bd3151438ee", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "d80d7a1d94635a9a5ce9305bf22133261181963691b70407d3212b0557e0d458", + "output_sha256": "f04d026e2ba2a0f5b7d7043b15ba913e60871b429828345b00314bd3151438ee", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "3c1a59254dd7797cdbe77a227eac7a15a006369026625cfb85d60753b32d31c5", + "output_sha256": "55e8f7833f37b58eb8fed5d68942ffc4fe5efe5a1690b22078fa23ef6a49d43f", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "3c1a59254dd7797cdbe77a227eac7a15a006369026625cfb85d60753b32d31c5", + "output_sha256": "55e8f7833f37b58eb8fed5d68942ffc4fe5efe5a1690b22078fa23ef6a49d43f", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "3c1a59254dd7797cdbe77a227eac7a15a006369026625cfb85d60753b32d31c5", + "output_sha256": "55e8f7833f37b58eb8fed5d68942ffc4fe5efe5a1690b22078fa23ef6a49d43f", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "42a0abe417221a6383219ba07c3a2afe852e0331d879b46a6d43e16a5cff220c", + "output_sha256": "d55ce007379ff24d98434b745f2ae1972ec4b662574ab6fc65fbaa12e55fdf5b", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "42a0abe417221a6383219ba07c3a2afe852e0331d879b46a6d43e16a5cff220c", + "output_sha256": "d55ce007379ff24d98434b745f2ae1972ec4b662574ab6fc65fbaa12e55fdf5b", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "42a0abe417221a6383219ba07c3a2afe852e0331d879b46a6d43e16a5cff220c", + "output_sha256": "d55ce007379ff24d98434b745f2ae1972ec4b662574ab6fc65fbaa12e55fdf5b", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "9664f6ac043ff969961c469b15aa5c9a0523bb53468dde28473123a231574f55", + "output_sha256": "3b277ef2b72e684b60a742994377fd6629c88b9ec0f96579cba448d5ecb9738f", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "9664f6ac043ff969961c469b15aa5c9a0523bb53468dde28473123a231574f55", + "output_sha256": "3b277ef2b72e684b60a742994377fd6629c88b9ec0f96579cba448d5ecb9738f", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "9664f6ac043ff969961c469b15aa5c9a0523bb53468dde28473123a231574f55", + "output_sha256": "3b277ef2b72e684b60a742994377fd6629c88b9ec0f96579cba448d5ecb9738f", + "complete": true, + "error": null + } + } + ] + }, + { + "mode": "bf16", + "pool": "distinct", + "prompt": "distinct", + "temperature": 0.0, + "streams": 16, + "failed": 0, + "alone": { + "equal": 16, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 16, + "unequal": 0, + "failed": 0 + }, + "token_hash_comparisons": [ + { + "concurrent": { + "tokens": 64, + "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0", + "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0", + "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0", + "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "d99ee3b3ea17e26e8c5bbdadd3aa09e747b018ae41cf6eabe0b24a55eecfce4f", + "output_sha256": "ce549c26dbcd1281bf48df56c9e67359c959db7d0b1dec8bd78a0736364660e2", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "d99ee3b3ea17e26e8c5bbdadd3aa09e747b018ae41cf6eabe0b24a55eecfce4f", + "output_sha256": "ce549c26dbcd1281bf48df56c9e67359c959db7d0b1dec8bd78a0736364660e2", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "d99ee3b3ea17e26e8c5bbdadd3aa09e747b018ae41cf6eabe0b24a55eecfce4f", + "output_sha256": "ce549c26dbcd1281bf48df56c9e67359c959db7d0b1dec8bd78a0736364660e2", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "62bdf45358eadd229505c80c2e1d91b089d5eee70eb3644df4ec7347c496ad89", + "output_sha256": "502fedcdf2669b346ef9c7a01dc4004453e243072f995334caa465e57be9c309", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "62bdf45358eadd229505c80c2e1d91b089d5eee70eb3644df4ec7347c496ad89", + "output_sha256": "502fedcdf2669b346ef9c7a01dc4004453e243072f995334caa465e57be9c309", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "62bdf45358eadd229505c80c2e1d91b089d5eee70eb3644df4ec7347c496ad89", + "output_sha256": "502fedcdf2669b346ef9c7a01dc4004453e243072f995334caa465e57be9c309", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "d10f504ba0ea3e42aabf8d121c59354a0f6c4159a313f96f50cf28c2f109ec10", + "output_sha256": "c5f9d75515c1bdd5eef1de9bf1d825efead755ffb6c40d9fa211e9b73639a2e3", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "d10f504ba0ea3e42aabf8d121c59354a0f6c4159a313f96f50cf28c2f109ec10", + "output_sha256": "c5f9d75515c1bdd5eef1de9bf1d825efead755ffb6c40d9fa211e9b73639a2e3", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "d10f504ba0ea3e42aabf8d121c59354a0f6c4159a313f96f50cf28c2f109ec10", + "output_sha256": "c5f9d75515c1bdd5eef1de9bf1d825efead755ffb6c40d9fa211e9b73639a2e3", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "d80d7a1d94635a9a5ce9305bf22133261181963691b70407d3212b0557e0d458", + "output_sha256": "f04d026e2ba2a0f5b7d7043b15ba913e60871b429828345b00314bd3151438ee", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "d80d7a1d94635a9a5ce9305bf22133261181963691b70407d3212b0557e0d458", + "output_sha256": "f04d026e2ba2a0f5b7d7043b15ba913e60871b429828345b00314bd3151438ee", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "d80d7a1d94635a9a5ce9305bf22133261181963691b70407d3212b0557e0d458", + "output_sha256": "f04d026e2ba2a0f5b7d7043b15ba913e60871b429828345b00314bd3151438ee", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "3c1a59254dd7797cdbe77a227eac7a15a006369026625cfb85d60753b32d31c5", + "output_sha256": "55e8f7833f37b58eb8fed5d68942ffc4fe5efe5a1690b22078fa23ef6a49d43f", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "3c1a59254dd7797cdbe77a227eac7a15a006369026625cfb85d60753b32d31c5", + "output_sha256": "55e8f7833f37b58eb8fed5d68942ffc4fe5efe5a1690b22078fa23ef6a49d43f", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "3c1a59254dd7797cdbe77a227eac7a15a006369026625cfb85d60753b32d31c5", + "output_sha256": "55e8f7833f37b58eb8fed5d68942ffc4fe5efe5a1690b22078fa23ef6a49d43f", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "42a0abe417221a6383219ba07c3a2afe852e0331d879b46a6d43e16a5cff220c", + "output_sha256": "d55ce007379ff24d98434b745f2ae1972ec4b662574ab6fc65fbaa12e55fdf5b", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "42a0abe417221a6383219ba07c3a2afe852e0331d879b46a6d43e16a5cff220c", + "output_sha256": "d55ce007379ff24d98434b745f2ae1972ec4b662574ab6fc65fbaa12e55fdf5b", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "42a0abe417221a6383219ba07c3a2afe852e0331d879b46a6d43e16a5cff220c", + "output_sha256": "d55ce007379ff24d98434b745f2ae1972ec4b662574ab6fc65fbaa12e55fdf5b", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "9664f6ac043ff969961c469b15aa5c9a0523bb53468dde28473123a231574f55", + "output_sha256": "3b277ef2b72e684b60a742994377fd6629c88b9ec0f96579cba448d5ecb9738f", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "9664f6ac043ff969961c469b15aa5c9a0523bb53468dde28473123a231574f55", + "output_sha256": "3b277ef2b72e684b60a742994377fd6629c88b9ec0f96579cba448d5ecb9738f", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "9664f6ac043ff969961c469b15aa5c9a0523bb53468dde28473123a231574f55", + "output_sha256": "3b277ef2b72e684b60a742994377fd6629c88b9ec0f96579cba448d5ecb9738f", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "0ace27ad2ee234e4f86b8b4ea226079f934b5d6ac697f139c548365d46114d2d", + "output_sha256": "84870d12db10190beaf10456c4035bce579618d8ac812a4af9b0874661871f15", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "0ace27ad2ee234e4f86b8b4ea226079f934b5d6ac697f139c548365d46114d2d", + "output_sha256": "84870d12db10190beaf10456c4035bce579618d8ac812a4af9b0874661871f15", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "0ace27ad2ee234e4f86b8b4ea226079f934b5d6ac697f139c548365d46114d2d", + "output_sha256": "84870d12db10190beaf10456c4035bce579618d8ac812a4af9b0874661871f15", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "c5d52f49d6581ab2e678675c597127a56f589f693cf4d0455d1ef7ac2ae33c89", + "output_sha256": "a04e5befcd334988c9bff65bb9ea6ee33200b1aea661712d5fef487911207cc1", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "c5d52f49d6581ab2e678675c597127a56f589f693cf4d0455d1ef7ac2ae33c89", + "output_sha256": "a04e5befcd334988c9bff65bb9ea6ee33200b1aea661712d5fef487911207cc1", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "c5d52f49d6581ab2e678675c597127a56f589f693cf4d0455d1ef7ac2ae33c89", + "output_sha256": "a04e5befcd334988c9bff65bb9ea6ee33200b1aea661712d5fef487911207cc1", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "8608689cd2e9edeca8d4801f688a5cd0e27f7f8ccb1a636ed5812b028e476b19", + "output_sha256": "dc1f8ff0a23086bd2ddbcc50eb36bfb033df04fe704801ad6af334ae595ffc65", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "8608689cd2e9edeca8d4801f688a5cd0e27f7f8ccb1a636ed5812b028e476b19", + "output_sha256": "dc1f8ff0a23086bd2ddbcc50eb36bfb033df04fe704801ad6af334ae595ffc65", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "8608689cd2e9edeca8d4801f688a5cd0e27f7f8ccb1a636ed5812b028e476b19", + "output_sha256": "dc1f8ff0a23086bd2ddbcc50eb36bfb033df04fe704801ad6af334ae595ffc65", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "1a82e4d07000d781ed10b7c58538e5bd2c57847f9968ec30213dfd60ce9564af", + "output_sha256": "34486c101f28ec19c8894b939e2064aff1babd6892965120ab4b649893fa2812", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "1a82e4d07000d781ed10b7c58538e5bd2c57847f9968ec30213dfd60ce9564af", + "output_sha256": "34486c101f28ec19c8894b939e2064aff1babd6892965120ab4b649893fa2812", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "1a82e4d07000d781ed10b7c58538e5bd2c57847f9968ec30213dfd60ce9564af", + "output_sha256": "34486c101f28ec19c8894b939e2064aff1babd6892965120ab4b649893fa2812", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "46f174b1c36d840fad6f2ed83bd37eb796356898f9ba875b67c0660ffa707030", + "output_sha256": "8c4162384e0679f452d097b0ad26d0a66f1ac2a66b43afe7c8da0a3e920dcc07", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "46f174b1c36d840fad6f2ed83bd37eb796356898f9ba875b67c0660ffa707030", + "output_sha256": "8c4162384e0679f452d097b0ad26d0a66f1ac2a66b43afe7c8da0a3e920dcc07", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "46f174b1c36d840fad6f2ed83bd37eb796356898f9ba875b67c0660ffa707030", + "output_sha256": "8c4162384e0679f452d097b0ad26d0a66f1ac2a66b43afe7c8da0a3e920dcc07", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "3cae46679e7612f5f841dfeb0ebf0838917d9cb75fb59cdfe739e3100ecd99f8", + "output_sha256": "58f7c4c5c1e9afff767a6c368a2d0b1f0b5766ec2cda284c9aacbef8923ef17b", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "3cae46679e7612f5f841dfeb0ebf0838917d9cb75fb59cdfe739e3100ecd99f8", + "output_sha256": "58f7c4c5c1e9afff767a6c368a2d0b1f0b5766ec2cda284c9aacbef8923ef17b", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "3cae46679e7612f5f841dfeb0ebf0838917d9cb75fb59cdfe739e3100ecd99f8", + "output_sha256": "58f7c4c5c1e9afff767a6c368a2d0b1f0b5766ec2cda284c9aacbef8923ef17b", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "973c168f90290876d28e6d880599f226cd948b3fdaeff1fb6d27654ebb18843f", + "output_sha256": "0726d2d6052b8b2efbbb6baa9af4c82fa34cb9e8e657fb6c0d97da49347d0bde", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "973c168f90290876d28e6d880599f226cd948b3fdaeff1fb6d27654ebb18843f", + "output_sha256": "0726d2d6052b8b2efbbb6baa9af4c82fa34cb9e8e657fb6c0d97da49347d0bde", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "973c168f90290876d28e6d880599f226cd948b3fdaeff1fb6d27654ebb18843f", + "output_sha256": "0726d2d6052b8b2efbbb6baa9af4c82fa34cb9e8e657fb6c0d97da49347d0bde", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "c4877a862694f9147f705940504b4c687c2d1dd7b1189d494f474ded0ba7a7f7", + "output_sha256": "f5333ec66a788fc1c0fe6c7de053d2660699f9738ed5afa365f9d1a62589be06", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "c4877a862694f9147f705940504b4c687c2d1dd7b1189d494f474ded0ba7a7f7", + "output_sha256": "f5333ec66a788fc1c0fe6c7de053d2660699f9738ed5afa365f9d1a62589be06", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "c4877a862694f9147f705940504b4c687c2d1dd7b1189d494f474ded0ba7a7f7", + "output_sha256": "f5333ec66a788fc1c0fe6c7de053d2660699f9738ed5afa365f9d1a62589be06", + "complete": true, + "error": null + } + } + ] + }, + { + "mode": "8bit", + "pool": "same", + "prompt": "code", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "alone": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "token_hash_comparisons": [ + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + } + ] + }, + { + "mode": "8bit", + "pool": "same", + "prompt": "code", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "alone": { + "equal": 4, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "token_hash_comparisons": [ + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + } + ] + }, + { + "mode": "8bit", + "pool": "same", + "prompt": "code", + "temperature": 0.0, + "streams": 8, + "failed": 0, + "alone": { + "equal": 8, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "token_hash_comparisons": [ + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + } + ] + }, + { + "mode": "8bit", + "pool": "same", + "prompt": "code", + "temperature": 0.0, + "streams": 16, + "failed": 0, + "alone": { + "equal": 16, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "token_hash_comparisons": [ + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "f4703afc84db904cdc537b5824699afc179c7360c2df654fdd127d833b9e49ab", + "output_sha256": "2c08095f07eb56d8b5eb49130251817dca7af38e201e871bcb3d3fbc5af44656", + "complete": true, + "error": null + } + } + ] + }, + { + "mode": "8bit", + "pool": "same", + "prompt": "chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "alone": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "token_hash_comparisons": [ + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + } + ] + }, + { + "mode": "8bit", + "pool": "same", + "prompt": "chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "alone": { + "equal": 4, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "token_hash_comparisons": [ + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + } + ] + }, + { + "mode": "8bit", + "pool": "same", + "prompt": "chat", + "temperature": 0.0, + "streams": 8, + "failed": 0, + "alone": { + "equal": 8, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "token_hash_comparisons": [ + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + } + ] + }, + { + "mode": "8bit", + "pool": "same", + "prompt": "chat", + "temperature": 0.0, + "streams": 16, + "failed": 0, + "alone": { + "equal": 16, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "token_hash_comparisons": [ + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "5bf28232030fe2c057df524a9cf7d076531e17182b069b95eb522eb1aa701b0a", + "output_sha256": "26b700f66721be28cba310ccd5d7cf056463bbddc9d348888a4ed39a7ecf4156", + "complete": true, + "error": null + } + } + ] + }, + { + "mode": "8bit", + "pool": "distinct", + "prompt": "distinct", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "alone": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 16, + "unequal": 0, + "failed": 0 + }, + "token_hash_comparisons": [ + { + "concurrent": { + "tokens": 64, + "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0", + "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0", + "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0", + "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7", + "complete": true, + "error": null + } + } + ] + }, + { + "mode": "8bit", + "pool": "distinct", + "prompt": "distinct", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "alone": { + "equal": 4, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 16, + "unequal": 0, + "failed": 0 + }, + "token_hash_comparisons": [ + { + "concurrent": { + "tokens": 64, + "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0", + "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0", + "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0", + "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "d99ee3b3ea17e26e8c5bbdadd3aa09e747b018ae41cf6eabe0b24a55eecfce4f", + "output_sha256": "ce549c26dbcd1281bf48df56c9e67359c959db7d0b1dec8bd78a0736364660e2", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "d99ee3b3ea17e26e8c5bbdadd3aa09e747b018ae41cf6eabe0b24a55eecfce4f", + "output_sha256": "ce549c26dbcd1281bf48df56c9e67359c959db7d0b1dec8bd78a0736364660e2", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "d99ee3b3ea17e26e8c5bbdadd3aa09e747b018ae41cf6eabe0b24a55eecfce4f", + "output_sha256": "ce549c26dbcd1281bf48df56c9e67359c959db7d0b1dec8bd78a0736364660e2", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "62bdf45358eadd229505c80c2e1d91b089d5eee70eb3644df4ec7347c496ad89", + "output_sha256": "502fedcdf2669b346ef9c7a01dc4004453e243072f995334caa465e57be9c309", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "62bdf45358eadd229505c80c2e1d91b089d5eee70eb3644df4ec7347c496ad89", + "output_sha256": "502fedcdf2669b346ef9c7a01dc4004453e243072f995334caa465e57be9c309", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "62bdf45358eadd229505c80c2e1d91b089d5eee70eb3644df4ec7347c496ad89", + "output_sha256": "502fedcdf2669b346ef9c7a01dc4004453e243072f995334caa465e57be9c309", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "d10f504ba0ea3e42aabf8d121c59354a0f6c4159a313f96f50cf28c2f109ec10", + "output_sha256": "c5f9d75515c1bdd5eef1de9bf1d825efead755ffb6c40d9fa211e9b73639a2e3", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "d10f504ba0ea3e42aabf8d121c59354a0f6c4159a313f96f50cf28c2f109ec10", + "output_sha256": "c5f9d75515c1bdd5eef1de9bf1d825efead755ffb6c40d9fa211e9b73639a2e3", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "d10f504ba0ea3e42aabf8d121c59354a0f6c4159a313f96f50cf28c2f109ec10", + "output_sha256": "c5f9d75515c1bdd5eef1de9bf1d825efead755ffb6c40d9fa211e9b73639a2e3", + "complete": true, + "error": null + } + } + ] + }, + { + "mode": "8bit", + "pool": "distinct", + "prompt": "distinct", + "temperature": 0.0, + "streams": 8, + "failed": 0, + "alone": { + "equal": 8, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 16, + "unequal": 0, + "failed": 0 + }, + "token_hash_comparisons": [ + { + "concurrent": { + "tokens": 64, + "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0", + "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0", + "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0", + "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "d99ee3b3ea17e26e8c5bbdadd3aa09e747b018ae41cf6eabe0b24a55eecfce4f", + "output_sha256": "ce549c26dbcd1281bf48df56c9e67359c959db7d0b1dec8bd78a0736364660e2", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "d99ee3b3ea17e26e8c5bbdadd3aa09e747b018ae41cf6eabe0b24a55eecfce4f", + "output_sha256": "ce549c26dbcd1281bf48df56c9e67359c959db7d0b1dec8bd78a0736364660e2", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "d99ee3b3ea17e26e8c5bbdadd3aa09e747b018ae41cf6eabe0b24a55eecfce4f", + "output_sha256": "ce549c26dbcd1281bf48df56c9e67359c959db7d0b1dec8bd78a0736364660e2", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "62bdf45358eadd229505c80c2e1d91b089d5eee70eb3644df4ec7347c496ad89", + "output_sha256": "502fedcdf2669b346ef9c7a01dc4004453e243072f995334caa465e57be9c309", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "62bdf45358eadd229505c80c2e1d91b089d5eee70eb3644df4ec7347c496ad89", + "output_sha256": "502fedcdf2669b346ef9c7a01dc4004453e243072f995334caa465e57be9c309", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "62bdf45358eadd229505c80c2e1d91b089d5eee70eb3644df4ec7347c496ad89", + "output_sha256": "502fedcdf2669b346ef9c7a01dc4004453e243072f995334caa465e57be9c309", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "d10f504ba0ea3e42aabf8d121c59354a0f6c4159a313f96f50cf28c2f109ec10", + "output_sha256": "c5f9d75515c1bdd5eef1de9bf1d825efead755ffb6c40d9fa211e9b73639a2e3", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "d10f504ba0ea3e42aabf8d121c59354a0f6c4159a313f96f50cf28c2f109ec10", + "output_sha256": "c5f9d75515c1bdd5eef1de9bf1d825efead755ffb6c40d9fa211e9b73639a2e3", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "d10f504ba0ea3e42aabf8d121c59354a0f6c4159a313f96f50cf28c2f109ec10", + "output_sha256": "c5f9d75515c1bdd5eef1de9bf1d825efead755ffb6c40d9fa211e9b73639a2e3", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "d80d7a1d94635a9a5ce9305bf22133261181963691b70407d3212b0557e0d458", + "output_sha256": "f04d026e2ba2a0f5b7d7043b15ba913e60871b429828345b00314bd3151438ee", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "d80d7a1d94635a9a5ce9305bf22133261181963691b70407d3212b0557e0d458", + "output_sha256": "f04d026e2ba2a0f5b7d7043b15ba913e60871b429828345b00314bd3151438ee", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "d80d7a1d94635a9a5ce9305bf22133261181963691b70407d3212b0557e0d458", + "output_sha256": "f04d026e2ba2a0f5b7d7043b15ba913e60871b429828345b00314bd3151438ee", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "3c1a59254dd7797cdbe77a227eac7a15a006369026625cfb85d60753b32d31c5", + "output_sha256": "55e8f7833f37b58eb8fed5d68942ffc4fe5efe5a1690b22078fa23ef6a49d43f", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "3c1a59254dd7797cdbe77a227eac7a15a006369026625cfb85d60753b32d31c5", + "output_sha256": "55e8f7833f37b58eb8fed5d68942ffc4fe5efe5a1690b22078fa23ef6a49d43f", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "3c1a59254dd7797cdbe77a227eac7a15a006369026625cfb85d60753b32d31c5", + "output_sha256": "55e8f7833f37b58eb8fed5d68942ffc4fe5efe5a1690b22078fa23ef6a49d43f", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "42a0abe417221a6383219ba07c3a2afe852e0331d879b46a6d43e16a5cff220c", + "output_sha256": "d55ce007379ff24d98434b745f2ae1972ec4b662574ab6fc65fbaa12e55fdf5b", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "42a0abe417221a6383219ba07c3a2afe852e0331d879b46a6d43e16a5cff220c", + "output_sha256": "d55ce007379ff24d98434b745f2ae1972ec4b662574ab6fc65fbaa12e55fdf5b", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "42a0abe417221a6383219ba07c3a2afe852e0331d879b46a6d43e16a5cff220c", + "output_sha256": "d55ce007379ff24d98434b745f2ae1972ec4b662574ab6fc65fbaa12e55fdf5b", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "9664f6ac043ff969961c469b15aa5c9a0523bb53468dde28473123a231574f55", + "output_sha256": "3b277ef2b72e684b60a742994377fd6629c88b9ec0f96579cba448d5ecb9738f", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "9664f6ac043ff969961c469b15aa5c9a0523bb53468dde28473123a231574f55", + "output_sha256": "3b277ef2b72e684b60a742994377fd6629c88b9ec0f96579cba448d5ecb9738f", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "9664f6ac043ff969961c469b15aa5c9a0523bb53468dde28473123a231574f55", + "output_sha256": "3b277ef2b72e684b60a742994377fd6629c88b9ec0f96579cba448d5ecb9738f", + "complete": true, + "error": null + } + } + ] + }, + { + "mode": "8bit", + "pool": "distinct", + "prompt": "distinct", + "temperature": 0.0, + "streams": 16, + "failed": 0, + "alone": { + "equal": 16, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 16, + "unequal": 0, + "failed": 0 + }, + "token_hash_comparisons": [ + { + "concurrent": { + "tokens": 64, + "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0", + "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0", + "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0", + "output_sha256": "39854434c68ec704b9304067ada4a049b2ed0245c900c55b97c1cc58656564a7", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "d99ee3b3ea17e26e8c5bbdadd3aa09e747b018ae41cf6eabe0b24a55eecfce4f", + "output_sha256": "ce549c26dbcd1281bf48df56c9e67359c959db7d0b1dec8bd78a0736364660e2", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "d99ee3b3ea17e26e8c5bbdadd3aa09e747b018ae41cf6eabe0b24a55eecfce4f", + "output_sha256": "ce549c26dbcd1281bf48df56c9e67359c959db7d0b1dec8bd78a0736364660e2", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "d99ee3b3ea17e26e8c5bbdadd3aa09e747b018ae41cf6eabe0b24a55eecfce4f", + "output_sha256": "ce549c26dbcd1281bf48df56c9e67359c959db7d0b1dec8bd78a0736364660e2", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "62bdf45358eadd229505c80c2e1d91b089d5eee70eb3644df4ec7347c496ad89", + "output_sha256": "502fedcdf2669b346ef9c7a01dc4004453e243072f995334caa465e57be9c309", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "62bdf45358eadd229505c80c2e1d91b089d5eee70eb3644df4ec7347c496ad89", + "output_sha256": "502fedcdf2669b346ef9c7a01dc4004453e243072f995334caa465e57be9c309", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "62bdf45358eadd229505c80c2e1d91b089d5eee70eb3644df4ec7347c496ad89", + "output_sha256": "502fedcdf2669b346ef9c7a01dc4004453e243072f995334caa465e57be9c309", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "d10f504ba0ea3e42aabf8d121c59354a0f6c4159a313f96f50cf28c2f109ec10", + "output_sha256": "c5f9d75515c1bdd5eef1de9bf1d825efead755ffb6c40d9fa211e9b73639a2e3", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "d10f504ba0ea3e42aabf8d121c59354a0f6c4159a313f96f50cf28c2f109ec10", + "output_sha256": "c5f9d75515c1bdd5eef1de9bf1d825efead755ffb6c40d9fa211e9b73639a2e3", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "d10f504ba0ea3e42aabf8d121c59354a0f6c4159a313f96f50cf28c2f109ec10", + "output_sha256": "c5f9d75515c1bdd5eef1de9bf1d825efead755ffb6c40d9fa211e9b73639a2e3", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "d80d7a1d94635a9a5ce9305bf22133261181963691b70407d3212b0557e0d458", + "output_sha256": "f04d026e2ba2a0f5b7d7043b15ba913e60871b429828345b00314bd3151438ee", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "d80d7a1d94635a9a5ce9305bf22133261181963691b70407d3212b0557e0d458", + "output_sha256": "f04d026e2ba2a0f5b7d7043b15ba913e60871b429828345b00314bd3151438ee", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "d80d7a1d94635a9a5ce9305bf22133261181963691b70407d3212b0557e0d458", + "output_sha256": "f04d026e2ba2a0f5b7d7043b15ba913e60871b429828345b00314bd3151438ee", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "3c1a59254dd7797cdbe77a227eac7a15a006369026625cfb85d60753b32d31c5", + "output_sha256": "55e8f7833f37b58eb8fed5d68942ffc4fe5efe5a1690b22078fa23ef6a49d43f", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "3c1a59254dd7797cdbe77a227eac7a15a006369026625cfb85d60753b32d31c5", + "output_sha256": "55e8f7833f37b58eb8fed5d68942ffc4fe5efe5a1690b22078fa23ef6a49d43f", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "3c1a59254dd7797cdbe77a227eac7a15a006369026625cfb85d60753b32d31c5", + "output_sha256": "55e8f7833f37b58eb8fed5d68942ffc4fe5efe5a1690b22078fa23ef6a49d43f", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "42a0abe417221a6383219ba07c3a2afe852e0331d879b46a6d43e16a5cff220c", + "output_sha256": "d55ce007379ff24d98434b745f2ae1972ec4b662574ab6fc65fbaa12e55fdf5b", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "42a0abe417221a6383219ba07c3a2afe852e0331d879b46a6d43e16a5cff220c", + "output_sha256": "d55ce007379ff24d98434b745f2ae1972ec4b662574ab6fc65fbaa12e55fdf5b", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "42a0abe417221a6383219ba07c3a2afe852e0331d879b46a6d43e16a5cff220c", + "output_sha256": "d55ce007379ff24d98434b745f2ae1972ec4b662574ab6fc65fbaa12e55fdf5b", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "9664f6ac043ff969961c469b15aa5c9a0523bb53468dde28473123a231574f55", + "output_sha256": "3b277ef2b72e684b60a742994377fd6629c88b9ec0f96579cba448d5ecb9738f", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "9664f6ac043ff969961c469b15aa5c9a0523bb53468dde28473123a231574f55", + "output_sha256": "3b277ef2b72e684b60a742994377fd6629c88b9ec0f96579cba448d5ecb9738f", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "9664f6ac043ff969961c469b15aa5c9a0523bb53468dde28473123a231574f55", + "output_sha256": "3b277ef2b72e684b60a742994377fd6629c88b9ec0f96579cba448d5ecb9738f", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "0ace27ad2ee234e4f86b8b4ea226079f934b5d6ac697f139c548365d46114d2d", + "output_sha256": "84870d12db10190beaf10456c4035bce579618d8ac812a4af9b0874661871f15", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "0ace27ad2ee234e4f86b8b4ea226079f934b5d6ac697f139c548365d46114d2d", + "output_sha256": "84870d12db10190beaf10456c4035bce579618d8ac812a4af9b0874661871f15", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "0ace27ad2ee234e4f86b8b4ea226079f934b5d6ac697f139c548365d46114d2d", + "output_sha256": "84870d12db10190beaf10456c4035bce579618d8ac812a4af9b0874661871f15", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "c5d52f49d6581ab2e678675c597127a56f589f693cf4d0455d1ef7ac2ae33c89", + "output_sha256": "a04e5befcd334988c9bff65bb9ea6ee33200b1aea661712d5fef487911207cc1", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "c5d52f49d6581ab2e678675c597127a56f589f693cf4d0455d1ef7ac2ae33c89", + "output_sha256": "a04e5befcd334988c9bff65bb9ea6ee33200b1aea661712d5fef487911207cc1", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "c5d52f49d6581ab2e678675c597127a56f589f693cf4d0455d1ef7ac2ae33c89", + "output_sha256": "a04e5befcd334988c9bff65bb9ea6ee33200b1aea661712d5fef487911207cc1", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "8608689cd2e9edeca8d4801f688a5cd0e27f7f8ccb1a636ed5812b028e476b19", + "output_sha256": "dc1f8ff0a23086bd2ddbcc50eb36bfb033df04fe704801ad6af334ae595ffc65", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "8608689cd2e9edeca8d4801f688a5cd0e27f7f8ccb1a636ed5812b028e476b19", + "output_sha256": "dc1f8ff0a23086bd2ddbcc50eb36bfb033df04fe704801ad6af334ae595ffc65", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "8608689cd2e9edeca8d4801f688a5cd0e27f7f8ccb1a636ed5812b028e476b19", + "output_sha256": "dc1f8ff0a23086bd2ddbcc50eb36bfb033df04fe704801ad6af334ae595ffc65", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "1a82e4d07000d781ed10b7c58538e5bd2c57847f9968ec30213dfd60ce9564af", + "output_sha256": "34486c101f28ec19c8894b939e2064aff1babd6892965120ab4b649893fa2812", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "1a82e4d07000d781ed10b7c58538e5bd2c57847f9968ec30213dfd60ce9564af", + "output_sha256": "34486c101f28ec19c8894b939e2064aff1babd6892965120ab4b649893fa2812", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "1a82e4d07000d781ed10b7c58538e5bd2c57847f9968ec30213dfd60ce9564af", + "output_sha256": "34486c101f28ec19c8894b939e2064aff1babd6892965120ab4b649893fa2812", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "46f174b1c36d840fad6f2ed83bd37eb796356898f9ba875b67c0660ffa707030", + "output_sha256": "8c4162384e0679f452d097b0ad26d0a66f1ac2a66b43afe7c8da0a3e920dcc07", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "46f174b1c36d840fad6f2ed83bd37eb796356898f9ba875b67c0660ffa707030", + "output_sha256": "8c4162384e0679f452d097b0ad26d0a66f1ac2a66b43afe7c8da0a3e920dcc07", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "46f174b1c36d840fad6f2ed83bd37eb796356898f9ba875b67c0660ffa707030", + "output_sha256": "8c4162384e0679f452d097b0ad26d0a66f1ac2a66b43afe7c8da0a3e920dcc07", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "3cae46679e7612f5f841dfeb0ebf0838917d9cb75fb59cdfe739e3100ecd99f8", + "output_sha256": "58f7c4c5c1e9afff767a6c368a2d0b1f0b5766ec2cda284c9aacbef8923ef17b", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "3cae46679e7612f5f841dfeb0ebf0838917d9cb75fb59cdfe739e3100ecd99f8", + "output_sha256": "58f7c4c5c1e9afff767a6c368a2d0b1f0b5766ec2cda284c9aacbef8923ef17b", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "3cae46679e7612f5f841dfeb0ebf0838917d9cb75fb59cdfe739e3100ecd99f8", + "output_sha256": "58f7c4c5c1e9afff767a6c368a2d0b1f0b5766ec2cda284c9aacbef8923ef17b", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "973c168f90290876d28e6d880599f226cd948b3fdaeff1fb6d27654ebb18843f", + "output_sha256": "0726d2d6052b8b2efbbb6baa9af4c82fa34cb9e8e657fb6c0d97da49347d0bde", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "973c168f90290876d28e6d880599f226cd948b3fdaeff1fb6d27654ebb18843f", + "output_sha256": "0726d2d6052b8b2efbbb6baa9af4c82fa34cb9e8e657fb6c0d97da49347d0bde", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "973c168f90290876d28e6d880599f226cd948b3fdaeff1fb6d27654ebb18843f", + "output_sha256": "0726d2d6052b8b2efbbb6baa9af4c82fa34cb9e8e657fb6c0d97da49347d0bde", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 64, + "token_sha256": "c4877a862694f9147f705940504b4c687c2d1dd7b1189d494f474ded0ba7a7f7", + "output_sha256": "f5333ec66a788fc1c0fe6c7de053d2660699f9738ed5afa365f9d1a62589be06", + "complete": true, + "error": null + }, + "alone": { + "tokens": 64, + "token_sha256": "c4877a862694f9147f705940504b4c687c2d1dd7b1189d494f474ded0ba7a7f7", + "output_sha256": "f5333ec66a788fc1c0fe6c7de053d2660699f9738ed5afa365f9d1a62589be06", + "complete": true, + "error": null + }, + "serial": { + "tokens": 64, + "token_sha256": "c4877a862694f9147f705940504b4c687c2d1dd7b1189d494f474ded0ba7a7f7", + "output_sha256": "f5333ec66a788fc1c0fe6c7de053d2660699f9738ed5afa365f9d1a62589be06", + "complete": true, + "error": null + } + } + ] + } + ] +} From 2dbbc0e8f5bbadf1a33351769439026f318ddb8f Mon Sep 17 00:00:00 2001 From: BlivionIaG <15323295+BlivionIaG@users.noreply.github.com> Date: Wed, 28 Oct 2026 10:00:17 +0100 Subject: [PATCH 35/39] docs: record independent drafter reference and partial seam probe --- docs/receipts/lfm2-dspark-review/README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/receipts/lfm2-dspark-review/README.md b/docs/receipts/lfm2-dspark-review/README.md index 9db7cad86d..9010848c9c 100644 --- a/docs/receipts/lfm2-dspark-review/README.md +++ b/docs/receipts/lfm2-dspark-review/README.md @@ -30,6 +30,12 @@ multiplication, and `--drafter` refuses a target restricted to one-row verificat 0 known findings; all 17 planted controls red, 0 missed; 0 limit problems. - `tf-kernel-contract --only tile_qmm`: 6 kernels passed gates 1-4, 0 failures, 0 known findings; 0 limit problems. +- `tf-lfm2-dspark MODEL DRAFTER 256`: 9/9 drafts equal the independent fp32 + reference's picks; maximum confidence difference 0.00106; 20 steady drafter + rounds had zero host allocations and resizes; exit 0. +- An additional `tf-lfm2-exact` probe was stopped before completion after 68 + successful checks, including the 2,047/2,049-token seams. The full tool did not + complete, so this is not a full forward/cache qualification receipt. ## Served equality From 8bd05874d98d9449ad2f8ddb8796bf26e229874b Mon Sep 17 00:00:00 2001 From: BlivionIaG <15323295+BlivionIaG@users.noreply.github.com> Date: Wed, 28 Oct 2026 11:12:03 +0100 Subject: [PATCH 36/39] docs: record completed M4 and M5 DSpark qualification --- docs/receipts/lfm2-dspark-review/README.md | 69 +- .../extended-qualification.json | 53517 ++++++++++++++++ .../lfm2-dspark-review/qualification.py | 220 + 3 files changed, 53796 insertions(+), 10 deletions(-) create mode 100644 docs/receipts/lfm2-dspark-review/extended-qualification.json create mode 100644 docs/receipts/lfm2-dspark-review/qualification.py diff --git a/docs/receipts/lfm2-dspark-review/README.md b/docs/receipts/lfm2-dspark-review/README.md index 9010848c9c..17dc78407e 100644 --- a/docs/receipts/lfm2-dspark-review/README.md +++ b/docs/receipts/lfm2-dspark-review/README.md @@ -33,9 +33,8 @@ multiplication, and `--drafter` refuses a target restricted to one-row verificat - `tf-lfm2-dspark MODEL DRAFTER 256`: 9/9 drafts equal the independent fp32 reference's picks; maximum confidence difference 0.00106; 20 steady drafter rounds had zero host allocations and resizes; exit 0. -- An additional `tf-lfm2-exact` probe was stopped before completion after 68 - successful checks, including the 2,047/2,049-token seams. The full tool did not - complete, so this is not a full forward/cache qualification receipt. +- Completed `tf-lfm2-exact`: 72/72 checks on both M4 Pro and M5 Ultra, zero + failures or skipped checks. This supersedes the earlier interrupted run. ## Served equality @@ -77,10 +76,60 @@ The measured reduction during load is 508,166,576 bytes (484.6 MiB). This compar the original pre-allocation head with the corrected stack rebased onto current main; it is not an isolated comparison of #1393's commit on the same base. -## Remaining qualification - -No new long-context, cancellation/replacement stress, resumed-serving, sampled -serving or alternating speed qualification is claimed here. The maintainer's -requested current-main performance cells and other-family A/B remain to be run. -M1, M2, M3, M5, CUDA and HIP were not run for this revision. The focused Metal -contract checks above are not the full cross-family `kernel-contract` gate. +## Extended qualification on the published stack + +Source `2dbbc0e8f`, main baseline `d0c9dc9fb`. Additional device: Apple M5 Ultra, +256 GiB RAM, macOS 27.0, Metal compiler 32023.921.6, Zig 0.17.0. +[Extended evidence](extended-qualification.json) retains request hashes, counters, +per-round throughput and latency. Generated text and token arrays are omitted. +[Reproduction harness](qualification.py) supports `functional`, `long`, +`performance`, `performance qwen`, and `cold`; set `TF_QUAL_MAIN`, +`TF_QUAL_CANDIDATE`, `TF_QUAL_MODEL`, `TF_QUAL_DRAFTER`, and `TF_QUAL_OUT` +for the local installations. Run one model process at a time. + +Both devices passed real two-turn resumed-versus-fresh chat equality (3,517 cached +prompt tokens), 29 seeded sampled equality cases across 1/4/8/16 streams, and +cancellation/replacement stress with Metal API validation enabled. M4 completed +143 requests with 84 cancellations at six slots and 252 with 134 at sixteen; +M5 completed 453 with 247 and 852 with 459 respectively. Zero wrong replies or +request errors. Server shutdowns were graceful. + +M5 plain/drafted serving hashes agree at 4,096, 16,384, 65,536 and 130,816 prompt +tokens. Independent fp32 drafter checks agree on all nine picks at histories +65,535 and 130,815; maximum confidence differences 0.00532 and 0.00351. +Twenty steady rounds had zero host allocations/resizes. M5 also passes the same +11 LFM2 and six tile_qmm contracts, with all 17 LFM2 planted controls red. + +### Performance remains an open issue + +Three alternating rounds per arm, 256 generated tokens, 1/4/8/16 streams, +32-token warmups and uncached prompts. Aggregate decode rate uses the span from +the earliest first token to the latest last token. The `chat` fixture label in +the JSON means prose submitted to raw completions, not a templated chat request. +All cross-arm output hashes agree. These results do **not** clear a no-loss gate. + +On M5, drafted versus main-plain median decode changes for code at 1/4/8/16 +streams are +19.44%, -0.96%, -2.20%, -4.23%; for prose they are +35.89%, +-0.72%, -1.25%, -1.17%. Candidate-plain closely follows main-plain. +On M4, changes are approximately -6.0%, -13.9%, -5.6%, +0.7% for code and +-6.4%, -13.4%, -7.8%, -8.0% for prose. M4 also shows variation between plain +arms, so these measurements do not isolate every source of the loss. + +M5 cold prefill medians at 2k/8k/32k/64k tokens are 0.1527/0.6027/2.9244/7.3276 +seconds on main-plain versus 0.1634/0.6327/3.0466/7.5652 with the drafter. +Prompt counts were checked and cached-token counts were zero. + +Other-family Qwen3.8-27B plus DFlash2 was tested at one and four streams with +three alternating rounds on both devices. M4 results vary with each executable's +independently measured persistent cost record: fresh records give code changes +of -7.13%/+3.71% and prose -6.47%/+0.83%. The original records and fresh-record +runs are both retained. No timings were copied between executables. M5 changes are +0.20%/-0.47% for code and +0.05%/+0.70% for prose. +No blanket other-family performance pass is claimed. + +Two isolated, uncommitted experiments (a larger absorb staging buffer and +removing forced per-stream copy pricing) did not consistently remove the loss. +Neither experiment is part of this PR. Further pricing work requires its own +validation; the rejected shared/lone guards have not been restored. + +M1, M2, M3, CUDA and HIP were not rerun for this revision. Focused Metal checks +are not a full cross-family kernel-contract receipt. All nine GitHub checks passed on `2dbbc0e8f` before this receipt-only update. diff --git a/docs/receipts/lfm2-dspark-review/extended-qualification.json b/docs/receipts/lfm2-dspark-review/extended-qualification.json new file mode 100644 index 0000000000..db8d93df99 --- /dev/null +++ b/docs/receipts/lfm2-dspark-review/extended-qualification.json @@ -0,0 +1,53517 @@ +{ + "source": "2dbbc0e8f5bbadf1a33351769439026f318ddb8f", + "main": "d0c9dc9fb", + "date": "2026-10-28", + "devices": { + "M4 Pro": { + "functional": { + "resumed": { + "usage": { + "prompt_tokens": 3605, + "completion_tokens": 64, + "total_tokens": 3669, + "prompt_tokens_details": { + "cached_tokens": 3517 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 106.39393036340272, + "seconds": 0.592139042, + "prefill_seconds": 0.181146458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.181769458, + "sampling": "greedy", + "drafts": true, + "token_sha": "a064a0a2711c", + "token_sha256": "a064a0a2711ce3e5053bd664c3540fc534b5f6044a0aa69b70bce596081d429e", + "min_rows": 2 + }, + "speculative": { + "rounds": 31, + "drafted": 55, + "accepted": 33, + "acceptance_rate": 0.6, + "tokens_per_round": 2.064516129032258 + }, + "wall": 0.7745752497576177 + }, + "resume_request": { + "messages": [ + { + "role": "user" + }, + { + "role": "assistant" + }, + { + "role": "user" + } + ], + "max_tokens": 64 + }, + "stress_reference_hashes": [ + { + "prompt": "quicksort-raw", + "sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0" + }, + { + "prompt": "rust-reverse-words", + "sha256": "d9499a1af105f9d0477f4fda4f2afde556c782008fe552d4a7cd0ebf71e80101" + }, + { + "prompt": "multiply-17-23", + "sha256": "3ac253c3ae100e4ca9328988d95e9324ab6de6f85787f191d13c718bfae1f4b4" + }, + { + "prompt": "sleep-tips", + "sha256": "076c7d4d0802aa0ee09b60f396deab8a6cfc5683e2f17b6b90f587b410d20ba5" + }, + { + "prompt": "js-debounce-raw", + "sha256": "d80d7a1d94635a9a5ce9305bf22133261181963691b70407d3212b0557e0d458" + }, + { + "prompt": "linear-equation", + "sha256": "9277e1d182a9cfc39ec6df2bb5f33c9acb4ca3db6922f5113769afeb3c2ee38b" + }, + { + "prompt": "fishing-village", + "sha256": "522136a9089e45996543679366846beba531361185bdc02baa0f336231b8848e" + }, + { + "prompt": "sql-left-join", + "sha256": "45afcf2bc8f7bdc6b1a1ae93f7a06c53e7a98db5e4d9f283753159d8e2707a74" + } + ], + "stress": [ + { + "slots": 6, + "seconds": 60, + "completed": 143, + "cancelled": 84, + "wrong": 0, + "errors": [] + }, + { + "slots": 16, + "seconds": 90, + "completed": 252, + "cancelled": 134, + "wrong": 0, + "errors": [] + } + ], + "sampled": [ + { + "streams": 1, + "hashes": [ + [ + "9958f5194528a17558001a973978b8a2e40b11e332c197e750bfbc15c7813a1c", + "9958f5194528a17558001a973978b8a2e40b11e332c197e750bfbc15c7813a1c", + "9958f5194528a17558001a973978b8a2e40b11e332c197e750bfbc15c7813a1c" + ] + ] + }, + { + "streams": 4, + "hashes": [ + [ + "9958f5194528a17558001a973978b8a2e40b11e332c197e750bfbc15c7813a1c", + "9958f5194528a17558001a973978b8a2e40b11e332c197e750bfbc15c7813a1c", + "9958f5194528a17558001a973978b8a2e40b11e332c197e750bfbc15c7813a1c" + ], + [ + "6dd87860c3c401b29c868be19918f593428854f18e08fce2d154b368e387bab1", + "6dd87860c3c401b29c868be19918f593428854f18e08fce2d154b368e387bab1", + "6dd87860c3c401b29c868be19918f593428854f18e08fce2d154b368e387bab1" + ], + [ + "aa087fd45921ba6768548dc4aefef6f6e157ec09202a4b93049839b47a42320d", + "aa087fd45921ba6768548dc4aefef6f6e157ec09202a4b93049839b47a42320d", + "aa087fd45921ba6768548dc4aefef6f6e157ec09202a4b93049839b47a42320d" + ], + [ + "8eefe260a824cded48e3c98eed082d0aaf915874a2d9ee331cc4ddf4412a9738", + "8eefe260a824cded48e3c98eed082d0aaf915874a2d9ee331cc4ddf4412a9738", + "8eefe260a824cded48e3c98eed082d0aaf915874a2d9ee331cc4ddf4412a9738" + ] + ] + }, + { + "streams": 8, + "hashes": [ + [ + "9958f5194528a17558001a973978b8a2e40b11e332c197e750bfbc15c7813a1c", + "9958f5194528a17558001a973978b8a2e40b11e332c197e750bfbc15c7813a1c", + "9958f5194528a17558001a973978b8a2e40b11e332c197e750bfbc15c7813a1c" + ], + [ + "6dd87860c3c401b29c868be19918f593428854f18e08fce2d154b368e387bab1", + "6dd87860c3c401b29c868be19918f593428854f18e08fce2d154b368e387bab1", + "6dd87860c3c401b29c868be19918f593428854f18e08fce2d154b368e387bab1" + ], + [ + "aa087fd45921ba6768548dc4aefef6f6e157ec09202a4b93049839b47a42320d", + "aa087fd45921ba6768548dc4aefef6f6e157ec09202a4b93049839b47a42320d", + "aa087fd45921ba6768548dc4aefef6f6e157ec09202a4b93049839b47a42320d" + ], + [ + "8eefe260a824cded48e3c98eed082d0aaf915874a2d9ee331cc4ddf4412a9738", + "8eefe260a824cded48e3c98eed082d0aaf915874a2d9ee331cc4ddf4412a9738", + "8eefe260a824cded48e3c98eed082d0aaf915874a2d9ee331cc4ddf4412a9738" + ], + [ + "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366", + "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366", + "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366" + ], + [ + "c83059ed7335ef8175d1f39441293d08ea656914525ec3ac36691e1927b275cf", + "c83059ed7335ef8175d1f39441293d08ea656914525ec3ac36691e1927b275cf", + "c83059ed7335ef8175d1f39441293d08ea656914525ec3ac36691e1927b275cf" + ], + [ + "c590ac015a46de196f04884bcffeb3849d9d68e18ff15f742621253248b7300e", + "c590ac015a46de196f04884bcffeb3849d9d68e18ff15f742621253248b7300e", + "c590ac015a46de196f04884bcffeb3849d9d68e18ff15f742621253248b7300e" + ], + [ + "28a8ff176c5da609ba9614ae38d5bf6cd7d83da8eba6627e93f3cd2a546d4dfb", + "28a8ff176c5da609ba9614ae38d5bf6cd7d83da8eba6627e93f3cd2a546d4dfb", + "28a8ff176c5da609ba9614ae38d5bf6cd7d83da8eba6627e93f3cd2a546d4dfb" + ] + ] + }, + { + "streams": 16, + "hashes": [ + [ + "9958f5194528a17558001a973978b8a2e40b11e332c197e750bfbc15c7813a1c", + "9958f5194528a17558001a973978b8a2e40b11e332c197e750bfbc15c7813a1c", + "9958f5194528a17558001a973978b8a2e40b11e332c197e750bfbc15c7813a1c" + ], + [ + "6dd87860c3c401b29c868be19918f593428854f18e08fce2d154b368e387bab1", + "6dd87860c3c401b29c868be19918f593428854f18e08fce2d154b368e387bab1", + "6dd87860c3c401b29c868be19918f593428854f18e08fce2d154b368e387bab1" + ], + [ + "aa087fd45921ba6768548dc4aefef6f6e157ec09202a4b93049839b47a42320d", + "aa087fd45921ba6768548dc4aefef6f6e157ec09202a4b93049839b47a42320d", + "aa087fd45921ba6768548dc4aefef6f6e157ec09202a4b93049839b47a42320d" + ], + [ + "8eefe260a824cded48e3c98eed082d0aaf915874a2d9ee331cc4ddf4412a9738", + "8eefe260a824cded48e3c98eed082d0aaf915874a2d9ee331cc4ddf4412a9738", + "8eefe260a824cded48e3c98eed082d0aaf915874a2d9ee331cc4ddf4412a9738" + ], + [ + "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366", + "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366", + "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366" + ], + [ + "c83059ed7335ef8175d1f39441293d08ea656914525ec3ac36691e1927b275cf", + "c83059ed7335ef8175d1f39441293d08ea656914525ec3ac36691e1927b275cf", + "c83059ed7335ef8175d1f39441293d08ea656914525ec3ac36691e1927b275cf" + ], + [ + "c590ac015a46de196f04884bcffeb3849d9d68e18ff15f742621253248b7300e", + "c590ac015a46de196f04884bcffeb3849d9d68e18ff15f742621253248b7300e", + "c590ac015a46de196f04884bcffeb3849d9d68e18ff15f742621253248b7300e" + ], + [ + "28a8ff176c5da609ba9614ae38d5bf6cd7d83da8eba6627e93f3cd2a546d4dfb", + "28a8ff176c5da609ba9614ae38d5bf6cd7d83da8eba6627e93f3cd2a546d4dfb", + "28a8ff176c5da609ba9614ae38d5bf6cd7d83da8eba6627e93f3cd2a546d4dfb" + ], + [ + "41d624ff3fbe39c070157c12610d6700673ec826973228dfddae8eb6a1cd223c", + "41d624ff3fbe39c070157c12610d6700673ec826973228dfddae8eb6a1cd223c", + "41d624ff3fbe39c070157c12610d6700673ec826973228dfddae8eb6a1cd223c" + ], + [ + "33d1280cae02dd02cfbeeb7de9a33323d9ea23f88247f2a986d03e213a3f8d0b", + "33d1280cae02dd02cfbeeb7de9a33323d9ea23f88247f2a986d03e213a3f8d0b", + "33d1280cae02dd02cfbeeb7de9a33323d9ea23f88247f2a986d03e213a3f8d0b" + ], + [ + "51d5ebc4e7862b6c45c836a85da47074cb651620b0015c91c5d82d3d70ccd6b2", + "51d5ebc4e7862b6c45c836a85da47074cb651620b0015c91c5d82d3d70ccd6b2", + "51d5ebc4e7862b6c45c836a85da47074cb651620b0015c91c5d82d3d70ccd6b2" + ], + [ + "c0afe7bf19f848ac3e8f99531b58d5608aa470f2c03ea63f8298901920a8c3d3", + "c0afe7bf19f848ac3e8f99531b58d5608aa470f2c03ea63f8298901920a8c3d3", + "c0afe7bf19f848ac3e8f99531b58d5608aa470f2c03ea63f8298901920a8c3d3" + ], + [ + "68172f4de066139fda82f3d6eb277ed21660aed3c34d5e488232e41e7ccae9de", + "68172f4de066139fda82f3d6eb277ed21660aed3c34d5e488232e41e7ccae9de", + "68172f4de066139fda82f3d6eb277ed21660aed3c34d5e488232e41e7ccae9de" + ], + [ + "a3c0804235311a54b522bd30daf9126c0ba02cd75f0de742b076ceeeea848e74", + "a3c0804235311a54b522bd30daf9126c0ba02cd75f0de742b076ceeeea848e74", + "a3c0804235311a54b522bd30daf9126c0ba02cd75f0de742b076ceeeea848e74" + ], + [ + "fecd90b849b2fd6c1ec01f018d24f3556603d49132e3935c0abe2cc7a03288c8", + "fecd90b849b2fd6c1ec01f018d24f3556603d49132e3935c0abe2cc7a03288c8", + "fecd90b849b2fd6c1ec01f018d24f3556603d49132e3935c0abe2cc7a03288c8" + ], + [ + "2d8f298739a79e14184852c9c8774acf3746569b579eaa3f1b79e9d096c410d8", + "2d8f298739a79e14184852c9c8774acf3746569b579eaa3f1b79e9d096c410d8", + "2d8f298739a79e14184852c9c8774acf3746569b579eaa3f1b79e9d096c410d8" + ] + ] + } + ], + "fresh": { + "usage": { + "prompt_tokens": 3605, + "completion_tokens": 64, + "total_tokens": 3669, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 94.76795634919337, + "seconds": 0.664781667, + "prefill_seconds": 4.404813625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 4.405911625, + "sampling": "greedy", + "drafts": true, + "token_sha": "a064a0a2711c", + "token_sha256": "a064a0a2711ce3e5053bd664c3540fc534b5f6044a0aa69b70bce596081d429e", + "min_rows": 2 + }, + "speculative": { + "rounds": 30, + "drafted": 63, + "accepted": 34, + "acceptance_rate": 0.5396825396825397, + "tokens_per_round": 2.1333333333333333 + }, + "wall": 5.071483042091131 + }, + "resume_equal": true + }, + "performance-lfm2": [ + { + "round": 0, + "arm": "main-plain", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 134.9239307001102, + "ttft_max": 0.08557750005275011, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 134.91792528327625, + "seconds": 1.890037958, + "prefill_seconds": 0.085051458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.085222875, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 204, + "drafted": 90, + "accepted": 51, + "acceptance_rate": 0.5666666666666667, + "tokens_per_round": 1.2549019607843137 + }, + "sent": 3605043.487071625, + "first": 3605043.572649125, + "last": 3605045.462602958, + "chunks": 205 + } + ] + }, + { + "round": 0, + "arm": "main-plain", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 228.88303862287736, + "ttft_max": 0.3615560829639435, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 59.630612906824005, + "seconds": 4.276327, + "prefill_seconds": 0.091884916, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.267615917, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 215, + "drafted": 51, + "accepted": 40, + "acceptance_rate": 0.7843137254901961, + "tokens_per_round": 1.1906976744186046 + }, + "sent": 3605045.4638635, + "first": 3605045.732146916, + "last": 3605050.008385625, + "chunks": 216 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 57.22032523118682, + "seconds": 4.456458417, + "prefill_seconds": 0.087317125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.087474584, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 215, + "drafted": 51, + "accepted": 40, + "acceptance_rate": 0.7843137254901961, + "tokens_per_round": 1.1906976744186046 + }, + "sent": 3605045.463993666, + "first": 3605045.551974833, + "last": 3605050.008344083, + "chunks": 216 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 60.960532403069806, + "seconds": 4.183034333, + "prefill_seconds": 0.093315625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.360946209, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 215, + "drafted": 51, + "accepted": 40, + "acceptance_rate": 0.7843137254901961, + "tokens_per_round": 1.1906976744186046 + }, + "sent": 3605045.46392825, + "first": 3605045.825484333, + "last": 3605050.008399416, + "chunks": 216 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 58.37623247178504, + "seconds": 4.368216125, + "prefill_seconds": 0.088234584, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.175705708, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 215, + "drafted": 51, + "accepted": 40, + "acceptance_rate": 0.7843137254901961, + "tokens_per_round": 1.1906976744186046 + }, + "sent": 3605045.463787041, + "first": 3605045.64022925, + "last": 3605050.008372833, + "chunks": 216 + } + ] + }, + { + "round": 0, + "arm": "main-plain", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 266.03897119309573, + "ttft_max": 0.7605068329721689, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 34.09353827701317, + "seconds": 7.479423166, + "prefill_seconds": 0.0945975, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.2810115, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605050.009457791, + "first": 3605050.291337958, + "last": 3605057.771036541, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 36.42370539324265, + "seconds": 7.000935167, + "prefill_seconds": 0.096240334, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.759415667, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605050.009511625, + "first": 3605050.770018458, + "last": 3605057.770923125, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 35.929790863783346, + "seconds": 7.097174625, + "prefill_seconds": 0.096306334, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.663202458, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605050.009567791, + "first": 3605050.673718083, + "last": 3605057.771226, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 34.53131103085494, + "seconds": 7.384602333, + "prefill_seconds": 0.094837, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.375764167, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605050.009616541, + "first": 3605050.38616375, + "last": 3605057.770763208, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 33.66776444448369, + "seconds": 7.574010458, + "prefill_seconds": 0.093538167, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.186413208, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605050.00977625, + "first": 3605050.196752333, + "last": 3605057.770742291, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 34.98275518635287, + "seconds": 7.289305792, + "prefill_seconds": 0.095362542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.471108958, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605050.009734708, + "first": 3605050.481606, + "last": 3605057.771092125, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 33.25701120968319, + "seconds": 7.667556125, + "prefill_seconds": 0.092666459, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.0928595, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605050.009661, + "first": 3605050.1031765, + "last": 3605057.770667708, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 35.44872150295576, + "seconds": 7.193489333, + "prefill_seconds": 0.095840875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.566871667, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605050.0093875, + "first": 3605050.577402208, + "last": 3605057.770988958, + "chunks": 229 + } + ] + }, + { + "round": 0, + "arm": "main-plain", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 311.1771970060855, + "ttft_max": 1.4885342502966523, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.010677199833463, + "seconds": 12.743196917, + "prefill_seconds": 0.092467916, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.458635584, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605057.774299166, + "first": 3605058.23469325, + "last": 3605070.977982791, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.157114695311563, + "seconds": 12.650620084, + "prefill_seconds": 0.092833334, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.551451666, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605057.77475, + "first": 3605058.327514458, + "last": 3605070.978081708, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.61350805080629, + "seconds": 12.370529042, + "prefill_seconds": 0.09319175, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.831520625, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605057.7743725, + "first": 3605058.607713041, + "last": 3605070.978289375, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.306786367739754, + "seconds": 12.557378375, + "prefill_seconds": 0.093261458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.644682167, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605057.775298666, + "first": 3605058.420757, + "last": 3605070.978255583, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.585725861822993, + "seconds": 13.019685959, + "prefill_seconds": 0.091256791, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.18242575, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605057.774426458, + "first": 3605057.958124208, + "last": 3605070.978151041, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.44945779897582, + "seconds": 13.110905334, + "prefill_seconds": 0.090963583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.091134833, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605057.774480916, + "first": 3605057.866821083, + "last": 3605070.977915166, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.77102097803939, + "seconds": 12.276719583, + "prefill_seconds": 0.09383925, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.925197417, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605057.775256541, + "first": 3605058.701541375, + "last": 3605070.978314, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.459505653813892, + "seconds": 12.46364425, + "prefill_seconds": 0.093743833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.738352916, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605057.774827583, + "first": 3605058.514517041, + "last": 3605070.978100291, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.86659477677835, + "seconds": 12.835616917, + "prefill_seconds": 0.092491125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.366140709, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605057.774540583, + "first": 3605058.1421745, + "last": 3605070.978321666, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.724519686382095, + "seconds": 12.928071459, + "prefill_seconds": 0.091558708, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.273895042, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605057.774587541, + "first": 3605058.049665916, + "last": 3605070.978064916, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.424400573077474, + "seconds": 11.902316666, + "prefill_seconds": 0.09373375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.299457542, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605057.774637, + "first": 3605059.076021416, + "last": 3605070.978304916, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.767101066869646, + "seconds": 11.714927, + "prefill_seconds": 0.0936205, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.486714584, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605057.774866916, + "first": 3605059.263401166, + "last": 3605070.978264833, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.930350667736132, + "seconds": 12.183264583, + "prefill_seconds": 0.093447667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.01861425, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605057.774904541, + "first": 3605058.795012791, + "last": 3605070.978209041, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.092359658728103, + "seconds": 12.08968575, + "prefill_seconds": 0.09357025, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.112186208, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605057.77468375, + "first": 3605058.888590125, + "last": 3605070.978239833, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.594453198550667, + "seconds": 11.808587958, + "prefill_seconds": 0.093753917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.3930715, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605057.774950875, + "first": 3605059.169768458, + "last": 3605070.978273, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.25698395196336, + "seconds": 11.996057417, + "prefill_seconds": 0.093664167, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.205769666, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605057.774183791, + "first": 3605058.9823115, + "last": 3605070.978227666, + "chunks": 256 + } + ] + }, + { + "round": 0, + "arm": "main-plain", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 157.14330724343966, + "ttft_max": 0.09867354203015566, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 157.13458006784825, + "seconds": 1.62281275, + "prefill_seconds": 0.098100917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.098283125, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 144, + "drafted": 180, + "accepted": 111, + "acceptance_rate": 0.6166666666666667, + "tokens_per_round": 1.7777777777777777 + }, + "sent": 3605070.980492208, + "first": 3605071.07916575, + "last": 3605072.701888375, + "chunks": 145 + } + ] + }, + { + "round": 0, + "arm": "main-plain", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 255.09658594530208, + "ttft_max": 0.4042953331954777, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 69.04753394701699, + "seconds": 3.693108, + "prefill_seconds": 0.103401542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.403547042, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 166, + "drafted": 99, + "accepted": 89, + "acceptance_rate": 0.898989898989899, + "tokens_per_round": 1.5421686746987953 + }, + "sent": 3605072.703041333, + "first": 3605073.107336666, + "last": 3605076.800417541, + "chunks": 167 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 65.40786452620499, + "seconds": 3.89861375, + "prefill_seconds": 0.09968425, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.19807675, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 166, + "drafted": 99, + "accepted": 89, + "acceptance_rate": 0.898989898989899, + "tokens_per_round": 1.5421686746987953 + }, + "sent": 3605072.702960125, + "first": 3605072.901815625, + "last": 3605076.800355916, + "chunks": 167 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 67.16709150848577, + "seconds": 3.796502041, + "prefill_seconds": 0.102110291, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.300229167, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 166, + "drafted": 99, + "accepted": 89, + "acceptance_rate": 0.898989898989899, + "tokens_per_round": 1.5421686746987953 + }, + "sent": 3605072.703119333, + "first": 3605073.003936041, + "last": 3605076.800587125, + "chunks": 167 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 63.77710860351041, + "seconds": 3.998299791, + "prefill_seconds": 0.098301959, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.098602667, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 166, + "drafted": 99, + "accepted": 89, + "acceptance_rate": 0.898989898989899, + "tokens_per_round": 1.5421686746987953 + }, + "sent": 3605072.702881125, + "first": 3605072.802101625, + "last": 3605076.800324375, + "chunks": 167 + } + ] + }, + { + "round": 0, + "arm": "main-plain", + "prompt": "chat", + "streams": 8, + "aggregate_decode_tps": 278.7918956411484, + "ttft_max": 0.8300348329357803, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 34.85159345841741, + "seconds": 7.316738625, + "prefill_seconds": 0.101190292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.101419667, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605076.801830875, + "first": 3605076.904127708, + "last": 3605084.22079525, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 38.09187846980368, + "seconds": 6.694340375, + "prefill_seconds": 0.104884833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.723694375, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605076.802039125, + "first": 3605077.526568416, + "last": 3605084.221311625, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 38.70093873857531, + "seconds": 6.588987459, + "prefill_seconds": 0.105350917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.829023, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605076.8018925, + "first": 3605077.631927333, + "last": 3605084.221138708, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 36.38489569109685, + "seconds": 7.008402667, + "prefill_seconds": 0.10326725, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.409696084, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605076.802100708, + "first": 3605077.212518, + "last": 3605084.220947666, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 35.85689134832941, + "seconds": 7.111603667, + "prefill_seconds": 0.102814666, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.306445875, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605076.802176333, + "first": 3605077.109206125, + "last": 3605084.221413583, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 36.934623923240295, + "seconds": 6.904090875, + "prefill_seconds": 0.104301042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.513925792, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605076.801991083, + "first": 3605077.316830916, + "last": 3605084.221197833, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 35.34592633607471, + "seconds": 7.214409875, + "prefill_seconds": 0.102321, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.203729291, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605076.801947291, + "first": 3605077.006408833, + "last": 3605084.22084075, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 37.504499161057446, + "seconds": 6.79918425, + "prefill_seconds": 0.10486425, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.618898666, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605076.801746666, + "first": 3605077.421678791, + "last": 3605084.220826541, + "chunks": 196 + } + ] + }, + { + "round": 0, + "arm": "main-plain", + "prompt": "chat", + "streams": 16, + "aggregate_decode_tps": 312.84454795576346, + "ttft_max": 1.64287120802328, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.680087870793056, + "seconds": 12.330701958, + "prefill_seconds": 0.103426583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.807823167, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605084.223774208, + "first": 3605085.034211333, + "last": 3605097.365090666, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.78272141148618, + "seconds": 11.70652625, + "prefill_seconds": 0.104171917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.432023125, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605084.225127125, + "first": 3605085.658580458, + "last": 3605097.365670583, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.339825642826977, + "seconds": 12.536980625, + "prefill_seconds": 0.102136875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.6015465, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605084.224564125, + "first": 3605084.827927208, + "last": 3605097.365589625, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 22.17720418879119, + "seconds": 11.498293375, + "prefill_seconds": 0.104055375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.639223542, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605084.22396625, + "first": 3605085.866837458, + "last": 3605097.365761166, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.97827367921699, + "seconds": 11.602367125, + "prefill_seconds": 0.104170167, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.5356085, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605084.223871666, + "first": 3605085.76279375, + "last": 3605097.365196416, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.508063343255248, + "seconds": 12.434133625, + "prefill_seconds": 0.102846541, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.7043205, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605084.224685291, + "first": 3605084.930803125, + "last": 3605097.365631, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.854728882925233, + "seconds": 12.227442583, + "prefill_seconds": 0.103323125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.911108458, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605084.224626083, + "first": 3605085.137543958, + "last": 3605097.365725875, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.401224171004174, + "seconds": 11.91520625, + "prefill_seconds": 0.10408875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.223428792, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605084.224805583, + "first": 3605085.449904833, + "last": 3605097.365363208, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.704330976395816, + "seconds": 12.941317333, + "prefill_seconds": 0.099322166, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.197725875, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605084.224089458, + "first": 3605084.423510833, + "last": 3605097.36485925, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.85722196718048, + "seconds": 12.841675458, + "prefill_seconds": 0.099694667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.297431583, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605084.224746333, + "first": 3605084.523204583, + "last": 3605097.365503916, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.175482988453066, + "seconds": 12.639102625, + "prefill_seconds": 0.10217, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.499395083, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605084.2242065, + "first": 3605084.725776375, + "last": 3605097.365019708, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.554265314859858, + "seconds": 13.040633125, + "prefill_seconds": 0.098349833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.098607125, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605084.224469916, + "first": 3605084.32414025, + "last": 3605097.364759083, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.032647235315533, + "seconds": 12.124008792, + "prefill_seconds": 0.103512084, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.014515417, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605084.224311875, + "first": 3605085.241059291, + "last": 3605097.365551041, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.01370291147283, + "seconds": 12.741270375, + "prefill_seconds": 0.10039225, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.397747709, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605084.224878875, + "first": 3605084.623611958, + "last": 3605097.365279708, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.21588669643829, + "seconds": 12.019294958, + "prefill_seconds": 0.10470325, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.119084083, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605084.22439125, + "first": 3605085.345772541, + "last": 3605097.3653205, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.59063720686525, + "seconds": 11.810675042, + "prefill_seconds": 0.104535833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.327713417, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605084.223643833, + "first": 3605085.554434666, + "last": 3605097.365423625, + "chunks": 256 + } + ] + }, + { + "round": 0, + "arm": "candidate-plain", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 135.59348465320016, + "ttft_max": 0.08575020777061582, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 135.5871612465971, + "seconds": 1.880709041, + "prefill_seconds": 0.085088958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.085327083, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 204, + "drafted": 90, + "accepted": 51, + "acceptance_rate": 0.5666666666666667, + "tokens_per_round": 1.2549019607843137 + }, + "sent": 3605102.731765583, + "first": 3605102.817515791, + "last": 3605104.698137125, + "chunks": 205 + } + ] + }, + { + "round": 0, + "arm": "candidate-plain", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 225.62437397965493, + "ttft_max": 0.34385187504813075, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 58.62565036902639, + "seconds": 4.349631917, + "prefill_seconds": 0.085715541, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.256372, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 215, + "drafted": 51, + "accepted": 40, + "acceptance_rate": 0.7843137254901961, + "tokens_per_round": 1.1906976744186046 + }, + "sent": 3605104.700027083, + "first": 3605104.957033833, + "last": 3605109.306794875, + "chunks": 216 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 57.49275017434879, + "seconds": 4.435341834, + "prefill_seconds": 0.085297834, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.170642833, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 215, + "drafted": 51, + "accepted": 40, + "acceptance_rate": 0.7843137254901961, + "tokens_per_round": 1.1906976744186046 + }, + "sent": 3605104.700094708, + "first": 3605104.871325708, + "last": 3605109.306695375, + "chunks": 216 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 59.82188035125414, + "seconds": 4.262654375, + "prefill_seconds": 0.086960625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.343317417, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 215, + "drafted": 51, + "accepted": 40, + "acceptance_rate": 0.7843137254901961, + "tokens_per_round": 1.1906976744186046 + }, + "sent": 3605104.700153458, + "first": 3605105.044005333, + "last": 3605109.306778791, + "chunks": 216 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 56.40822558708206, + "seconds": 4.520617292, + "prefill_seconds": 0.085123083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.08533825, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 215, + "drafted": 51, + "accepted": 40, + "acceptance_rate": 0.7843137254901961, + "tokens_per_round": 1.1906976744186046 + }, + "sent": 3605104.699931625, + "first": 3605104.786006708, + "last": 3605109.306667166, + "chunks": 216 + } + ] + }, + { + "round": 0, + "arm": "candidate-plain", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 239.06832963446098, + "ttft_max": 0.8974757078103721, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 29.885293065711632, + "seconds": 8.532625042, + "prefill_seconds": 0.109820125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.110008416, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 221, + "drafted": 39, + "accepted": 34, + "acceptance_rate": 0.8717948717948718, + "tokens_per_round": 1.158371040723982 + }, + "sent": 3605109.308800958, + "first": 3605109.419628375, + "last": 3605117.952543958, + "chunks": 222 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 31.982160390065538, + "seconds": 7.973194959, + "prefill_seconds": 0.113419125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.66929175, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 221, + "drafted": 39, + "accepted": 34, + "acceptance_rate": 0.8717948717948718, + "tokens_per_round": 1.158371040723982 + }, + "sent": 3605109.308735791, + "first": 3605109.979150333, + "last": 3605117.952352875, + "chunks": 222 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 30.682415268592585, + "seconds": 8.310949375, + "prefill_seconds": 0.110996166, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.331544792, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 221, + "drafted": 39, + "accepted": 34, + "acceptance_rate": 0.8717948717948718, + "tokens_per_round": 1.158371040723982 + }, + "sent": 3605109.309026541, + "first": 3605109.641333083, + "last": 3605117.952246875, + "chunks": 222 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 31.533558569504233, + "seconds": 8.086623, + "prefill_seconds": 0.112515709, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.555912916, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 221, + "drafted": 39, + "accepted": 34, + "acceptance_rate": 0.8717948717948718, + "tokens_per_round": 1.158371040723982 + }, + "sent": 3605109.308859125, + "first": 3605109.865747875, + "last": 3605117.952323583, + "chunks": 222 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 31.100853907471972, + "seconds": 8.199131791, + "prefill_seconds": 0.111869375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.443369, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 221, + "drafted": 39, + "accepted": 34, + "acceptance_rate": 0.8717948717948718, + "tokens_per_round": 1.158371040723982 + }, + "sent": 3605109.30925225, + "first": 3605109.753270708, + "last": 3605117.952603083, + "chunks": 222 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 32.44146962529053, + "seconds": 7.86030975, + "prefill_seconds": 0.112904125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.782198375, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 221, + "drafted": 39, + "accepted": 34, + "acceptance_rate": 0.8717948717948718, + "tokens_per_round": 1.158371040723982 + }, + "sent": 3605109.308911541, + "first": 3605110.092090125, + "last": 3605117.952276583, + "chunks": 222 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 30.278069487121737, + "seconds": 8.421937208, + "prefill_seconds": 0.110679667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.220623334, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 221, + "drafted": 39, + "accepted": 34, + "acceptance_rate": 0.8717948717948718, + "tokens_per_round": 1.158371040723982 + }, + "sent": 3605109.308964416, + "first": 3605109.530324958, + "last": 3605117.952205375, + "chunks": 222 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 32.9190156055533, + "seconds": 7.746282667, + "prefill_seconds": 0.114026709, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.896113292, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 221, + "drafted": 39, + "accepted": 34, + "acceptance_rate": 0.8717948717948718, + "tokens_per_round": 1.158371040723982 + }, + "sent": 3605109.308652208, + "first": 3605110.206127916, + "last": 3605117.952753625, + "chunks": 222 + } + ] + }, + { + "round": 0, + "arm": "candidate-plain", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 291.00522378471396, + "ttft_max": 1.6490971660241485, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.628776725782256, + "seconds": 12.9911305, + "prefill_seconds": 0.103253458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.129239958, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605117.954696041, + "first": 3605119.086046041, + "last": 3605132.077600958, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.876988695385773, + "seconds": 13.5085105, + "prefill_seconds": 0.102846959, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.61226975, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605117.955237666, + "first": 3605118.568501666, + "last": 3605132.077354166, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.02239297451555, + "seconds": 13.4052535, + "prefill_seconds": 0.103267709, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.715408334, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605117.955115041, + "first": 3605118.671778916, + "last": 3605132.077752458, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.473994241940837, + "seconds": 13.094386125, + "prefill_seconds": 0.103817167, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.026012584, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605117.955405916, + "first": 3605118.982773208, + "last": 3605132.077111625, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.455898082539104, + "seconds": 13.816721292, + "prefill_seconds": 0.10144075, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.304203416, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605117.954764791, + "first": 3605118.26023325, + "last": 3605132.076844166, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.32141160864201, + "seconds": 13.918141541, + "prefill_seconds": 0.101377208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.20284425, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605117.954831125, + "first": 3605118.158791583, + "last": 3605132.076891458, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.73436891317599, + "seconds": 13.611347208, + "prefill_seconds": 0.102797667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.50943175, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605117.955307625, + "first": 3605118.465649166, + "last": 3605132.077550958, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.59395354800949, + "seconds": 13.714135584, + "prefill_seconds": 0.102592667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.406669708, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605117.955194083, + "first": 3605118.362816333, + "last": 3605132.077146791, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.188910288888422, + "seconds": 14.019531459, + "prefill_seconds": 0.101305167, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.101564583, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605117.954890958, + "first": 3605118.057434458, + "last": 3605132.076825458, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.947034626440626, + "seconds": 12.783855083, + "prefill_seconds": 0.103593084, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.336430334, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605117.954939458, + "first": 3605119.293258, + "last": 3605132.077039208, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.109461251593757, + "seconds": 12.680598292, + "prefill_seconds": 0.103316959, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.439643042, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605117.955498541, + "first": 3605119.39657675, + "last": 3605132.077682041, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.17055943357387, + "seconds": 13.30164625, + "prefill_seconds": 0.103561666, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.81862925, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605117.954983666, + "first": 3605118.775327708, + "last": 3605132.077278, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.320957071673337, + "seconds": 13.198103958, + "prefill_seconds": 0.103588, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.922272042, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605117.95503025, + "first": 3605118.8789235, + "last": 3605132.077801333, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.27512247938112, + "seconds": 12.576989375, + "prefill_seconds": 0.103615666, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.543083667, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605117.95506975, + "first": 3605119.500232166, + "last": 3605132.077231583, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.78663726380439, + "seconds": 12.887485458, + "prefill_seconds": 0.103617125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.232856375, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605117.955829083, + "first": 3605119.189652875, + "last": 3605132.077437666, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.443263874905824, + "seconds": 12.473546375, + "prefill_seconds": 0.103488792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.64631775, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605117.95461375, + "first": 3605119.603710916, + "last": 3605132.077716833, + "chunks": 229 + } + ] + }, + { + "round": 0, + "arm": "candidate-plain", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 151.98627432059348, + "ttft_max": 0.10828037513419986, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 151.98426635034073, + "seconds": 1.67780525, + "prefill_seconds": 0.10773275, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.107901625, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 144, + "drafted": 180, + "accepted": 111, + "acceptance_rate": 0.6166666666666667, + "tokens_per_round": 1.7777777777777777 + }, + "sent": 3605132.079570916, + "first": 3605132.187851291, + "last": 3605133.865634375, + "chunks": 145 + } + ] + }, + { + "round": 0, + "arm": "candidate-plain", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 232.97216394450493, + "ttft_max": 0.4620435410179198, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 58.24617511786785, + "seconds": 4.377969875, + "prefill_seconds": 0.112383917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.112573583, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 166, + "drafted": 99, + "accepted": 89, + "acceptance_rate": 0.898989898989899, + "tokens_per_round": 1.5421686746987953 + }, + "sent": 3605133.868006958, + "first": 3605133.981091541, + "last": 3605138.359214, + "chunks": 167 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 63.291191598297736, + "seconds": 4.028996667, + "prefill_seconds": 0.118243792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.461524042, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 166, + "drafted": 99, + "accepted": 89, + "acceptance_rate": 0.898989898989899, + "tokens_per_round": 1.5421686746987953 + }, + "sent": 3605133.868063, + "first": 3605134.330106541, + "last": 3605138.359297, + "chunks": 167 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 59.809433662600966, + "seconds": 4.263541458, + "prefill_seconds": 0.11442525, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.226997416, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 166, + "drafted": 99, + "accepted": 89, + "acceptance_rate": 0.898989898989899, + "tokens_per_round": 1.5421686746987953 + }, + "sent": 3605133.868110208, + "first": 3605134.095524666, + "last": 3605138.359019625, + "chunks": 167 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 61.48683856136229, + "seconds": 4.147229, + "prefill_seconds": 0.11631375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.343276458, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 166, + "drafted": 99, + "accepted": 89, + "acceptance_rate": 0.898989898989899, + "tokens_per_round": 1.5421686746987953 + }, + "sent": 3605133.867932791, + "first": 3605134.2118565, + "last": 3605138.359159125, + "chunks": 167 + } + ] + }, + { + "round": 0, + "arm": "candidate-plain", + "prompt": "chat", + "streams": 8, + "aggregate_decode_tps": 273.27415594074364, + "ttft_max": 0.904660833068192, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 35.21827568262825, + "seconds": 7.240558916, + "prefill_seconds": 0.112328125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.335218041, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605138.361518583, + "first": 3605138.697389458, + "last": 3605145.938332166, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 35.7781916277839, + "seconds": 7.12724675, + "prefill_seconds": 0.113313125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.448416083, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605138.361304, + "first": 3605138.8107295, + "last": 3605145.93808775, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 36.35921233541174, + "seconds": 7.01335325, + "prefill_seconds": 0.113931333, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.562334125, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605138.361365958, + "first": 3605138.924684708, + "last": 3605145.9379155, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 37.58056725385068, + "seconds": 6.785421792, + "prefill_seconds": 0.11387375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.790247917, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605138.361588166, + "first": 3605139.152628875, + "last": 3605145.937980125, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 38.22085397307286, + "seconds": 6.671750458, + "prefill_seconds": 0.113680292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.903884875, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605138.3616655, + "first": 3605139.266326333, + "last": 3605145.938176333, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 34.16111921070131, + "seconds": 7.464626625, + "prefill_seconds": 0.110947917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.11120125, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605138.361424166, + "first": 3605138.473301333, + "last": 3605145.937860666, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 34.68028146516437, + "seconds": 7.3528815, + "prefill_seconds": 0.111733208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.222910333, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605138.361467666, + "first": 3605138.585063458, + "last": 3605145.938006166, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 36.960386954623175, + "seconds": 6.899278417, + "prefill_seconds": 0.1140825, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.67635775, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605138.361225083, + "first": 3605139.038731333, + "last": 3605145.93793625, + "chunks": 196 + } + ] + }, + { + "round": 0, + "arm": "candidate-plain", + "prompt": "chat", + "streams": 16, + "aggregate_decode_tps": 305.3379633728433, + "ttft_max": 1.633806750178337, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.822927297339806, + "seconds": 12.863892208, + "prefill_seconds": 0.101247709, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.59563525, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 202, + "drafted": 53, + "accepted": 53, + "acceptance_rate": 1.0, + "tokens_per_round": 1.2673267326732673 + }, + "sent": 3605145.940995291, + "first": 3605146.538419875, + "last": 3605159.402413541, + "chunks": 203 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.370879482298406, + "seconds": 11.93212475, + "prefill_seconds": 0.104167625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.526834291, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 202, + "drafted": 53, + "accepted": 53, + "acceptance_rate": 1.0, + "tokens_per_round": 1.2673267326732673 + }, + "sent": 3605145.941094541, + "first": 3605147.470343208, + "last": 3605159.402632, + "chunks": 203 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.142649520347533, + "seconds": 12.659704958, + "prefill_seconds": 0.102535625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.799723584, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 202, + "drafted": 53, + "accepted": 53, + "acceptance_rate": 1.0, + "tokens_per_round": 1.2673267326732673 + }, + "sent": 3605145.941823625, + "first": 3605146.7426835, + "last": 3605159.402381083, + "chunks": 203 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.477246794794308, + "seconds": 12.452845959, + "prefill_seconds": 0.103823583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.006469416, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 202, + "drafted": 53, + "accepted": 53, + "acceptance_rate": 1.0, + "tokens_per_round": 1.2673267326732673 + }, + "sent": 3605145.941530375, + "first": 3605146.949606708, + "last": 3605159.402607833, + "chunks": 203 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.00382553794195, + "seconds": 12.1406455, + "prefill_seconds": 0.104434792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.318557541, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 202, + "drafted": 53, + "accepted": 53, + "acceptance_rate": 1.0, + "tokens_per_round": 1.2673267326732673 + }, + "sent": 3605145.941182916, + "first": 3605147.261812166, + "last": 3605159.402363333, + "chunks": 203 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.668129306462152, + "seconds": 12.965137458, + "prefill_seconds": 0.100130583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.49446125, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 202, + "drafted": 53, + "accepted": 53, + "acceptance_rate": 1.0, + "tokens_per_round": 1.2673267326732673 + }, + "sent": 3605145.941742708, + "first": 3605146.437193875, + "last": 3605159.402312875, + "chunks": 203 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.185936302711863, + "seconds": 12.036286542, + "prefill_seconds": 0.104347459, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.422695625, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 202, + "drafted": 53, + "accepted": 53, + "acceptance_rate": 1.0, + "tokens_per_round": 1.2673267326732673 + }, + "sent": 3605145.942140416, + "first": 3605147.366185, + "last": 3605159.402436958, + "chunks": 203 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.517372686097072, + "seconds": 13.065283125, + "prefill_seconds": 0.099564083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.394375834, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 202, + "drafted": 53, + "accepted": 53, + "acceptance_rate": 1.0, + "tokens_per_round": 1.2673267326732673 + }, + "sent": 3605145.941583333, + "first": 3605146.337003916, + "last": 3605159.40296525, + "chunks": 203 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.084778292035907, + "seconds": 13.361433709, + "prefill_seconds": 0.09827075, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.098502, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 202, + "drafted": 53, + "accepted": 53, + "acceptance_rate": 1.0, + "tokens_per_round": 1.2673267326732673 + }, + "sent": 3605145.941264916, + "first": 3605146.040833125, + "last": 3605159.402510166, + "chunks": 203 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.560151994421332, + "seconds": 11.827374875, + "prefill_seconds": 0.104764667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.6315635, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 202, + "drafted": 53, + "accepted": 53, + "acceptance_rate": 1.0, + "tokens_per_round": 1.2673267326732673 + }, + "sent": 3605145.941330583, + "first": 3605147.575137333, + "last": 3605159.402668125, + "chunks": 203 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.649004277684163, + "seconds": 12.34926375, + "prefill_seconds": 0.1036195, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.110109584, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 202, + "drafted": 53, + "accepted": 53, + "acceptance_rate": 1.0, + "tokens_per_round": 1.2673267326732673 + }, + "sent": 3605145.941380125, + "first": 3605147.053187208, + "last": 3605159.403075916, + "chunks": 203 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.82467850167173, + "seconds": 12.245087, + "prefill_seconds": 0.104157708, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.21415075, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 202, + "drafted": 53, + "accepted": 53, + "acceptance_rate": 1.0, + "tokens_per_round": 1.2673267326732673 + }, + "sent": 3605145.94190325, + "first": 3605147.157471791, + "last": 3605159.402822958, + "chunks": 203 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.307930746690957, + "seconds": 12.55667075, + "prefill_seconds": 0.103049709, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.902659542, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 202, + "drafted": 53, + "accepted": 53, + "acceptance_rate": 1.0, + "tokens_per_round": 1.2673267326732673 + }, + "sent": 3605145.941425291, + "first": 3605146.845705916, + "last": 3605159.402448333, + "chunks": 203 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.225912789847122, + "seconds": 13.263349459, + "prefill_seconds": 0.098078833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.196348167, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 202, + "drafted": 53, + "accepted": 53, + "acceptance_rate": 1.0, + "tokens_per_round": 1.2673267326732673 + }, + "sent": 3605145.941657666, + "first": 3605146.138916416, + "last": 3605159.402203416, + "chunks": 203 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.980784034931435, + "seconds": 12.762261959, + "prefill_seconds": 0.101674958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.697299833, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 202, + "drafted": 53, + "accepted": 53, + "acceptance_rate": 1.0, + "tokens_per_round": 1.2673267326732673 + }, + "sent": 3605145.941471583, + "first": 3605146.640102875, + "last": 3605159.402835125, + "chunks": 203 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.36980007410188, + "seconds": 13.164823541, + "prefill_seconds": 0.0985185, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.294843917, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 202, + "drafted": 53, + "accepted": 53, + "acceptance_rate": 1.0, + "tokens_per_round": 1.2673267326732673 + }, + "sent": 3605145.940892041, + "first": 3605146.237452041, + "last": 3605159.402982875, + "chunks": 203 + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 124.69219413734717, + "ttft_max": 0.09309358289465308, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 124.68141557488158, + "seconds": 2.045212583, + "prefill_seconds": 0.092243959, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.092521084, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 2 + }, + "speculative": { + "rounds": 86, + "drafted": 275, + "accepted": 169, + "acceptance_rate": 0.6145454545454545, + "tokens_per_round": 2.9767441860465116 + }, + "sent": 3605165.7339885, + "first": 3605165.827082083, + "last": 3605167.872117875, + "chunks": 87 + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 208.12402438377157, + "ttft_max": 0.4569550408050418, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.27837233945698, + "seconds": 4.78618225, + "prefill_seconds": 0.114795834, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.224114042, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 218, + "drafted": 46, + "accepted": 37, + "acceptance_rate": 0.8043478260869565, + "tokens_per_round": 1.1743119266055047 + }, + "sent": 3605167.875097708, + "first": 3605168.099788708, + "last": 3605172.8859015, + "chunks": 219 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 55.99561521394278, + "seconds": 4.553928, + "prefill_seconds": 0.116606292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.456425666, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 218, + "drafted": 46, + "accepted": 37, + "acceptance_rate": 0.8043478260869565, + "tokens_per_round": 1.1743119266055047 + }, + "sent": 3605167.875204875, + "first": 3605168.332159916, + "last": 3605172.885936333, + "chunks": 219 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.03033548440874, + "seconds": 4.900987042, + "prefill_seconds": 0.109144042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.109344458, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 218, + "drafted": 46, + "accepted": 37, + "acceptance_rate": 0.8043478260869565, + "tokens_per_round": 1.1743119266055047 + }, + "sent": 3605167.875152166, + "first": 3605167.985012458, + "last": 3605172.885886541, + "chunks": 219 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 54.59856064422458, + "seconds": 4.670452792, + "prefill_seconds": 0.115727375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.339796125, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 218, + "drafted": 46, + "accepted": 37, + "acceptance_rate": 0.8043478260869565, + "tokens_per_round": 1.1743119266055047 + }, + "sent": 3605167.875021041, + "first": 3605168.215677, + "last": 3605172.885920875, + "chunks": 219 + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 251.6911281708875, + "ttft_max": 0.8647510409355164, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 34.220280315461096, + "seconds": 7.451721542, + "prefill_seconds": 0.109500625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.75442575, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 227, + "drafted": 31, + "accepted": 28, + "acceptance_rate": 0.9032258064516129, + "tokens_per_round": 1.1277533039647578 + }, + "sent": 3605172.889201708, + "first": 3605173.644758291, + "last": 3605181.09655925, + "chunks": 228 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 32.784678873897505, + "seconds": 7.778023417, + "prefill_seconds": 0.107079834, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.428348125, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 227, + "drafted": 31, + "accepted": 28, + "acceptance_rate": 0.9032258064516129, + "tokens_per_round": 1.1277533039647578 + }, + "sent": 3605172.888983666, + "first": 3605173.318376625, + "last": 3605181.096337708, + "chunks": 228 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 33.24804015841713, + "seconds": 7.669625, + "prefill_seconds": 0.108443959, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.536531417, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 227, + "drafted": 31, + "accepted": 28, + "acceptance_rate": 0.9032258064516129, + "tokens_per_round": 1.1277533039647578 + }, + "sent": 3605172.889049125, + "first": 3605173.426806583, + "last": 3605181.096312666, + "chunks": 228 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 34.729339277725074, + "seconds": 7.342495, + "prefill_seconds": 0.109349667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.863685958, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 227, + "drafted": 31, + "accepted": 28, + "acceptance_rate": 0.9032258064516129, + "tokens_per_round": 1.1277533039647578 + }, + "sent": 3605172.8893795, + "first": 3605173.754130541, + "last": 3605181.096878333, + "chunks": 228 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 33.724893118953005, + "seconds": 7.561180375, + "prefill_seconds": 0.1084465, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.644938125, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 227, + "drafted": 31, + "accepted": 28, + "acceptance_rate": 0.9032258064516129, + "tokens_per_round": 1.1277533039647578 + }, + "sent": 3605172.889145291, + "first": 3605173.535275375, + "last": 3605181.09632675, + "chunks": 228 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 32.33946614750428, + "seconds": 7.885102334, + "prefill_seconds": 0.106294208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.321225417, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 227, + "drafted": 31, + "accepted": 28, + "acceptance_rate": 0.9032258064516129, + "tokens_per_round": 1.1277533039647578 + }, + "sent": 3605172.889298375, + "first": 3605173.211260166, + "last": 3605181.096401833, + "chunks": 228 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 31.909315309593254, + "seconds": 7.991396792, + "prefill_seconds": 0.11322875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.215124917, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 227, + "drafted": 31, + "accepted": 28, + "acceptance_rate": 0.9032258064516129, + "tokens_per_round": 1.1277533039647578 + }, + "sent": 3605172.889103083, + "first": 3605173.104958333, + "last": 3605181.096496208, + "chunks": 228 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 31.4635540533525, + "seconds": 8.10461525, + "prefill_seconds": 0.1015975, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.101866833, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 227, + "drafted": 31, + "accepted": 28, + "acceptance_rate": 0.9032258064516129, + "tokens_per_round": 1.1277533039647578 + }, + "sent": 3605172.888902375, + "first": 3605172.991705875, + "last": 3605181.096285333, + "chunks": 228 + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 313.42934103469173, + "ttft_max": 1.5213572499342263, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.304617493868918, + "seconds": 11.96923625, + "prefill_seconds": 0.094936375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.136720792, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605181.1012335, + "first": 3605182.239094333, + "last": 3605194.208778666, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.18264648591888, + "seconds": 12.634616584, + "prefill_seconds": 0.094024083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.471642333, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605181.100237125, + "first": 3605181.573688583, + "last": 3605194.208247916, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.649907776155853, + "seconds": 11.778341166, + "prefill_seconds": 0.095603625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.327519875, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605181.100312666, + "first": 3605182.430033666, + "last": 3605194.208865708, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.88634233985453, + "seconds": 12.822870875, + "prefill_seconds": 0.093289083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.283379084, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605181.100461416, + "first": 3605181.385283833, + "last": 3605194.208107541, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.47560058642282, + "seconds": 11.873940334, + "prefill_seconds": 0.095297208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.231966833, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605181.100956208, + "first": 3605182.334405541, + "last": 3605194.208390541, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.33497099959012, + "seconds": 12.539973625, + "prefill_seconds": 0.094606625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.566118125, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605181.100694333, + "first": 3605181.668279125, + "last": 3605194.208471791, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.82723891494671, + "seconds": 11.682650334, + "prefill_seconds": 0.095696292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.423259709, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605181.100372083, + "first": 3605182.525757166, + "last": 3605194.208583583, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.742696023980194, + "seconds": 12.916169083, + "prefill_seconds": 0.100347583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.19014025, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605181.100417375, + "first": 3605181.291986041, + "last": 3605194.208053541, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.033635098473024, + "seconds": 12.728593625, + "prefill_seconds": 0.094364917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.377678292, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605181.101027916, + "first": 3605181.479648083, + "last": 3605194.208136083, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.971089533527618, + "seconds": 12.159597125, + "prefill_seconds": 0.095635834, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.946418208, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605181.100824541, + "first": 3605182.048714083, + "last": 3605194.208658458, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.646486217524153, + "seconds": 12.350769875, + "prefill_seconds": 0.0944225, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.755228792, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605181.100615916, + "first": 3605181.857476375, + "last": 3605194.208165041, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.590524741306723, + "seconds": 13.016496667, + "prefill_seconds": 0.089569291, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.089765958, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605181.10051575, + "first": 3605181.191606166, + "last": 3605194.208012375, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.136996552358998, + "seconds": 12.064154875, + "prefill_seconds": 0.095451583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.04185975, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605181.101147708, + "first": 3605182.144160458, + "last": 3605194.208894166, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.489745150967604, + "seconds": 12.445249959, + "prefill_seconds": 0.094742375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.660845583, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605181.1007485, + "first": 3605181.763080083, + "last": 3605194.20826575, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.80747677102762, + "seconds": 12.255210125, + "prefill_seconds": 0.095575875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.850826625, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605181.10057275, + "first": 3605181.953056416, + "last": 3605194.208488833, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 22.00764844769439, + "seconds": 11.586880834, + "prefill_seconds": 0.095791084, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.518658584, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605181.100157833, + "first": 3605182.621515083, + "last": 3605194.208810041, + "chunks": 256 + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 144.5383970515986, + "ttft_max": 0.09813770884647965, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 144.53195240137714, + "seconds": 1.76431575, + "prefill_seconds": 0.097527583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.097719208, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 2 + }, + "speculative": { + "rounds": 78, + "drafted": 285, + "accepted": 177, + "acceptance_rate": 0.6210526315789474, + "tokens_per_round": 3.282051282051282 + }, + "sent": 3605194.211645666, + "first": 3605194.309783375, + "last": 3605196.074020458, + "chunks": 79 + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 224.85815545136234, + "ttft_max": 0.46384895825758576, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 56.21441182975977, + "seconds": 4.536203292, + "prefill_seconds": 0.109698167, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.109858792, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 166, + "drafted": 97, + "accepted": 89, + "acceptance_rate": 0.9175257731958762, + "tokens_per_round": 1.5421686746987953 + }, + "sent": 3605196.076701458, + "first": 3605196.18695575, + "last": 3605200.72311825, + "chunks": 167 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 60.964521487528344, + "seconds": 4.182760625, + "prefill_seconds": 0.119314042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.463325, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 166, + "drafted": 97, + "accepted": 89, + "acceptance_rate": 0.9175257731958762, + "tokens_per_round": 1.5421686746987953 + }, + "sent": 3605196.076561208, + "first": 3605196.540410166, + "last": 3605200.723148791, + "chunks": 167 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 59.27368123032637, + "seconds": 4.302078, + "prefill_seconds": 0.118036208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.344047292, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 166, + "drafted": 97, + "accepted": 89, + "acceptance_rate": 0.9175257731958762, + "tokens_per_round": 1.5421686746987953 + }, + "sent": 3605196.076630166, + "first": 3605196.421134458, + "last": 3605200.723136666, + "chunks": 167 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 57.690874674474806, + "seconds": 4.420109791, + "prefill_seconds": 0.116078292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.226017625, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 166, + "drafted": 97, + "accepted": 89, + "acceptance_rate": 0.9175257731958762, + "tokens_per_round": 1.5421686746987953 + }, + "sent": 3605196.076477958, + "first": 3605196.30318375, + "last": 3605200.723097416, + "chunks": 167 + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 8, + "aggregate_decode_tps": 228.3324076029578, + "ttft_max": 1.0712138330563903, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 29.879040126457436, + "seconds": 8.534410708, + "prefill_seconds": 0.13294175, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.52166925, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 65, + "accepted": 60, + "acceptance_rate": 0.9230769230769231, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605200.726163041, + "first": 3605201.248800291, + "last": 3605209.783159708, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 28.54197716851694, + "seconds": 8.934209375, + "prefill_seconds": 0.121683084, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.121890709, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 65, + "accepted": 60, + "acceptance_rate": 0.9230769230769231, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605200.726410625, + "first": 3605200.848901708, + "last": 3605209.783069583, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 30.860278678874316, + "seconds": 8.263049166, + "prefill_seconds": 0.136530834, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.793128791, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 65, + "accepted": 60, + "acceptance_rate": 0.9230769230769231, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605200.72622, + "first": 3605201.520281958, + "last": 3605209.783206916, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 31.932770373201418, + "seconds": 7.985527, + "prefill_seconds": 0.1394635, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.070516375, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 65, + "accepted": 60, + "acceptance_rate": 0.9230769230769231, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605200.726513375, + "first": 3605201.797727208, + "last": 3605209.7831435, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 31.384495157857014, + "seconds": 8.125031125, + "prefill_seconds": 0.138017042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.931052709, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 65, + "accepted": 60, + "acceptance_rate": 0.9230769230769231, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605200.726452208, + "first": 3605201.658244208, + "last": 3605209.7832445, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 28.98285892631777, + "seconds": 8.798303875, + "prefill_seconds": 0.13589625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.257799375, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 65, + "accepted": 60, + "acceptance_rate": 0.9230769230769231, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605200.726282916, + "first": 3605200.984814458, + "last": 3605209.783097333, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 29.42073754897265, + "seconds": 8.667355792, + "prefill_seconds": 0.131006333, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.38884425, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 65, + "accepted": 60, + "acceptance_rate": 0.9230769230769231, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605200.726354375, + "first": 3605201.115897041, + "last": 3605209.783179041, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 30.358845572752436, + "seconds": 8.399528875, + "prefill_seconds": 0.134879375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.656546625, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 65, + "accepted": 60, + "acceptance_rate": 0.9230769230769231, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605200.72609275, + "first": 3605201.383693708, + "last": 3605209.783224708, + "chunks": 196 + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 16, + "aggregate_decode_tps": 282.3386996870639, + "ttft_max": 1.8823581668548286, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.226622070138276, + "seconds": 13.262860167, + "prefill_seconds": 0.119095916, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.296116708, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605209.787171458, + "first": 3605211.085496583, + "last": 3605224.34824125, + "chunks": 254 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 17.79779946245813, + "seconds": 14.327613958, + "prefill_seconds": 0.122848958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.231967375, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605209.787227958, + "first": 3605210.020624291, + "last": 3605224.34815675, + "chunks": 254 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.055507778747845, + "seconds": 13.38195775, + "prefill_seconds": 0.119004625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.177078458, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605209.788033583, + "first": 3605210.966369083, + "last": 3605224.348362791, + "chunks": 254 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.5742003097195, + "seconds": 13.027352125, + "prefill_seconds": 0.11742125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.531489708, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605209.787339583, + "first": 3605211.321101208, + "last": 3605224.348435458, + "chunks": 254 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.24437998196221, + "seconds": 13.976906875, + "prefill_seconds": 0.118000417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.582595542, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605209.787291625, + "first": 3605210.371332125, + "last": 3605224.348315875, + "chunks": 254 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.399385883911336, + "seconds": 13.144746, + "prefill_seconds": 0.118168417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.414581041, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605209.787738208, + "first": 3605211.203957833, + "last": 3605224.3483335, + "chunks": 254 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.400278522490467, + "seconds": 13.858485875, + "prefill_seconds": 0.118439167, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.700956625, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605209.787458583, + "first": 3605210.489796333, + "last": 3605224.348448625, + "chunks": 254 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 17.942747804926768, + "seconds": 14.211870042, + "prefill_seconds": 0.1157755, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.347734791, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605209.787552291, + "first": 3605210.136408458, + "last": 3605224.348429083, + "chunks": 254 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.11232199448928, + "seconds": 12.678794625, + "prefill_seconds": 0.115700042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.880046584, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605209.787384666, + "first": 3605211.669742833, + "last": 3605224.34841975, + "chunks": 254 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.93046255050799, + "seconds": 12.794484792, + "prefill_seconds": 0.116433, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.764330792, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605209.787492458, + "first": 3605211.554018, + "last": 3605224.348388208, + "chunks": 254 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.722089689626134, + "seconds": 13.620274458, + "prefill_seconds": 0.119076291, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.939076041, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605209.787769583, + "first": 3605210.728119333, + "last": 3605224.348404166, + "chunks": 254 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.750736349295554, + "seconds": 12.910911041, + "prefill_seconds": 0.116433375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.647919833, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605209.787423583, + "first": 3605211.437522208, + "last": 3605224.3483775, + "chunks": 254 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.091644716210116, + "seconds": 14.09490425, + "prefill_seconds": 0.116940458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.464571833, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605209.787609208, + "first": 3605210.253391583, + "last": 3605224.348343875, + "chunks": 254 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.559834143429526, + "seconds": 13.739346916, + "prefill_seconds": 0.119137375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.819974375, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605209.787643166, + "first": 3605210.608930375, + "last": 3605224.34844175, + "chunks": 254 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 17.646478088315806, + "seconds": 14.450475541, + "prefill_seconds": 0.109028625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.109224, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605209.787675125, + "first": 3605209.89771975, + "last": 3605224.348114458, + "chunks": 254 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.88748538643221, + "seconds": 13.501003166, + "prefill_seconds": 0.119320625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.058378042, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605209.787071333, + "first": 3605210.847375166, + "last": 3605224.348277583, + "chunks": 254 + } + ] + }, + { + "round": 1, + "arm": "candidate-plain", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 135.3705299959295, + "ttft_max": 0.08670658292248845, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 135.36170933161313, + "seconds": 1.883841459, + "prefill_seconds": 0.085781333, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.086096792, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 204, + "drafted": 90, + "accepted": 51, + "acceptance_rate": 0.5666666666666667, + "tokens_per_round": 1.2549019607843137 + }, + "sent": 3605230.084579083, + "first": 3605230.171285666, + "last": 3605232.055004375, + "chunks": 205 + } + ] + }, + { + "round": 1, + "arm": "candidate-plain", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 247.18854085067397, + "ttft_max": 0.3422554158605635, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 64.4663127278345, + "seconds": 3.955554292, + "prefill_seconds": 0.085400125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.255194791, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 215, + "drafted": 51, + "accepted": 40, + "acceptance_rate": 0.7843137254901961, + "tokens_per_round": 1.1906976744186046 + }, + "sent": 3605232.05815025, + "first": 3605232.314051291, + "last": 3605236.269604041, + "chunks": 216 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 63.10377137843549, + "seconds": 4.040962916, + "prefill_seconds": 0.085076667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.169815167, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 215, + "drafted": 51, + "accepted": 40, + "acceptance_rate": 0.7843137254901961, + "tokens_per_round": 1.1906976744186046 + }, + "sent": 3605232.0582255, + "first": 3605232.228662958, + "last": 3605236.269935791, + "chunks": 216 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 61.80205211183789, + "seconds": 4.126076583, + "prefill_seconds": 0.084549458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.084750125, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 215, + "drafted": 51, + "accepted": 40, + "acceptance_rate": 0.7843137254901961, + "tokens_per_round": 1.1906976744186046 + }, + "sent": 3605232.058285291, + "first": 3605232.143530916, + "last": 3605236.269508, + "chunks": 216 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 65.90286679271894, + "seconds": 3.869330917, + "prefill_seconds": 0.086262041, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.341408667, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 215, + "drafted": 51, + "accepted": 40, + "acceptance_rate": 0.7843137254901961, + "tokens_per_round": 1.1906976744186046 + }, + "sent": 3605232.058066125, + "first": 3605232.400321541, + "last": 3605236.269820875, + "chunks": 216 + } + ] + }, + { + "round": 1, + "arm": "candidate-plain", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 268.44417944890495, + "ttft_max": 0.7584767080843449, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 35.30311403855571, + "seconds": 7.223158833, + "prefill_seconds": 0.095596542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.468107459, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605236.273302125, + "first": 3605236.74224275, + "last": 3605243.965309791, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 36.268515877396204, + "seconds": 7.0308915, + "prefill_seconds": 0.09632875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.66032425, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605236.273358083, + "first": 3605236.9345115, + "last": 3605243.965373625, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 33.97195388726913, + "seconds": 7.50619175, + "prefill_seconds": 0.093056167, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.185104417, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605236.27363875, + "first": 3605236.459160666, + "last": 3605243.965230541, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 34.39914261753111, + "seconds": 7.412975458, + "prefill_seconds": 0.093215666, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.278315125, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605236.27341825, + "first": 3605236.552370416, + "last": 3605243.965326041, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 34.842132802854294, + "seconds": 7.318725333, + "prefill_seconds": 0.09426725, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.372486833, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605236.27355425, + "first": 3605236.646647666, + "last": 3605243.965269583, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 33.55590475435463, + "seconds": 7.599258666, + "prefill_seconds": 0.091830958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.092044417, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605236.273475458, + "first": 3605236.366028375, + "last": 3605243.965252666, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 35.77833657849277, + "seconds": 7.127217875, + "prefill_seconds": 0.095941375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.563977417, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605236.273594833, + "first": 3605236.83817375, + "last": 3605243.965293083, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 36.776712264682, + "seconds": 6.933735625, + "prefill_seconds": 0.097161791, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.757486292, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605236.273225458, + "first": 3605237.031702166, + "last": 3605243.965334, + "chunks": 229 + } + ] + }, + { + "round": 1, + "arm": "candidate-plain", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 319.2847676851127, + "ttft_max": 1.4483123333193362, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.80600240787437, + "seconds": 11.694027875, + "prefill_seconds": 0.091334042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.173616291, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605243.969443166, + "first": 3605245.145292416, + "last": 3605256.839270791, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 22.150983325961008, + "seconds": 11.511904291, + "prefill_seconds": 0.091012292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.355748375, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605243.96950325, + "first": 3605245.327496291, + "last": 3605256.839313541, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.145950469632904, + "seconds": 12.0590465, + "prefill_seconds": 0.090774708, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.808576667, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605243.970349458, + "first": 3605244.780217958, + "last": 3605256.839328166, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.2376016783258, + "seconds": 12.600307292, + "prefill_seconds": 0.089279583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.267583958, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605243.969966791, + "first": 3605244.238831666, + "last": 3605256.839117791, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.833228656571702, + "seconds": 12.2400615, + "prefill_seconds": 0.090448792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.627604958, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605243.970547625, + "first": 3605244.599190125, + "last": 3605256.839249541, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.98798123254718, + "seconds": 12.14981075, + "prefill_seconds": 0.090244583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.717820292, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605243.970049083, + "first": 3605244.689422625, + "last": 3605256.83936075, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.68041589249006, + "seconds": 12.330506375, + "prefill_seconds": 0.09017325, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.537172167, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605243.969576333, + "first": 3605244.508701625, + "last": 3605256.839198958, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.955637312919926, + "seconds": 12.778344084, + "prefill_seconds": 0.089494417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.089734333, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605243.969684041, + "first": 3605244.060805875, + "last": 3605256.839061666, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.095216256935384, + "seconds": 12.68958725, + "prefill_seconds": 0.088746, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.178434625, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605243.96963825, + "first": 3605244.149560791, + "last": 3605256.8390755, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.637063857646872, + "seconds": 11.785332875, + "prefill_seconds": 0.091055709, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.08228675, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605243.969914375, + "first": 3605245.053956541, + "last": 3605256.839345041, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 22.32804931103493, + "seconds": 11.420612542, + "prefill_seconds": 0.091313042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.446868375, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605243.97046325, + "first": 3605245.418775583, + "last": 3605256.83936725, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.383060046456592, + "seconds": 12.5103885, + "prefill_seconds": 0.089931041, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.357453166, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605243.969826875, + "first": 3605244.32876, + "last": 3605256.839146, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.9772229504082, + "seconds": 11.602921833, + "prefill_seconds": 0.09111725, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.264695667, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605243.97027275, + "first": 3605245.236458291, + "last": 3605256.839352625, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.530285809656654, + "seconds": 12.420674625, + "prefill_seconds": 0.089750042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.447048958, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605243.970631333, + "first": 3605244.41854875, + "last": 3605256.83918125, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.471178787040312, + "seconds": 11.876385667, + "prefill_seconds": 0.091522541, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.99120025, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605243.969740833, + "first": 3605244.962909166, + "last": 3605256.8392915, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.307010309878976, + "seconds": 11.967892083, + "prefill_seconds": 0.091147875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.899696625, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605243.969366083, + "first": 3605244.871349166, + "last": 3605256.839337791, + "chunks": 256 + } + ] + }, + { + "round": 1, + "arm": "candidate-plain", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 157.47416781759355, + "ttft_max": 0.0967360413633287, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 157.4648286095686, + "seconds": 1.61940925, + "prefill_seconds": 0.096069542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.096265166, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 144, + "drafted": 180, + "accepted": 111, + "acceptance_rate": 0.6166666666666667, + "tokens_per_round": 1.7777777777777777 + }, + "sent": 3605256.844516, + "first": 3605256.941252041, + "last": 3605258.56056525, + "chunks": 145 + } + ] + }, + { + "round": 1, + "arm": "candidate-plain", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 254.6708458908824, + "ttft_max": 0.3996269996277988, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 65.28964545138331, + "seconds": 3.905672917, + "prefill_seconds": 0.099539084, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.19903375, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 166, + "drafted": 99, + "accepted": 89, + "acceptance_rate": 0.898989898989899, + "tokens_per_round": 1.5421686746987953 + }, + "sent": 3605258.564169375, + "first": 3605258.763831125, + "last": 3605262.669454333, + "chunks": 167 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 66.9946382614655, + "seconds": 3.80627475, + "prefill_seconds": 0.099390625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.298365042, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 166, + "drafted": 99, + "accepted": 89, + "acceptance_rate": 0.898989898989899, + "tokens_per_round": 1.5421686746987953 + }, + "sent": 3605258.564251458, + "first": 3605258.863197083, + "last": 3605262.669443208, + "chunks": 167 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 68.81561334793373, + "seconds": 3.705554417, + "prefill_seconds": 0.100711833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.399111875, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 166, + "drafted": 99, + "accepted": 89, + "acceptance_rate": 0.898989898989899, + "tokens_per_round": 1.5421686746987953 + }, + "sent": 3605258.564329916, + "first": 3605258.963956916, + "last": 3605262.669422791, + "chunks": 167 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 63.66673319241069, + "seconds": 4.005231417, + "prefill_seconds": 0.099261291, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.09953825, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 166, + "drafted": 99, + "accepted": 89, + "acceptance_rate": 0.898989898989899, + "tokens_per_round": 1.5421686746987953 + }, + "sent": 3605258.564030833, + "first": 3605258.664284458, + "last": 3605262.669395041, + "chunks": 167 + } + ] + }, + { + "round": 1, + "arm": "candidate-plain", + "prompt": "chat", + "streams": 8, + "aggregate_decode_tps": 285.70738648598456, + "ttft_max": 0.8014092496596277, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 35.716172818794945, + "seconds": 7.13962275, + "prefill_seconds": 0.098861958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.0990905, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605262.67330575, + "first": 3605262.773175375, + "last": 3605269.912740333, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 38.99465037162661, + "seconds": 6.539358542, + "prefill_seconds": 0.10073975, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.699211458, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605262.673546166, + "first": 3605263.37350375, + "last": 3605269.912882291, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 37.82942755620118, + "seconds": 6.740784, + "prefill_seconds": 0.100308, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.497804958, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605262.673493833, + "first": 3605263.172063958, + "last": 3605269.91280275, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 37.274715233487235, + "seconds": 6.841098541, + "prefill_seconds": 0.099807333, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.397588125, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605262.673645791, + "first": 3605263.07171825, + "last": 3605269.9129505, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 36.73872080653342, + "seconds": 6.940905791, + "prefill_seconds": 0.099730791, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.297757625, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605262.673439583, + "first": 3605262.971927291, + "last": 3605269.912967, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 36.21838064353387, + "seconds": 7.040624, + "prefill_seconds": 0.098990417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.198081833, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605262.673371833, + "first": 3605262.872175791, + "last": 3605269.913055333, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 39.61007249398499, + "seconds": 6.437756458, + "prefill_seconds": 0.101606083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.800571709, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605262.673717208, + "first": 3605263.475126458, + "last": 3605269.913224208, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 38.403006695548854, + "seconds": 6.640105084, + "prefill_seconds": 0.100696167, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.598439417, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605262.673213166, + "first": 3605263.272753958, + "last": 3605269.913347791, + "chunks": 196 + } + ] + }, + { + "round": 1, + "arm": "candidate-plain", + "prompt": "chat", + "streams": 16, + "aggregate_decode_tps": 303.7043595048409, + "ttft_max": 1.6232123747467995, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.707553939051333, + "seconds": 12.939200917, + "prefill_seconds": 0.100283042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.591923375, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 219, + "drafted": 36, + "accepted": 36, + "acceptance_rate": 1.0, + "tokens_per_round": 1.1689497716894977 + }, + "sent": 3605269.917221625, + "first": 3605270.510649541, + "last": 3605283.449745416, + "chunks": 220 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.1210335801187, + "seconds": 13.336099167, + "prefill_seconds": 0.097765375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.195100708, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 219, + "drafted": 36, + "accepted": 36, + "acceptance_rate": 1.0, + "tokens_per_round": 1.1689497716894977 + }, + "sent": 3605269.917575666, + "first": 3605270.113639666, + "last": 3605283.44979175, + "chunks": 220 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.01970233116774, + "seconds": 12.737452125, + "prefill_seconds": 0.101138208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.793557834, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 219, + "drafted": 36, + "accepted": 36, + "acceptance_rate": 1.0, + "tokens_per_round": 1.1689497716894977 + }, + "sent": 3605269.917355333, + "first": 3605270.712412625, + "last": 3605283.449932916, + "chunks": 220 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.346740438964275, + "seconds": 12.532719959, + "prefill_seconds": 0.103530875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.998317875, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 219, + "drafted": 36, + "accepted": 36, + "acceptance_rate": 1.0, + "tokens_per_round": 1.1689497716894977 + }, + "sent": 3605269.917632041, + "first": 3605270.917182083, + "last": 3605283.449850291, + "chunks": 220 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.693519640614777, + "seconds": 12.322698334, + "prefill_seconds": 0.10384725, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.208413917, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 219, + "drafted": 36, + "accepted": 36, + "acceptance_rate": 1.0, + "tokens_per_round": 1.1689497716894977 + }, + "sent": 3605269.918093333, + "first": 3605271.127308625, + "last": 3605283.449959291, + "chunks": 220 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.86200798449543, + "seconds": 12.838581084, + "prefill_seconds": 0.100614541, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.692508416, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 219, + "drafted": 36, + "accepted": 36, + "acceptance_rate": 1.0, + "tokens_per_round": 1.1689497716894977 + }, + "sent": 3605269.917294666, + "first": 3605270.611324958, + "last": 3605283.449966333, + "chunks": 220 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.26308732203688, + "seconds": 13.237753416, + "prefill_seconds": 0.09834925, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.293452084, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 219, + "drafted": 36, + "accepted": 36, + "acceptance_rate": 1.0, + "tokens_per_round": 1.1689497716894977 + }, + "sent": 3605269.917457625, + "first": 3605270.211989041, + "last": 3605283.449669833, + "chunks": 220 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.180066868031723, + "seconds": 12.636231667, + "prefill_seconds": 0.101215833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.894750166, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 219, + "drafted": 36, + "accepted": 36, + "acceptance_rate": 1.0, + "tokens_per_round": 1.1689497716894977 + }, + "sent": 3605269.917404291, + "first": 3605270.813657875, + "last": 3605283.449834666, + "chunks": 220 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.228878666832493, + "seconds": 12.011939208, + "prefill_seconds": 0.102896834, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.518830125, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 219, + "drafted": 36, + "accepted": 36, + "acceptance_rate": 1.0, + "tokens_per_round": 1.1689497716894977 + }, + "sent": 3605269.917793791, + "first": 3605271.437999583, + "last": 3605283.44995075, + "chunks": 220 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.412562914304186, + "seconds": 11.908896708, + "prefill_seconds": 0.103050708, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.621631667, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 219, + "drafted": 36, + "accepted": 36, + "acceptance_rate": 1.0, + "tokens_per_round": 1.1689497716894977 + }, + "sent": 3605269.917860541, + "first": 3605271.541072916, + "last": 3605283.449972708, + "chunks": 220 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.55593888604215, + "seconds": 13.039517125, + "prefill_seconds": 0.099267583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.491657291, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 219, + "drafted": 36, + "accepted": 36, + "acceptance_rate": 1.0, + "tokens_per_round": 1.1689497716894977 + }, + "sent": 3605269.917944833, + "first": 3605270.410338416, + "last": 3605283.449818, + "chunks": 220 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.981866052192267, + "seconds": 13.433874167, + "prefill_seconds": 0.097258833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.097454292, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 219, + "drafted": 36, + "accepted": 36, + "acceptance_rate": 1.0, + "tokens_per_round": 1.1689497716894977 + }, + "sent": 3605269.917525875, + "first": 3605270.015855375, + "last": 3605283.449766875, + "chunks": 220 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.87036352214025, + "seconds": 12.218282625, + "prefill_seconds": 0.104461041, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.312728458, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 219, + "drafted": 36, + "accepted": 36, + "acceptance_rate": 1.0, + "tokens_per_round": 1.1689497716894977 + }, + "sent": 3605269.917714166, + "first": 3605271.231622583, + "last": 3605283.449863208, + "chunks": 220 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.520543693562686, + "seconds": 12.426571333, + "prefill_seconds": 0.106122416, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.104378167, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 219, + "drafted": 36, + "accepted": 36, + "acceptance_rate": 1.0, + "tokens_per_round": 1.1689497716894977 + }, + "sent": 3605269.917754708, + "first": 3605271.023299541, + "last": 3605283.449915125, + "chunks": 220 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.048586514158238, + "seconds": 12.11482775, + "prefill_seconds": 0.103449667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.416263459, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 219, + "drafted": 36, + "accepted": 36, + "acceptance_rate": 1.0, + "tokens_per_round": 1.1689497716894977 + }, + "sent": 3605269.917914916, + "first": 3605271.335092041, + "last": 3605283.449897208, + "chunks": 220 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.408236402670443, + "seconds": 13.13875175, + "prefill_seconds": 0.099055875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.392426458, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 219, + "drafted": 36, + "accepted": 36, + "acceptance_rate": 1.0, + "tokens_per_round": 1.1689497716894977 + }, + "sent": 3605269.917140958, + "first": 3605270.311076083, + "last": 3605283.449698625, + "chunks": 220 + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 126.78640117388437, + "ttft_max": 0.09193537477403879, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 126.76524222550795, + "seconds": 2.011592417, + "prefill_seconds": 0.090170916, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.090489583, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 2 + }, + "speculative": { + "rounds": 85, + "drafted": 283, + "accepted": 170, + "acceptance_rate": 0.6007067137809188, + "tokens_per_round": 3.011764705882353 + }, + "sent": 3605290.036222666, + "first": 3605290.128158041, + "last": 3605292.13941475, + "chunks": 86 + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 200.43096587895258, + "ttft_max": 0.4380846251733601, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.650528654504505, + "seconds": 5.034498292, + "prefill_seconds": 0.104293291, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.104680292, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 197, + "drafted": 73, + "accepted": 58, + "acceptance_rate": 0.7945205479452054, + "tokens_per_round": 1.299492385786802 + }, + "sent": 3605292.143276458, + "first": 3605292.248608291, + "last": 3605297.2830055, + "chunks": 198 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.474920914161814, + "seconds": 4.859464208, + "prefill_seconds": 0.111324375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.325527875, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 200, + "drafted": 74, + "accepted": 55, + "acceptance_rate": 0.7432432432432432, + "tokens_per_round": 1.28 + }, + "sent": 3605292.143338916, + "first": 3605292.469470583, + "last": 3605297.328782541, + "chunks": 201 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.61383015029026, + "seconds": 4.756235458, + "prefill_seconds": 0.112080791, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.43766275, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 201, + "drafted": 69, + "accepted": 54, + "acceptance_rate": 0.782608695652174, + "tokens_per_round": 1.2736318407960199 + }, + "sent": 3605292.143407208, + "first": 3605292.581491833, + "last": 3605297.337642291, + "chunks": 202 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.42472493865987, + "seconds": 4.958704209, + "prefill_seconds": 0.109472208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.214242583, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 199, + "drafted": 71, + "accepted": 56, + "acceptance_rate": 0.7887323943661971, + "tokens_per_round": 1.2864321608040201 + }, + "sent": 3605292.143192375, + "first": 3605292.358079333, + "last": 3605297.316705916, + "chunks": 200 + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 255.46984362436277, + "ttft_max": 0.9172884998843074, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 34.96757134858678, + "seconds": 7.292471, + "prefill_seconds": 0.115811167, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.800858792, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 227, + "drafted": 32, + "accepted": 28, + "acceptance_rate": 0.875, + "tokens_per_round": 1.1277533039647578 + }, + "sent": 3605297.341384333, + "first": 3605298.143214708, + "last": 3605305.435554958, + "chunks": 228 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 35.53055484152029, + "seconds": 7.176921417, + "prefill_seconds": 0.115590667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.91638425, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 227, + "drafted": 32, + "accepted": 28, + "acceptance_rate": 0.875, + "tokens_per_round": 1.1277533039647578 + }, + "sent": 3605297.341507458, + "first": 3605298.258795958, + "last": 3605305.435580625, + "chunks": 228 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 33.38492431371953, + "seconds": 7.638178167, + "prefill_seconds": 0.114445042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.455078458, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 227, + "drafted": 32, + "accepted": 28, + "acceptance_rate": 0.875, + "tokens_per_round": 1.1277533039647578 + }, + "sent": 3605297.341628541, + "first": 3605297.797422375, + "last": 3605305.435521375, + "chunks": 228 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 31.934024719695735, + "seconds": 7.985213334, + "prefill_seconds": 0.107942584, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.108172083, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 227, + "drafted": 32, + "accepted": 28, + "acceptance_rate": 0.875, + "tokens_per_round": 1.1277533039647578 + }, + "sent": 3605297.341680875, + "first": 3605297.450301958, + "last": 3605305.435538041, + "chunks": 228 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 34.42087377624303, + "seconds": 7.408295375, + "prefill_seconds": 0.11484875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.685039542, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 227, + "drafted": 32, + "accepted": 28, + "acceptance_rate": 0.875, + "tokens_per_round": 1.1277533039647578 + }, + "sent": 3605297.341740041, + "first": 3605298.027373458, + "last": 3605305.43556775, + "chunks": 228 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 33.895674014438306, + "seconds": 7.523083916, + "prefill_seconds": 0.115094834, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.570190875, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 227, + "drafted": 32, + "accepted": 28, + "acceptance_rate": 0.875, + "tokens_per_round": 1.1277533039647578 + }, + "sent": 3605297.341450916, + "first": 3605297.912514875, + "last": 3605305.435588875, + "chunks": 228 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 32.892222827396125, + "seconds": 7.7525925, + "prefill_seconds": 0.113420959, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.340682666, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 227, + "drafted": 32, + "accepted": 28, + "acceptance_rate": 0.875, + "tokens_per_round": 1.1277533039647578 + }, + "sent": 3605297.341554, + "first": 3605297.682979916, + "last": 3605305.435493916, + "chunks": 228 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 32.41796563335079, + "seconds": 7.866008709, + "prefill_seconds": 0.119201834, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.227344958, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 227, + "drafted": 32, + "accepted": 28, + "acceptance_rate": 0.875, + "tokens_per_round": 1.1277533039647578 + }, + "sent": 3605297.34129825, + "first": 3605297.569545375, + "last": 3605305.435476666, + "chunks": 228 + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 313.8774581034392, + "ttft_max": 1.5110153751447797, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.060272004503304, + "seconds": 12.711692042, + "prefill_seconds": 0.093187083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.374716458, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605305.44135625, + "first": 3605305.817483791, + "last": 3605318.530290208, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.67025183008689, + "seconds": 12.336569583, + "prefill_seconds": 0.094140792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.749637917, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605305.441405125, + "first": 3605306.192615291, + "last": 3605318.529260125, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.513701085871652, + "seconds": 12.430716375, + "prefill_seconds": 0.094110208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.655543625, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605305.442013791, + "first": 3605306.098466166, + "last": 3605318.529441541, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.829214784946384, + "seconds": 12.242420208, + "prefill_seconds": 0.094183209, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.843790334, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605305.441461916, + "first": 3605306.2868475, + "last": 3605318.530216875, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.359571456579687, + "seconds": 12.524821583, + "prefill_seconds": 0.093567333, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.561439209, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605305.441520416, + "first": 3605306.004342125, + "last": 3605318.530478416, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.495463716658133, + "seconds": 11.862968083, + "prefill_seconds": 0.095485417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.223262791, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605305.441578041, + "first": 3605306.666461125, + "last": 3605318.529598458, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 22.02659359839882, + "seconds": 11.576914917, + "prefill_seconds": 0.095079667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.509074917, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605305.441627291, + "first": 3605306.952642666, + "last": 3605318.529668291, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.156408017755954, + "seconds": 12.05308575, + "prefill_seconds": 0.094809959, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.033162541, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605305.441862375, + "first": 3605306.476218875, + "last": 3605318.52974675, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.99136372992379, + "seconds": 12.147852959, + "prefill_seconds": 0.094577583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.938381959, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605305.44167525, + "first": 3605306.381410791, + "last": 3605318.529809208, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.669564145303198, + "seconds": 11.767657083, + "prefill_seconds": 0.095331208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.318576959, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605305.441896666, + "first": 3605306.76181175, + "last": 3605318.529422416, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.619602263921294, + "seconds": 12.997205375, + "prefill_seconds": 0.088972792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.089189584, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605305.44171625, + "first": 3605305.531774958, + "last": 3605318.528930666, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.847292069824867, + "seconds": 11.671927083, + "prefill_seconds": 0.095725, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.414164333, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605305.441935625, + "first": 3605306.857545041, + "last": 3605318.529885, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.32386163636074, + "seconds": 11.958434375, + "prefill_seconds": 0.094735083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.127798667, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605305.441755666, + "first": 3605306.57101725, + "last": 3605318.530401416, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.914469759301284, + "seconds": 12.804759709, + "prefill_seconds": 0.092802833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.281539584, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605305.441971875, + "first": 3605305.724298875, + "last": 3605318.529047875, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.208637605857678, + "seconds": 12.618366709, + "prefill_seconds": 0.093288458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.467903583, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605305.441798791, + "first": 3605305.910813125, + "last": 3605318.529313708, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.77116191646302, + "seconds": 12.897572792, + "prefill_seconds": 0.099672334, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.188832667, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605305.441279833, + "first": 3605305.63146675, + "last": 3605318.528985041, + "chunks": 256 + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 147.83629271145213, + "ttft_max": 0.096926209051162, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 147.82846866597168, + "seconds": 1.724972208, + "prefill_seconds": 0.096228583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.096440459, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 2 + }, + "speculative": { + "rounds": 78, + "drafted": 284, + "accepted": 177, + "acceptance_rate": 0.6232394366197183, + "tokens_per_round": 3.282051282051282 + }, + "sent": 3605318.535977041, + "first": 3605318.63290325, + "last": 3605320.357784166, + "chunks": 79 + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 220.9503576822706, + "ttft_max": 0.4242655001580715, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 59.40146559139531, + "seconds": 4.292823375, + "prefill_seconds": 0.109193292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.423730833, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 10, + "accepted": 3, + "acceptance_rate": 0.3, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 3605320.361777708, + "first": 3605320.786043208, + "last": 3605325.078767416, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 55.23800969974208, + "seconds": 4.616386459, + "prefill_seconds": 0.099986875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.100170916, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 10, + "accepted": 3, + "acceptance_rate": 0.3, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 3605320.361634458, + "first": 3605320.462448, + "last": 3605325.078736916, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 57.92797484039848, + "seconds": 4.402018208, + "prefill_seconds": 0.108133708, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.314518667, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 10, + "accepted": 3, + "acceptance_rate": 0.3, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 3605320.361707958, + "first": 3605320.67684525, + "last": 3605325.078869583, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 56.53916666194564, + "seconds": 4.510147833, + "prefill_seconds": 0.106229667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.206412666, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 10, + "accepted": 3, + "acceptance_rate": 0.3, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 3605320.361551875, + "first": 3605320.568685541, + "last": 3605325.0787525, + "chunks": 253 + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 8, + "aggregate_decode_tps": 257.0465470267213, + "ttft_max": 0.8670476251281798, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 35.559741330359046, + "seconds": 7.171030791, + "prefill_seconds": 0.110246708, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.865938416, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 216, + "drafted": 43, + "accepted": 39, + "acceptance_rate": 0.9069767441860465, + "tokens_per_round": 1.1851851851851851 + }, + "sent": 3605325.082999916, + "first": 3605325.950047541, + "last": 3605333.120935583, + "chunks": 217 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 33.0487811674307, + "seconds": 7.715867, + "prefill_seconds": 0.107012416, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.321258417, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 216, + "drafted": 43, + "accepted": 39, + "acceptance_rate": 0.9069767441860465, + "tokens_per_round": 1.1851851851851851 + }, + "sent": 3605325.08327425, + "first": 3605325.405153541, + "last": 3605333.1209695, + "chunks": 217 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 34.0007043812591, + "seconds": 7.499844625, + "prefill_seconds": 0.108330958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.537242959, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 216, + "drafted": 43, + "accepted": 39, + "acceptance_rate": 0.9069767441860465, + "tokens_per_round": 1.1851851851851851 + }, + "sent": 3605325.083354583, + "first": 3605325.621150041, + "last": 3605333.120979291, + "chunks": 217 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 32.59696730918858, + "seconds": 7.822813625, + "prefill_seconds": 0.113354958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.214276167, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 216, + "drafted": 43, + "accepted": 39, + "acceptance_rate": 0.9069767441860465, + "tokens_per_round": 1.1851851851851851 + }, + "sent": 3605325.083070083, + "first": 3605325.298106916, + "last": 3605333.120946875, + "chunks": 217 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 35.02128968234008, + "seconds": 7.28128525, + "prefill_seconds": 0.109204875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.755724292, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 216, + "drafted": 43, + "accepted": 39, + "acceptance_rate": 0.9069767441860465, + "tokens_per_round": 1.1851851851851851 + }, + "sent": 3605325.083523375, + "first": 3605325.83974725, + "last": 3605333.121043833, + "chunks": 217 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 33.51667722496765, + "seconds": 7.60815275, + "prefill_seconds": 0.107665042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.428864333, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 216, + "drafted": 43, + "accepted": 39, + "acceptance_rate": 0.9069767441860465, + "tokens_per_round": 1.1851851851851851 + }, + "sent": 3605325.083146, + "first": 3605325.512829125, + "last": 3605333.120899958, + "chunks": 217 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 34.50375520787656, + "seconds": 7.390499917, + "prefill_seconds": 0.109379, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.6465175, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 216, + "drafted": 43, + "accepted": 39, + "acceptance_rate": 0.9069767441860465, + "tokens_per_round": 1.1851851851851851 + }, + "sent": 3605325.083463166, + "first": 3605325.730562708, + "last": 3605333.121033916, + "chunks": 217 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 32.13133897047426, + "seconds": 7.936177208, + "prefill_seconds": 0.100699916, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.100950125, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 216, + "drafted": 43, + "accepted": 39, + "acceptance_rate": 0.9069767441860465, + "tokens_per_round": 1.1851851851851851 + }, + "sent": 3605325.082914666, + "first": 3605325.184738041, + "last": 3605333.121023791, + "chunks": 217 + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 16, + "aggregate_decode_tps": 263.9872023279576, + "ttft_max": 2.16144795762375, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 16.500083985330427, + "seconds": 15.454466791, + "prefill_seconds": 0.122913333, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.1231215, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 5, + "accepted": 4, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 3605333.126206875, + "first": 3605333.25041925, + "last": 3605348.704820291, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.817667675727197, + "seconds": 13.551094875, + "prefill_seconds": 0.134893458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 2.025968417, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 5, + "accepted": 4, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 3605333.126846, + "first": 3605335.154130541, + "last": 3605348.705543875, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 16.9412854102989, + "seconds": 15.051986542, + "prefill_seconds": 0.133306958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.525254, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 5, + "accepted": 4, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 3605333.126274166, + "first": 3605333.6529565, + "last": 3605348.704856916, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 17.577823638472832, + "seconds": 14.506915375, + "prefill_seconds": 0.136994667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.070291541, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 5, + "accepted": 4, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 3605333.126759583, + "first": 3605334.198122, + "last": 3605348.704980125, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.00578384558076, + "seconds": 13.416968333, + "prefill_seconds": 0.13424025, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 2.160254292, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 5, + "accepted": 4, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 3605333.126926333, + "first": 3605335.288374291, + "last": 3605348.705384041, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 17.747479878980165, + "seconds": 14.368237166, + "prefill_seconds": 0.138685917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.208875292, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 5, + "accepted": 4, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 3605333.126330583, + "first": 3605334.33682525, + "last": 3605348.705066833, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 17.09438622618824, + "seconds": 14.917177875, + "prefill_seconds": 0.134858, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.660051667, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 5, + "accepted": 4, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 3605333.126624666, + "first": 3605333.787864, + "last": 3605348.704940416, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 17.252081909394978, + "seconds": 14.780824792, + "prefill_seconds": 0.136416833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.796385917, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 5, + "accepted": 4, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 3605333.126385708, + "first": 3605333.924240208, + "last": 3605348.7050225, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 17.41324414474544, + "seconds": 14.644026, + "prefill_seconds": 0.136864667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.933301083, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 5, + "accepted": 4, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 3605333.126715541, + "first": 3605334.061117541, + "last": 3605348.705056041, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.092909645749767, + "seconds": 14.093918833, + "prefill_seconds": 0.137012834, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.483268125, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 5, + "accepted": 4, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 3605333.126442833, + "first": 3605334.611230625, + "last": 3605348.705081875, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 16.792604524165665, + "seconds": 15.185256083, + "prefill_seconds": 0.131768125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.392004542, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 5, + "accepted": 4, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 3605333.126482833, + "first": 3605333.519721708, + "last": 3605348.705042458, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.449357599261333, + "seconds": 13.821619459, + "prefill_seconds": 0.135470416, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.755477959, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 5, + "accepted": 4, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 3605333.126891791, + "first": 3605334.883549125, + "last": 3605348.705171125, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.270250536285104, + "seconds": 13.957115667, + "prefill_seconds": 0.13683125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.620021208, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 5, + "accepted": 4, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 3605333.126673666, + "first": 3605334.748088833, + "last": 3605348.705274708, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 16.648131928168212, + "seconds": 15.317033833, + "prefill_seconds": 0.137435125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.260257041, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 5, + "accepted": 4, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 3605333.126563791, + "first": 3605333.387854375, + "last": 3605348.704874083, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 17.91874193182425, + "seconds": 14.230909791, + "prefill_seconds": 0.137384084, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.346198542, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 5, + "accepted": 4, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 3605333.126526375, + "first": 3605334.474237, + "last": 3605348.705713916, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.632244499725477, + "seconds": 13.685951792, + "prefill_seconds": 0.135675834, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.891081125, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 5, + "accepted": 4, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 3605333.126121875, + "first": 3605335.019221583, + "last": 3605348.705492625, + "chunks": 252 + } + ] + }, + { + "round": 1, + "arm": "main-plain", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 135.21683007811936, + "ttft_max": 0.0871472917497158, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 135.2024288303004, + "seconds": 1.886060792, + "prefill_seconds": 0.086245667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.086543083, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 204, + "drafted": 90, + "accepted": 51, + "acceptance_rate": 0.5666666666666667, + "tokens_per_round": 1.2549019607843137 + }, + "sent": 3605354.644582916, + "first": 3605354.731730208, + "last": 3605356.617590125, + "chunks": 205 + } + ] + }, + { + "round": 1, + "arm": "main-plain", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 241.76743130633574, + "ttft_max": 0.35046541690826416, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 61.718282989438244, + "seconds": 4.131676833, + "prefill_seconds": 0.086965875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.17304075, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 215, + "drafted": 51, + "accepted": 40, + "acceptance_rate": 0.7843137254901961, + "tokens_per_round": 1.1906976744186046 + }, + "sent": 3605356.622275875, + "first": 3605356.795996958, + "last": 3605360.927636875, + "chunks": 216 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 63.05896439746511, + "seconds": 4.04383425, + "prefill_seconds": 0.087845292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.26081375, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 215, + "drafted": 51, + "accepted": 40, + "acceptance_rate": 0.7843137254901961, + "tokens_per_round": 1.1906976744186046 + }, + "sent": 3605356.622413541, + "first": 3605356.883842, + "last": 3605360.927687291, + "chunks": 216 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 64.47625109491958, + "seconds": 3.954944583, + "prefill_seconds": 0.08896475, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.349774417, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 215, + "drafted": 51, + "accepted": 40, + "acceptance_rate": 0.7843137254901961, + "tokens_per_round": 1.1906976744186046 + }, + "sent": 3605356.622349708, + "first": 3605356.972815125, + "last": 3605360.927937083, + "chunks": 216 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 60.44555424989418, + "seconds": 4.218672542, + "prefill_seconds": 0.085891709, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.086074041, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 215, + "drafted": 51, + "accepted": 40, + "acceptance_rate": 0.7843137254901961, + "tokens_per_round": 1.1906976744186046 + }, + "sent": 3605356.622183791, + "first": 3605356.709006541, + "last": 3605360.927656833, + "chunks": 216 + } + ] + }, + { + "round": 1, + "arm": "main-plain", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 270.7456803820994, + "ttft_max": 0.7304916670545936, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 35.10999689684354, + "seconds": 7.262888708, + "prefill_seconds": 0.091176541, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.361503833, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605360.933122416, + "first": 3605361.295646916, + "last": 3605368.558554916, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 34.25366513350512, + "seconds": 7.444458834, + "prefill_seconds": 0.09023775, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.179989584, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605360.933404833, + "first": 3605361.114011958, + "last": 3605368.558453083, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 35.55645373853483, + "seconds": 7.171693833, + "prefill_seconds": 0.091205792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.452702834, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605360.933335958, + "first": 3605361.386909875, + "last": 3605368.558520791, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 33.843515292583646, + "seconds": 7.534678292, + "prefill_seconds": 0.089659083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.089863125, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605360.933188916, + "first": 3605361.023834791, + "last": 3605368.558379916, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 36.49754156803731, + "seconds": 6.986771959, + "prefill_seconds": 0.092288, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.637563583, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605360.933269291, + "first": 3605361.571819208, + "last": 3605368.558564291, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 36.9856298318258, + "seconds": 6.894569625, + "prefill_seconds": 0.09220275, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.729732584, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605360.933553041, + "first": 3605361.664044708, + "last": 3605368.558581041, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 34.67458892637427, + "seconds": 7.354088625, + "prefill_seconds": 0.090460375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.270407417, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605360.933460708, + "first": 3605361.204480708, + "last": 3605368.558538208, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 36.021696924128804, + "seconds": 7.079066834, + "prefill_seconds": 0.092644084, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.545317208, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605360.93303275, + "first": 3605361.479543083, + "last": 3605368.558492, + "chunks": 229 + } + ] + }, + { + "round": 1, + "arm": "main-plain", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 317.14545460069326, + "ttft_max": 1.5035453750751913, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.264583340348953, + "seconds": 12.583530375, + "prefill_seconds": 0.094196833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.373004334, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605368.564750375, + "first": 3605368.939365208, + "last": 3605381.523284958, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.114033885301385, + "seconds": 12.677715542, + "prefill_seconds": 0.093041542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.278851875, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605368.565608166, + "first": 3605368.845154833, + "last": 3605381.523831416, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 22.08067244265804, + "seconds": 11.548561334, + "prefill_seconds": 0.093428917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.407884875, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605368.565175666, + "first": 3605369.975220458, + "last": 3605381.523954666, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.581051025094403, + "seconds": 12.390037792, + "prefill_seconds": 0.100268333, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.566598, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605368.564819791, + "first": 3605369.133161083, + "last": 3605381.523207875, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.221400404997944, + "seconds": 12.016172125, + "prefill_seconds": 0.093754625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.940406041, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605368.56523525, + "first": 3605369.507174916, + "last": 3605381.523527625, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 22.26102090220382, + "seconds": 11.45500025, + "prefill_seconds": 0.093674042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.501510833, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605368.565353291, + "first": 3605370.068898666, + "last": 3605381.523995208, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.823640472034473, + "seconds": 12.863429417, + "prefill_seconds": 0.092897375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.093108458, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605368.564889291, + "first": 3605368.659426416, + "last": 3605381.523334791, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.05599916202474, + "seconds": 12.110562792, + "prefill_seconds": 0.093667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.84692175, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605368.565460083, + "first": 3605369.413415208, + "last": 3605381.524185916, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.967498850440553, + "seconds": 12.770753208, + "prefill_seconds": 0.092665459, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.185828791, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605368.564947375, + "first": 3605368.752121291, + "last": 3605381.522796333, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.387795624969122, + "seconds": 11.922687334, + "prefill_seconds": 0.093571708, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.033904875, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605368.565401291, + "first": 3605369.6007535, + "last": 3605381.524110041, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.736696626974393, + "seconds": 12.297040584, + "prefill_seconds": 0.093499584, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.660084125, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605368.565037166, + "first": 3605369.226532875, + "last": 3605381.523677875, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.894685794925884, + "seconds": 12.204060042, + "prefill_seconds": 0.093236083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.753312333, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605368.564991208, + "first": 3605369.319736166, + "last": 3605381.523785666, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.415943287029275, + "seconds": 12.490238458, + "prefill_seconds": 0.093332875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.466354083, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605368.565313833, + "first": 3605369.032799916, + "last": 3605381.523712625, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.556892300832946, + "seconds": 11.829163334, + "prefill_seconds": 0.093681209, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.127521958, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605368.565076666, + "first": 3605369.694434791, + "last": 3605381.524069916, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.729100870895973, + "seconds": 11.73541425, + "prefill_seconds": 0.093859833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.221267625, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605368.565120791, + "first": 3605369.78833975, + "last": 3605381.524150666, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.903448234267138, + "seconds": 11.642002541, + "prefill_seconds": 0.09342325, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.314510833, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605368.564657416, + "first": 3605369.881750375, + "last": 3605381.523902958, + "chunks": 256 + } + ] + }, + { + "round": 1, + "arm": "main-plain", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 155.2756224781027, + "ttft_max": 0.09816666599363089, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 155.26933117741075, + "seconds": 1.642307583, + "prefill_seconds": 0.097623791, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.097776958, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 144, + "drafted": 180, + "accepted": 111, + "acceptance_rate": 0.6166666666666667, + "tokens_per_round": 1.7777777777777777 + }, + "sent": 3605381.528660625, + "first": 3605381.626827291, + "last": 3605383.269068333, + "chunks": 145 + } + ] + }, + { + "round": 1, + "arm": "main-plain", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 261.3078294249977, + "ttft_max": 0.3997202501632273, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 67.02797961517537, + "seconds": 3.804381416, + "prefill_seconds": 0.099036, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.1972735, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 166, + "drafted": 99, + "accepted": 89, + "acceptance_rate": 0.898989898989899, + "tokens_per_round": 1.5421686746987953 + }, + "sent": 3605383.273448833, + "first": 3605383.471414916, + "last": 3605387.275758833, + "chunks": 167 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 68.83537927939683, + "seconds": 3.704490375, + "prefill_seconds": 0.09988475, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.297147083, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 166, + "drafted": 99, + "accepted": 89, + "acceptance_rate": 0.898989898989899, + "tokens_per_round": 1.5421686746987953 + }, + "sent": 3605383.273581875, + "first": 3605383.571316625, + "last": 3605387.275774583, + "chunks": 167 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 70.7808129870301, + "seconds": 3.60267125, + "prefill_seconds": 0.101852208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.398936292, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 166, + "drafted": 99, + "accepted": 89, + "acceptance_rate": 0.898989898989899, + "tokens_per_round": 1.5421686746987953 + }, + "sent": 3605383.273515041, + "first": 3605383.673235291, + "last": 3605387.275792125, + "chunks": 167 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 65.32720632146649, + "seconds": 3.903427291, + "prefill_seconds": 0.097979833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.098211333, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 166, + "drafted": 99, + "accepted": 89, + "acceptance_rate": 0.898989898989899, + "tokens_per_round": 1.5421686746987953 + }, + "sent": 3605383.273371041, + "first": 3605383.372349958, + "last": 3605387.275720166, + "chunks": 167 + } + ] + }, + { + "round": 1, + "arm": "main-plain", + "prompt": "chat", + "streams": 8, + "aggregate_decode_tps": 280.2334342413751, + "ttft_max": 0.8117147092707455, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 37.084801340102665, + "seconds": 6.876132291, + "prefill_seconds": 0.102005167, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.501668542, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605387.280753, + "first": 3605387.783466291, + "last": 3605394.659875291, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 38.228965469386054, + "seconds": 6.670334833, + "prefill_seconds": 0.102883166, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.70730875, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605387.281004958, + "first": 3605387.989273833, + "last": 3605394.659580416, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 36.54287143821144, + "seconds": 6.978105167, + "prefill_seconds": 0.101316292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.39971725, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605387.280939791, + "first": 3605387.681480041, + "last": 3605394.659770625, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 37.64820566621007, + "seconds": 6.773231167, + "prefill_seconds": 0.102894541, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.604616459, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605387.280877833, + "first": 3605387.886363625, + "last": 3605394.659825291, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 38.83195706222774, + "seconds": 6.566756334, + "prefill_seconds": 0.103611958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.81094475, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605387.281200166, + "first": 3605388.092914875, + "last": 3605394.659544541, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 35.516121262368586, + "seconds": 7.179838083, + "prefill_seconds": 0.09941125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.198144292, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605387.280812791, + "first": 3605387.479661916, + "last": 3605394.659483416, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 36.01983336947377, + "seconds": 7.079433083, + "prefill_seconds": 0.100446917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.29840925, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605387.281063625, + "first": 3605387.580137125, + "last": 3605394.6596125, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 35.03117754607157, + "seconds": 7.279230042, + "prefill_seconds": 0.098551667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.098811084, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605387.280672125, + "first": 3605387.38023, + "last": 3605394.659435708, + "chunks": 196 + } + ] + }, + { + "round": 1, + "arm": "main-plain", + "prompt": "chat", + "streams": 16, + "aggregate_decode_tps": 278.70302133474246, + "ttft_max": 1.760789040941745, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.50306570402351, + "seconds": 13.7815, + "prefill_seconds": 0.110498333, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.959904333, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 221, + "drafted": 36, + "accepted": 34, + "acceptance_rate": 0.9444444444444444, + "tokens_per_round": 1.158371040723982 + }, + "sent": 3605394.665737375, + "first": 3605395.626808416, + "last": 3605409.408287583, + "chunks": 222 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.808313204366144, + "seconds": 13.557834625, + "prefill_seconds": 0.112213583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.183590458, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 221, + "drafted": 36, + "accepted": 34, + "acceptance_rate": 0.9444444444444444, + "tokens_per_round": 1.158371040723982 + }, + "sent": 3605394.665193875, + "first": 3605395.850536708, + "last": 3605409.40838575, + "chunks": 222 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.96723081167307, + "seconds": 13.444239833, + "prefill_seconds": 0.113519625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.297130625, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 221, + "drafted": 36, + "accepted": 34, + "acceptance_rate": 0.9444444444444444, + "tokens_per_round": 1.158371040723982 + }, + "sent": 3605394.665262833, + "first": 3605395.964034166, + "last": 3605409.408241791, + "chunks": 222 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.129585362089532, + "seconds": 13.330137333, + "prefill_seconds": 0.114137583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.411010958, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 221, + "drafted": 36, + "accepted": 34, + "acceptance_rate": 0.9444444444444444, + "tokens_per_round": 1.158371040723982 + }, + "sent": 3605394.666038291, + "first": 3605396.078187125, + "last": 3605409.408339708, + "chunks": 222 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 17.542866690280768, + "seconds": 14.535822708, + "prefill_seconds": 0.103120084, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.205699625, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 221, + "drafted": 36, + "accepted": 34, + "acceptance_rate": 0.9444444444444444, + "tokens_per_round": 1.158371040723982 + }, + "sent": 3605394.665567166, + "first": 3605394.872240125, + "last": 3605409.408193541, + "chunks": 222 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.46713693925592, + "seconds": 13.098998625, + "prefill_seconds": 0.11613475, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.642015666, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 221, + "drafted": 36, + "accepted": 34, + "acceptance_rate": 0.9444444444444444, + "tokens_per_round": 1.158371040723982 + }, + "sent": 3605394.666172666, + "first": 3605396.309409916, + "last": 3605409.408378, + "chunks": 222 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 17.801062083252255, + "seconds": 14.324987959, + "prefill_seconds": 0.105807542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.416487666, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 221, + "drafted": 36, + "accepted": 34, + "acceptance_rate": 0.9444444444444444, + "tokens_per_round": 1.158371040723982 + }, + "sent": 3605394.665355333, + "first": 3605395.083124083, + "last": 3605409.408067333, + "chunks": 222 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.654002241477198, + "seconds": 13.669988708, + "prefill_seconds": 0.111501334, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.071457708, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 221, + "drafted": 36, + "accepted": 34, + "acceptance_rate": 0.9444444444444444, + "tokens_per_round": 1.158371040723982 + }, + "sent": 3605394.66611875, + "first": 3605395.738288291, + "last": 3605409.408216, + "chunks": 222 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 17.93483167995468, + "seconds": 14.218142916, + "prefill_seconds": 0.1068725, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.523246875, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 221, + "drafted": 36, + "accepted": 34, + "acceptance_rate": 0.9444444444444444, + "tokens_per_round": 1.158371040723982 + }, + "sent": 3605394.665833375, + "first": 3605395.189994625, + "last": 3605409.408132833, + "chunks": 222 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 17.670543782090437, + "seconds": 14.430795291, + "prefill_seconds": 0.105029583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.31069625, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 221, + "drafted": 36, + "accepted": 34, + "acceptance_rate": 0.9444444444444444, + "tokens_per_round": 1.158371040723982 + }, + "sent": 3605394.665481, + "first": 3605394.977281125, + "last": 3605409.408325666, + "chunks": 222 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.296077272876726, + "seconds": 13.215121208, + "prefill_seconds": 0.1150405, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.52606125, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 221, + "drafted": 36, + "accepted": 34, + "acceptance_rate": 0.9444444444444444, + "tokens_per_round": 1.158371040723982 + }, + "sent": 3605394.666239916, + "first": 3605396.193254041, + "last": 3605409.408366625, + "chunks": 222 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 17.419306216699344, + "seconds": 14.63892975, + "prefill_seconds": 0.102313917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.102579709, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 221, + "drafted": 36, + "accepted": 34, + "acceptance_rate": 0.9444444444444444, + "tokens_per_round": 1.158371040723982 + }, + "sent": 3605394.665622041, + "first": 3605394.76914725, + "last": 3605409.408009791, + "chunks": 222 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.07181697629501, + "seconds": 14.110368666, + "prefill_seconds": 0.107833959, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.631082958, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 221, + "drafted": 36, + "accepted": 34, + "acceptance_rate": 0.9444444444444444, + "tokens_per_round": 1.158371040723982 + }, + "sent": 3605394.666305125, + "first": 3605395.297861458, + "last": 3605409.408104041, + "chunks": 222 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.35592830677946, + "seconds": 13.891969708, + "prefill_seconds": 0.109609208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.849399459, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 221, + "drafted": 36, + "accepted": 34, + "acceptance_rate": 0.9444444444444444, + "tokens_per_round": 1.158371040723982 + }, + "sent": 3605394.665948833, + "first": 3605395.516287375, + "last": 3605409.408301083, + "chunks": 222 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.643012259790854, + "seconds": 12.981715667, + "prefill_seconds": 0.1172645, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.759228, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 221, + "drafted": 36, + "accepted": 34, + "acceptance_rate": 0.9444444444444444, + "tokens_per_round": 1.158371040723982 + }, + "sent": 3605394.66590175, + "first": 3605396.426690791, + "last": 3605409.408348791, + "chunks": 222 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.212233857674743, + "seconds": 14.001577291, + "prefill_seconds": 0.108812375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.739878791, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 221, + "drafted": 36, + "accepted": 34, + "acceptance_rate": 0.9444444444444444, + "tokens_per_round": 1.158371040723982 + }, + "sent": 3605394.665112375, + "first": 3605395.406704, + "last": 3605409.408167958, + "chunks": 222 + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 134.84870970275804, + "ttft_max": 0.08884287532418966, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 134.8328003400667, + "seconds": 1.891231209, + "prefill_seconds": 0.088050667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.08823575, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 2 + }, + "speculative": { + "rounds": 86, + "drafted": 274, + "accepted": 169, + "acceptance_rate": 0.6167883211678832, + "tokens_per_round": 2.9767441860465116 + }, + "sent": 3605415.381412625, + "first": 3605415.4702555, + "last": 3605417.361263583, + "chunks": 87 + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 232.29052564817403, + "ttft_max": 0.39637737488374114, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 59.44672236125179, + "seconds": 4.28955525, + "prefill_seconds": 0.101351875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.195082083, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 218, + "drafted": 48, + "accepted": 37, + "acceptance_rate": 0.7708333333333334, + "tokens_per_round": 1.1743119266055047 + }, + "sent": 3605417.367488291, + "first": 3605417.563336, + "last": 3605421.85298975, + "chunks": 219 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 62.36256925225751, + "seconds": 4.08899125, + "prefill_seconds": 0.100780375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.395675208, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 218, + "drafted": 48, + "accepted": 37, + "acceptance_rate": 0.7708333333333334, + "tokens_per_round": 1.1743119266055047 + }, + "sent": 3605417.367552958, + "first": 3605417.763930333, + "last": 3605421.853057125, + "chunks": 219 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 60.86286961475636, + "seconds": 4.189746583, + "prefill_seconds": 0.0998135, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.294881542, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 218, + "drafted": 48, + "accepted": 37, + "acceptance_rate": 0.7708333333333334, + "tokens_per_round": 1.1743119266055047 + }, + "sent": 3605417.367618916, + "first": 3605417.6631545, + "last": 3605421.853040291, + "chunks": 219 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 58.0744488389433, + "seconds": 4.390915542, + "prefill_seconds": 0.093563875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.093795125, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 218, + "drafted": 48, + "accepted": 37, + "acceptance_rate": 0.7708333333333334, + "tokens_per_round": 1.1743119266055047 + }, + "sent": 3605417.367393208, + "first": 3605417.462004166, + "last": 3605421.852839708, + "chunks": 219 + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 269.42968125685593, + "ttft_max": 0.7622620002366602, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 36.41528397005163, + "seconds": 7.002554208, + "prefill_seconds": 0.0957725, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.665132083, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 227, + "drafted": 31, + "accepted": 28, + "acceptance_rate": 0.9032258064516129, + "tokens_per_round": 1.1277533039647578 + }, + "sent": 3605421.858775541, + "first": 3605422.524708875, + "last": 3605429.527125458, + "chunks": 228 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 34.53646371845895, + "seconds": 7.383500583, + "prefill_seconds": 0.093928333, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.284276834, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 227, + "drafted": 31, + "accepted": 28, + "acceptance_rate": 0.9032258064516129, + "tokens_per_round": 1.1277533039647578 + }, + "sent": 3605421.858546583, + "first": 3605422.143683916, + "last": 3605429.527139208, + "chunks": 228 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 36.92196447130425, + "seconds": 6.906458084, + "prefill_seconds": 0.096173416, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.761258875, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 227, + "drafted": 31, + "accepted": 28, + "acceptance_rate": 0.9032258064516129, + "tokens_per_round": 1.1277533039647578 + }, + "sent": 3605421.858615583, + "first": 3605422.620877583, + "last": 3605429.527359791, + "chunks": 228 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 34.98216209825066, + "seconds": 7.289429375, + "prefill_seconds": 0.094067792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.378250666, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 227, + "drafted": 31, + "accepted": 28, + "acceptance_rate": 0.9032258064516129, + "tokens_per_round": 1.1277533039647578 + }, + "sent": 3605421.858727333, + "first": 3605422.237760875, + "last": 3605429.527248791, + "chunks": 228 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 35.446830935158296, + "seconds": 7.193873, + "prefill_seconds": 0.09555475, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.473801166, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 227, + "drafted": 31, + "accepted": 28, + "acceptance_rate": 0.9032258064516129, + "tokens_per_round": 1.1277533039647578 + }, + "sent": 3605421.858674, + "first": 3605422.333324208, + "last": 3605429.527215875, + "chunks": 228 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 34.10270598285169, + "seconds": 7.4774125, + "prefill_seconds": 0.093924667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.190319, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 227, + "drafted": 31, + "accepted": 28, + "acceptance_rate": 0.9032258064516129, + "tokens_per_round": 1.1277533039647578 + }, + "sent": 3605421.858834375, + "first": 3605422.049752291, + "last": 3605429.527175708, + "chunks": 228 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 35.92392179763727, + "seconds": 7.098334125, + "prefill_seconds": 0.095578625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.569345542, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 227, + "drafted": 31, + "accepted": 28, + "acceptance_rate": 0.9032258064516129, + "tokens_per_round": 1.1277533039647578 + }, + "sent": 3605421.859005666, + "first": 3605422.428939458, + "last": 3605429.527267541, + "chunks": 228 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 33.67961369351017, + "seconds": 7.57134575, + "prefill_seconds": 0.096190333, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.096446583, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 227, + "drafted": 31, + "accepted": 28, + "acceptance_rate": 0.9032258064516129, + "tokens_per_round": 1.1277533039647578 + }, + "sent": 3605421.858450958, + "first": 3605421.955810916, + "last": 3605429.527082541, + "chunks": 228 + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 307.8591382227815, + "ttft_max": 1.5631438749842346, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.962722485692893, + "seconds": 12.773808792, + "prefill_seconds": 0.096862583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.567743333, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605429.533959458, + "first": 3605430.103254916, + "last": 3605442.877523375, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.66563851479748, + "seconds": 12.966779584, + "prefill_seconds": 0.095515, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.3748605, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605429.534009333, + "first": 3605429.910252666, + "last": 3605442.877322541, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.379971505472856, + "seconds": 13.157914083, + "prefill_seconds": 0.093905833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.183785542, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605429.534398958, + "first": 3605429.719103583, + "last": 3605442.876903291, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.10659876382601, + "seconds": 12.081529708, + "prefill_seconds": 0.100075125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.259563292, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605429.5340755, + "first": 3605430.795726333, + "last": 3605442.877671583, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.28345692918465, + "seconds": 11.981136375, + "prefill_seconds": 0.1004525, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.359942458, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605429.534133125, + "first": 3605430.89622525, + "last": 3605442.877421458, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.933271174510256, + "seconds": 12.181564834, + "prefill_seconds": 0.099594042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.159546209, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605429.534439041, + "first": 3605430.695691541, + "last": 3605442.877127708, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.242652886598563, + "seconds": 13.251811042, + "prefill_seconds": 0.089549083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.089800042, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605429.534185125, + "first": 3605429.625141666, + "last": 3605442.876929125, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.52187078626089, + "seconds": 13.062272709, + "prefill_seconds": 0.095650833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.279312375, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605429.534242083, + "first": 3605429.814729958, + "last": 3605442.877101833, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.59748518639467, + "seconds": 12.380152125, + "prefill_seconds": 0.099545667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.961020708, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605429.534475958, + "first": 3605430.497009958, + "last": 3605442.877838291, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.763545246040287, + "seconds": 12.281139708, + "prefill_seconds": 0.099029959, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.060062583, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605429.534515041, + "first": 3605430.596032833, + "last": 3605442.877956, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.812497437850745, + "seconds": 12.870664125, + "prefill_seconds": 0.096127583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.470914666, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605429.534545208, + "first": 3605430.006379833, + "last": 3605442.876971416, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.116567752228978, + "seconds": 12.676118667, + "prefill_seconds": 0.097671334, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.665405875, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605429.5342855, + "first": 3605430.200931208, + "last": 3605442.877004958, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.647095139435265, + "seconds": 11.779871542, + "prefill_seconds": 0.10088975, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.561067417, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605429.534324333, + "first": 3605431.097468208, + "last": 3605442.877751208, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.46325171461922, + "seconds": 11.880772, + "prefill_seconds": 0.100381917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.460275792, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605429.5345785, + "first": 3605430.99657175, + "last": 3605442.8774815, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.43321193835248, + "seconds": 12.479682625, + "prefill_seconds": 0.098590583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.861649792, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605429.534357583, + "first": 3605430.397450916, + "last": 3605442.8771395, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.273075142073512, + "seconds": 12.5782595, + "prefill_seconds": 0.097901292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.763198375, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605429.533873541, + "first": 3605430.29883175, + "last": 3605442.877249875, + "chunks": 256 + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 145.3755917478066, + "ttft_max": 0.09841450024396181, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 145.36730782270743, + "seconds": 1.754177083, + "prefill_seconds": 0.097816084, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.098008292, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 2 + }, + "speculative": { + "rounds": 78, + "drafted": 285, + "accepted": 177, + "acceptance_rate": 0.6210526315789474, + "tokens_per_round": 3.282051282051282 + }, + "sent": 3605442.883334708, + "first": 3605442.981749208, + "last": 3605444.735826333, + "chunks": 79 + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 218.4538282582019, + "ttft_max": 0.44721245765686035, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 57.43069322667104, + "seconds": 4.440134459, + "prefill_seconds": 0.11613075, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.330156041, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 11, + "accepted": 4, + "acceptance_rate": 0.36363636363636365, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 3605444.741474708, + "first": 3605445.072298333, + "last": 3605449.512541, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 54.61526033658863, + "seconds": 4.669024709, + "prefill_seconds": 0.101002, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.101313875, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 11, + "accepted": 4, + "acceptance_rate": 0.36363636363636365, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 3605444.741541416, + "first": 3605444.843362125, + "last": 3605449.51230525, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 55.96697684991846, + "seconds": 4.55625825, + "prefill_seconds": 0.112759209, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.214062, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 11, + "accepted": 4, + "acceptance_rate": 0.36363636363636365, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 3605444.741598166, + "first": 3605444.9561535, + "last": 3605449.512485458, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 58.97507051495962, + "seconds": 4.323860875, + "prefill_seconds": 0.116266625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.446445666, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 11, + "accepted": 4, + "acceptance_rate": 0.36363636363636365, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 3605444.741394, + "first": 3605445.188606458, + "last": 3605449.512434791, + "chunks": 252 + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 8, + "aggregate_decode_tps": 276.2903641691943, + "ttft_max": 0.8495748750865459, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 34.54092742950388, + "seconds": 7.382546416, + "prefill_seconds": 0.099146834, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.099410292, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 63, + "accepted": 60, + "acceptance_rate": 0.9523809523809523, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605449.518375708, + "first": 3605449.618810166, + "last": 3605457.001380083, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 37.823851195836035, + "seconds": 6.741777792, + "prefill_seconds": 0.108070917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.740279375, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 63, + "accepted": 60, + "acceptance_rate": 0.9523809523809523, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605449.518509833, + "first": 3605450.259891375, + "last": 3605457.002291791, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 36.09481406821825, + "seconds": 7.064726792, + "prefill_seconds": 0.106603333, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.417260542, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 63, + "accepted": 60, + "acceptance_rate": 0.9523809523809523, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605449.518622416, + "first": 3605449.936744791, + "last": 3605457.002059708, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 35.03749687564972, + "seconds": 7.277917167, + "prefill_seconds": 0.104644917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.2040505, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 63, + "accepted": 60, + "acceptance_rate": 0.9523809523809523, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605449.518439166, + "first": 3605449.72347175, + "last": 3605457.002032125, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 36.649998499347845, + "seconds": 6.957708334, + "prefill_seconds": 0.107106167, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.524294333, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 63, + "accepted": 60, + "acceptance_rate": 0.9523809523809523, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605449.51878775, + "first": 3605450.043845541, + "last": 3605457.002192833, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 38.441681761612095, + "seconds": 6.633424666, + "prefill_seconds": 0.108358833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.848550417, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 63, + "accepted": 60, + "acceptance_rate": 0.9523809523809523, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605449.518678541, + "first": 3605450.368253416, + "last": 3605457.002250708, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 37.227168873286004, + "seconds": 6.849835959, + "prefill_seconds": 0.107952625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.632268834, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 63, + "accepted": 60, + "acceptance_rate": 0.9523809523809523, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605449.518562333, + "first": 3605450.151802666, + "last": 3605457.002346708, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 35.55849484955847, + "seconds": 7.171282167, + "prefill_seconds": 0.106634083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.310710417, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 63, + "accepted": 60, + "acceptance_rate": 0.9523809523809523, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605449.51828825, + "first": 3605449.830112083, + "last": 3605457.001886833, + "chunks": 196 + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 16, + "aggregate_decode_tps": 305.7837592707918, + "ttft_max": 1.7033216250129044, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.227481228525594, + "seconds": 12.606611625, + "prefill_seconds": 0.107409542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.833712583, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605457.00861875, + "first": 3605457.844106125, + "last": 3605470.450826708, + "chunks": 254 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.328889265817107, + "seconds": 11.955615542, + "prefill_seconds": 0.109789625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.484054917, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605457.008686833, + "first": 3605458.4952175, + "last": 3605470.450810958, + "chunks": 254 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.567234730667614, + "seconds": 13.031989625, + "prefill_seconds": 0.104422084, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.408458958, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605457.00904975, + "first": 3605457.418656166, + "last": 3605470.450537833, + "chunks": 254 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.134836567906717, + "seconds": 12.065387834, + "prefill_seconds": 0.108807208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.374314042, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605457.009264791, + "first": 3605458.385420666, + "last": 3605470.450794625, + "chunks": 254 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.580029918216074, + "seconds": 12.390652541, + "prefill_seconds": 0.108638917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.0496385, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605457.008748625, + "first": 3605458.060122333, + "last": 3605470.45078075, + "chunks": 254 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.726250061702938, + "seconds": 12.926937416, + "prefill_seconds": 0.105059, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.513551125, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605457.008799458, + "first": 3605457.5237645, + "last": 3605470.450574833, + "chunks": 254 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.111733770131554, + "seconds": 13.342588541, + "prefill_seconds": 0.098031875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.098241875, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605457.009110625, + "first": 3605457.108064458, + "last": 3605470.4505095, + "chunks": 254 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.411757826194172, + "seconds": 13.136368292, + "prefill_seconds": 0.103489208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.304175709, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605457.008959458, + "first": 3605457.314250791, + "last": 3605470.450658833, + "chunks": 254 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.761268916543667, + "seconds": 12.28248625, + "prefill_seconds": 0.108184833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.15796075, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605457.009216583, + "first": 3605458.168288208, + "last": 3605470.450773291, + "chunks": 254 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.723004468900438, + "seconds": 11.738707708, + "prefill_seconds": 0.108524208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.700951291, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605457.008847083, + "first": 3605458.712168708, + "last": 3605470.450819875, + "chunks": 254 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.945919433429196, + "seconds": 12.174208958, + "prefill_seconds": 0.10830875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.265652833, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605457.009310833, + "first": 3605458.276619291, + "last": 3605470.450755083, + "chunks": 254 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.889079449391613, + "seconds": 12.821106208, + "prefill_seconds": 0.105834375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.619311958, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605457.009351708, + "first": 3605457.629581875, + "last": 3605470.450625041, + "chunks": 254 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.40119374239388, + "seconds": 12.499268583, + "prefill_seconds": 0.107347, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.940954208, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605457.0090025, + "first": 3605457.951464666, + "last": 3605470.450725125, + "chunks": 254 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.260036328797145, + "seconds": 13.239850416, + "prefill_seconds": 0.102721667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.200804834, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605457.008900416, + "first": 3605457.210753041, + "last": 3605470.450522791, + "chunks": 254 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.056534521066187, + "seconds": 12.714060833, + "prefill_seconds": 0.107087416, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.726305417, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605457.009167583, + "first": 3605457.736689583, + "last": 3605470.450684333, + "chunks": 254 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.524041935164966, + "seconds": 11.847217208, + "prefill_seconds": 0.108408375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.592457708, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 253, + "drafted": 2, + "accepted": 2, + "acceptance_rate": 1.0, + "tokens_per_round": 1.0118577075098814 + }, + "sent": 3605457.008531416, + "first": 3605458.603612666, + "last": 3605470.450765791, + "chunks": 254 + } + ] + }, + { + "round": 2, + "arm": "main-plain", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 134.04015127905572, + "ttft_max": 0.08502129185944796, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 134.03280939207315, + "seconds": 1.90251925, + "prefill_seconds": 0.084423209, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.084615291, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 204, + "drafted": 90, + "accepted": 51, + "acceptance_rate": 0.5666666666666667, + "tokens_per_round": 1.2549019607843137 + }, + "sent": 3605475.610554166, + "first": 3605475.695575458, + "last": 3605477.5979905, + "chunks": 205 + } + ] + }, + { + "round": 2, + "arm": "main-plain", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 245.54320777721466, + "ttft_max": 0.34452699963003397, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 64.03927407833346, + "seconds": 3.981931458, + "prefill_seconds": 0.085632083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.25774, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 215, + "drafted": 51, + "accepted": 40, + "acceptance_rate": 0.7843137254901961, + "tokens_per_round": 1.1906976744186046 + }, + "sent": 3605477.614962666, + "first": 3605477.873537583, + "last": 3605481.855391875, + "chunks": 216 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 65.45556544127109, + "seconds": 3.895772625, + "prefill_seconds": 0.086155875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.343880375, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 215, + "drafted": 51, + "accepted": 40, + "acceptance_rate": 0.7843137254901961, + "tokens_per_round": 1.1906976744186046 + }, + "sent": 3605477.6151625, + "first": 3605477.9596895, + "last": 3605481.855459041, + "chunks": 216 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 62.691247363564464, + "seconds": 4.067553458, + "prefill_seconds": 0.086513208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.172089333, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 215, + "drafted": 51, + "accepted": 40, + "acceptance_rate": 0.7843137254901961, + "tokens_per_round": 1.1906976744186046 + }, + "sent": 3605477.615059958, + "first": 3605477.787903041, + "last": 3605481.855448625, + "chunks": 216 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 61.385409103510334, + "seconds": 4.154081625, + "prefill_seconds": 0.085394667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.085709541, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 215, + "drafted": 51, + "accepted": 40, + "acceptance_rate": 0.7843137254901961, + "tokens_per_round": 1.1906976744186046 + }, + "sent": 3605477.614839208, + "first": 3605477.7014425, + "last": 3605481.855497541, + "chunks": 216 + } + ] + }, + { + "round": 2, + "arm": "main-plain", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 270.7423117196063, + "ttft_max": 0.7330167503096163, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 33.84593855412964, + "seconds": 7.534138833, + "prefill_seconds": 0.0900295, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.090248, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605481.861976458, + "first": 3605481.953038875, + "last": 3605489.487249041, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 35.1149320635995, + "seconds": 7.261867958, + "prefill_seconds": 0.090984167, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.362650291, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605481.862190875, + "first": 3605482.225547375, + "last": 3605489.487622208, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 35.562696398712745, + "seconds": 7.170434917, + "prefill_seconds": 0.091634833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.454281416, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605481.862037, + "first": 3605482.317202208, + "last": 3605489.487771833, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 36.99559208154767, + "seconds": 6.892713041, + "prefill_seconds": 0.092998417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.732097667, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605481.862138625, + "first": 3605482.595155375, + "last": 3605489.487726291, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 34.68031094366792, + "seconds": 7.35287525, + "prefill_seconds": 0.090826291, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.271782708, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605481.862229875, + "first": 3605482.134551708, + "last": 3605489.487819208, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 36.02547021935962, + "seconds": 7.078325375, + "prefill_seconds": 0.092130791, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.546408625, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605481.862090458, + "first": 3605482.409339833, + "last": 3605489.487566125, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 36.503284092463026, + "seconds": 6.985672833, + "prefill_seconds": 0.092687833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.6390715, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605481.862369958, + "first": 3605482.502132416, + "last": 3605489.48766875, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 34.258055273838764, + "seconds": 7.443504833, + "prefill_seconds": 0.090699459, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.180952834, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605481.861897625, + "first": 3605482.04376525, + "last": 3605489.487878875, + "chunks": 229 + } + ] + }, + { + "round": 2, + "arm": "main-plain", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 292.93412642405116, + "ttft_max": 1.4647241248749197, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.926920256904737, + "seconds": 13.472873375, + "prefill_seconds": 0.09103375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.546364292, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 29, + "accepted": 27, + "acceptance_rate": 0.9310344827586207, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605489.494451375, + "first": 3605490.042471791, + "last": 3605503.515362625, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.733000571158392, + "seconds": 12.922515209, + "prefill_seconds": 0.091383459, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.096374833, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 29, + "accepted": 27, + "acceptance_rate": 0.9310344827586207, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605489.494896916, + "first": 3605490.59284475, + "last": 3605503.5152445, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.550024820973466, + "seconds": 13.746612334, + "prefill_seconds": 0.090700209, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.272539583, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 29, + "accepted": 27, + "acceptance_rate": 0.9310344827586207, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605489.494518625, + "first": 3605489.76867875, + "last": 3605503.515130208, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.456901603825724, + "seconds": 13.105889375, + "prefill_seconds": 0.091986333, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.91311975, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 29, + "accepted": 27, + "acceptance_rate": 0.9310344827586207, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605489.494632416, + "first": 3605490.409464791, + "last": 3605503.515424291, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.308820640498432, + "seconds": 13.927713041, + "prefill_seconds": 0.091075125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.0913155, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 29, + "accepted": 27, + "acceptance_rate": 0.9310344827586207, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605489.49495075, + "first": 3605489.587432416, + "last": 3605503.515098, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.594456981695206, + "seconds": 13.0138845, + "prefill_seconds": 0.091970959, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.005073208, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 29, + "accepted": 27, + "acceptance_rate": 0.9310344827586207, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605489.494578333, + "first": 3605490.501446208, + "last": 3605503.515478458, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.30949796549612, + "seconds": 12.555701792, + "prefill_seconds": 0.091528292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.462739375, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 29, + "accepted": 27, + "acceptance_rate": 0.9310344827586207, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605489.494995833, + "first": 3605490.959719958, + "last": 3605503.515442125, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.056723214752623, + "seconds": 13.38110425, + "prefill_seconds": 0.091652, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.637867125, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 29, + "accepted": 27, + "acceptance_rate": 0.9310344827586207, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605489.495036, + "first": 3605490.134127541, + "last": 3605503.5153105, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.67458408863247, + "seconds": 13.654922583, + "prefill_seconds": 0.091694166, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.364163875, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 29, + "accepted": 27, + "acceptance_rate": 0.9310344827586207, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605489.494694291, + "first": 3605489.860316083, + "last": 3605503.515465708, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.321338124166765, + "seconds": 13.197843667, + "prefill_seconds": 0.091791208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.821177041, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 29, + "accepted": 27, + "acceptance_rate": 0.9310344827586207, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605489.494736916, + "first": 3605490.3174595, + "last": 3605503.515433416, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.80005716768568, + "seconds": 13.563788542, + "prefill_seconds": 0.091140583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.455208083, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 29, + "accepted": 27, + "acceptance_rate": 0.9310344827586207, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605489.49508525, + "first": 3605489.951429166, + "last": 3605503.515194958, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.187885356109707, + "seconds": 13.289635375, + "prefill_seconds": 0.091527333, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.729372166, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 29, + "accepted": 27, + "acceptance_rate": 0.9310344827586207, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605489.49484925, + "first": 3605490.225666791, + "last": 3605503.515388125, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.162431911887484, + "seconds": 12.647283875, + "prefill_seconds": 0.091754458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.3712595, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 29, + "accepted": 27, + "acceptance_rate": 0.9310344827586207, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605489.494791166, + "first": 3605490.868178291, + "last": 3605503.515450375, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.01728296135841, + "seconds": 12.738991625, + "prefill_seconds": 0.0919845, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.279713875, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 29, + "accepted": 27, + "acceptance_rate": 0.9310344827586207, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605489.495518708, + "first": 3605490.776427416, + "last": 3605503.515411041, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 18.428504408437014, + "seconds": 13.837259625, + "prefill_seconds": 0.090444042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.181785917, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 29, + "accepted": 27, + "acceptance_rate": 0.9310344827586207, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605489.49517225, + "first": 3605489.677895625, + "last": 3605503.515071041, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.87381691536153, + "seconds": 12.830952458, + "prefill_seconds": 0.091586458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.187832792, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 29, + "accepted": 27, + "acceptance_rate": 0.9310344827586207, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605489.494358, + "first": 3605490.68442925, + "last": 3605503.515375041, + "chunks": 229 + } + ] + }, + { + "round": 2, + "arm": "main-plain", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 134.8330796716681, + "ttft_max": 0.13193579204380512, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 134.82593874550517, + "seconds": 1.891327458, + "prefill_seconds": 0.131301792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.131517041, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 144, + "drafted": 180, + "accepted": 111, + "acceptance_rate": 0.6166666666666667, + "tokens_per_round": 1.7777777777777777 + }, + "sent": 3605503.525222083, + "first": 3605503.657157875, + "last": 3605505.548385166, + "chunks": 145 + } + ] + }, + { + "round": 2, + "arm": "main-plain", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 225.21937567407258, + "ttft_max": 0.5493802912533283, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 58.06139216612445, + "seconds": 4.391902958, + "prefill_seconds": 0.136901375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.27295775, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 166, + "drafted": 99, + "accepted": 89, + "acceptance_rate": 0.898989898989899, + "tokens_per_round": 1.5421686746987953 + }, + "sent": 3605505.554539708, + "first": 3605505.828181583, + "last": 3605510.220012125, + "chunks": 167 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 56.306342560787606, + "seconds": 4.528797084, + "prefill_seconds": 0.135906542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.13614225, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 166, + "drafted": 99, + "accepted": 89, + "acceptance_rate": 0.898989898989899, + "tokens_per_round": 1.5421686746987953 + }, + "sent": 3605505.554665291, + "first": 3605505.69124725, + "last": 3605510.220045833, + "chunks": 167 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 61.95063353421815, + "seconds": 4.116180666, + "prefill_seconds": 0.137669792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.548800833, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 166, + "drafted": 99, + "accepted": 89, + "acceptance_rate": 0.898989898989899, + "tokens_per_round": 1.5421686746987953 + }, + "sent": 3605505.55460375, + "first": 3605506.103984041, + "last": 3605510.220164875, + "chunks": 167 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 59.94556237694263, + "seconds": 4.2538595, + "prefill_seconds": 0.138106334, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.411087208, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 166, + "drafted": 99, + "accepted": 89, + "acceptance_rate": 0.898989898989899, + "tokens_per_round": 1.5421686746987953 + }, + "sent": 3605505.5544525, + "first": 3605505.96628925, + "last": 3605510.22013575, + "chunks": 167 + } + ] + }, + { + "round": 2, + "arm": "main-plain", + "prompt": "chat", + "streams": 8, + "aggregate_decode_tps": 259.820332969546, + "ttft_max": 0.902233749628067, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 35.54192665919949, + "seconds": 7.174625125, + "prefill_seconds": 0.114212292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.785898625, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605510.226858166, + "first": 3605511.013980875, + "last": 3605518.18909125, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 34.447547902386255, + "seconds": 7.402558833, + "prefill_seconds": 0.1132025, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.558141875, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605510.226640083, + "first": 3605510.785963583, + "last": 3605518.188963833, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 34.985114146738646, + "seconds": 7.288814292, + "prefill_seconds": 0.1137675, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.671693375, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605510.226808916, + "first": 3605510.899731583, + "last": 3605518.1890325, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 33.9289249370816, + "seconds": 7.515711166, + "prefill_seconds": 0.112677292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.445253042, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605510.22669775, + "first": 3605510.672740375, + "last": 3605518.188532375, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 32.94573820483554, + "seconds": 7.739999584, + "prefill_seconds": 0.110556541, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.220648625, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605510.226751083, + "first": 3605510.448116708, + "last": 3605518.188665958, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 36.12177692157362, + "seconds": 7.059453375, + "prefill_seconds": 0.115266209, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.901208792, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605510.227011333, + "first": 3605511.129245083, + "last": 3605518.188815666, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 32.481980621250365, + "seconds": 7.8505065, + "prefill_seconds": 0.109736375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.110060875, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605510.226935625, + "first": 3605510.33751175, + "last": 3605518.188263375, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 33.42890668679567, + "seconds": 7.628128625, + "prefill_seconds": 0.111950875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.332594958, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605510.226550291, + "first": 3605510.560035833, + "last": 3605518.188732541, + "chunks": 196 + } + ] + }, + { + "round": 2, + "arm": "main-plain", + "prompt": "chat", + "streams": 16, + "aggregate_decode_tps": 306.84608203546827, + "ttft_max": 1.7599479579366744, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.17835935085708, + "seconds": 13.296236416, + "prefill_seconds": 0.107219209, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.107445625, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 237, + "drafted": 18, + "accepted": 18, + "acceptance_rate": 1.0, + "tokens_per_round": 1.080168776371308 + }, + "sent": 3605518.195807791, + "first": 3605518.304832041, + "last": 3605531.601008625, + "chunks": 238 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.832565639618313, + "seconds": 12.857640541, + "prefill_seconds": 0.110754666, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.54585225, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 237, + "drafted": 18, + "accepted": 18, + "acceptance_rate": 1.0, + "tokens_per_round": 1.080168776371308 + }, + "sent": 3605518.195857416, + "first": 3605518.743459541, + "last": 3605531.601181958, + "chunks": 238 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.897513191293374, + "seconds": 11.645157958, + "prefill_seconds": 0.108737208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.757228209, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 237, + "drafted": 18, + "accepted": 18, + "acceptance_rate": 1.0, + "tokens_per_round": 1.080168776371308 + }, + "sent": 3605518.196149458, + "first": 3605519.956097416, + "last": 3605531.601200083, + "chunks": 238 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.36116420187307, + "seconds": 12.523841833, + "prefill_seconds": 0.111492834, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.879644042, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 237, + "drafted": 18, + "accepted": 18, + "acceptance_rate": 1.0, + "tokens_per_round": 1.080168776371308 + }, + "sent": 3605518.195917958, + "first": 3605519.077346291, + "last": 3605531.601344041, + "chunks": 238 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.497621232863647, + "seconds": 13.0785185, + "prefill_seconds": 0.109325875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.325160334, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 237, + "drafted": 18, + "accepted": 18, + "acceptance_rate": 1.0, + "tokens_per_round": 1.080168776371308 + }, + "sent": 3605518.196680708, + "first": 3605518.522573291, + "last": 3605531.601083916, + "chunks": 238 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.33601166000266, + "seconds": 13.187828208, + "prefill_seconds": 0.108400667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.215765959, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 237, + "drafted": 18, + "accepted": 18, + "acceptance_rate": 1.0, + "tokens_per_round": 1.080168776371308 + }, + "sent": 3605518.195975708, + "first": 3605518.413255375, + "last": 3605531.601117, + "chunks": 238 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.694897027351384, + "seconds": 11.753916125, + "prefill_seconds": 0.108971333, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.648525292, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 237, + "drafted": 18, + "accepted": 18, + "acceptance_rate": 1.0, + "tokens_per_round": 1.080168776371308 + }, + "sent": 3605518.196330583, + "first": 3605519.847375333, + "last": 3605531.601384875, + "chunks": 238 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.18146261034833, + "seconds": 12.63535775, + "prefill_seconds": 0.111334958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.76831725, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 237, + "drafted": 18, + "accepted": 18, + "acceptance_rate": 1.0, + "tokens_per_round": 1.080168776371308 + }, + "sent": 3605518.1960735, + "first": 3605518.965816791, + "last": 3605531.601354958, + "chunks": 238 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.299143038827552, + "seconds": 11.972312667, + "prefill_seconds": 0.109716708, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.430249917, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 237, + "drafted": 18, + "accepted": 18, + "acceptance_rate": 1.0, + "tokens_per_round": 1.080168776371308 + }, + "sent": 3605518.196623625, + "first": 3605519.628945291, + "last": 3605531.601218833, + "chunks": 238 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.105705712451527, + "seconds": 12.082040917, + "prefill_seconds": 0.110033333, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.320596875, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 237, + "drafted": 18, + "accepted": 18, + "acceptance_rate": 1.0, + "tokens_per_round": 1.080168776371308 + }, + "sent": 3605518.196563083, + "first": 3605519.519230083, + "last": 3605531.601266041, + "chunks": 238 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.91525583241108, + "seconds": 12.192057417, + "prefill_seconds": 0.1104875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.210580166, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 237, + "drafted": 18, + "accepted": 18, + "acceptance_rate": 1.0, + "tokens_per_round": 1.080168776371308 + }, + "sent": 3605518.196260583, + "first": 3605519.409184083, + "last": 3605531.601251291, + "chunks": 238 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.663188221835757, + "seconds": 12.968395416, + "prefill_seconds": 0.110114292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.435185958, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 237, + "drafted": 18, + "accepted": 18, + "acceptance_rate": 1.0, + "tokens_per_round": 1.080168776371308 + }, + "sent": 3605518.196021708, + "first": 3605518.632716416, + "last": 3605531.60104925, + "chunks": 238 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.72746324258728, + "seconds": 12.3025185, + "prefill_seconds": 0.110307333, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.100176625, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 237, + "drafted": 18, + "accepted": 18, + "acceptance_rate": 1.0, + "tokens_per_round": 1.080168776371308 + }, + "sent": 3605518.19639025, + "first": 3605519.298645125, + "last": 3605531.601400708, + "chunks": 238 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.00525320846724, + "seconds": 12.746651959, + "prefill_seconds": 0.110993667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.656783625, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 237, + "drafted": 18, + "accepted": 18, + "acceptance_rate": 1.0, + "tokens_per_round": 1.080168776371308 + }, + "sent": 3605518.196212041, + "first": 3605518.854473916, + "last": 3605531.601234666, + "chunks": 238 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.495638724570185, + "seconds": 11.8628715, + "prefill_seconds": 0.109426, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.539636958, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 237, + "drafted": 18, + "accepted": 18, + "acceptance_rate": 1.0, + "tokens_per_round": 1.080168776371308 + }, + "sent": 3605518.196869666, + "first": 3605519.738365708, + "last": 3605531.601364166, + "chunks": 238 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.543276273335106, + "seconds": 12.412820458, + "prefill_seconds": 0.111021292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.990023, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 237, + "drafted": 18, + "accepted": 18, + "acceptance_rate": 1.0, + "tokens_per_round": 1.080168776371308 + }, + "sent": 3605518.195726041, + "first": 3605519.188370916, + "last": 3605531.6011585, + "chunks": 238 + } + ] + }, + { + "round": 2, + "arm": "candidate-plain", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 131.02762955084603, + "ttft_max": 0.09522395813837647, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 131.01969958356412, + "seconds": 1.946272208, + "prefill_seconds": 0.094651084, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.094815916, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 204, + "drafted": 90, + "accepted": 51, + "acceptance_rate": 0.5666666666666667, + "tokens_per_round": 1.2549019607843137 + }, + "sent": 3605536.725807625, + "first": 3605536.821031583, + "last": 3605538.767186, + "chunks": 205 + } + ] + }, + { + "round": 2, + "arm": "candidate-plain", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 223.55292221131992, + "ttft_max": 0.41851866664364934, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 57.185937215559534, + "seconds": 4.45913825, + "prefill_seconds": 0.103478292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.205440625, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 215, + "drafted": 51, + "accepted": 40, + "acceptance_rate": 0.7843137254901961, + "tokens_per_round": 1.1906976744186046 + }, + "sent": 3605538.77615325, + "first": 3605538.982185041, + "last": 3605543.441316208, + "chunks": 216 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 60.04708515180622, + "seconds": 4.246667417, + "prefill_seconds": 0.107146875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.417955625, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 215, + "drafted": 51, + "accepted": 40, + "acceptance_rate": 0.7843137254901961, + "tokens_per_round": 1.1906976744186046 + }, + "sent": 3605538.776206083, + "first": 3605539.19472475, + "last": 3605543.441306208, + "chunks": 216 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 55.88888894368192, + "seconds": 4.56262425, + "prefill_seconds": 0.101644542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.101953875, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 215, + "drafted": 51, + "accepted": 40, + "acceptance_rate": 0.7843137254901961, + "tokens_per_round": 1.1906976744186046 + }, + "sent": 3605538.77628525, + "first": 3605538.87870325, + "last": 3605543.441272791, + "chunks": 216 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 58.56940889280687, + "seconds": 4.353808666, + "prefill_seconds": 0.105337417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.310781958, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 215, + "drafted": 51, + "accepted": 40, + "acceptance_rate": 0.7843137254901961, + "tokens_per_round": 1.1906976744186046 + }, + "sent": 3605538.77606675, + "first": 3605539.087549666, + "last": 3605543.44138125, + "chunks": 216 + } + ] + }, + { + "round": 2, + "arm": "candidate-plain", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 267.110955393874, + "ttft_max": 0.7607742501422763, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 36.088886715693185, + "seconds": 7.065887125, + "prefill_seconds": 0.096627542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.663958834, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605543.448670208, + "first": 3605544.11365275, + "last": 3605551.179667666, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 35.120935183051415, + "seconds": 7.260626708, + "prefill_seconds": 0.095467708, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.469219375, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605543.448989833, + "first": 3605543.918952083, + "last": 3605551.179549, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 33.80224652784556, + "seconds": 7.543877292, + "prefill_seconds": 0.093102875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.185995834, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605543.448920625, + "first": 3605543.635653291, + "last": 3605551.1797455, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 33.39017811252395, + "seconds": 7.636976333, + "prefill_seconds": 0.092668541, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.092922833, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605543.448741083, + "first": 3605543.542469833, + "last": 3605551.179358875, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 35.60209569225996, + "seconds": 7.162499708, + "prefill_seconds": 0.098108875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.567303125, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605543.448808416, + "first": 3605544.017031583, + "last": 3605551.179516375, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 34.66521645381622, + "seconds": 7.356076958, + "prefill_seconds": 0.094272583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.373741791, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605543.448862708, + "first": 3605543.823425708, + "last": 3605551.1794025, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 36.586875211863024, + "seconds": 6.969712459, + "prefill_seconds": 0.096179833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.760045167, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605543.44908225, + "first": 3605544.2098565, + "last": 3605551.179606166, + "chunks": 229 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 34.22654626729075, + "seconds": 7.450357334, + "prefill_seconds": 0.093555, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.279560833, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 228, + "drafted": 30, + "accepted": 27, + "acceptance_rate": 0.9, + "tokens_per_round": 1.1228070175438596 + }, + "sent": 3605543.448587083, + "first": 3605543.729159208, + "last": 3605551.179385541, + "chunks": 229 + } + ] + }, + { + "round": 2, + "arm": "candidate-plain", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 304.5591714806901, + "ttft_max": 1.5759439589455724, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.896696675450308, + "seconds": 12.816197792, + "prefill_seconds": 0.098274708, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.673495459, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605551.187829666, + "first": 3605551.862587583, + "last": 3605564.679362333, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.454755059424507, + "seconds": 13.107335416, + "prefill_seconds": 0.096827833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.381084167, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605551.187347875, + "first": 3605551.5699955, + "last": 3605564.678450041, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.173570848218482, + "seconds": 13.299557084, + "prefill_seconds": 0.094394833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.188357292, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605551.187417791, + "first": 3605551.377271083, + "last": 3605564.676923666, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.401400417421726, + "seconds": 11.915108125, + "prefill_seconds": 0.100877667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.57352325, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605551.187671166, + "first": 3605552.763615125, + "last": 3605564.679279708, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.59868609461536, + "seconds": 13.011076292, + "prefill_seconds": 0.097481084, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.478319084, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605551.187617791, + "first": 3605551.667259833, + "last": 3605564.678642791, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.701212860267265, + "seconds": 12.318118833, + "prefill_seconds": 0.100245917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.170799667, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605551.18805875, + "first": 3605552.3602425, + "last": 3605564.679030583, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.745475464432126, + "seconds": 12.914350959, + "prefill_seconds": 0.097031708, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.575222417, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605551.187489, + "first": 3605551.764302416, + "last": 3605564.678928875, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.312291263568483, + "seconds": 13.204026209, + "prefill_seconds": 0.095629917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.284033292, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605551.187895708, + "first": 3605551.472959958, + "last": 3605564.676970541, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.044453199714567, + "seconds": 12.117207208, + "prefill_seconds": 0.100306625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.371618125, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605551.187939375, + "first": 3605552.5613415, + "last": 3605564.678822625, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.53412080141582, + "seconds": 12.418354916, + "prefill_seconds": 0.099302875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.070665625, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605551.187554916, + "first": 3605552.259993458, + "last": 3605564.678601375, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.053464962880685, + "seconds": 12.716006958, + "prefill_seconds": 0.0985685, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.772066042, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605551.187994291, + "first": 3605551.96115275, + "last": 3605564.677346375, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.87184934034128, + "seconds": 12.217412834, + "prefill_seconds": 0.100775084, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.271502834, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605551.188120791, + "first": 3605552.4610205, + "last": 3605564.679188333, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.211309971776885, + "seconds": 12.616698292, + "prefill_seconds": 0.099337709, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.871298708, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605551.188180583, + "first": 3605552.06051425, + "last": 3605564.677739375, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.03846760643467, + "seconds": 13.393935125, + "prefill_seconds": 0.093820291, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.094052667, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605551.187727583, + "first": 3605551.282950791, + "last": 3605564.678359958, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.22169114824537, + "seconds": 12.0160075, + "prefill_seconds": 0.101354042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.472934666, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605551.187773083, + "first": 3605552.6626945, + "last": 3605564.67910925, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.37291427891917, + "seconds": 12.516618708, + "prefill_seconds": 0.100147458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.971464084, + "sampling": "greedy", + "drafts": true, + "token_sha": "f66e3fd7e509", + "token_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605551.187247375, + "first": 3605552.160674291, + "last": 3605564.67800675, + "chunks": 256 + } + ] + }, + { + "round": 2, + "arm": "candidate-plain", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 154.95886207861588, + "ttft_max": 0.10082387505099177, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 154.94833976171662, + "seconds": 1.645709792, + "prefill_seconds": 0.099591459, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.100275959, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 144, + "drafted": 180, + "accepted": 111, + "acceptance_rate": 0.6166666666666667, + "tokens_per_round": 1.7777777777777777 + }, + "sent": 3605564.688457666, + "first": 3605564.789281541, + "last": 3605566.434879583, + "chunks": 145 + } + ] + }, + { + "round": 2, + "arm": "candidate-plain", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 242.36047700354084, + "ttft_max": 0.42187445890158415, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 60.59083789787342, + "seconds": 4.208557083, + "prefill_seconds": 0.102118834, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.102311542, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 166, + "drafted": 99, + "accepted": 89, + "acceptance_rate": 0.898989898989899, + "tokens_per_round": 1.5421686746987953 + }, + "sent": 3605566.441818333, + "first": 3605566.544721125, + "last": 3605570.753210666, + "chunks": 167 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 65.55965406317821, + "seconds": 3.889587333, + "prefill_seconds": 0.1074365, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.421266333, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 166, + "drafted": 99, + "accepted": 89, + "acceptance_rate": 0.898989898989899, + "tokens_per_round": 1.5421686746987953 + }, + "sent": 3605566.441888166, + "first": 3605566.863762625, + "last": 3605570.753328125, + "chunks": 167 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 62.141115395800036, + "seconds": 4.103563291, + "prefill_seconds": 0.105009958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.2073145, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 166, + "drafted": 99, + "accepted": 89, + "acceptance_rate": 0.898989898989899, + "tokens_per_round": 1.5421686746987953 + }, + "sent": 3605566.441946958, + "first": 3605566.649737666, + "last": 3605570.753248791, + "chunks": 167 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 63.79769276269591, + "seconds": 3.99700975, + "prefill_seconds": 0.106556875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.313817375, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 166, + "drafted": 99, + "accepted": 89, + "acceptance_rate": 0.898989898989899, + "tokens_per_round": 1.5421686746987953 + }, + "sent": 3605566.441734333, + "first": 3605566.756301208, + "last": 3605570.753289875, + "chunks": 167 + } + ] + }, + { + "round": 2, + "arm": "candidate-plain", + "prompt": "chat", + "streams": 8, + "aggregate_decode_tps": 263.8200654034127, + "ttft_max": 0.9031712496653199, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 33.962132787976955, + "seconds": 7.508362375, + "prefill_seconds": 0.112015833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.335270667, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605570.760726458, + "first": 3605571.09699175, + "last": 3605578.605317541, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 35.573171744850306, + "seconds": 7.168323416, + "prefill_seconds": 0.114088625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.675248375, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605570.760784666, + "first": 3605571.437124833, + "last": 3605578.605774666, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 35.01562509172935, + "seconds": 7.282463167, + "prefill_seconds": 0.112924792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.561342583, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605570.760840583, + "first": 3605571.323135333, + "last": 3605578.605375083, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 36.14625113408863, + "seconds": 7.0546735, + "prefill_seconds": 0.113693125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.788898458, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605570.761060916, + "first": 3605571.550822083, + "last": 3605578.605628708, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 34.481382512777, + "seconds": 7.395295125, + "prefill_seconds": 0.11306225, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.448364625, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605570.7608885, + "first": 3605571.210063208, + "last": 3605578.605350541, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 36.73751667190106, + "seconds": 6.941133291, + "prefill_seconds": 0.113508459, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.902484542, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605570.761161208, + "first": 3605571.664332458, + "last": 3605578.605549208, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 32.97946084302359, + "seconds": 7.732085167, + "prefill_seconds": 0.111460375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.111665417, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605570.760939416, + "first": 3605570.873231666, + "last": 3605578.605298208, + "chunks": 196 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 33.462928365466354, + "seconds": 7.620373125, + "prefill_seconds": 0.111713334, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.22331875, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 195, + "drafted": 62, + "accepted": 60, + "acceptance_rate": 0.967741935483871, + "tokens_per_round": 1.3128205128205128 + }, + "sent": 3605570.760643791, + "first": 3605570.98495875, + "last": 3605578.605269208, + "chunks": 196 + } + ] + }, + { + "round": 2, + "arm": "candidate-plain", + "prompt": "chat", + "streams": 16, + "aggregate_decode_tps": 310.3552343905495, + "ttft_max": 1.649341749958694, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.601231295597064, + "seconds": 11.804882625, + "prefill_seconds": 0.103076792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.438742334, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605578.613517666, + "first": 3605580.054471916, + "last": 3605591.860033416, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.514480305695376, + "seconds": 12.430244208, + "prefill_seconds": 0.106008791, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.813621208, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605578.6139825, + "first": 3605579.428936916, + "last": 3605591.859972666, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.17141942444813, + "seconds": 12.641648792, + "prefill_seconds": 0.103951, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.602347542, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605578.613593041, + "first": 3605579.217488416, + "last": 3605591.859112083, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.69574863463283, + "seconds": 12.946956459, + "prefill_seconds": 0.0992075, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.297069625, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605578.613781833, + "first": 3605578.912178, + "last": 3605591.860200125, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.41421071156215, + "seconds": 11.907980333, + "prefill_seconds": 0.104237667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.3356565, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605578.6137215, + "first": 3605579.951374291, + "last": 3605591.86023075, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.045202774397332, + "seconds": 12.116775625, + "prefill_seconds": 0.104969083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.12696575, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605578.614247583, + "first": 3605579.742488625, + "last": 3605591.86000325, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.688762015476815, + "seconds": 12.325532084, + "prefill_seconds": 0.104711167, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.918210667, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605578.613651375, + "first": 3605579.533632958, + "last": 3605591.859162416, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.795983017424774, + "seconds": 11.6994035, + "prefill_seconds": 0.105472208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.544115209, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605578.614026916, + "first": 3605580.159944208, + "last": 3605591.859847833, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.341035948504953, + "seconds": 12.536234666, + "prefill_seconds": 0.10541625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.707677917, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605578.613929291, + "first": 3605579.322897208, + "last": 3605591.859859708, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.398889956353763, + "seconds": 13.145082042, + "prefill_seconds": 0.098952833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.099173125, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605578.613878041, + "first": 3605578.714004916, + "last": 3605591.859769666, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.228452308918147, + "seconds": 12.012180459, + "prefill_seconds": 0.104630334, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.231449042, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605578.614070458, + "first": 3605579.847168583, + "last": 3605591.859674458, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.849658943156147, + "seconds": 12.846568333, + "prefill_seconds": 0.100359583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.397427875, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605578.613832041, + "first": 3605579.012549083, + "last": 3605591.860064916, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 21.990573014412327, + "seconds": 11.595877917, + "prefill_seconds": 0.103537833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.647648167, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605578.614167625, + "first": 3605580.263509375, + "last": 3605591.859724208, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.864530693861955, + "seconds": 12.221698333, + "prefill_seconds": 0.103841959, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.022063, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605578.614118708, + "first": 3605579.637514458, + "last": 3605591.859193, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 20.006909837448564, + "seconds": 12.7455965, + "prefill_seconds": 0.100966709, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.498359167, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605578.614321291, + "first": 3605579.113519083, + "last": 3605591.859611208, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 19.54596923465485, + "seconds": 13.046168084, + "prefill_seconds": 0.098901375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.197920209, + "sampling": "greedy", + "drafts": true, + "token_sha": "5a6a547bd3c4", + "token_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 3605578.613433208, + "first": 3605578.812915958, + "last": 3605591.859001875, + "chunks": 256 + } + ] + } + ], + "performance-qwen": [ + { + "round": 0, + "arm": "main-drafted", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 30.344307936698552, + "ttft_max": 0.23980883276090026, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 30.344048558503754, + "seconds": 8.403624833, + "prefill_seconds": 0.239020458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.239324791, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 55, + "drafted": 335, + "accepted": 200, + "acceptance_rate": 0.5970149253731343, + "tokens_per_round": 4.654545454545454 + }, + "sent": 3604496.127048333, + "first": 3604496.366857166, + "last": 3604504.770410166, + "chunks": 56 + } + ] + }, + { + "round": 0, + "arm": "main-drafted", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 30.93870088644765, + "ttft_max": 25.042438166681677, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 31.830604264701808, + "seconds": 8.011158, + "prefill_seconds": 0.229463833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 8.644388292, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 53, + "drafted": 329, + "accepted": 202, + "acceptance_rate": 0.6139817629179332, + "tokens_per_round": 4.830188679245283 + }, + "sent": 3604504.77142875, + "first": 3604513.416526083, + "last": 3604521.427605083, + "chunks": 54 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 31.158479446950714, + "seconds": 8.183968041, + "prefill_seconds": 0.230723791, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.231085125, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 55, + "drafted": 327, + "accepted": 200, + "acceptance_rate": 0.6116207951070336, + "tokens_per_round": 4.654545454545454 + }, + "sent": 3604504.771560125, + "first": 3604505.003225333, + "last": 3604513.187041583, + "chunks": 56 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 32.17277731613354, + "seconds": 7.925955459, + "prefill_seconds": 0.23254725, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 16.888045542, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 53, + "drafted": 323, + "accepted": 202, + "acceptance_rate": 0.6253869969040248, + "tokens_per_round": 4.830188679245283 + }, + "sent": 3604504.771489125, + "first": 3604521.660191041, + "last": 3604529.586075875, + "chunks": 54 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 31.25806670338704, + "seconds": 8.157894166, + "prefill_seconds": 0.227744292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 25.041650917, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 54, + "drafted": 328, + "accepted": 201, + "acceptance_rate": 0.6128048780487805, + "tokens_per_round": 4.7407407407407405 + }, + "sent": 3604504.771343333, + "first": 3604529.8137815, + "last": 3604537.971642583, + "chunks": 55 + } + ] + }, + { + "round": 0, + "arm": "main-drafted", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 23.065841730942, + "ttft_max": 0.43003095779567957, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 23.06554077318274, + "seconds": 11.055452916, + "prefill_seconds": 0.429262125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.429609584, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 82, + "drafted": 345, + "accepted": 173, + "acceptance_rate": 0.5014492753623189, + "tokens_per_round": 3.1219512195121952 + }, + "sent": 3604537.973906, + "first": 3604538.403936958, + "last": 3604549.459245625, + "chunks": 83 + } + ] + }, + { + "round": 0, + "arm": "main-drafted", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 22.55082188387638, + "ttft_max": 34.96590099995956, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 22.962253892898396, + "seconds": 11.105181625, + "prefill_seconds": 0.425930625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 11.91523375, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 83, + "drafted": 356, + "accepted": 172, + "acceptance_rate": 0.48314606741573035, + "tokens_per_round": 3.0843373493975905 + }, + "sent": 3604549.460459541, + "first": 3604561.376253166, + "last": 3604572.48135725, + "chunks": 84 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 23.04879780747181, + "seconds": 11.06348375, + "prefill_seconds": 0.425502417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.425864083, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 83, + "drafted": 356, + "accepted": 172, + "acceptance_rate": 0.48314606741573035, + "tokens_per_round": 3.0843373493975905 + }, + "sent": 3604549.460593666, + "first": 3604549.88709, + "last": 3604560.950315375, + "chunks": 84 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 23.037278457924852, + "seconds": 11.069015833, + "prefill_seconds": 0.430014417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 23.45031175, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 81, + "drafted": 360, + "accepted": 174, + "acceptance_rate": 0.48333333333333334, + "tokens_per_round": 3.1604938271604937 + }, + "sent": 3604549.4605245, + "first": 3604572.911377833, + "last": 3604583.980308791, + "chunks": 82 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 23.84952037711046, + "seconds": 10.692038916, + "prefill_seconds": 0.445968, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 34.965197042, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 81, + "drafted": 348, + "accepted": 174, + "acceptance_rate": 0.5, + "tokens_per_round": 3.1604938271604937 + }, + "sent": 3604549.460381583, + "first": 3604584.426282583, + "last": 3604595.118257416, + "chunks": 82 + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 31.3400196637318, + "ttft_max": 0.22861679224297404, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 31.33964026993998, + "seconds": 8.136660083, + "prefill_seconds": 0.227769375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.228073333, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 53, + "drafted": 375, + "accepted": 202, + "acceptance_rate": 0.5386666666666666, + "tokens_per_round": 4.830188679245283 + }, + "sent": 3604649.728608666, + "first": 3604649.957225458, + "last": 3604658.093787041, + "chunks": 54 + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 48.70154286002622, + "ttft_max": 0.8868492497131228, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 12.442917404598337, + "seconds": 20.493586167, + "prefill_seconds": 0.660301083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.66058425, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 67, + "drafted": 217, + "accepted": 188, + "acceptance_rate": 0.8663594470046083, + "tokens_per_round": 3.8208955223880596 + }, + "sent": 3604658.0960795, + "first": 3604658.757237458, + "last": 3604679.250707458, + "chunks": 68 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 12.581526389590532, + "seconds": 20.267811083, + "prefill_seconds": 0.886135833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.886447709, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 67, + "drafted": 217, + "accepted": 188, + "acceptance_rate": 0.8663594470046083, + "tokens_per_round": 3.8208955223880596 + }, + "sent": 3604658.096224625, + "first": 3604658.983073875, + "last": 3604679.250772125, + "chunks": 68 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 12.308087453809105, + "seconds": 20.718084833, + "prefill_seconds": 0.435774209, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.436020416, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 67, + "drafted": 218, + "accepted": 188, + "acceptance_rate": 0.8623853211009175, + "tokens_per_round": 3.8208955223880596 + }, + "sent": 3604658.096145875, + "first": 3604658.532716958, + "last": 3604679.2507495, + "chunks": 68 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 12.175402937052157, + "seconds": 20.943865375, + "prefill_seconds": 0.209930417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.210243625, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 67, + "drafted": 220, + "accepted": 188, + "acceptance_rate": 0.8545454545454545, + "tokens_per_round": 3.8208955223880596 + }, + "sent": 3604658.095991625, + "first": 3604658.306877125, + "last": 3604679.250664625, + "chunks": 68 + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 24.00490857949334, + "ttft_max": 0.4215462077409029, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 24.004729686343946, + "seconds": 10.622906541, + "prefill_seconds": 0.420831, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.421155459, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 75, + "drafted": 462, + "accepted": 180, + "acceptance_rate": 0.38961038961038963, + "tokens_per_round": 3.4133333333333336 + }, + "sent": 3604679.252754833, + "first": 3604679.674301041, + "last": 3604690.297128416, + "chunks": 76 + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 41.78288826705579, + "ttft_max": 1.6581098344177008, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.628013650766608, + "seconds": 23.993194625, + "prefill_seconds": 0.836824834, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.837248625, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 2 + }, + "speculative": { + "rounds": 97, + "drafted": 220, + "accepted": 158, + "acceptance_rate": 0.7181818181818181, + "tokens_per_round": 2.6391752577319587 + }, + "sent": 3604690.2980735, + "first": 3604691.1358885, + "last": 3604715.129133166, + "chunks": 98 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 11.004252625792686, + "seconds": 23.1728595, + "prefill_seconds": 1.238961458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.657580292, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 2 + }, + "speculative": { + "rounds": 97, + "drafted": 216, + "accepted": 158, + "acceptance_rate": 0.7314814814814815, + "tokens_per_round": 2.6391752577319587 + }, + "sent": 3604690.298201916, + "first": 3604691.95631175, + "last": 3604715.129071458, + "chunks": 98 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.445826937596213, + "seconds": 24.411662334, + "prefill_seconds": 0.418261625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.418658166, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 2 + }, + "speculative": { + "rounds": 97, + "drafted": 220, + "accepted": 158, + "acceptance_rate": 0.7181818181818181, + "tokens_per_round": 2.6391752577319587 + }, + "sent": 3604690.298135, + "first": 3604690.717330833, + "last": 3604715.128903916, + "chunks": 98 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.813981575196546, + "seconds": 23.580583916, + "prefill_seconds": 0.831247542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.249867292, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 2 + }, + "speculative": { + "rounds": 97, + "drafted": 216, + "accepted": 158, + "acceptance_rate": 0.7314814814814815, + "tokens_per_round": 2.6391752577319587 + }, + "sent": 3604690.297998291, + "first": 3604691.548578375, + "last": 3604715.12923825, + "chunks": 98 + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 36.11988572345143, + "ttft_max": 0.19940866669639945, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 36.11952098561528, + "seconds": 7.059894291, + "prefill_seconds": 0.198697, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.198979292, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 54, + "drafted": 355, + "accepted": 201, + "acceptance_rate": 0.5661971830985916, + "tokens_per_round": 4.7407407407407405 + }, + "sent": 3604725.818032958, + "first": 3604726.017441625, + "last": 3604733.077264625, + "chunks": 55 + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 32.39324544693206, + "ttft_max": 20.72907174983993, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 29.33421034201612, + "seconds": 8.692921917, + "prefill_seconds": 0.205756625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.20602725, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 53, + "drafted": 375, + "accepted": 202, + "acceptance_rate": 0.5386666666666666, + "tokens_per_round": 4.830188679245283 + }, + "sent": 3604733.078777125, + "first": 3604733.28554225, + "last": 3604741.978387208, + "chunks": 54 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 24.466282928760503, + "seconds": 10.422506792, + "prefill_seconds": 0.24281125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 17.766874792, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 62, + "drafted": 262, + "accepted": 193, + "acceptance_rate": 0.7366412213740458, + "tokens_per_round": 4.129032258064516 + }, + "sent": 3604733.0789195, + "first": 3604750.846356583, + "last": 3604761.2687955, + "chunks": 63 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 23.25422850288127, + "seconds": 10.965747583, + "prefill_seconds": 0.251704791, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 20.728427125, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 1 + }, + "speculative": { + "rounds": 67, + "drafted": 277, + "accepted": 188, + "acceptance_rate": 0.6787003610108303, + "tokens_per_round": 3.8208955223880596 + }, + "sent": 3604733.078850041, + "first": 3604753.807921791, + "last": 3604764.773588166, + "chunks": 68 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 30.390343707071754, + "seconds": 8.390823166, + "prefill_seconds": 0.234339208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 9.133264833, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 53, + "drafted": 368, + "accepted": 202, + "acceptance_rate": 0.5489130434782609, + "tokens_per_round": 4.830188679245283 + }, + "sent": 3604733.078688416, + "first": 3604742.212719333, + "last": 3604750.603482375, + "chunks": 54 + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 22.499851186148447, + "ttft_max": 0.4421482081525028, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 22.499807345399603, + "seconds": 11.333430375, + "prefill_seconds": 0.441396833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.44167975, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 75, + "drafted": 460, + "accepted": 180, + "acceptance_rate": 0.391304347826087, + "tokens_per_round": 3.4133333333333336 + }, + "sent": 3604764.775536208, + "first": 3604765.217684416, + "last": 3604776.551092708, + "chunks": 76 + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 39.84032356389979, + "ttft_max": 1.768473374657333, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.13453022532727, + "seconds": 25.16150175, + "prefill_seconds": 0.885118083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.885429667, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 2 + }, + "speculative": { + "rounds": 96, + "drafted": 220, + "accepted": 159, + "acceptance_rate": 0.7227272727272728, + "tokens_per_round": 2.6666666666666665 + }, + "sent": 3604776.552390458, + "first": 3604777.438389916, + "last": 3604802.599801291, + "chunks": 97 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.3173151293953, + "seconds": 24.715732417, + "prefill_seconds": 0.886504542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.331123084, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 2 + }, + "speculative": { + "rounds": 96, + "drafted": 218, + "accepted": 159, + "acceptance_rate": 0.7293577981651376, + "tokens_per_round": 2.6666666666666665 + }, + "sent": 3604776.552447875, + "first": 3604777.8841745, + "last": 3604802.599754083, + "chunks": 97 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 9.960122290447101, + "seconds": 25.602095292, + "prefill_seconds": 0.444373291, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.444644625, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 2 + }, + "speculative": { + "rounds": 96, + "drafted": 220, + "accepted": 159, + "acceptance_rate": 0.7227272727272728, + "tokens_per_round": 2.6666666666666665 + }, + "sent": 3604776.55252475, + "first": 3604776.997629208, + "last": 3604802.599635708, + "chunks": 97 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.502846912495857, + "seconds": 24.279131375, + "prefill_seconds": 1.323141792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.767754791, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 96, + "drafted": 217, + "accepted": 159, + "acceptance_rate": 0.7327188940092166, + "tokens_per_round": 2.6666666666666665 + }, + "sent": 3604776.552310333, + "first": 3604778.320783708, + "last": 3604802.599830916, + "chunks": 97 + } + ] + }, + { + "round": 1, + "arm": "main-drafted", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 36.20655991593265, + "ttft_max": 0.19942687498405576, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 36.20599550053157, + "seconds": 7.043032417, + "prefill_seconds": 0.198634458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.198939541, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 55, + "drafted": 310, + "accepted": 200, + "acceptance_rate": 0.6451612903225806, + "tokens_per_round": 4.654545454545454 + }, + "sent": 3604812.635286291, + "first": 3604812.834713166, + "last": 3604819.877635791, + "chunks": 56 + } + ] + }, + { + "round": 1, + "arm": "main-drafted", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 47.047737556009366, + "ttft_max": 0.841904915869236, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 12.294832987952992, + "seconds": 20.74042, + "prefill_seconds": 0.411415292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.411670375, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 1 + }, + "speculative": { + "rounds": 78, + "drafted": 210, + "accepted": 177, + "acceptance_rate": 0.8428571428571429, + "tokens_per_round": 3.282051282051282 + }, + "sent": 3604819.879005583, + "first": 3604820.291204, + "last": 3604841.03153175, + "chunks": 79 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 12.448224614983888, + "seconds": 20.484848875, + "prefill_seconds": 0.640674458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.841417875, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 1 + }, + "speculative": { + "rounds": 79, + "drafted": 213, + "accepted": 176, + "acceptance_rate": 0.8262910798122066, + "tokens_per_round": 3.240506329113924 + }, + "sent": 3604819.879153375, + "first": 3604820.721058291, + "last": 3604841.205771791, + "chunks": 80 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 11.996807508383107, + "seconds": 21.255654875, + "prefill_seconds": 0.625017334, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.625289167, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 1 + }, + "speculative": { + "rounds": 83, + "drafted": 219, + "accepted": 172, + "acceptance_rate": 0.7853881278538812, + "tokens_per_round": 3.0843373493975905 + }, + "sent": 3604819.879072541, + "first": 3604820.504795458, + "last": 3604841.760405958, + "chunks": 84 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 12.17110093138091, + "seconds": 20.951268208, + "prefill_seconds": 0.200510875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.200836458, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 1 + }, + "speculative": { + "rounds": 78, + "drafted": 212, + "accepted": 177, + "acceptance_rate": 0.8349056603773585, + "tokens_per_round": 3.282051282051282 + }, + "sent": 3604819.878924333, + "first": 3604820.080298625, + "last": 3604841.031495291, + "chunks": 79 + } + ] + }, + { + "round": 1, + "arm": "main-drafted", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 21.34347493314762, + "ttft_max": 0.4058342077769339, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 21.34320473816755, + "seconds": 11.947596583, + "prefill_seconds": 0.405067875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.405350209, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 83, + "drafted": 360, + "accepted": 172, + "acceptance_rate": 0.4777777777777778, + "tokens_per_round": 3.0843373493975905 + }, + "sent": 3604841.762581833, + "first": 3604842.168416041, + "last": 3604854.115861375, + "chunks": 84 + } + ] + }, + { + "round": 1, + "arm": "main-drafted", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 41.72305834262606, + "ttft_max": 1.741847958881408, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.618586816004251, + "seconds": 24.014495, + "prefill_seconds": 0.4395565, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.439896125, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 95, + "drafted": 231, + "accepted": 160, + "acceptance_rate": 0.6926406926406926, + "tokens_per_round": 2.694736842105263 + }, + "sent": 3604854.117007833, + "first": 3604854.557580416, + "last": 3604878.571969666, + "chunks": 96 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.85651813294774, + "seconds": 23.488193625, + "prefill_seconds": 0.872232791, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.312090042, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 97, + "drafted": 217, + "accepted": 158, + "acceptance_rate": 0.728110599078341, + "tokens_per_round": 2.6391752577319587 + }, + "sent": 3604854.117268125, + "first": 3604855.429762958, + "last": 3604878.917895041, + "chunks": 98 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.65955182687684, + "seconds": 23.9222065, + "prefill_seconds": 0.877797084, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.878160459, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 97, + "drafted": 227, + "accepted": 158, + "acceptance_rate": 0.6960352422907489, + "tokens_per_round": 2.6391752577319587 + }, + "sent": 3604854.117089583, + "first": 3604854.995781625, + "last": 3604878.917951916, + "chunks": 98 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 11.017110258099947, + "seconds": 23.145815375, + "prefill_seconds": 1.301217375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.741100167, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 98, + "drafted": 212, + "accepted": 157, + "acceptance_rate": 0.7405660377358491, + "tokens_per_round": 2.6122448979591835 + }, + "sent": 3604854.116930541, + "first": 3604855.8587785, + "last": 3604879.004493958, + "chunks": 99 + } + ] + }, + { + "round": 2, + "arm": "main-drafted", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 36.12399860205154, + "ttft_max": 0.20252579217776656, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 36.123212024726776, + "seconds": 7.059172917, + "prefill_seconds": 0.200018875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.200550916, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 55, + "drafted": 310, + "accepted": 200, + "acceptance_rate": 0.6451612903225806, + "tokens_per_round": 4.654545454545454 + }, + "sent": 3604889.219564666, + "first": 3604889.422090458, + "last": 3604896.481109666, + "chunks": 56 + } + ] + }, + { + "round": 2, + "arm": "main-drafted", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 49.44176860435623, + "ttft_max": 0.8374648750759661, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 12.837368778818945, + "seconds": 19.863883666, + "prefill_seconds": 0.409651375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.409944125, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 1 + }, + "speculative": { + "rounds": 82, + "drafted": 216, + "accepted": 173, + "acceptance_rate": 0.8009259259259259, + "tokens_per_round": 3.1219512195121952 + }, + "sent": 3604896.482677875, + "first": 3604896.893221833, + "last": 3604916.757026166, + "chunks": 83 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 12.754208356907291, + "seconds": 19.993400834, + "prefill_seconds": 0.636964958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.836840709, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 1 + }, + "speculative": { + "rounds": 84, + "drafted": 216, + "accepted": 171, + "acceptance_rate": 0.7916666666666666, + "tokens_per_round": 3.0476190476190474 + }, + "sent": 3604896.482735666, + "first": 3604897.320200541, + "last": 3604917.313493791, + "chunks": 85 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 12.619169315959981, + "seconds": 20.207352292, + "prefill_seconds": 0.622653792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.622908709, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 1 + }, + "speculative": { + "rounds": 84, + "drafted": 217, + "accepted": 171, + "acceptance_rate": 0.7880184331797235, + "tokens_per_round": 3.0476190476190474 + }, + "sent": 3604896.482804208, + "first": 3604897.106206625, + "last": 3604917.313513666, + "chunks": 85 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 12.894398186944702, + "seconds": 19.776029583, + "prefill_seconds": 0.199632, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.199933875, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 1 + }, + "speculative": { + "rounds": 81, + "drafted": 214, + "accepted": 174, + "acceptance_rate": 0.8130841121495327, + "tokens_per_round": 3.1604938271604937 + }, + "sent": 3604896.4826005, + "first": 3604896.683183708, + "last": 3604916.45914225, + "chunks": 82 + } + ] + }, + { + "round": 2, + "arm": "main-drafted", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 23.096668462332214, + "ttft_max": 0.4583138329908252, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 23.096494306274323, + "seconds": 11.040636584, + "prefill_seconds": 0.457591875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.457907792, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 84, + "drafted": 338, + "accepted": 171, + "acceptance_rate": 0.5059171597633136, + "tokens_per_round": 3.0476190476190474 + }, + "sent": 3604917.315005208, + "first": 3604917.773319041, + "last": 3604928.813872375, + "chunks": 85 + } + ] + }, + { + "round": 2, + "arm": "main-drafted", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 39.58191601199035, + "ttft_max": 1.6596389156766236, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.31685269722128, + "seconds": 24.71684025, + "prefill_seconds": 0.831686959, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.248863583, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 99, + "drafted": 219, + "accepted": 156, + "acceptance_rate": 0.7123287671232876, + "tokens_per_round": 2.585858585858586 + }, + "sent": 3604928.815203625, + "first": 3604930.064585041, + "last": 3604954.781297375, + "chunks": 100 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.39658399943459, + "seconds": 24.527287041, + "prefill_seconds": 1.242135334, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.659193041, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 101, + "drafted": 214, + "accepted": 154, + "acceptance_rate": 0.719626168224299, + "tokens_per_round": 2.5346534653465347 + }, + "sent": 3604928.81539575, + "first": 3604930.475034666, + "last": 3604955.00224225, + "chunks": 102 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.146833534859063, + "seconds": 25.13099275, + "prefill_seconds": 0.83436225, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.834691042, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 99, + "drafted": 232, + "accepted": 156, + "acceptance_rate": 0.6724137931034483, + "tokens_per_round": 2.585858585858586 + }, + "sent": 3604928.815273833, + "first": 3604929.650407416, + "last": 3604954.781352958, + "chunks": 100 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 9.981063525080605, + "seconds": 25.548379625, + "prefill_seconds": 0.416843875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.417150625, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 99, + "drafted": 235, + "accepted": 156, + "acceptance_rate": 0.6638297872340425, + "tokens_per_round": 2.585858585858586 + }, + "sent": 3604928.815126666, + "first": 3604929.2328985, + "last": 3604954.781213166, + "chunks": 100 + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 30.31858681923586, + "ttft_max": 0.19988291710615158, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 30.318100784216046, + "seconds": 8.410817083, + "prefill_seconds": 0.19909825, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.199392041, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 52, + "drafted": 376, + "accepted": 203, + "acceptance_rate": 0.5398936170212766, + "tokens_per_round": 4.923076923076923 + }, + "sent": 3604965.120085958, + "first": 3604965.319968875, + "last": 3604973.730651125, + "chunks": 53 + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 42.716145327090175, + "ttft_max": 1.1741919172927737, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.813385107043793, + "seconds": 23.581884625, + "prefill_seconds": 0.575820834, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.57606975, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 65, + "drafted": 242, + "accepted": 190, + "acceptance_rate": 0.7851239669421488, + "tokens_per_round": 3.9384615384615387 + }, + "sent": 3604973.732724208, + "first": 3604974.309460291, + "last": 3604997.891232416, + "chunks": 66 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.952668570562086, + "seconds": 23.281997292, + "prefill_seconds": 0.875755958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.876025834, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 65, + "drafted": 239, + "accepted": 190, + "acceptance_rate": 0.7949790794979079, + "tokens_per_round": 3.9384615384615387 + }, + "sent": 3604973.732836041, + "first": 3604974.609402333, + "last": 3604997.891275625, + "chunks": 66 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 11.094450179357176, + "seconds": 22.984464834, + "prefill_seconds": 0.894214334, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.1735915, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 65, + "drafted": 239, + "accepted": 190, + "acceptance_rate": 0.7949790794979079, + "tokens_per_round": 3.9384615384615387 + }, + "sent": 3604973.732783083, + "first": 3604974.906975, + "last": 3604997.891289791, + "chunks": 66 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.679046785710884, + "seconds": 23.878535708, + "prefill_seconds": 0.27908075, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.279380125, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 65, + "drafted": 244, + "accepted": 190, + "acceptance_rate": 0.7786885245901639, + "tokens_per_round": 3.9384615384615387 + }, + "sent": 3604973.732628458, + "first": 3604974.012730708, + "last": 3604997.891204208, + "chunks": 66 + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 23.72393278235333, + "ttft_max": 0.3962868331000209, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 23.72372098737278, + "seconds": 10.748735417, + "prefill_seconds": 0.395573708, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.395857292, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 75, + "drafted": 469, + "accepted": 180, + "acceptance_rate": 0.3837953091684435, + "tokens_per_round": 3.4133333333333336 + }, + "sent": 3604997.893435625, + "first": 3604998.289722458, + "last": 3605009.038361916, + "chunks": 76 + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 37.63360072289366, + "ttft_max": 1.6586603340692818, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 9.523506182264372, + "seconds": 26.775852834, + "prefill_seconds": 0.412750042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.413028125, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 95, + "drafted": 226, + "accepted": 160, + "acceptance_rate": 0.7079646017699115, + "tokens_per_round": 2.694736842105263 + }, + "sent": 3605009.039835666, + "first": 3605009.453469416, + "last": 3605036.229273375, + "chunks": 96 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 9.861389867154255, + "seconds": 25.858423958, + "prefill_seconds": 1.24508075, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.658040666, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 97, + "drafted": 219, + "accepted": 158, + "acceptance_rate": 0.7214611872146118, + "tokens_per_round": 2.6391752577319587 + }, + "sent": 3605009.039891291, + "first": 3605010.698551625, + "last": 3605036.556908375, + "chunks": 98 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 9.674276576325884, + "seconds": 26.358560042, + "prefill_seconds": 0.830139, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.830437917, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 95, + "drafted": 225, + "accepted": 160, + "acceptance_rate": 0.7111111111111111, + "tokens_per_round": 2.694736842105263 + }, + "sent": 3605009.039945166, + "first": 3605009.870907166, + "last": 3605036.229306958, + "chunks": 96 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 9.70758935738823, + "seconds": 26.268107417, + "prefill_seconds": 1.248113042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.248405541, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 97, + "drafted": 219, + "accepted": 158, + "acceptance_rate": 0.7214611872146118, + "tokens_per_round": 2.6391752577319587 + }, + "sent": 3605009.039748541, + "first": 3605010.288857416, + "last": 3605036.556876958, + "chunks": 98 + } + ] + } + ], + "performance-qwen-fresh-cost-records": [ + { + "round": 0, + "arm": "main-drafted", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 36.15852812983743, + "ttft_max": 0.19951058318838477, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 36.15814210107307, + "seconds": 7.0523535, + "prefill_seconds": 0.198717, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.198999834, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 54, + "drafted": 355, + "accepted": 201, + "acceptance_rate": 0.5661971830985916, + "tokens_per_round": 4.7407407407407405 + }, + "sent": 3606027.273496208, + "first": 3606027.473006791, + "last": 3606034.525285, + "chunks": 55 + } + ] + }, + { + "round": 0, + "arm": "main-drafted", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 51.576538557442284, + "ttft_max": 0.8468245826661587, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 12.894425707530571, + "seconds": 19.775987375, + "prefill_seconds": 0.199771833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.200094417, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 82, + "drafted": 210, + "accepted": 173, + "acceptance_rate": 0.8238095238095238, + "tokens_per_round": 3.1219512195121952 + }, + "sent": 3606034.526158833, + "first": 3606034.726779708, + "last": 3606054.502661083, + "chunks": 83 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 13.32983899880677, + "seconds": 19.1300135, + "prefill_seconds": 0.846160167, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.846441083, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 82, + "drafted": 210, + "accepted": 173, + "acceptance_rate": 0.8238095238095238, + "tokens_per_round": 3.1219512195121952 + }, + "sent": 3606034.526308833, + "first": 3606035.373133416, + "last": 3606054.5032135, + "chunks": 83 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 13.178562543878034, + "seconds": 19.349606541, + "prefill_seconds": 0.626363083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.626663041, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 82, + "drafted": 210, + "accepted": 173, + "acceptance_rate": 0.8238095238095238, + "tokens_per_round": 3.1219512195121952 + }, + "sent": 3606034.526230625, + "first": 3606035.153333916, + "last": 3606054.503172208, + "chunks": 83 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 13.034482495107342, + "seconds": 19.563492459, + "prefill_seconds": 0.412400709, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.412706292, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 82, + "drafted": 210, + "accepted": 173, + "acceptance_rate": 0.8238095238095238, + "tokens_per_round": 3.1219512195121952 + }, + "sent": 3606034.526077, + "first": 3606034.93936725, + "last": 3606054.503073791, + "chunks": 83 + } + ] + }, + { + "round": 0, + "arm": "main-drafted", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 26.04380517925345, + "ttft_max": 0.3833406246267259, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 26.043570663924534, + "seconds": 9.791284125, + "prefill_seconds": 0.382626125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.382920334, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 75, + "drafted": 448, + "accepted": 180, + "acceptance_rate": 0.4017857142857143, + "tokens_per_round": 3.4133333333333336 + }, + "sent": 3606054.50418025, + "first": 3606054.887520875, + "last": 3606064.678716833, + "chunks": 76 + } + ] + }, + { + "round": 0, + "arm": "main-drafted", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 25.201173385079827, + "ttft_max": 31.02874766662717, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 25.97921266330204, + "seconds": 9.815539959, + "prefill_seconds": 0.372683292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 20.837619541, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 75, + "drafted": 463, + "accepted": 180, + "acceptance_rate": 0.38876889848812096, + "tokens_per_round": 3.4133333333333336 + }, + "sent": 3606064.67963875, + "first": 3606085.517932916, + "last": 3606095.333423208, + "chunks": 76 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 25.956132195884578, + "seconds": 9.824268041, + "prefill_seconds": 0.375099959, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 31.028073875, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 75, + "drafted": 463, + "accepted": 180, + "acceptance_rate": 0.38876889848812096, + "tokens_per_round": 3.4133333333333336 + }, + "sent": 3606064.679766916, + "first": 3606095.708514583, + "last": 3606105.5327065, + "chunks": 76 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 25.998247972971406, + "seconds": 9.80835325, + "prefill_seconds": 0.377797958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.378097291, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 75, + "drafted": 463, + "accepted": 180, + "acceptance_rate": 0.38876889848812096, + "tokens_per_round": 3.4133333333333336 + }, + "sent": 3606064.67970125, + "first": 3606065.058400625, + "last": 3606074.866651333, + "chunks": 76 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 25.735948028532654, + "seconds": 9.908319667, + "prefill_seconds": 0.370257542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 10.556647125, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 75, + "drafted": 463, + "accepted": 180, + "acceptance_rate": 0.38876889848812096, + "tokens_per_round": 3.4133333333333336 + }, + "sent": 3606064.679569333, + "first": 3606075.236987916, + "last": 3606085.145233291, + "chunks": 76 + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 31.075094534881394, + "ttft_max": 0.20538291614502668, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 31.072944666747166, + "seconds": 8.206496125, + "prefill_seconds": 0.203727334, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.204398334, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 55, + "drafted": 365, + "accepted": 200, + "acceptance_rate": 0.547945205479452, + "tokens_per_round": 4.654545454545454 + }, + "sent": 3606141.369936875, + "first": 3606141.575319791, + "last": 3606149.781248166, + "chunks": 56 + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 41.92490293167692, + "ttft_max": 0.9278107918798923, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.826532270883073, + "seconds": 23.553248041, + "prefill_seconds": 0.228371666, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.228656208, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 66, + "drafted": 247, + "accepted": 189, + "acceptance_rate": 0.7651821862348178, + "tokens_per_round": 3.878787878787879 + }, + "sent": 3606149.7824475, + "first": 3606150.011670291, + "last": 3606173.564840375, + "chunks": 67 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.93183481567368, + "seconds": 23.326367833, + "prefill_seconds": 0.455304625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.455602625, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 66, + "drafted": 245, + "accepted": 189, + "acceptance_rate": 0.7714285714285715, + "tokens_per_round": 3.878787878787879 + }, + "sent": 3606149.782563416, + "first": 3606150.238595125, + "last": 3606173.564934041, + "chunks": 67 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.791077036134208, + "seconds": 23.63063475, + "prefill_seconds": 0.698629208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.927254084, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 1 + }, + "speculative": { + "rounds": 69, + "drafted": 236, + "accepted": 186, + "acceptance_rate": 0.788135593220339, + "tokens_per_round": 3.710144927536232 + }, + "sent": 3606149.782507666, + "first": 3606150.710318458, + "last": 3606174.340885833, + "chunks": 70 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.683139854491918, + "seconds": 23.869387041, + "prefill_seconds": 0.68825425, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.688510666, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 69, + "drafted": 239, + "accepted": 186, + "acceptance_rate": 0.7782426778242678, + "tokens_per_round": 3.710144927536232 + }, + "sent": 3606149.782372166, + "first": 3606150.471545041, + "last": 3606174.340864916, + "chunks": 70 + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 21.680902825813735, + "ttft_max": 0.4451714172028005, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 21.680688840498675, + "seconds": 11.761618917, + "prefill_seconds": 0.444441917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.444756875, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 75, + "drafted": 463, + "accepted": 180, + "acceptance_rate": 0.38876889848812096, + "tokens_per_round": 3.4133333333333336 + }, + "sent": 3606174.342055833, + "first": 3606174.78722725, + "last": 3606186.548730083, + "chunks": 76 + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 35.18775727391821, + "ttft_max": 1.921789416577667, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 9.267201928012476, + "seconds": 27.516396209, + "prefill_seconds": 1.4708, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.92109, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 94, + "drafted": 224, + "accepted": 161, + "acceptance_rate": 0.71875, + "tokens_per_round": 2.723404255319149 + }, + "sent": 3606186.550534833, + "first": 3606188.47232425, + "last": 3606215.988622041, + "chunks": 95 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 8.900161211192339, + "seconds": 28.651166417, + "prefill_seconds": 0.450010667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.4503955, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 92, + "drafted": 237, + "accepted": 163, + "acceptance_rate": 0.6877637130801688, + "tokens_per_round": 2.782608695652174 + }, + "sent": 3606186.550593958, + "first": 3606187.001462875, + "last": 3606215.65254575, + "chunks": 93 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 8.942051678556043, + "seconds": 28.516945458, + "prefill_seconds": 0.47025075, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.920595625, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 2 + }, + "speculative": { + "rounds": 94, + "drafted": 245, + "accepted": 161, + "acceptance_rate": 0.6571428571428571, + "tokens_per_round": 2.723404255319149 + }, + "sent": 3606186.550662875, + "first": 3606187.47172825, + "last": 3606215.988588166, + "chunks": 95 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 9.103197704543579, + "seconds": 28.012134667, + "prefill_seconds": 0.975065292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.425407291, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 2 + }, + "speculative": { + "rounds": 94, + "drafted": 234, + "accepted": 161, + "acceptance_rate": 0.688034188034188, + "tokens_per_round": 2.723404255319149 + }, + "sent": 3606186.550445166, + "first": 3606187.976557666, + "last": 3606215.988817541, + "chunks": 95 + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 32.82984563370725, + "ttft_max": 0.2014766251668334, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 32.83012336049014, + "seconds": 7.767256833, + "prefill_seconds": 0.200702875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.200988792, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 55, + "drafted": 354, + "accepted": 200, + "acceptance_rate": 0.5649717514124294, + "tokens_per_round": 4.654545454545454 + }, + "sent": 3606225.58774575, + "first": 3606225.789222375, + "last": 3606233.556544916, + "chunks": 56 + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 39.476843897617066, + "ttft_max": 0.9977558329701424, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.06532199933952, + "seconds": 25.334509916, + "prefill_seconds": 0.743525709, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.743825583, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 64, + "drafted": 241, + "accepted": 191, + "acceptance_rate": 0.7925311203319502, + "tokens_per_round": 4.0 + }, + "sent": 3606233.558710833, + "first": 3606234.30318025, + "last": 3606259.637624083, + "chunks": 65 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 9.966010124797743, + "seconds": 25.586969791, + "prefill_seconds": 0.491034333, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.491316208, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 64, + "drafted": 242, + "accepted": 191, + "acceptance_rate": 0.7892561983471075, + "tokens_per_round": 4.0 + }, + "sent": 3606233.558772708, + "first": 3606234.050711375, + "last": 3606259.63764625, + "chunks": 65 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 9.869207552850355, + "seconds": 25.83794075, + "prefill_seconds": 0.240055, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.240312917, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 64, + "drafted": 244, + "accepted": 191, + "acceptance_rate": 0.7827868852459017, + "tokens_per_round": 4.0 + }, + "sent": 3606233.558825416, + "first": 3606233.799724416, + "last": 3606259.637587541, + "chunks": 65 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.166927138522293, + "seconds": 25.081324625, + "prefill_seconds": 0.75667025, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.996916125, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 1 + }, + "speculative": { + "rounds": 64, + "drafted": 239, + "accepted": 191, + "acceptance_rate": 0.799163179916318, + "tokens_per_round": 4.0 + }, + "sent": 3606233.558633875, + "first": 3606234.556389708, + "last": 3606259.637656208, + "chunks": 65 + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 19.40846506022387, + "ttft_max": 0.4472993751987815, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 19.408321894553616, + "seconds": 13.138693875, + "prefill_seconds": 0.446586166, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.4468845, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 2 + }, + "speculative": { + "rounds": 74, + "drafted": 471, + "accepted": 181, + "acceptance_rate": 0.3842887473460722, + "tokens_per_round": 3.4594594594594597 + }, + "sent": 3606259.639965625, + "first": 3606260.087265, + "last": 3606273.225861958, + "chunks": 75 + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 37.247923082251035, + "ttft_max": 2.0259827501140535, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 9.487299469359217, + "seconds": 26.878038458, + "prefill_seconds": 1.035085542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.035508875, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 2 + }, + "speculative": { + "rounds": 96, + "drafted": 233, + "accepted": 159, + "acceptance_rate": 0.6824034334763949, + "tokens_per_round": 2.6666666666666665 + }, + "sent": 3606273.227919, + "first": 3606274.264037375, + "last": 3606301.141950958, + "chunks": 97 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 9.66677787410069, + "seconds": 26.379006875, + "prefill_seconds": 1.005173042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.534582791, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 2 + }, + "speculative": { + "rounds": 96, + "drafted": 229, + "accepted": 159, + "acceptance_rate": 0.6943231441048034, + "tokens_per_round": 2.6666666666666665 + }, + "sent": 3606273.228045125, + "first": 3606274.76313025, + "last": 3606301.141967875, + "chunks": 97 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 9.311984043601145, + "seconds": 27.384067542, + "prefill_seconds": 0.529000083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.529438167, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 2 + }, + "speculative": { + "rounds": 96, + "drafted": 233, + "accepted": 159, + "acceptance_rate": 0.6824034334763949, + "tokens_per_round": 2.6666666666666665 + }, + "sent": 3606273.227982375, + "first": 3606273.757919333, + "last": 3606301.141920083, + "chunks": 97 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 9.85000733950893, + "seconds": 25.888305583, + "prefill_seconds": 1.495880584, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 2.02529125, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 96, + "drafted": 228, + "accepted": 159, + "acceptance_rate": 0.6973684210526315, + "tokens_per_round": 2.6666666666666665 + }, + "sent": 3606273.227833291, + "first": 3606275.253816041, + "last": 3606301.1419965, + "chunks": 97 + } + ] + }, + { + "round": 1, + "arm": "main-drafted", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 35.350466980056616, + "ttft_max": 0.20126425009220839, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 35.3500488025168, + "seconds": 7.213568542, + "prefill_seconds": 0.200582458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.200864167, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 54, + "drafted": 355, + "accepted": 201, + "acceptance_rate": 0.5661971830985916, + "tokens_per_round": 4.7407407407407405 + }, + "sent": 3606311.548035666, + "first": 3606311.749299916, + "last": 3606318.962783125, + "chunks": 55 + } + ] + }, + { + "round": 1, + "arm": "main-drafted", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 40.423617449409754, + "ttft_max": 0.8631084999069571, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.975599933959169, + "seconds": 23.233354125, + "prefill_seconds": 0.207475292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.20785825, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 65, + "drafted": 243, + "accepted": 190, + "acceptance_rate": 0.7818930041152263, + "tokens_per_round": 3.9384615384615387 + }, + "sent": 3606318.964425125, + "first": 3606319.172912041, + "last": 3606342.406194541, + "chunks": 66 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.398315459161685, + "seconds": 24.523202917, + "prefill_seconds": 0.424516458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.424947459, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 1 + }, + "speculative": { + "rounds": 68, + "drafted": 244, + "accepted": 187, + "acceptance_rate": 0.7663934426229508, + "tokens_per_round": 3.764705882352941 + }, + "sent": 3606318.964586583, + "first": 3606319.389932416, + "last": 3606343.913043416, + "chunks": 69 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.375067787026046, + "seconds": 24.578152667, + "prefill_seconds": 0.654687291, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.862650709, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 70, + "drafted": 234, + "accepted": 185, + "acceptance_rate": 0.7905982905982906, + "tokens_per_round": 3.657142857142857 + }, + "sent": 3606318.964503083, + "first": 3606319.827611583, + "last": 3606344.405685958, + "chunks": 71 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.283350014062258, + "seconds": 24.797366583, + "prefill_seconds": 0.435474292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.643465125, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 70, + "drafted": 236, + "accepted": 185, + "acceptance_rate": 0.7838983050847458, + "tokens_per_round": 3.657142857142857 + }, + "sent": 3606318.964343125, + "first": 3606319.608362333, + "last": 3606344.405664041, + "chunks": 71 + } + ] + }, + { + "round": 1, + "arm": "main-drafted", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 23.181142415827715, + "ttft_max": 0.4115130412392318, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 23.18076293026933, + "seconds": 11.00050075, + "prefill_seconds": 0.410683709, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.410999875, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 75, + "drafted": 468, + "accepted": 180, + "acceptance_rate": 0.38461538461538464, + "tokens_per_round": 3.4133333333333336 + }, + "sent": 3606344.407623125, + "first": 3606344.819136166, + "last": 3606355.819456833, + "chunks": 76 + } + ] + }, + { + "round": 1, + "arm": "main-drafted", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 41.04567326689068, + "ttft_max": 1.7338110408745706, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.261494365126726, + "seconds": 24.85018175, + "prefill_seconds": 0.422173833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.422573417, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 2 + }, + "speculative": { + "rounds": 97, + "drafted": 223, + "accepted": 158, + "acceptance_rate": 0.7085201793721974, + "tokens_per_round": 2.6391752577319587 + }, + "sent": 3606355.821066083, + "first": 3606356.244260083, + "last": 3606381.094373708, + "chunks": 98 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.446103423829815, + "seconds": 24.411016209, + "prefill_seconds": 0.439312583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.861861041, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 2 + }, + "speculative": { + "rounds": 97, + "drafted": 223, + "accepted": 158, + "acceptance_rate": 0.7085201793721974, + "tokens_per_round": 2.6391752577319587 + }, + "sent": 3606355.821124958, + "first": 3606356.683676166, + "last": 3606381.094626, + "chunks": 98 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.64010986129906, + "seconds": 23.965917958, + "prefill_seconds": 0.884405125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.30694925, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 2 + }, + "speculative": { + "rounds": 97, + "drafted": 219, + "accepted": 158, + "acceptance_rate": 0.7214611872146118, + "tokens_per_round": 2.6391752577319587 + }, + "sent": 3606355.821194625, + "first": 3606357.128658333, + "last": 3606381.094574708, + "chunks": 98 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.832718965943481, + "seconds": 23.5397965, + "prefill_seconds": 1.310551042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.7330955, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 2 + }, + "speculative": { + "rounds": 97, + "drafted": 219, + "accepted": 158, + "acceptance_rate": 0.7214611872146118, + "tokens_per_round": 2.6391752577319587 + }, + "sent": 3606355.8209675, + "first": 3606357.554778541, + "last": 3606381.094475583, + "chunks": 98 + } + ] + }, + { + "round": 2, + "arm": "main-drafted", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 28.223626529326367, + "ttft_max": 0.2059693750925362, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 28.223380789937625, + "seconds": 9.035062167, + "prefill_seconds": 0.205207625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.205514375, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 53, + "drafted": 376, + "accepted": 202, + "acceptance_rate": 0.5372340425531915, + "tokens_per_round": 4.830188679245283 + }, + "sent": 3606391.650607583, + "first": 3606391.856576958, + "last": 3606400.891560458, + "chunks": 54 + } + ] + }, + { + "round": 2, + "arm": "main-drafted", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 28.761086081267262, + "ttft_max": 27.405207625124604, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 28.326025086672857, + "seconds": 9.002322042, + "prefill_seconds": 0.273832209, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.274136208, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 53, + "drafted": 376, + "accepted": 202, + "acceptance_rate": 0.5372340425531915, + "tokens_per_round": 4.830188679245283 + }, + "sent": 3606400.894081041, + "first": 3606401.168842916, + "last": 3606410.171049916, + "chunks": 54 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 29.82616056363124, + "seconds": 8.549541583, + "prefill_seconds": 0.265464208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 18.628984333, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 53, + "drafted": 368, + "accepted": 202, + "acceptance_rate": 0.5489130434782609, + "tokens_per_round": 4.830188679245283 + }, + "sent": 3606400.893924166, + "first": 3606419.523642625, + "last": 3606428.073088041, + "chunks": 54 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 30.596510163274633, + "seconds": 8.334283833, + "prefill_seconds": 0.226095667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 27.404542583, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 53, + "drafted": 368, + "accepted": 202, + "acceptance_rate": 0.5489130434782609, + "tokens_per_round": 4.830188679245283 + }, + "sent": 3606400.894007083, + "first": 3606428.299214708, + "last": 3606436.633428541, + "chunks": 54 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 28.817921085343936, + "seconds": 8.8486605, + "prefill_seconds": 0.238501375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 9.51489725, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 52, + "drafted": 377, + "accepted": 203, + "acceptance_rate": 0.5384615384615384, + "tokens_per_round": 4.923076923076923 + }, + "sent": 3606400.893823, + "first": 3606410.409605375, + "last": 3606419.258174583, + "chunks": 53 + } + ] + }, + { + "round": 2, + "arm": "main-drafted", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 22.158009935284845, + "ttft_max": 0.43920179177075624, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 22.157811137460392, + "seconds": 11.508356959, + "prefill_seconds": 0.438467209, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.438798917, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 75, + "drafted": 459, + "accepted": 180, + "acceptance_rate": 0.39215686274509803, + "tokens_per_round": 3.4133333333333336 + }, + "sent": 3606436.634665083, + "first": 3606437.073866875, + "last": 3606448.582120583, + "chunks": 76 + } + ] + }, + { + "round": 2, + "arm": "main-drafted", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 36.93984744629104, + "ttft_max": 2.149744499940425, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 9.849580616640145, + "seconds": 25.889427167, + "prefill_seconds": 1.308605375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 2.149030417, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 96, + "drafted": 227, + "accepted": 159, + "acceptance_rate": 0.7004405286343612, + "tokens_per_round": 2.6666666666666665 + }, + "sent": 3606448.584418208, + "first": 3606450.734162708, + "last": 3606476.623452708, + "chunks": 97 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 9.531338599247102, + "seconds": 26.753849666, + "prefill_seconds": 0.444140084, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.284677042, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 96, + "drafted": 231, + "accepted": 159, + "acceptance_rate": 0.6883116883116883, + "tokens_per_round": 2.6666666666666665 + }, + "sent": 3606448.584539041, + "first": 3606449.869655166, + "last": 3606476.623400333, + "chunks": 97 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 9.688840220673356, + "seconds": 26.318939542, + "prefill_seconds": 0.879069125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.71950625, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 96, + "drafted": 227, + "accepted": 159, + "acceptance_rate": 0.7004405286343612, + "tokens_per_round": 2.6666666666666665 + }, + "sent": 3606448.58448275, + "first": 3606450.304553666, + "last": 3606476.62342825, + "chunks": 97 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 9.234971686120044, + "seconds": 27.612429, + "prefill_seconds": 0.425563125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.425944542, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 99, + "drafted": 227, + "accepted": 156, + "acceptance_rate": 0.6872246696035242, + "tokens_per_round": 2.585858585858586 + }, + "sent": 3606448.584335333, + "first": 3606449.010994333, + "last": 3606476.623366333, + "chunks": 100 + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 34.74238473493653, + "ttft_max": 0.20315800001844764, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 34.741093727197175, + "seconds": 7.340010709, + "prefill_seconds": 0.202288333, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.202571875, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 55, + "drafted": 355, + "accepted": 200, + "acceptance_rate": 0.5633802816901409, + "tokens_per_round": 4.654545454545454 + }, + "sent": 3606487.465374333, + "first": 3606487.668532333, + "last": 3606495.008270291, + "chunks": 56 + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 45.67148543407574, + "ttft_max": 0.8624916248954833, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 11.417828712467486, + "seconds": 22.333493208, + "prefill_seconds": 0.199364125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.199731458, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 70, + "drafted": 231, + "accepted": 186, + "acceptance_rate": 0.8051948051948052, + "tokens_per_round": 3.657142857142857 + }, + "sent": 3606495.010333583, + "first": 3606495.210814833, + "last": 3606517.544128833, + "chunks": 71 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 11.529024323617588, + "seconds": 22.118090208, + "prefill_seconds": 0.41482, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.415123, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 70, + "drafted": 227, + "accepted": 186, + "acceptance_rate": 0.8193832599118943, + "tokens_per_round": 3.657142857142857 + }, + "sent": 3606495.010474125, + "first": 3606495.426240125, + "last": 3606517.5441555, + "chunks": 71 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 11.766654527744414, + "seconds": 21.671410459, + "prefill_seconds": 0.662173417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.861817625, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 1 + }, + "speculative": { + "rounds": 70, + "drafted": 224, + "accepted": 186, + "acceptance_rate": 0.8303571428571429, + "tokens_per_round": 3.657142857142857 + }, + "sent": 3606495.010416916, + "first": 3606495.872908541, + "last": 3606517.544200166, + "chunks": 71 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 11.64494984033345, + "seconds": 21.897904542, + "prefill_seconds": 0.435676917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.6354045, + "sampling": "greedy", + "drafts": true, + "token_sha": "2b28aa2108de", + "token_sha256": "2b28aa2108de022e66b60c3a8797e6939c63743429793c6366264ad9c88f6d76", + "min_rows": 2 + }, + "speculative": { + "rounds": 70, + "drafted": 225, + "accepted": 186, + "acceptance_rate": 0.8266666666666667, + "tokens_per_round": 3.657142857142857 + }, + "sent": 3606495.010198375, + "first": 3606495.646404666, + "last": 3606517.544224625, + "chunks": 71 + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 23.125861831112022, + "ttft_max": 0.43125058291479945, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 23.12566704818981, + "seconds": 11.026708958, + "prefill_seconds": 0.430470709, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.430793375, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 75, + "drafted": 448, + "accepted": 180, + "acceptance_rate": 0.4017857142857143, + "tokens_per_round": 3.4133333333333336 + }, + "sent": 3606517.5461745, + "first": 3606517.977425083, + "last": 3606529.004041166, + "chunks": 76 + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 40.05300163872442, + "ttft_max": 1.7167558339424431, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.360270922105201, + "seconds": 24.613255958, + "prefill_seconds": 0.852961125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.289912375, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 2 + }, + "speculative": { + "rounds": 97, + "drafted": 218, + "accepted": 158, + "acceptance_rate": 0.7247706422018348, + "tokens_per_round": 2.6391752577319587 + }, + "sent": 3606529.005617333, + "first": 3606530.296233, + "last": 3606554.909576625, + "chunks": 98 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.542903217856782, + "seconds": 24.186886167, + "prefill_seconds": 1.279218041, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.716137417, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 1 + }, + "speculative": { + "rounds": 97, + "drafted": 217, + "accepted": 158, + "acceptance_rate": 0.728110599078341, + "tokens_per_round": 2.6391752577319587 + }, + "sent": 3606529.005735666, + "first": 3606530.7224915, + "last": 3606554.909267916, + "chunks": 98 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.185256518927337, + "seconds": 25.036188291, + "prefill_seconds": 0.866474084, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.866868041, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 2 + }, + "speculative": { + "rounds": 97, + "drafted": 222, + "accepted": 158, + "acceptance_rate": 0.7117117117117117, + "tokens_per_round": 2.6391752577319587 + }, + "sent": 3606529.005681125, + "first": 3606529.873125625, + "last": 3606554.90949975, + "chunks": 98 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 10.013385130654662, + "seconds": 25.465913542, + "prefill_seconds": 0.4365775, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.436960042, + "sampling": "greedy", + "drafts": true, + "token_sha": "a948470d6fea", + "token_sha256": "a948470d6fea2076d204ad98750a02e149e0baeeed82c9eafae64716c0a493c7", + "min_rows": 2 + }, + "speculative": { + "rounds": 97, + "drafted": 222, + "accepted": 158, + "acceptance_rate": 0.7117117117117117, + "tokens_per_round": 2.6391752577319587 + }, + "sent": 3606529.005529375, + "first": 3606529.443320458, + "last": 3606554.909149625, + "chunks": 98 + } + ] + } + ] + }, + "M5 Ultra": { + "functional": { + "resumed": { + "usage": { + "prompt_tokens": 3605, + "completion_tokens": 64, + "total_tokens": 3669, + "prompt_tokens_details": { + "cached_tokens": 3517 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 545.6868379633086, + "seconds": 0.115450833, + "prefill_seconds": 0.0341725, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.034537708, + "sampling": "greedy", + "drafts": true, + "token_sha": "2fed3f850c53", + "token_sha256": "2fed3f850c53a2a46620eb8b0facca0a7db4890e9462c4db2a39a5597f462a91", + "min_rows": 2 + }, + "speculative": { + "rounds": 11, + "drafted": 58, + "accepted": 52, + "acceptance_rate": 0.896551724137931, + "tokens_per_round": 5.818181818181818 + }, + "wall": 0.15039120900000036 + }, + "resume_request": { + "messages": [ + { + "role": "user" + }, + { + "role": "assistant" + }, + { + "role": "user" + } + ], + "max_tokens": 64 + }, + "stress_reference_hashes": [ + { + "prompt": "quicksort-raw", + "sha256": "9e4feb5a9aadd228c02f26c10e22d5a858be52f9690de23130431897ab4a95d0" + }, + { + "prompt": "rust-reverse-words", + "sha256": "d9499a1af105f9d0477f4fda4f2afde556c782008fe552d4a7cd0ebf71e80101" + }, + { + "prompt": "multiply-17-23", + "sha256": "72bba60a60f4df7bf0b2a34dab50868cd071c0b99c2ac9d3b2f0cc99a8c6ab84" + }, + { + "prompt": "sleep-tips", + "sha256": "dc9d14472ff7c82357f0686ee61236b54726924051b1222778247438e54345db" + }, + { + "prompt": "js-debounce-raw", + "sha256": "d80d7a1d94635a9a5ce9305bf22133261181963691b70407d3212b0557e0d458" + }, + { + "prompt": "linear-equation", + "sha256": "8ebb10c3e63f3fd3a265072c4a9ef09d88058a4fc50e488f523ce44cb0afdc0d" + }, + { + "prompt": "fishing-village", + "sha256": "c1100e452897c3c18c70f946d0ae4f05a345e7f90b10bece58e0cd584daf0ade" + }, + { + "prompt": "sql-left-join", + "sha256": "d04f354206e1c4c56ff5956161df55ffea050df8d858b655c26a2f8673b741b4" + } + ], + "stress": [ + { + "slots": 6, + "seconds": 60, + "completed": 453, + "cancelled": 247, + "wrong": 0, + "errors": [] + }, + { + "slots": 16, + "seconds": 90, + "completed": 852, + "cancelled": 459, + "wrong": 0, + "errors": [] + } + ], + "sampled": [ + { + "streams": 1, + "hashes": [ + [ + "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d", + "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d", + "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d" + ] + ] + }, + { + "streams": 4, + "hashes": [ + [ + "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d", + "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d", + "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d" + ], + [ + "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5", + "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5", + "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5" + ], + [ + "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770", + "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770", + "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770" + ], + [ + "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133", + "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133", + "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133" + ] + ] + }, + { + "streams": 8, + "hashes": [ + [ + "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d", + "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d", + "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d" + ], + [ + "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5", + "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5", + "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5" + ], + [ + "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770", + "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770", + "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770" + ], + [ + "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133", + "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133", + "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133" + ], + [ + "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366", + "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366", + "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366" + ], + [ + "ba44af1f43a9f23b0bbadc69ea88414df1e7625f0aa2ade7acd73666362ec4ff", + "ba44af1f43a9f23b0bbadc69ea88414df1e7625f0aa2ade7acd73666362ec4ff", + "ba44af1f43a9f23b0bbadc69ea88414df1e7625f0aa2ade7acd73666362ec4ff" + ], + [ + "36b7dc4d8daac1d982e69cb8a13b97aa3d76403555565c13e19ff839f6a50c31", + "36b7dc4d8daac1d982e69cb8a13b97aa3d76403555565c13e19ff839f6a50c31", + "36b7dc4d8daac1d982e69cb8a13b97aa3d76403555565c13e19ff839f6a50c31" + ], + [ + "4e295b6a9b36a30ee500d0c93fb4d23083bf86a26a0d2dbdc720516e39c3b2cf", + "4e295b6a9b36a30ee500d0c93fb4d23083bf86a26a0d2dbdc720516e39c3b2cf", + "4e295b6a9b36a30ee500d0c93fb4d23083bf86a26a0d2dbdc720516e39c3b2cf" + ] + ] + }, + { + "streams": 16, + "hashes": [ + [ + "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d", + "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d", + "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d" + ], + [ + "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5", + "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5", + "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5" + ], + [ + "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770", + "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770", + "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770" + ], + [ + "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133", + "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133", + "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133" + ], + [ + "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366", + "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366", + "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366" + ], + [ + "ba44af1f43a9f23b0bbadc69ea88414df1e7625f0aa2ade7acd73666362ec4ff", + "ba44af1f43a9f23b0bbadc69ea88414df1e7625f0aa2ade7acd73666362ec4ff", + "ba44af1f43a9f23b0bbadc69ea88414df1e7625f0aa2ade7acd73666362ec4ff" + ], + [ + "36b7dc4d8daac1d982e69cb8a13b97aa3d76403555565c13e19ff839f6a50c31", + "36b7dc4d8daac1d982e69cb8a13b97aa3d76403555565c13e19ff839f6a50c31", + "36b7dc4d8daac1d982e69cb8a13b97aa3d76403555565c13e19ff839f6a50c31" + ], + [ + "4e295b6a9b36a30ee500d0c93fb4d23083bf86a26a0d2dbdc720516e39c3b2cf", + "4e295b6a9b36a30ee500d0c93fb4d23083bf86a26a0d2dbdc720516e39c3b2cf", + "4e295b6a9b36a30ee500d0c93fb4d23083bf86a26a0d2dbdc720516e39c3b2cf" + ], + [ + "16e209d6f6ca13b44c6c2ac86018d6117f48739fe679c8b10f219711ff9cb986", + "16e209d6f6ca13b44c6c2ac86018d6117f48739fe679c8b10f219711ff9cb986", + "16e209d6f6ca13b44c6c2ac86018d6117f48739fe679c8b10f219711ff9cb986" + ], + [ + "45cab8cc38d33c168e96169e6cd61ecb339df2e61855515953cdeb5314d7e351", + "45cab8cc38d33c168e96169e6cd61ecb339df2e61855515953cdeb5314d7e351", + "45cab8cc38d33c168e96169e6cd61ecb339df2e61855515953cdeb5314d7e351" + ], + [ + "0bfbd2b2a91cab827879cb2bd2aa696bf6bd49516c2aa5439c83ecb386ea7369", + "0bfbd2b2a91cab827879cb2bd2aa696bf6bd49516c2aa5439c83ecb386ea7369", + "0bfbd2b2a91cab827879cb2bd2aa696bf6bd49516c2aa5439c83ecb386ea7369" + ], + [ + "1e81d9e26277be61b38974b207ca6f31e045350d3e939c4ec700e40603553f0a", + "1e81d9e26277be61b38974b207ca6f31e045350d3e939c4ec700e40603553f0a", + "1e81d9e26277be61b38974b207ca6f31e045350d3e939c4ec700e40603553f0a" + ], + [ + "97ba371a7ee308d84fd24f6cf3e574b817a1fe7a56728a122cfe9ab4aae75363", + "97ba371a7ee308d84fd24f6cf3e574b817a1fe7a56728a122cfe9ab4aae75363", + "97ba371a7ee308d84fd24f6cf3e574b817a1fe7a56728a122cfe9ab4aae75363" + ], + [ + "6884f675a0b28c8d331d9d245919814741dcefeaf12900e783d1b525d4490398", + "6884f675a0b28c8d331d9d245919814741dcefeaf12900e783d1b525d4490398", + "6884f675a0b28c8d331d9d245919814741dcefeaf12900e783d1b525d4490398" + ], + [ + "7b81161c5817e10474e598a19f3fc2008bdece4a9e4c68e1dc52024e4d5d32db", + "7b81161c5817e10474e598a19f3fc2008bdece4a9e4c68e1dc52024e4d5d32db", + "7b81161c5817e10474e598a19f3fc2008bdece4a9e4c68e1dc52024e4d5d32db" + ], + [ + "e5997fd3392b57373c76457f8c69bd05899ad99c92abf281efc274e148dd74fd", + "e5997fd3392b57373c76457f8c69bd05899ad99c92abf281efc274e148dd74fd", + "e5997fd3392b57373c76457f8c69bd05899ad99c92abf281efc274e148dd74fd" + ] + ] + } + ], + "fresh": { + "usage": { + "prompt_tokens": 3605, + "completion_tokens": 64, + "total_tokens": 3669, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 597.1205482317299, + "seconds": 0.105506334, + "prefill_seconds": 0.308241792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.310870084, + "sampling": "greedy", + "drafts": true, + "token_sha": "2fed3f850c53", + "token_sha256": "2fed3f850c53a2a46620eb8b0facca0a7db4890e9462c4db2a39a5597f462a91", + "min_rows": 2 + }, + "speculative": { + "rounds": 10, + "drafted": 59, + "accepted": 53, + "acceptance_rate": 0.8983050847457628, + "tokens_per_round": 6.4 + }, + "wall": 0.41713370900001223 + }, + "resume_equal": true + }, + "long-context": [ + { + "length": 4096, + "plain": { + "usage": { + "prompt_tokens": 4096, + "completion_tokens": 64, + "total_tokens": 4160, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 251.27072892278412, + "seconds": 0.250725583, + "prefill_seconds": 0.298777459, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.299359667, + "sampling": "greedy", + "drafts": false, + "token_sha": "8394ececc0cc", + "token_sha256": "8394ececc0ccdeea4fd63598ff8309e489e7309932166b11e366f6706b8a4b3a", + "min_rows": 1 + }, + "speculative": { + "rounds": 63, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.0158730158730158 + }, + "wall": 0.5511507499999997 + }, + "drafted": { + "usage": { + "prompt_tokens": 4096, + "completion_tokens": 64, + "total_tokens": 4160, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 313.5117331542196, + "seconds": 0.200949417, + "prefill_seconds": 0.302226708, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.302388917, + "sampling": "greedy", + "drafts": true, + "token_sha": "8394ececc0cc", + "token_sha256": "8394ececc0ccdeea4fd63598ff8309e489e7309932166b11e366f6706b8a4b3a", + "min_rows": 2 + }, + "speculative": { + "rounds": 26, + "drafted": 68, + "accepted": 38, + "acceptance_rate": 0.5588235294117647, + "tokens_per_round": 2.4615384615384617 + }, + "wall": 0.5038303750000002 + }, + "equal": true + }, + { + "length": 16384, + "plain": { + "usage": { + "prompt_tokens": 16384, + "completion_tokens": 64, + "total_tokens": 16448, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 231.32519503146526, + "seconds": 0.272343875, + "prefill_seconds": 1.283367667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.283625625, + "sampling": "greedy", + "drafts": false, + "token_sha": "a2dcb347e183", + "token_sha256": "a2dcb347e183d9e8a0be485abb99d222ff060dfc83781384863e7bf16cf22da3", + "min_rows": 1 + }, + "speculative": { + "rounds": 63, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.0158730158730158 + }, + "wall": 1.5566373330000003 + }, + "drafted": { + "usage": { + "prompt_tokens": 16384, + "completion_tokens": 64, + "total_tokens": 16448, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 259.7309798859134, + "seconds": 0.242558666, + "prefill_seconds": 1.342481375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 1.34274125, + "sampling": "greedy", + "drafts": true, + "token_sha": "a2dcb347e183", + "token_sha256": "a2dcb347e183d9e8a0be485abb99d222ff060dfc83781384863e7bf16cf22da3", + "min_rows": 2 + }, + "speculative": { + "rounds": 32, + "drafted": 57, + "accepted": 31, + "acceptance_rate": 0.543859649122807, + "tokens_per_round": 2.0 + }, + "wall": 1.5860129589999996 + }, + "equal": true + }, + { + "length": 65536, + "plain": { + "usage": { + "prompt_tokens": 65536, + "completion_tokens": 64, + "total_tokens": 65600, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 157.88251974236329, + "seconds": 0.399030875, + "prefill_seconds": 7.318780875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 7.319519292, + "sampling": "greedy", + "drafts": false, + "token_sha": "c2ba0e4604c6", + "token_sha256": "c2ba0e4604c6beb356eefe87cdec6c938f5f4a484521bfd73e8f9e6b57bc22b6", + "min_rows": 1 + }, + "speculative": { + "rounds": 63, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.0158730158730158 + }, + "wall": 7.720276332999999 + }, + "drafted": { + "usage": { + "prompt_tokens": 65536, + "completion_tokens": 64, + "total_tokens": 65600, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 151.86195081442588, + "seconds": 0.414850459, + "prefill_seconds": 7.556797125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 7.557524875, + "sampling": "greedy", + "drafts": true, + "token_sha": "c2ba0e4604c6", + "token_sha256": "c2ba0e4604c6beb356eefe87cdec6c938f5f4a484521bfd73e8f9e6b57bc22b6", + "min_rows": 2 + }, + "speculative": { + "rounds": 36, + "drafted": 122, + "accepted": 27, + "acceptance_rate": 0.22131147540983606, + "tokens_per_round": 1.7777777777777777 + }, + "wall": 7.974062624999998 + }, + "equal": true + }, + { + "length": 130816, + "plain": { + "usage": { + "prompt_tokens": 130816, + "completion_tokens": 64, + "total_tokens": 130880, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 116.19828729405333, + "seconds": 0.542176666, + "prefill_seconds": 20.563776416, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 20.565105666, + "sampling": "greedy", + "drafts": false, + "token_sha": "3cc988610216", + "token_sha256": "3cc9886102163ddcfa01f3a3f20f723929d6cb01e51cb4bd3e960f05ca0569c3", + "min_rows": 1 + }, + "speculative": { + "rounds": 63, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.0158730158730158 + }, + "wall": 21.110205083000004 + }, + "drafted": { + "usage": { + "prompt_tokens": 130816, + "completion_tokens": 64, + "total_tokens": 130880, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 230.23142702247748, + "seconds": 0.273637708, + "prefill_seconds": 21.036062792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 21.037378792, + "sampling": "greedy", + "drafts": true, + "token_sha": "3cc988610216", + "token_sha256": "3cc9886102163ddcfa01f3a3f20f723929d6cb01e51cb4bd3e960f05ca0569c3", + "min_rows": 5 + }, + "speculative": { + "rounds": 12, + "drafted": 108, + "accepted": 51, + "acceptance_rate": 0.4722222222222222, + "tokens_per_round": 5.333333333333333 + }, + "wall": 21.314060583999996 + }, + "equal": true + } + ], + "performance-lfm2": [ + { + "round": 0, + "arm": "main-plain", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 289.9714374495252, + "ttft_max": 0.016745999999997707, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 289.9559952775242, + "seconds": 0.879443792, + "prefill_seconds": 0.016395833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.016501792, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 216, + "drafted": 100, + "accepted": 39, + "acceptance_rate": 0.39, + "tokens_per_round": 1.1851851851851851 + }, + "sent": 124.128642, + "first": 124.145388, + "last": 125.024784958, + "chunks": 217 + } + ] + }, + { + "round": 0, + "arm": "main-plain", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 652.2080388699854, + "ttft_max": 0.06611995800000159, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 166.96049804643943, + "seconds": 1.527307375, + "prefill_seconds": 0.01635325, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.049324708, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 223, + "drafted": 42, + "accepted": 32, + "acceptance_rate": 0.7619047619047619, + "tokens_per_round": 1.147982062780269 + }, + "sent": 125.0255105, + "first": 125.075392583, + "last": 126.602660541, + "chunks": 224 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 168.34147176327625, + "seconds": 1.51477825, + "prefill_seconds": 0.01636675, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.0656875, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 224, + "drafted": 46, + "accepted": 31, + "acceptance_rate": 0.6739130434782609, + "tokens_per_round": 1.1428571428571428 + }, + "sent": 125.025627875, + "first": 125.091747833, + "last": 126.606481416, + "chunks": 225 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 163.45264548507336, + "seconds": 1.560084875, + "prefill_seconds": 0.016379542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.016536916, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 223, + "drafted": 43, + "accepted": 32, + "acceptance_rate": 0.7441860465116279, + "tokens_per_round": 1.147982062780269 + }, + "sent": 125.025571583, + "first": 125.04256325, + "last": 126.602603041, + "chunks": 224 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 165.19216464411534, + "seconds": 1.54365675, + "prefill_seconds": 0.016455834, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.032961709, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 223, + "drafted": 43, + "accepted": 32, + "acceptance_rate": 0.7441860465116279, + "tokens_per_round": 1.147982062780269 + }, + "sent": 125.025436875, + "first": 125.059020333, + "last": 126.602634583, + "chunks": 224 + } + ] + }, + { + "round": 0, + "arm": "main-plain", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 830.9301245588248, + "ttft_max": 0.13169079099999692, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 104.19389804837425, + "seconds": 2.447360208, + "prefill_seconds": 0.016465708, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.016684167, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 126.607762625, + "first": 126.625097166, + "last": 129.0724045, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 104.89724081177302, + "seconds": 2.4309505, + "prefill_seconds": 0.016419292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.033021917, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 126.607816791, + "first": 126.641500916, + "last": 129.072416125, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 108.5459214960778, + "seconds": 2.349236125, + "prefill_seconds": 0.016382917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.114713417, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 126.608144291, + "first": 126.7233725, + "last": 129.072579958, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 107.79394994721177, + "seconds": 2.365624417, + "prefill_seconds": 0.016353916, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.098507667, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 126.607865791, + "first": 126.70698675, + "last": 129.072642708, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 107.05374686342388, + "seconds": 2.381981084, + "prefill_seconds": 0.016367334, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.082089875, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 126.608013291, + "first": 126.690629583, + "last": 129.072869166, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 105.60819370489229, + "seconds": 2.414585375, + "prefill_seconds": 0.0163805, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.049418458, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 126.607912666, + "first": 126.657892958, + "last": 129.072431375, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 108.95296996983765, + "seconds": 2.340459375, + "prefill_seconds": 0.016380709, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.131116792, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 236, + "drafted": 42, + "accepted": 19, + "acceptance_rate": 0.4523809523809524, + "tokens_per_round": 1.0847457627118644 + }, + "sent": 126.60806725, + "first": 126.739758041, + "last": 129.08017725, + "chunks": 237 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 106.3282934211501, + "seconds": 2.398232792, + "prefill_seconds": 0.016363417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.065817125, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 126.607685583, + "first": 126.674259291, + "last": 129.072453583, + "chunks": 235 + } + ] + }, + { + "round": 0, + "arm": "main-plain", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 828.3957621161079, + "ttft_max": 0.26417679200000066, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.4837114578651, + "seconds": 4.858650292, + "prefill_seconds": 0.016387708, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.082543416, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 20, + "accepted": 16, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 129.082258416, + "first": 129.166236416, + "last": 134.025300916, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.132941351539856, + "seconds": 4.891341125, + "prefill_seconds": 0.016411458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.049863042, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 20, + "accepted": 16, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 129.082833125, + "first": 129.133453208, + "last": 134.024820291, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.0191678536274, + "seconds": 4.809581333, + "prefill_seconds": 0.016347333, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.131707875, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 20, + "accepted": 16, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 129.082449458, + "first": 129.215360125, + "last": 134.025603666, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.66131800556297, + "seconds": 4.842263917, + "prefill_seconds": 0.016385584, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.098904375, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 20, + "accepted": 16, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 129.082311083, + "first": 129.182623833, + "last": 134.025126708, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.78590461813402, + "seconds": 4.924119833, + "prefill_seconds": 0.016984208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.0171665, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 20, + "accepted": 16, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 129.082358166, + "first": 129.100618625, + "last": 134.024723791, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.83961379629883, + "seconds": 4.82592475, + "prefill_seconds": 0.016392042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.115322833, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 20, + "accepted": 16, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 129.0824055, + "first": 129.199020583, + "last": 134.025367041, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.93609398148886, + "seconds": 4.727817333, + "prefill_seconds": 0.016383958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.212811334, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 20, + "accepted": 16, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 129.082654708, + "first": 129.297255958, + "last": 134.025415541, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.56612729160864, + "seconds": 4.760471083, + "prefill_seconds": 0.016407792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.180273792, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 20, + "accepted": 16, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 129.082776333, + "first": 129.264506041, + "last": 134.025465958, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.30800705704185, + "seconds": 4.874970666, + "prefill_seconds": 0.016384042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.0662055, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 20, + "accepted": 16, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 129.082710958, + "first": 129.1498505, + "last": 134.0255115, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.959096956934374, + "seconds": 4.907706541, + "prefill_seconds": 0.016405084, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.033492417, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 20, + "accepted": 16, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 129.082581166, + "first": 129.117039416, + "last": 134.024706166, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.38055057810182, + "seconds": 4.777020792, + "prefill_seconds": 0.016359792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.164066458, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 20, + "accepted": 16, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 129.082493833, + "first": 129.248097666, + "last": 134.025556375, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.750243175321984, + "seconds": 4.744164583, + "prefill_seconds": 0.016359167, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.196493458, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 20, + "accepted": 16, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 129.083095208, + "first": 129.2808835, + "last": 134.025245041, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 54.12412620540813, + "seconds": 4.711392458, + "prefill_seconds": 0.0163735, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.229135875, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 20, + "accepted": 16, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 129.082913458, + "first": 129.313639791, + "last": 134.025751041, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.19990749800366, + "seconds": 4.793241417, + "prefill_seconds": 0.016356667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.147830375, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 20, + "accepted": 16, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 129.082537041, + "first": 129.231729, + "last": 134.025192166, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 54.31266282813846, + "seconds": 4.695037708, + "prefill_seconds": 0.016368, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.245525542, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 20, + "accepted": 16, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 129.083007458, + "first": 129.330005458, + "last": 134.025707458, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 54.502041390310886, + "seconds": 4.678723833, + "prefill_seconds": 0.016355791, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.261876959, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 20, + "accepted": 16, + "acceptance_rate": 0.8, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 129.082185333, + "first": 129.346362125, + "last": 134.02580075, + "chunks": 240 + } + ] + }, + { + "round": 0, + "arm": "main-plain", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 261.8548695057814, + "ttft_max": 0.019914957999986882, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 261.84384531549165, + "seconds": 0.973862875, + "prefill_seconds": 0.019466083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.019607666, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 247, + "drafted": 72, + "accepted": 8, + "acceptance_rate": 0.1111111111111111, + "tokens_per_round": 1.0364372469635628 + }, + "sent": 134.028428208, + "first": 134.048343166, + "last": 135.022165041, + "chunks": 248 + } + ] + }, + { + "round": 0, + "arm": "main-plain", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 623.8913057741527, + "ttft_max": 0.07612033399999518, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 155.97151868320893, + "seconds": 1.634913875, + "prefill_seconds": 0.018871917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.019013209, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + }, + "sent": 135.024678125, + "first": 135.044165708, + "last": 136.679035208, + "chunks": 249 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 159.655644687928, + "seconds": 1.5971875, + "prefill_seconds": 0.018885958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.056715125, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + }, + "sent": 135.024729791, + "first": 135.081912541, + "last": 136.679057833, + "chunks": 249 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 157.79077236147523, + "seconds": 1.616064084, + "prefill_seconds": 0.018858833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.037831417, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + }, + "sent": 135.024784708, + "first": 135.06302475, + "last": 136.679047833, + "chunks": 249 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 161.55734099774537, + "seconds": 1.578386958, + "prefill_seconds": 0.0188095, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.075517208, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 248, + "drafted": 12, + "accepted": 7, + "acceptance_rate": 0.5833333333333334, + "tokens_per_round": 1.032258064516129 + }, + "sent": 135.024608416, + "first": 135.10072875, + "last": 136.679065875, + "chunks": 249 + } + ] + }, + { + "round": 0, + "arm": "main-plain", + "prompt": "chat", + "streams": 8, + "aggregate_decode_tps": 804.7903304448168, + "ttft_max": 0.15244674999999575, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 105.45497930497726, + "seconds": 2.4180935, + "prefill_seconds": 0.018895542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.132696, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 136.682299791, + "first": 136.816032333, + "last": 139.234285625, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 101.50799246585228, + "seconds": 2.512117458, + "prefill_seconds": 0.018809333, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.038682083, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 136.682349875, + "first": 136.721842625, + "last": 139.233927375, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 102.2725854447154, + "seconds": 2.493336791, + "prefill_seconds": 0.018816833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.057453708, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 136.682500916, + "first": 136.740661625, + "last": 139.233946833, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 100.75387442870228, + "seconds": 2.530920041, + "prefill_seconds": 0.019629541, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.019855542, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 136.682447458, + "first": 136.70303025, + "last": 139.23390925, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 106.1153164506986, + "seconds": 2.403046125, + "prefill_seconds": 0.018809541, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.151517333, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 30, + "accepted": 3, + "acceptance_rate": 0.1, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 136.682397375, + "first": 136.834844125, + "last": 139.237851958, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 103.04935504488036, + "seconds": 2.474542416, + "prefill_seconds": 0.018781333, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.076166791, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 136.682617583, + "first": 136.759446666, + "last": 139.234074125, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 104.6375463110905, + "seconds": 2.43698375, + "prefill_seconds": 0.018829917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.113811833, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 136.682559875, + "first": 136.797134958, + "last": 139.234247458, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 103.83638983320756, + "seconds": 2.455786458, + "prefill_seconds": 0.018852292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.094927917, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 136.682232541, + "first": 136.778290083, + "last": 139.234064083, + "chunks": 252 + } + ] + }, + { + "round": 0, + "arm": "main-plain", + "prompt": "chat", + "streams": 16, + "aggregate_decode_tps": 798.7138944430546, + "ttft_max": 0.30219858300000624, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.038958238356024, + "seconds": 4.900174958, + "prefill_seconds": 0.018812083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.225735791, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 139.241876291, + "first": 139.469755916, + "last": 144.370635541, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.645081133940984, + "seconds": 4.843757375, + "prefill_seconds": 0.018843625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.282080375, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 139.241926416, + "first": 139.526226541, + "last": 144.370152958, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.114745571121645, + "seconds": 5.08832275, + "prefill_seconds": 0.018857834, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.038370542, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 139.242242916, + "first": 139.281345041, + "last": 144.3696365, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.250450280175464, + "seconds": 4.975566041, + "prefill_seconds": 0.018787792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.150630333, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 139.242420083, + "first": 139.394317125, + "last": 144.370054333, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.44518779537533, + "seconds": 4.956731833, + "prefill_seconds": 0.01885675, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.169326625, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 139.241972833, + "first": 139.413183916, + "last": 144.370551041, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.67643014217816, + "seconds": 5.031925084, + "prefill_seconds": 0.018838334, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.09432575, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 139.2420175, + "first": 139.337810041, + "last": 144.37069625, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.48755177359346, + "seconds": 5.050749958, + "prefill_seconds": 0.018804167, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.07552175, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 139.242474, + "first": 139.318966, + "last": 144.369743, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 49.92971047107042, + "seconds": 5.107179625, + "prefill_seconds": 0.019435333, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.019633792, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 139.242060333, + "first": 139.262484125, + "last": 144.369621125, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.64220612545368, + "seconds": 4.937821583, + "prefill_seconds": 0.018907209, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.188229709, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 139.242191541, + "first": 139.432086583, + "last": 144.369915875, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.058301350286555, + "seconds": 4.994290708, + "prefill_seconds": 0.018909833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.13187375, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 139.242326416, + "first": 139.375525041, + "last": 144.370594041, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.840043719916856, + "seconds": 4.918977333, + "prefill_seconds": 0.0188405, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.207001916, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 139.24274875, + "first": 139.450944, + "last": 144.370281958, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.300370060335396, + "seconds": 5.069545208, + "prefill_seconds": 0.018811042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.056869542, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 139.24210275, + "first": 139.300158791, + "last": 144.369966, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.8499599998342, + "seconds": 4.82498, + "prefill_seconds": 0.018821833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.300614791, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 139.242850333, + "first": 139.545048916, + "last": 144.370439416, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.44091554383292, + "seconds": 4.862615333, + "prefill_seconds": 0.018820458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.263346125, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 139.2421485, + "first": 139.507378208, + "last": 144.370390166, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.23848569029113, + "seconds": 4.8814585, + "prefill_seconds": 0.018789917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.24454025, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 139.242551333, + "first": 139.488551541, + "last": 144.3701935, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.86513617793781, + "seconds": 5.013257, + "prefill_seconds": 0.018808041, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.113095333, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 139.241802291, + "first": 139.35662025, + "last": 144.369869333, + "chunks": 252 + } + ] + }, + { + "round": 0, + "arm": "candidate-plain", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 289.72587991488484, + "ttft_max": 0.017593750000003183, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 289.6930751741181, + "seconds": 0.880241959, + "prefill_seconds": 0.016860375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.017127334, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 216, + "drafted": 100, + "accepted": 39, + "acceptance_rate": 0.39, + "tokens_per_round": 1.1851851851851851 + }, + "sent": 147.223546083, + "first": 147.241139833, + "last": 148.121282125, + "chunks": 217 + } + ] + }, + { + "round": 0, + "arm": "candidate-plain", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 649.6021903057296, + "ttft_max": 0.06667979200000218, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 162.79983373680727, + "seconds": 1.566340666, + "prefill_seconds": 0.017153125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.017348375, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 223, + "drafted": 43, + "accepted": 32, + "acceptance_rate": 0.7441860465116279, + "tokens_per_round": 1.147982062780269 + }, + "sent": 148.12660075, + "first": 148.144440958, + "last": 149.710751083, + "chunks": 224 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 167.6232283395662, + "seconds": 1.521268875, + "prefill_seconds": 0.016347458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.066252792, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 224, + "drafted": 46, + "accepted": 31, + "acceptance_rate": 0.6739130434782609, + "tokens_per_round": 1.1428571428571428 + }, + "sent": 148.126719708, + "first": 148.1933995, + "last": 149.714632708, + "chunks": 225 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 164.51003069848784, + "seconds": 1.550057458, + "prefill_seconds": 0.0162855, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.0336135, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 223, + "drafted": 43, + "accepted": 32, + "acceptance_rate": 0.7441860465116279, + "tokens_per_round": 1.147982062780269 + }, + "sent": 148.126664666, + "first": 148.16073, + "last": 149.710764916, + "chunks": 224 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 166.2580215102581, + "seconds": 1.533760583, + "prefill_seconds": 0.0163095, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.049878458, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 223, + "drafted": 42, + "accepted": 32, + "acceptance_rate": 0.7619047619047619, + "tokens_per_round": 1.147982062780269 + }, + "sent": 148.126504958, + "first": 148.177041083, + "last": 149.710775, + "chunks": 224 + } + ] + }, + { + "round": 0, + "arm": "candidate-plain", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 829.8565857754016, + "ttft_max": 0.13205166699998472, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 104.7570662804786, + "seconds": 2.434203334, + "prefill_seconds": 0.016336958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.033825708, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 149.719728875, + "first": 149.754335333, + "last": 152.188516791, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 105.46240456557089, + "seconds": 2.41792325, + "prefill_seconds": 0.016293041, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.050100792, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 149.719781708, + "first": 149.770638708, + "last": 152.188533875, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 104.05825140106275, + "seconds": 2.4505505, + "prefill_seconds": 0.017265125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.017462875, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 149.719938416, + "first": 149.737996958, + "last": 152.188544291, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 106.17673990755674, + "seconds": 2.401655958, + "prefill_seconds": 0.016282125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.066412625, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 149.719830541, + "first": 149.786910916, + "last": 152.188621416, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 108.78403340480372, + "seconds": 2.344094, + "prefill_seconds": 0.016315416, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.131539833, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 236, + "drafted": 42, + "accepted": 19, + "acceptance_rate": 0.4523809523809524, + "tokens_per_round": 1.0847457627118644 + }, + "sent": 149.720132166, + "first": 149.852183833, + "last": 152.196253041, + "chunks": 237 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 108.38048753710204, + "seconds": 2.352822042, + "prefill_seconds": 0.016322125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.115258875, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 149.720000333, + "first": 149.835876291, + "last": 152.189067625, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 106.90075464692875, + "seconds": 2.385390083, + "prefill_seconds": 0.016310875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.082632, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 149.719877166, + "first": 149.8032345, + "last": 152.18876775, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 107.63435531635936, + "seconds": 2.369132042, + "prefill_seconds": 0.016312375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.098949667, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 149.719657166, + "first": 149.819553125, + "last": 152.188947875, + "chunks": 235 + } + ] + }, + { + "round": 0, + "arm": "candidate-plain", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 828.6755735521535, + "ttft_max": 0.26237383299999806, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.976483494829445, + "seconds": 4.906064875, + "prefill_seconds": 0.016308208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.033549875, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 152.20229475, + "first": 152.237289541, + "last": 157.143305666, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 54.506561542949406, + "seconds": 4.678335833, + "prefill_seconds": 0.016273417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.261174, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 152.202958583, + "first": 152.465332416, + "last": 157.143935, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.80460451006316, + "seconds": 4.922342375, + "prefill_seconds": 0.017116209, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.017279959, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 152.202357791, + "first": 152.220975375, + "last": 157.143273958, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.57511701564012, + "seconds": 4.759672292, + "prefill_seconds": 0.016269667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.179849709, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 152.202687208, + "first": 152.38391825, + "last": 157.144389083, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 54.31631183691749, + "seconds": 4.694722292, + "prefill_seconds": 0.016261083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.244935209, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 152.202408916, + "first": 152.44905775, + "last": 157.144159333, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.497494527736826, + "seconds": 4.857374667, + "prefill_seconds": 0.016310375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.082430375, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 152.202455875, + "first": 152.286180208, + "last": 157.144008375, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.323544098349856, + "seconds": 4.873523084, + "prefill_seconds": 0.016281834, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.066054958, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 152.202730708, + "first": 152.269866833, + "last": 157.144050166, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.392395322641754, + "seconds": 4.775961042, + "prefill_seconds": 0.016313, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.163628, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 152.202636541, + "first": 152.367643291, + "last": 157.1442535, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.75897038755569, + "seconds": 4.743394417, + "prefill_seconds": 0.016290917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.196119792, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 152.202501708, + "first": 152.400212541, + "last": 157.144343833, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.2098882511113, + "seconds": 4.792342333, + "prefill_seconds": 0.01627575, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.147333, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 152.202592625, + "first": 152.351327666, + "last": 157.144213208, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.85089150476571, + "seconds": 4.824894959, + "prefill_seconds": 0.016256834, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.114773875, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 152.202779791, + "first": 152.318726541, + "last": 157.143661333, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 54.129992851779946, + "seconds": 4.710881834, + "prefill_seconds": 0.016289166, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.228667375, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 152.202546791, + "first": 152.432780375, + "last": 157.144299583, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.0301257905778, + "seconds": 4.8085875, + "prefill_seconds": 0.016313, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.131061416, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 152.202824, + "first": 152.335047458, + "last": 157.144494458, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.149293060798676, + "seconds": 4.889807417, + "prefill_seconds": 0.016292833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.04980925, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 152.202868833, + "first": 152.253582125, + "last": 157.143424291, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.67394686960922, + "seconds": 4.841102958, + "prefill_seconds": 0.016283458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.098661333, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 152.20291025, + "first": 152.302458083, + "last": 157.143725, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.94291697444283, + "seconds": 4.727219333, + "prefill_seconds": 0.016285708, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.212410958, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 152.202220541, + "first": 152.416500125, + "last": 157.143870666, + "chunks": 240 + } + ] + }, + { + "round": 0, + "arm": "candidate-plain", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 261.4389358917785, + "ttft_max": 0.01995562499999437, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 261.42741066011195, + "seconds": 0.975414167, + "prefill_seconds": 0.019505167, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.019642667, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 247, + "drafted": 72, + "accepted": 8, + "acceptance_rate": 0.1111111111111111, + "tokens_per_round": 1.0364372469635628 + }, + "sent": 157.151165291, + "first": 157.171120916, + "last": 158.146492083, + "chunks": 248 + } + ] + }, + { + "round": 0, + "arm": "candidate-plain", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 621.9575399660765, + "ttft_max": 0.07646687499999416, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 159.14182827765322, + "seconds": 1.602344291, + "prefill_seconds": 0.018828959, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.057355417, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + }, + "sent": 158.15290625, + "first": 158.210693208, + "last": 159.812987583, + "chunks": 249 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 155.48832633280173, + "seconds": 1.6399945, + "prefill_seconds": 0.019593875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.0197315, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + }, + "sent": 158.152959, + "first": 158.173025416, + "last": 159.812993458, + "chunks": 249 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 161.03016363348462, + "seconds": 1.58355425, + "prefill_seconds": 0.018808125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.076165958, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 248, + "drafted": 12, + "accepted": 7, + "acceptance_rate": 0.5833333333333334, + "tokens_per_round": 1.032258064516129 + }, + "sent": 158.153034708, + "first": 158.229501583, + "last": 159.81300875, + "chunks": 249 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 157.29485342441458, + "seconds": 1.621159208, + "prefill_seconds": 0.018828042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.038547125, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + }, + "sent": 158.152837458, + "first": 158.191863208, + "last": 159.812975875, + "chunks": 249 + } + ] + }, + { + "round": 0, + "arm": "candidate-plain", + "prompt": "chat", + "streams": 8, + "aggregate_decode_tps": 803.6481953347434, + "ttft_max": 0.15192016599999647, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 100.61238418108967, + "seconds": 2.53447925, + "prefill_seconds": 0.019770458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.01991025, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 159.820251541, + "first": 159.840839125, + "last": 162.375272125, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 101.36327475042472, + "seconds": 2.515704042, + "prefill_seconds": 0.018790042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.038690417, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 159.820299, + "first": 159.859620833, + "last": 162.375289666, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 102.12461416990364, + "seconds": 2.496949458, + "prefill_seconds": 0.018780041, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.057416125, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 159.820439166, + "first": 159.878424416, + "last": 162.375407166, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 104.47681621037697, + "seconds": 2.440732875, + "prefill_seconds": 0.018787542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.113625458, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 159.820388875, + "first": 159.93475425, + "last": 162.3757295, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 105.93935860945503, + "seconds": 2.407037416, + "prefill_seconds": 0.018755042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.151151375, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 30, + "accepted": 3, + "acceptance_rate": 0.1, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 159.8203455, + "first": 159.972265666, + "last": 162.379263291, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 105.28566012405747, + "seconds": 2.42198225, + "prefill_seconds": 0.018747875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.132366375, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 159.82058875, + "first": 159.953500375, + "last": 162.375637625, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 103.68018231006161, + "seconds": 2.459486416, + "prefill_seconds": 0.018772292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.094867541, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 159.820525666, + "first": 159.915963208, + "last": 162.375684708, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 102.89784024999332, + "seconds": 2.478186125, + "prefill_seconds": 0.018772834, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.076230041, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 159.82018225, + "first": 159.897197333, + "last": 162.37579275, + "chunks": 252 + } + ] + }, + { + "round": 0, + "arm": "candidate-plain", + "prompt": "chat", + "streams": 16, + "aggregate_decode_tps": 798.2060644244926, + "ttft_max": 0.30242166599998654, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.082243715282694, + "seconds": 5.091624917, + "prefill_seconds": 0.018790083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.038452791, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 162.387256333, + "first": 162.426992083, + "last": 167.518563958, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.99798519348889, + "seconds": 4.904036167, + "prefill_seconds": 0.018809625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.225475875, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 162.387305541, + "first": 162.614833375, + "last": 167.519646833, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.21441254901633, + "seconds": 4.979067167, + "prefill_seconds": 0.018818959, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.150812333, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 162.387613125, + "first": 162.539740208, + "last": 167.51956775, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.39927234860693, + "seconds": 4.866479792, + "prefill_seconds": 0.018790792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.26276875, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 162.387708291, + "first": 162.652404833, + "last": 167.519421208, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.64125397401233, + "seconds": 5.035420334, + "prefill_seconds": 0.018788459, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.094459, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 162.387351458, + "first": 162.483348291, + "last": 167.519237833, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.800133781219394, + "seconds": 4.922767209, + "prefill_seconds": 0.018728084, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.2068905, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 162.387667333, + "first": 162.59601925, + "last": 167.519464416, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.603122245333005, + "seconds": 4.941561458, + "prefill_seconds": 0.0187735, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.188214166, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 162.387397375, + "first": 162.577278791, + "last": 167.519096166, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.02185605900568, + "seconds": 4.997858167, + "prefill_seconds": 0.018797625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.131972417, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 162.387799041, + "first": 162.520916083, + "last": 167.519039375, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.453946602893055, + "seconds": 5.054114042, + "prefill_seconds": 0.018778125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.075802416, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 162.387568333, + "first": 162.46453425, + "last": 167.519609166, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.197727318076026, + "seconds": 4.885270166, + "prefill_seconds": 0.018782167, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.2441205, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 162.387998958, + "first": 162.633622416, + "last": 167.519282833, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.80526185126259, + "seconds": 4.829064208, + "prefill_seconds": 0.018736541, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.300208875, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 162.387441125, + "first": 162.689862791, + "last": 167.51918725, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.40736524587146, + "seconds": 4.960378708, + "prefill_seconds": 0.018771959, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.169516292, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 162.387843791, + "first": 162.55851125, + "last": 167.519140791, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.26736818885785, + "seconds": 5.0728735, + "prefill_seconds": 0.018771375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.057141208, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 162.387483916, + "first": 162.445761958, + "last": 167.518660041, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.830627266884015, + "seconds": 5.0166605, + "prefill_seconds": 0.018771666, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.113197791, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 162.3877535, + "first": 162.502124708, + "last": 167.519378458, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.60158047773111, + "seconds": 4.847763084, + "prefill_seconds": 0.01871975, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.281475459, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 162.387527541, + "first": 162.671121, + "last": 167.519336791, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 49.89828698936166, + "seconds": 5.110395875, + "prefill_seconds": 0.01938875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.019613167, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 162.387183083, + "first": 162.408184791, + "last": 167.518552875, + "chunks": 252 + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 345.7276309402865, + "ttft_max": 0.019197541000011142, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 345.65026843416723, + "seconds": 0.737739916, + "prefill_seconds": 0.01819825, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.018542042, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 2 + }, + "speculative": { + "rounds": 89, + "drafted": 306, + "accepted": 166, + "acceptance_rate": 0.5424836601307189, + "tokens_per_round": 2.8764044943820224 + }, + "sent": 170.434448875, + "first": 170.453646416, + "last": 171.19122125, + "chunks": 90 + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 633.5514893351391, + "ttft_max": 0.0739344999999787, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 161.98510734078445, + "seconds": 1.574218792, + "prefill_seconds": 0.0179905, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.054373459, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 224, + "drafted": 40, + "accepted": 31, + "acceptance_rate": 0.775, + "tokens_per_round": 1.1428571428571428 + }, + "sent": 171.217551833, + "first": 171.273054125, + "last": 172.84715575, + "chunks": 225 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 160.54720369974092, + "seconds": 1.588317916, + "prefill_seconds": 0.017848833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.036408458, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 223, + "drafted": 41, + "accepted": 32, + "acceptance_rate": 0.7804878048780488, + "tokens_per_round": 1.147982062780269 + }, + "sent": 171.217838833, + "first": 171.255051541, + "last": 172.843258958, + "chunks": 224 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 158.761572570732, + "seconds": 1.606182125, + "prefill_seconds": 0.018322083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.018604875, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 223, + "drafted": 42, + "accepted": 32, + "acceptance_rate": 0.7619047619047619, + "tokens_per_round": 1.147982062780269 + }, + "sent": 171.217703708, + "first": 171.237184, + "last": 172.843268166, + "chunks": 224 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 164.94290112406395, + "seconds": 1.545989541, + "prefill_seconds": 0.018218917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.072570042, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 222, + "drafted": 41, + "accepted": 33, + "acceptance_rate": 0.8048780487804879, + "tokens_per_round": 1.1531531531531531 + }, + "sent": 171.217352083, + "first": 171.291286583, + "last": 172.83718675, + "chunks": 223 + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 812.1249994900322, + "ttft_max": 0.13950241699998855, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 102.18747313821937, + "seconds": 2.4954135, + "prefill_seconds": 0.017388125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.017560833, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 233, + "drafted": 27, + "accepted": 22, + "acceptance_rate": 0.8148148148148148, + "tokens_per_round": 1.0987124463519313 + }, + "sent": 172.856445875, + "first": 172.874706416, + "last": 175.370184833, + "chunks": 234 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 105.15704404096843, + "seconds": 2.424944542, + "prefill_seconds": 0.017013917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.087911917, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 233, + "drafted": 27, + "accepted": 22, + "acceptance_rate": 0.8148148148148148, + "tokens_per_round": 1.0987124463519313 + }, + "sent": 172.856716333, + "first": 172.945285125, + "last": 175.370540791, + "chunks": 234 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 103.70128001877501, + "seconds": 2.458986041, + "prefill_seconds": 0.017051292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.054009708, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 233, + "drafted": 27, + "accepted": 22, + "acceptance_rate": 0.8148148148148148, + "tokens_per_round": 1.0987124463519313 + }, + "sent": 172.856507541, + "first": 172.91115875, + "last": 175.370447958, + "chunks": 234 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 106.67751940439814, + "seconds": 2.390381792, + "prefill_seconds": 0.01699375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.138761167, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 237, + "drafted": 24, + "accepted": 18, + "acceptance_rate": 0.75, + "tokens_per_round": 1.080168776371308 + }, + "sent": 172.856785416, + "first": 172.996287833, + "last": 175.386635, + "chunks": 238 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 102.98779025304378, + "seconds": 2.476021666, + "prefill_seconds": 0.019396792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.036931875, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 233, + "drafted": 27, + "accepted": 22, + "acceptance_rate": 0.8148148148148148, + "tokens_per_round": 1.0987124463519313 + }, + "sent": 172.856606041, + "first": 172.89410975, + "last": 175.370159291, + "chunks": 234 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 106.43588215244323, + "seconds": 2.395808583, + "prefill_seconds": 0.017016833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.121752083, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 28, + "accepted": 21, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 172.856832125, + "first": 172.979291833, + "last": 175.375062791, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 104.42807278452388, + "seconds": 2.441872125, + "prefill_seconds": 0.017100292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.071068333, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 233, + "drafted": 27, + "accepted": 22, + "acceptance_rate": 0.8148148148148148, + "tokens_per_round": 1.0987124463519313 + }, + "sent": 172.856558916, + "first": 172.928262041, + "last": 175.370236125, + "chunks": 234 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 105.89930597631496, + "seconds": 2.407947792, + "prefill_seconds": 0.016990542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.104894583, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 233, + "drafted": 27, + "accepted": 22, + "acceptance_rate": 0.8148148148148148, + "tokens_per_round": 1.0987124463519313 + }, + "sent": 172.856355, + "first": 172.962278375, + "last": 175.370590625, + "chunks": 234 + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 794.6944987638544, + "ttft_max": 0.27339062500001887, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.011193681879966, + "seconds": 5.0988585, + "prefill_seconds": 0.017067916, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.051142125, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 175.396549583, + "first": 175.449037666, + "last": 180.54798075, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.34534065594796, + "seconds": 5.065016875, + "prefill_seconds": 0.016973375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.085040708, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 175.396940291, + "first": 175.483047708, + "last": 180.548542, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.096281589595456, + "seconds": 4.894783125, + "prefill_seconds": 0.017031459, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.254624417, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 175.396599125, + "first": 175.653388375, + "last": 180.548447583, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.91615750589105, + "seconds": 4.911765667, + "prefill_seconds": 0.017022167, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.2376825, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 175.397019541, + "first": 175.636356583, + "last": 180.54875, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.55892191507427, + "seconds": 4.94579775, + "prefill_seconds": 0.017085083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.203777167, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 175.396647041, + "first": 175.602322416, + "last": 180.548898041, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.859778745725706, + "seconds": 5.013785083, + "prefill_seconds": 0.017029166, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.136140291, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 175.396692166, + "first": 175.534216125, + "last": 180.548619, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.51630928871303, + "seconds": 5.047874708, + "prefill_seconds": 0.017060833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.102066541, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 175.397157708, + "first": 175.500112791, + "last": 180.548385375, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.38126740894436, + "seconds": 4.962898209, + "prefill_seconds": 0.0169445, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.186799916, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 175.397239166, + "first": 175.585225166, + "last": 180.548285625, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.27741474109795, + "seconds": 4.877823459, + "prefill_seconds": 0.016977833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.271606333, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 175.396979666, + "first": 175.670370291, + "last": 180.548828166, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.03211749181326, + "seconds": 4.996853208, + "prefill_seconds": 0.016988583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.152903666, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 175.396747, + "first": 175.551213166, + "last": 180.548215, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 49.844554216135364, + "seconds": 5.115904917, + "prefill_seconds": 0.01705125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.034082291, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 175.396788666, + "first": 175.431957333, + "last": 180.547865708, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.68731040786727, + "seconds": 5.030844958, + "prefill_seconds": 0.017073458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.119139792, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 175.396873458, + "first": 175.517188, + "last": 180.548184416, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 49.67913488754507, + "seconds": 5.132939625, + "prefill_seconds": 0.017062709, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.017221458, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 175.397074583, + "first": 175.41490575, + "last": 180.547812625, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.20732437437765, + "seconds": 4.979756375, + "prefill_seconds": 0.017061458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.169880459, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 175.396829125, + "first": 175.568279208, + "last": 180.548662125, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.73702453243566, + "seconds": 4.928772041, + "prefill_seconds": 0.017009958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.220804, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 175.397466333, + "first": 175.619330208, + "last": 180.548715958, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.17800811704017, + "seconds": 5.081907584, + "prefill_seconds": 0.017031125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.068205584, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 175.396474958, + "first": 175.466075916, + "last": 180.548954125, + "chunks": 253 + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 356.283045357237, + "ttft_max": 0.0198536249999961, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 356.2623676742896, + "seconds": 0.715764625, + "prefill_seconds": 0.019405291, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.019556458, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 2 + }, + "speculative": { + "rounds": 91, + "drafted": 283, + "accepted": 164, + "acceptance_rate": 0.5795053003533569, + "tokens_per_round": 2.8131868131868134 + }, + "sent": 180.559402541, + "first": 180.579256166, + "last": 181.29497925, + "chunks": 92 + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 617.4407646534572, + "ttft_max": 0.07852779099999907, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 154.376093024738, + "seconds": 1.651810167, + "prefill_seconds": 0.019667583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.019799375, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 249, + "drafted": 11, + "accepted": 6, + "acceptance_rate": 0.5454545454545454, + "tokens_per_round": 1.0281124497991967 + }, + "sent": 181.305110583, + "first": 181.325383083, + "last": 182.977196416, + "chunks": 250 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 158.08517319160634, + "seconds": 1.6130545, + "prefill_seconds": 0.019354208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.058541542, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 249, + "drafted": 11, + "accepted": 6, + "acceptance_rate": 0.5454545454545454, + "tokens_per_round": 1.0281124497991967 + }, + "sent": 181.305161208, + "first": 181.364150666, + "last": 182.977168083, + "chunks": 250 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 156.21058841312555, + "seconds": 1.63241175, + "prefill_seconds": 0.019416625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.039194125, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 249, + "drafted": 11, + "accepted": 6, + "acceptance_rate": 0.5454545454545454, + "tokens_per_round": 1.0281124497991967 + }, + "sent": 181.305215083, + "first": 181.344788708, + "last": 182.977363416, + "chunks": 250 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 160.00901903699273, + "seconds": 1.593660167, + "prefill_seconds": 0.019406792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.0779435, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 249, + "drafted": 17, + "accepted": 6, + "acceptance_rate": 0.35294117647058826, + "tokens_per_round": 1.0281124497991967 + }, + "sent": 181.305036, + "first": 181.383563791, + "last": 182.977279, + "chunks": 250 + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 8, + "aggregate_decode_tps": 793.7909080606516, + "ttft_max": 0.15909400000001028, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 99.37856571901393, + "seconds": 2.565945666, + "prefill_seconds": 0.019888792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.020052083, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 4, + "accepted": 1, + "acceptance_rate": 0.25, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 182.988145125, + "first": 183.008811875, + "last": 185.574721083, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 104.20306644084567, + "seconds": 2.447144875, + "prefill_seconds": 0.019436125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.138857125, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 4, + "accepted": 1, + "acceptance_rate": 0.25, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 182.988194666, + "first": 183.127767625, + "last": 185.575160208, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 100.99723881470433, + "seconds": 2.5248215, + "prefill_seconds": 0.019431667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.061271291, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 4, + "accepted": 1, + "acceptance_rate": 0.25, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 182.988337291, + "first": 183.050054916, + "last": 185.5750405, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 102.5767566060795, + "seconds": 2.485943292, + "prefill_seconds": 0.019458459, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.100055583, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 4, + "accepted": 1, + "acceptance_rate": 0.25, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 182.988463333, + "first": 183.0889355, + "last": 185.575254833, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 101.7821996501377, + "seconds": 2.505349667, + "prefill_seconds": 0.019409708, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.080589292, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 4, + "accepted": 1, + "acceptance_rate": 0.25, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 182.988240625, + "first": 183.069476125, + "last": 185.574807333, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 100.2295879774243, + "seconds": 2.544158917, + "prefill_seconds": 0.02180525, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.041865, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 4, + "accepted": 1, + "acceptance_rate": 0.25, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 182.988285916, + "first": 183.030620833, + "last": 185.574762083, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 103.38082320911384, + "seconds": 2.466608333, + "prefill_seconds": 0.019389542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.119433209, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 4, + "accepted": 1, + "acceptance_rate": 0.25, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 182.988398041, + "first": 183.108331958, + "last": 185.575021041, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 104.86764061650318, + "seconds": 2.431636666, + "prefill_seconds": 0.019388958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.158255167, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 6, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 182.988072916, + "first": 183.147166916, + "last": 185.578758166, + "chunks": 256 + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 16, + "aggregate_decode_tps": 789.3346380570592, + "ttft_max": 0.3117172080000046, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 49.52943356426058, + "seconds": 5.148453791, + "prefill_seconds": 0.019454542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.039253959, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 185.590695791, + "first": 185.631209958, + "last": 190.779631666, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.870230469382534, + "seconds": 5.012754958, + "prefill_seconds": 0.019362375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.17457875, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 185.590743708, + "first": 185.767068708, + "last": 190.779988041, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 49.904543704315714, + "seconds": 5.109755166, + "prefill_seconds": 0.019406417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.077998375, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 185.591052625, + "first": 185.670025708, + "last": 190.780126208, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.479579513021015, + "seconds": 5.051547625, + "prefill_seconds": 0.01936975, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.135929542, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 185.590789041, + "first": 185.728268083, + "last": 190.780533541, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.09559499773575, + "seconds": 5.090267917, + "prefill_seconds": 0.01942475, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.097430375, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 185.5912585, + "first": 185.689452666, + "last": 190.780443958, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 49.71594163114495, + "seconds": 5.1291395, + "prefill_seconds": 0.019402875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.05859625, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 185.591108541, + "first": 185.65061025, + "last": 190.77987325, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.285914114393485, + "seconds": 4.877030541, + "prefill_seconds": 0.019357291, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.309896666, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 185.591211708, + "first": 185.902928916, + "last": 190.780663958, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.07942152629365, + "seconds": 4.89636775, + "prefill_seconds": 0.019388875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.290604208, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 185.590877541, + "first": 185.88357275, + "last": 190.7804955, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.67053368265316, + "seconds": 4.9351145, + "prefill_seconds": 0.019424083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.251939834, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 185.591358833, + "first": 185.844793958, + "last": 190.78015475, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 49.34316665682364, + "seconds": 5.167888834, + "prefill_seconds": 0.01960475, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.019844209, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 185.590833791, + "first": 185.611753583, + "last": 190.779611083, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.67403630212287, + "seconds": 5.032162792, + "prefill_seconds": 0.019429083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.155263208, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 185.591302958, + "first": 185.747701375, + "last": 190.780263916, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.26752849289349, + "seconds": 4.973908583, + "prefill_seconds": 0.019454792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.213334541, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 185.591161458, + "first": 185.80593725, + "last": 190.780202541, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.28697180415821, + "seconds": 5.070895917, + "prefill_seconds": 0.019438792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.116738916, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 185.590919041, + "first": 185.708885791, + "last": 190.780083375, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.87409404691879, + "seconds": 4.9157485, + "prefill_seconds": 0.019384709, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.271264208, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 185.591003125, + "first": 185.864176083, + "last": 190.780342958, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.06723171619991, + "seconds": 4.993417333, + "prefill_seconds": 0.019408791, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.193930083, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 185.590959916, + "first": 185.786477416, + "last": 190.7806215, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.46810585048602, + "seconds": 4.954524667, + "prefill_seconds": 0.019427209, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.232649, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 185.5906235, + "first": 185.825365833, + "last": 190.780392083, + "chunks": 255 + } + ] + }, + { + "round": 1, + "arm": "candidate-plain", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 289.5446835251194, + "ttft_max": 0.01664883299997655, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 289.52884856931587, + "seconds": 0.88074125, + "prefill_seconds": 0.016308417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.016413334, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 216, + "drafted": 100, + "accepted": 39, + "acceptance_rate": 0.39, + "tokens_per_round": 1.1851851851851851 + }, + "sent": 192.945268875, + "first": 192.961917708, + "last": 193.842610791, + "chunks": 217 + } + ] + }, + { + "round": 1, + "arm": "candidate-plain", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 649.3601917375305, + "ttft_max": 0.06672287500001062, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 162.73396034928217, + "seconds": 1.566974708, + "prefill_seconds": 0.017142042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.017301542, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 223, + "drafted": 43, + "accepted": 32, + "acceptance_rate": 0.7441860465116279, + "tokens_per_round": 1.147982062780269 + }, + "sent": 193.855370208, + "first": 193.873108583, + "last": 195.440038791, + "chunks": 224 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 164.45010581596875, + "seconds": 1.550622292, + "prefill_seconds": 0.016355, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.033608292, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 223, + "drafted": 43, + "accepted": 32, + "acceptance_rate": 0.7441860465116279, + "tokens_per_round": 1.147982062780269 + }, + "sent": 193.85548125, + "first": 193.8894655, + "last": 195.440049791, + "chunks": 224 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 167.56919930261643, + "seconds": 1.521759375, + "prefill_seconds": 0.016328708, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.066281583, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 224, + "drafted": 46, + "accepted": 31, + "acceptance_rate": 0.6739130434782609, + "tokens_per_round": 1.1428571428571428 + }, + "sent": 193.855425916, + "first": 193.922148791, + "last": 195.4438855, + "chunks": 225 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 166.20392994080697, + "seconds": 1.53425975, + "prefill_seconds": 0.01636925, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.04996875, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 223, + "drafted": 42, + "accepted": 32, + "acceptance_rate": 0.7619047619047619, + "tokens_per_round": 1.147982062780269 + }, + "sent": 193.855305916, + "first": 193.905832083, + "last": 195.440065833, + "chunks": 224 + } + ] + }, + { + "round": 1, + "arm": "candidate-plain", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 830.0258741687314, + "ttft_max": 0.1323993340000129, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 104.77697039105169, + "seconds": 2.433740917, + "prefill_seconds": 0.016276833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.033837958, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 195.456540916, + "first": 195.490997, + "last": 197.9247045, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 104.08126446027792, + "seconds": 2.450008667, + "prefill_seconds": 0.017435542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.017601792, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 195.456586416, + "first": 195.474721708, + "last": 197.924690666, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 106.2004508521493, + "seconds": 2.40111975, + "prefill_seconds": 0.016355792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.066483375, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 195.456443291, + "first": 195.523651708, + "last": 197.924731416, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 108.80566369748101, + "seconds": 2.343628, + "prefill_seconds": 0.016292792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.131531875, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 236, + "drafted": 42, + "accepted": 19, + "acceptance_rate": 0.4523809523809524, + "tokens_per_round": 1.0847457627118644 + }, + "sent": 195.456493416, + "first": 195.58889275, + "last": 197.932476416, + "chunks": 237 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 108.40519500956965, + "seconds": 2.352285792, + "prefill_seconds": 0.016349209, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.115261166, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 195.456750041, + "first": 195.572604125, + "last": 197.92507575, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 106.92444446704627, + "seconds": 2.384861584, + "prefill_seconds": 0.016282792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.082664833, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 195.456802458, + "first": 195.53992875, + "last": 197.925125416, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 105.48317214793134, + "seconds": 2.417447208, + "prefill_seconds": 0.016295375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.050117875, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 195.456674125, + "first": 195.50729325, + "last": 197.924719583, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 107.65957033632907, + "seconds": 2.368577166, + "prefill_seconds": 0.016295583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.098914542, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 195.456369541, + "first": 195.55624475, + "last": 197.924863041, + "chunks": 235 + } + ] + }, + { + "round": 1, + "arm": "candidate-plain", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 831.5239140286818, + "ttft_max": 0.26237274999999727, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.768413276464976, + "seconds": 4.742561375, + "prefill_seconds": 0.016265, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.179626458, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 17, + "accepted": 16, + "acceptance_rate": 0.9411764705882353, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 197.945928541, + "first": 198.127401375, + "last": 202.870412958, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.04038814956047, + "seconds": 4.807657125, + "prefill_seconds": 0.016352458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.114574583, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 17, + "accepted": 16, + "acceptance_rate": 0.9411764705882353, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 197.946512666, + "first": 198.062228083, + "last": 202.870853291, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.15677557025132, + "seconds": 4.889105916, + "prefill_seconds": 0.016377709, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.033394542, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 17, + "accepted": 16, + "acceptance_rate": 0.9411764705882353, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 197.94597725, + "first": 197.980583041, + "last": 202.869653833, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 54.707620700272, + "seconds": 4.661142209, + "prefill_seconds": 0.016277583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.260805917, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 17, + "accepted": 16, + "acceptance_rate": 0.9411764705882353, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 197.946568125, + "first": 198.208940875, + "last": 202.870685583, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.95393048432777, + "seconds": 4.726254375, + "prefill_seconds": 0.016303875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.195915292, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 17, + "accepted": 16, + "acceptance_rate": 0.9411764705882353, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 197.946023791, + "first": 198.143698791, + "last": 202.870647958, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.861242596841535, + "seconds": 4.823950166, + "prefill_seconds": 0.01634625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.098226958, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 17, + "accepted": 16, + "acceptance_rate": 0.9411764705882353, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 197.946314666, + "first": 198.045870291, + "last": 202.870816041, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.98265410181364, + "seconds": 4.9054825, + "prefill_seconds": 0.016940541, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.017084917, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 17, + "accepted": 16, + "acceptance_rate": 0.9411764705882353, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 197.946113333, + "first": 197.964199458, + "last": 202.869641791, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 54.327417121993896, + "seconds": 4.693762625, + "prefill_seconds": 0.016323084, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.228316708, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 17, + "accepted": 16, + "acceptance_rate": 0.9411764705882353, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 197.946240291, + "first": 198.176346041, + "last": 202.87076675, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.221206566966686, + "seconds": 4.791323167, + "prefill_seconds": 0.016321584, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.130839125, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 17, + "accepted": 16, + "acceptance_rate": 0.9411764705882353, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 197.946069333, + "first": 198.078556458, + "last": 202.870587708, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 54.14007566234809, + "seconds": 4.7100045, + "prefill_seconds": 0.016314625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.212103375, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 17, + "accepted": 16, + "acceptance_rate": 0.9411764705882353, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 197.946365916, + "first": 198.160016541, + "last": 202.87048725, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.506466084817106, + "seconds": 4.856544708, + "prefill_seconds": 0.016302458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.065666708, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 17, + "accepted": 16, + "acceptance_rate": 0.9411764705882353, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 197.94646675, + "first": 198.013189166, + "last": 202.869760958, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.401544906694156, + "seconds": 4.77514275, + "prefill_seconds": 0.01628175, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.147076541, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 17, + "accepted": 16, + "acceptance_rate": 0.9411764705882353, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 197.946155583, + "first": 198.094844833, + "last": 202.870727166, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.68279198187996, + "seconds": 4.840290167, + "prefill_seconds": 0.016327667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.081921709, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 17, + "accepted": 16, + "acceptance_rate": 0.9411764705882353, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 197.946608208, + "first": 198.029519625, + "last": 202.8702055, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 54.517634027718074, + "seconds": 4.677385667, + "prefill_seconds": 0.016312833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.244603667, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 17, + "accepted": 16, + "acceptance_rate": 0.9411764705882353, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 197.946196291, + "first": 198.192671041, + "last": 202.870100875, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.33092992421609, + "seconds": 4.87283525, + "prefill_seconds": 0.016299958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.049342125, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 17, + "accepted": 16, + "acceptance_rate": 0.9411764705882353, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 197.946422875, + "first": 197.9968835, + "last": 202.869671625, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.5834534300851, + "seconds": 4.758931791, + "prefill_seconds": 0.016287667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.163394917, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 17, + "accepted": 16, + "acceptance_rate": 0.9411764705882353, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 197.945856791, + "first": 198.111126625, + "last": 202.870542458, + "chunks": 240 + } + ] + }, + { + "round": 1, + "arm": "candidate-plain", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 261.0430596423368, + "ttft_max": 0.019998125000000755, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 261.0291981241995, + "seconds": 0.976902208, + "prefill_seconds": 0.019545625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.0196965, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 247, + "drafted": 72, + "accepted": 8, + "acceptance_rate": 0.1111111111111111, + "tokens_per_round": 1.0364372469635628 + }, + "sent": 202.884711916, + "first": 202.904710041, + "last": 203.881560375, + "chunks": 248 + } + ] + }, + { + "round": 1, + "arm": "candidate-plain", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 620.6793785851976, + "ttft_max": 0.07659712499997795, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 160.68386640173455, + "seconds": 1.586967041, + "prefill_seconds": 0.018783417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.076120167, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 248, + "drafted": 12, + "accepted": 7, + "acceptance_rate": 0.5833333333333334, + "tokens_per_round": 1.032258064516129 + }, + "sent": 203.895535375, + "first": 203.9721325, + "last": 205.559074833, + "chunks": 249 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 158.80574759701076, + "seconds": 1.605735333, + "prefill_seconds": 0.018791208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.057380833, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + }, + "sent": 203.895588625, + "first": 203.953345291, + "last": 205.559063166, + "chunks": 249 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 155.16896340819196, + "seconds": 1.643369875, + "prefill_seconds": 0.019636334, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.019772292, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + }, + "sent": 203.895639375, + "first": 203.915714291, + "last": 205.559055458, + "chunks": 249 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 156.96882729897197, + "seconds": 1.624526375, + "prefill_seconds": 0.018837042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.038587417, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + }, + "sent": 203.895463125, + "first": 203.934550916, + "last": 205.559046166, + "chunks": 249 + } + ] + }, + { + "round": 1, + "arm": "candidate-plain", + "prompt": "chat", + "streams": 8, + "aggregate_decode_tps": 803.2640477075901, + "ttft_max": 0.15217400000000225, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 101.31499581811627, + "seconds": 2.516902833, + "prefill_seconds": 0.018763333, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.038594, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 205.573844833, + "first": 205.613133958, + "last": 208.13001625, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 105.23827830592693, + "seconds": 2.423072708, + "prefill_seconds": 0.018833083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.13241375, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 205.573894208, + "first": 205.707139458, + "last": 208.130425166, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 103.63429028995851, + "seconds": 2.460575542, + "prefill_seconds": 0.018792917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.09493575, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 205.573939083, + "first": 205.669533, + "last": 208.130074166, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 104.42832582906196, + "seconds": 2.441866208, + "prefill_seconds": 0.018770416, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.113580292, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 205.57424025, + "first": 205.688305958, + "last": 208.130318666, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 100.5653613853155, + "seconds": 2.535664333, + "prefill_seconds": 0.019734875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.019905292, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 205.574035875, + "first": 205.594377, + "last": 208.129995791, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 102.84966510914845, + "seconds": 2.479346916, + "prefill_seconds": 0.018774792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.076161583, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 205.573984791, + "first": 205.650730083, + "last": 208.130127583, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 102.07667828584013, + "seconds": 2.498122042, + "prefill_seconds": 0.018817875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.057407125, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 205.574106125, + "first": 205.631965333, + "last": 208.130049125, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 105.89272971052287, + "seconds": 2.408097333, + "prefill_seconds": 0.018806458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.151230584, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 30, + "accepted": 3, + "acceptance_rate": 0.1, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 205.573776875, + "first": 205.725950875, + "last": 208.134015125, + "chunks": 253 + } + ] + }, + { + "round": 1, + "arm": "candidate-plain", + "prompt": "chat", + "streams": 16, + "aggregate_decode_tps": 799.8179957692911, + "ttft_max": 0.30238558299998886, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.7115797791297, + "seconds": 4.93119725, + "prefill_seconds": 0.018804875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.188510916, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 208.149626, + "first": 208.340005708, + "last": 213.271452875, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.1077184684266, + "seconds": 4.893708792, + "prefill_seconds": 0.018815375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.225882042, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 208.150235666, + "first": 208.377575416, + "last": 213.271850708, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.369354695341165, + "seconds": 5.062602083, + "prefill_seconds": 0.018815208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.057438583, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 208.149681458, + "first": 208.208409458, + "last": 213.2711705, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.3088641120886, + "seconds": 4.874890792, + "prefill_seconds": 0.0187635, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.244614333, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 208.150010291, + "first": 208.396334166, + "last": 213.271311916, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.90841015481174, + "seconds": 4.91249875, + "prefill_seconds": 0.018757916, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.2071505, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 208.14973025, + "first": 208.35875975, + "last": 213.271764833, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.18287023633437, + "seconds": 5.081415208, + "prefill_seconds": 0.018837875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.038693083, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 208.149776708, + "first": 208.189589125, + "last": 213.27097, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.92015856856854, + "seconds": 4.818579666, + "prefill_seconds": 0.018806625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.300774125, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 208.150295125, + "first": 208.452680708, + "last": 213.271397083, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.745412377113176, + "seconds": 5.025084792, + "prefill_seconds": 0.018868375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.094998125, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 208.150162333, + "first": 208.246045458, + "last": 213.271578916, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.556738316460006, + "seconds": 5.043838042, + "prefill_seconds": 0.018774458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.07632475, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 208.150066833, + "first": 208.227183375, + "last": 213.270984583, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 49.99778033383685, + "seconds": 5.100226416, + "prefill_seconds": 0.019620083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.019882083, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 208.149950208, + "first": 208.170753625, + "last": 213.27092925, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.319368125751765, + "seconds": 4.96888425, + "prefill_seconds": 0.018726959, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.151211916, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 208.149820791, + "first": 208.302334666, + "last": 213.271914166, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.12634641303452, + "seconds": 4.987643708, + "prefill_seconds": 0.01880325, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.132510542, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 208.149907666, + "first": 208.283601583, + "last": 213.271712291, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.510550441416676, + "seconds": 4.856166958, + "prefill_seconds": 0.018748667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.263278042, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 208.149864916, + "first": 208.415094041, + "last": 213.271539125, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.514815451228166, + "seconds": 4.950032292, + "prefill_seconds": 0.018847875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.169946291, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 208.150120375, + "first": 208.321178375, + "last": 213.271666708, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.713996927962626, + "seconds": 4.837424875, + "prefill_seconds": 0.018780041, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.282029875, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 208.150433958, + "first": 208.433877958, + "last": 213.271623166, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.93547239679412, + "seconds": 5.00633425, + "prefill_seconds": 0.018738667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.113766375, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 208.149541958, + "first": 208.264785541, + "last": 213.271209083, + "chunks": 252 + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 356.51506556960055, + "ttft_max": 0.0180762499999787, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 356.4531651992678, + "seconds": 0.7153815, + "prefill_seconds": 0.0173945, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.017618917, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 2 + }, + "speculative": { + "rounds": 89, + "drafted": 298, + "accepted": 166, + "acceptance_rate": 0.5570469798657718, + "tokens_per_round": 2.8764044943820224 + }, + "sent": 216.2409795, + "first": 216.25905575, + "last": 216.974313041, + "chunks": 90 + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 645.4558312923031, + "ttft_max": 0.0689538329999948, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 163.5263986917888, + "seconds": 1.55938125, + "prefill_seconds": 0.017043042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.034629166, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 223, + "drafted": 41, + "accepted": 32, + "acceptance_rate": 0.7804878048780488, + "tokens_per_round": 1.147982062780269 + }, + "sent": 216.991414125, + "first": 217.026535, + "last": 218.585877125, + "chunks": 224 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 161.7575112179627, + "seconds": 1.57643375, + "prefill_seconds": 0.017402209, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.017555875, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 223, + "drafted": 42, + "accepted": 32, + "acceptance_rate": 0.7619047619047619, + "tokens_per_round": 1.147982062780269 + }, + "sent": 216.991469916, + "first": 217.009495541, + "last": 218.585886, + "chunks": 224 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 167.8327708426794, + "seconds": 1.519369541, + "prefill_seconds": 0.016975375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.068522416, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 222, + "drafted": 40, + "accepted": 33, + "acceptance_rate": 0.825, + "tokens_per_round": 1.1531531531531531 + }, + "sent": 216.991527583, + "first": 217.060481416, + "last": 218.57982675, + "chunks": 223 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 164.90833606031902, + "seconds": 1.546313583, + "prefill_seconds": 0.016955791, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.051565416, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 224, + "drafted": 40, + "accepted": 31, + "acceptance_rate": 0.775, + "tokens_per_round": 1.1428571428571428 + }, + "sent": 216.991326166, + "first": 217.04349525, + "last": 218.589774083, + "chunks": 225 + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 812.625302340236, + "ttft_max": 0.1390859169999885, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 105.95701780856994, + "seconds": 2.40663625, + "prefill_seconds": 0.0169995, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.10443525, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 233, + "drafted": 27, + "accepted": 22, + "acceptance_rate": 0.8148148148148148, + "tokens_per_round": 1.0987124463519313 + }, + "sent": 218.606616583, + "first": 218.712053958, + "last": 221.118782208, + "chunks": 234 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 102.25043516050957, + "seconds": 2.493876917, + "prefill_seconds": 0.017096458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.017283167, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 233, + "drafted": 27, + "accepted": 22, + "acceptance_rate": 0.8148148148148148, + "tokens_per_round": 1.0987124463519313 + }, + "sent": 218.606668125, + "first": 218.624661916, + "last": 221.1184945, + "chunks": 234 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 104.47972886410659, + "seconds": 2.440664833, + "prefill_seconds": 0.017086292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.070542625, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 233, + "drafted": 27, + "accepted": 22, + "acceptance_rate": 0.8148148148148148, + "tokens_per_round": 1.0987124463519313 + }, + "sent": 218.606761375, + "first": 218.678019125, + "last": 221.118684333, + "chunks": 234 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 103.75777254356842, + "seconds": 2.457647208, + "prefill_seconds": 0.016906459, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.053488041, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 233, + "drafted": 27, + "accepted": 22, + "acceptance_rate": 0.8148148148148148, + "tokens_per_round": 1.0987124463519313 + }, + "sent": 218.606812291, + "first": 218.660929833, + "last": 221.118551541, + "chunks": 234 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 103.04917456095748, + "seconds": 2.47454675, + "prefill_seconds": 0.019356792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.036653375, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 233, + "drafted": 27, + "accepted": 22, + "acceptance_rate": 0.8148148148148148, + "tokens_per_round": 1.0987124463519313 + }, + "sent": 218.606879208, + "first": 218.644024416, + "last": 221.118540875, + "chunks": 234 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 105.21514330611976, + "seconds": 2.4236055, + "prefill_seconds": 0.017024, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.087507292, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 233, + "drafted": 27, + "accepted": 22, + "acceptance_rate": 0.8148148148148148, + "tokens_per_round": 1.0987124463519313 + }, + "sent": 218.606715416, + "first": 218.695048, + "last": 221.118941916, + "chunks": 234 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 106.74145728307404, + "seconds": 2.388949959, + "prefill_seconds": 0.017080458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.138532334, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 237, + "drafted": 24, + "accepted": 18, + "acceptance_rate": 0.75, + "tokens_per_round": 1.080168776371308 + }, + "sent": 218.607053583, + "first": 218.7461395, + "last": 221.135044, + "chunks": 238 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 106.49642829772976, + "seconds": 2.3944465, + "prefill_seconds": 0.016995209, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.121442542, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 28, + "accepted": 21, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 218.606543458, + "first": 218.729046666, + "last": 221.123450958, + "chunks": 235 + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 796.1534147494743, + "ttft_max": 0.27384479200000555, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.26800940189429, + "seconds": 5.072808791, + "prefill_seconds": 0.016912166, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.06778725, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 221.152768583, + "first": 221.222254416, + "last": 226.295372291, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.37578979471439, + "seconds": 4.868661666, + "prefill_seconds": 0.017050542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.271505791, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 221.152817833, + "first": 221.426662625, + "last": 226.295984708, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.30002477833443, + "seconds": 4.970757833, + "prefill_seconds": 0.01705575, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.169690292, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 221.152910875, + "first": 221.324447916, + "last": 226.295665875, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.12491136915193, + "seconds": 4.987783708, + "prefill_seconds": 0.016988417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.152699916, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 221.152996875, + "first": 221.307384041, + "last": 226.295858625, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.77814942225688, + "seconds": 5.021845083, + "prefill_seconds": 0.017025625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.118747375, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 221.153127833, + "first": 221.273364333, + "last": 226.295630291, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 49.93494626963064, + "seconds": 5.106644125, + "prefill_seconds": 0.017024709, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.034276375, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 221.152865333, + "first": 221.188368333, + "last": 226.295026208, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.01157661760684, + "seconds": 4.902754667, + "prefill_seconds": 0.017041375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.237577584, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 221.153558125, + "first": 221.392554083, + "last": 226.295546625, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 49.76917431684454, + "seconds": 5.123653416, + "prefill_seconds": 0.01709925, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.017298625, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 221.15331425, + "first": 221.17134425, + "last": 226.294957958, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.95109289172138, + "seconds": 5.004799417, + "prefill_seconds": 0.0170235, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.135711625, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 221.15339475, + "first": 221.290385, + "last": 226.295898458, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.60725271892516, + "seconds": 5.038803458, + "prefill_seconds": 0.01699775, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.101790334, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 221.153193625, + "first": 221.256347375, + "last": 226.295453666, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.10126144743879, + "seconds": 5.089692208, + "prefill_seconds": 0.016966458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.05106175, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 221.152953625, + "first": 221.205344125, + "last": 226.294994416, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.65335031479425, + "seconds": 4.93675625, + "prefill_seconds": 0.017052166, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.203563667, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 221.153439041, + "first": 221.358512458, + "last": 226.295934875, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.831596943059246, + "seconds": 4.919778958, + "prefill_seconds": 0.017001833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.220550333, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 221.153038416, + "first": 221.375508458, + "last": 226.295724916, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.47540885983634, + "seconds": 4.95382175, + "prefill_seconds": 0.017001875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.186600666, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 221.153079916, + "first": 221.341449, + "last": 226.29581125, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.437527003547316, + "seconds": 5.055759375, + "prefill_seconds": 0.017075375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.084843917, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 221.153274916, + "first": 221.239331875, + "last": 226.295589208, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.19304069563798, + "seconds": 4.885708834, + "prefill_seconds": 0.017049458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.254538875, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 221.152695791, + "first": 221.409601916, + "last": 226.295761541, + "chunks": 253 + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 356.0100491221774, + "ttft_max": 0.020198124999978972, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 355.9871871193013, + "seconds": 0.716317916, + "prefill_seconds": 0.019727208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.01988125, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 2 + }, + "speculative": { + "rounds": 91, + "drafted": 283, + "accepted": 164, + "acceptance_rate": 0.5795053003533569, + "tokens_per_round": 2.8131868131868134 + }, + "sent": 226.314053625, + "first": 226.33425175, + "last": 227.050523666, + "chunks": 92 + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 618.4685158218351, + "ttft_max": 0.07836916599998744, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 160.27251778546173, + "seconds": 1.591040083, + "prefill_seconds": 0.019357709, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.077917208, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 249, + "drafted": 17, + "accepted": 6, + "acceptance_rate": 0.35294117647058826, + "tokens_per_round": 1.0281124497991967 + }, + "sent": 227.06915325, + "first": 227.147522416, + "last": 228.738690791, + "chunks": 250 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 158.34876392877254, + "seconds": 1.610369375, + "prefill_seconds": 0.019305875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.058574625, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 249, + "drafted": 11, + "accepted": 6, + "acceptance_rate": 0.5454545454545454, + "tokens_per_round": 1.0281124497991967 + }, + "sent": 227.069204916, + "first": 227.128161625, + "last": 228.738491125, + "chunks": 250 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 154.63508038201635, + "seconds": 1.649043667, + "prefill_seconds": 0.019737, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.019870542, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 249, + "drafted": 11, + "accepted": 6, + "acceptance_rate": 0.5454545454545454, + "tokens_per_round": 1.0281124497991967 + }, + "sent": 227.069281375, + "first": 227.089455666, + "last": 228.738462583, + "chunks": 250 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 156.47366856257813, + "seconds": 1.629667166, + "prefill_seconds": 0.019395083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.039290583, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 249, + "drafted": 11, + "accepted": 6, + "acceptance_rate": 0.5454545454545454, + "tokens_per_round": 1.0281124497991967 + }, + "sent": 227.069078708, + "first": 227.108854375, + "last": 228.738507458, + "chunks": 250 + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 8, + "aggregate_decode_tps": 794.2858917207209, + "ttft_max": 0.15848308399998245, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 101.0587355415921, + "seconds": 2.523285084, + "prefill_seconds": 0.019405292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.061060667, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 4, + "accepted": 1, + "acceptance_rate": 0.25, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 228.757089041, + "first": 228.818985875, + "last": 231.342461, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 103.44137453891526, + "seconds": 2.465164458, + "prefill_seconds": 0.019400916, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.119085792, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 4, + "accepted": 1, + "acceptance_rate": 0.25, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 228.757139041, + "first": 228.877125083, + "last": 231.342672666, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 101.83898746886234, + "seconds": 2.503952625, + "prefill_seconds": 0.019355083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.08043925, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 4, + "accepted": 1, + "acceptance_rate": 0.25, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 228.757230458, + "first": 228.838339333, + "last": 231.342510416, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 100.29101108382824, + "seconds": 2.54260075, + "prefill_seconds": 0.021773917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.041741625, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 4, + "accepted": 1, + "acceptance_rate": 0.25, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 228.757292416, + "first": 228.799579583, + "last": 231.342144, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 104.26003787647788, + "seconds": 2.445807667, + "prefill_seconds": 0.019376375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.138402208, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 3, + "accepted": 1, + "acceptance_rate": 0.3333333333333333, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 228.757185333, + "first": 228.896502166, + "last": 231.342379708, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 102.63346743176672, + "seconds": 2.484569667, + "prefill_seconds": 0.019376791, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.099703125, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 4, + "accepted": 1, + "acceptance_rate": 0.25, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 228.757397791, + "first": 228.857733291, + "last": 231.342557458, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 104.92297573491601, + "seconds": 2.43035425, + "prefill_seconds": 0.019320458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.157714333, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 6, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 228.757346791, + "first": 228.915829875, + "last": 231.34614375, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 99.43941680009635, + "seconds": 2.564375458, + "prefill_seconds": 0.019845875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.020029833, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 4, + "accepted": 1, + "acceptance_rate": 0.25, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 228.757016125, + "first": 228.777799, + "last": 231.342134333, + "chunks": 255 + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 16, + "aggregate_decode_tps": 789.771383093008, + "ttft_max": 0.31042429199999333, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 49.5585013758026, + "seconds": 5.145434041, + "prefill_seconds": 0.019434375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.03891575, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 231.366020625, + "first": 231.406101041, + "last": 236.551513208, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 49.37207354814985, + "seconds": 5.164863083, + "prefill_seconds": 0.019504125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.019657625, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 231.366069541, + "first": 231.386665333, + "last": 236.551489041, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.314192230717474, + "seconds": 4.874394292, + "prefill_seconds": 0.019387959, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.309051, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 231.367078958, + "first": 231.67750325, + "last": 236.552717291, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.10618255422472, + "seconds": 4.893853042, + "prefill_seconds": 0.019403834, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.289837834, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 231.366399625, + "first": 231.658110041, + "last": 236.552315041, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.123972476181336, + "seconds": 5.087386083, + "prefill_seconds": 0.019403625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.09677175, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 231.366507041, + "first": 231.464250583, + "last": 236.552484833, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 49.745306071496096, + "seconds": 5.126111791, + "prefill_seconds": 0.019392042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.058307, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 231.366205541, + "first": 231.425497333, + "last": 236.552256583, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.898896549288594, + "seconds": 5.009931792, + "prefill_seconds": 0.019383541, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.174048792, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 231.366115833, + "first": 231.541814125, + "last": 236.551894041, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.2947249526529, + "seconds": 4.971271417, + "prefill_seconds": 0.019353375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.212683375, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 231.366611583, + "first": 231.580536958, + "last": 236.552053875, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.31405424947672, + "seconds": 5.068166416, + "prefill_seconds": 0.019370708, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.1160415, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 231.366161416, + "first": 231.483617208, + "last": 236.552527041, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.09530423884891, + "seconds": 4.990673875, + "prefill_seconds": 0.01935975, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.193347583, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 231.366554291, + "first": 231.561176666, + "last": 236.552657791, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.901169968812894, + "seconds": 4.913184041, + "prefill_seconds": 0.0193665, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.27052375, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 231.366248708, + "first": 231.638703708, + "last": 236.552156041, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.495257347179034, + "seconds": 4.951912334, + "prefill_seconds": 0.019391792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.231982292, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 231.366697958, + "first": 231.59992825, + "last": 236.552439875, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.69738391616706, + "seconds": 4.932551334, + "prefill_seconds": 0.019402542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.251379375, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 231.366291166, + "first": 231.619342458, + "last": 236.552565583, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 49.93336384761851, + "seconds": 5.106805958, + "prefill_seconds": 0.019342875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.077485708, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 231.366456791, + "first": 231.444845416, + "last": 236.551823916, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.70264840649132, + "seconds": 5.029323083, + "prefill_seconds": 0.019391417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.154691708, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 231.366335916, + "first": 231.522416166, + "last": 236.55221075, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.50744466196342, + "seconds": 5.048760667, + "prefill_seconds": 0.019405917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.135417708, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 231.36594425, + "first": 231.50303225, + "last": 236.552390583, + "chunks": 255 + } + ] + }, + { + "round": 1, + "arm": "main-plain", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 290.8510658622243, + "ttft_max": 0.01669720799998231, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 290.8404504890468, + "seconds": 0.876769375, + "prefill_seconds": 0.016364292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.01647625, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 216, + "drafted": 100, + "accepted": 39, + "acceptance_rate": 0.39, + "tokens_per_round": 1.1851851851851851 + }, + "sent": 239.119373958, + "first": 239.136071166, + "last": 240.012808541, + "chunks": 217 + } + ] + }, + { + "round": 1, + "arm": "main-plain", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 651.741970635724, + "ttft_max": 0.06697054099998923, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 168.2030078750511, + "seconds": 1.516025208, + "prefill_seconds": 0.016288875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.066519584, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 224, + "drafted": 46, + "accepted": 31, + "acceptance_rate": 0.6739130434782609, + "tokens_per_round": 1.1428571428571428 + }, + "sent": 240.033585875, + "first": 240.100556416, + "last": 241.616544208, + "chunks": 225 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 165.06071405913494, + "seconds": 1.544886083, + "prefill_seconds": 0.016394459, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.033796, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 223, + "drafted": 43, + "accepted": 32, + "acceptance_rate": 0.7441860465116279, + "tokens_per_round": 1.147982062780269 + }, + "sent": 240.033701625, + "first": 240.067897041, + "last": 241.612781625, + "chunks": 224 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 163.32746514404067, + "seconds": 1.561280583, + "prefill_seconds": 0.017268041, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.01742925, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 223, + "drafted": 43, + "accepted": 32, + "acceptance_rate": 0.7441860465116279, + "tokens_per_round": 1.147982062780269 + }, + "sent": 240.033644458, + "first": 240.051507666, + "last": 241.612744708, + "chunks": 224 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 166.82790179935552, + "seconds": 1.528521292, + "prefill_seconds": 0.016369292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.05020025, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 223, + "drafted": 42, + "accepted": 32, + "acceptance_rate": 0.7619047619047619, + "tokens_per_round": 1.147982062780269 + }, + "sent": 240.033513, + "first": 240.084273625, + "last": 241.6128545, + "chunks": 224 + } + ] + }, + { + "round": 1, + "arm": "main-plain", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 830.8281925407863, + "ttft_max": 0.13214558399999987, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 105.58861198447467, + "seconds": 2.415033167, + "prefill_seconds": 0.016334791, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.050086541, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 241.637057125, + "first": 241.687804708, + "last": 244.102791208, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 108.92960785183546, + "seconds": 2.340961333, + "prefill_seconds": 0.01633, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.131459333, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 236, + "drafted": 42, + "accepted": 19, + "acceptance_rate": 0.4523809523809524, + "tokens_per_round": 1.0847457627118644 + }, + "sent": 241.637417541, + "first": 241.769563125, + "last": 244.110488041, + "chunks": 237 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 107.77019400800891, + "seconds": 2.366145875, + "prefill_seconds": 0.016328375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.098779375, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 241.637356291, + "first": 241.736815583, + "last": 244.103246708, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 106.30634603982891, + "seconds": 2.398727917, + "prefill_seconds": 0.01633, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.066277916, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 241.637215333, + "first": 241.704144916, + "last": 244.103204708, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 107.03474678681029, + "seconds": 2.382403917, + "prefill_seconds": 0.016336375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.082566, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 241.63729, + "first": 241.720486041, + "last": 244.103041875, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 108.52098436526774, + "seconds": 2.349775958, + "prefill_seconds": 0.016411959, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.11517475, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 241.6371155, + "first": 241.753231291, + "last": 244.102989125, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 104.1788219264758, + "seconds": 2.447714375, + "prefill_seconds": 0.0172515, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.0174245, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 241.637166333, + "first": 241.65510675, + "last": 244.102778208, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 104.87980361772534, + "seconds": 2.431354667, + "prefill_seconds": 0.016368875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.033824042, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 241.636985416, + "first": 241.67147825, + "last": 244.102807333, + "chunks": 235 + } + ] + }, + { + "round": 1, + "arm": "main-plain", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 830.8764623507997, + "ttft_max": 0.2633158749999893, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.2876866740977, + "seconds": 4.876865209, + "prefill_seconds": 0.016316958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.0502095, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 238, + "drafted": 20, + "accepted": 17, + "acceptance_rate": 0.85, + "tokens_per_round": 1.0756302521008403 + }, + "sent": 244.132334416, + "first": 244.183997083, + "last": 249.060898041, + "chunks": 239 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 54.0965041857833, + "seconds": 4.713798125, + "prefill_seconds": 0.016320625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.213210958, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 238, + "drafted": 20, + "accepted": 17, + "acceptance_rate": 0.85, + "tokens_per_round": 1.0756302521008403 + }, + "sent": 244.132382875, + "first": 244.347323291, + "last": 249.061706125, + "chunks": 239 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.91079714136565, + "seconds": 4.730035791, + "prefill_seconds": 0.016310959, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.196879916, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 238, + "drafted": 20, + "accepted": 17, + "acceptance_rate": 0.85, + "tokens_per_round": 1.0756302521008403 + }, + "sent": 244.1327245, + "first": 244.331000041, + "last": 249.061462708, + "chunks": 239 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.46293821308609, + "seconds": 4.860574125, + "prefill_seconds": 0.016295833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.066502208, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 238, + "drafted": 20, + "accepted": 17, + "acceptance_rate": 0.85, + "tokens_per_round": 1.0756302521008403 + }, + "sent": 244.13243375, + "first": 244.20029975, + "last": 249.061247041, + "chunks": 239 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.11353376479628, + "seconds": 4.893162708, + "prefill_seconds": 0.016336459, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.033903458, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 238, + "drafted": 20, + "accepted": 17, + "acceptance_rate": 0.85, + "tokens_per_round": 1.0756302521008403 + }, + "sent": 244.1327845, + "first": 244.167669583, + "last": 249.060805666, + "chunks": 239 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.357004782800345, + "seconds": 4.779128833, + "prefill_seconds": 0.016427583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.148012458, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 238, + "drafted": 20, + "accepted": 17, + "acceptance_rate": 0.85, + "tokens_per_round": 1.0756302521008403 + }, + "sent": 244.132483083, + "first": 244.282021833, + "last": 249.061749333, + "chunks": 239 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.94034400629201, + "seconds": 4.909478458, + "prefill_seconds": 0.017405792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.017608125, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 238, + "drafted": 20, + "accepted": 17, + "acceptance_rate": 0.85, + "tokens_per_round": 1.0756302521008403 + }, + "sent": 244.13298025, + "first": 244.1513485, + "last": 249.060786416, + "chunks": 239 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.540147133077845, + "seconds": 4.762781084, + "prefill_seconds": 0.016341667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.164288708, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 238, + "drafted": 20, + "accepted": 17, + "acceptance_rate": 0.85, + "tokens_per_round": 1.0756302521008403 + }, + "sent": 244.132836833, + "first": 244.298358583, + "last": 249.061825791, + "chunks": 239 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.639884162991194, + "seconds": 4.844235584, + "prefill_seconds": 0.016357167, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.082845584, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 238, + "drafted": 20, + "accepted": 17, + "acceptance_rate": 0.85, + "tokens_per_round": 1.0756302521008403 + }, + "sent": 244.132530791, + "first": 244.216666208, + "last": 249.060937625, + "chunks": 239 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.99553494433352, + "seconds": 4.811726125, + "prefill_seconds": 0.016302583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.115342583, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 238, + "drafted": 20, + "accepted": 17, + "acceptance_rate": 0.85, + "tokens_per_round": 1.0756302521008403 + }, + "sent": 244.132880583, + "first": 244.249265375, + "last": 249.061494625, + "chunks": 239 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.815375363532894, + "seconds": 4.8281395, + "prefill_seconds": 0.016290834, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.0989905, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 238, + "drafted": 20, + "accepted": 17, + "acceptance_rate": 0.85, + "tokens_per_round": 1.0756302521008403 + }, + "sent": 244.132577625, + "first": 244.232958875, + "last": 249.061538416, + "chunks": 239 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 54.66295212256913, + "seconds": 4.664951125, + "prefill_seconds": 0.01633075, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.262054, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 238, + "drafted": 20, + "accepted": 17, + "acceptance_rate": 0.85, + "tokens_per_round": 1.0756302521008403 + }, + "sent": 244.132931541, + "first": 244.396247416, + "last": 249.061641625, + "chunks": 239 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 54.28344066227193, + "seconds": 4.697565167, + "prefill_seconds": 0.01627325, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.229426625, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 238, + "drafted": 20, + "accepted": 17, + "acceptance_rate": 0.85, + "tokens_per_round": 1.0756302521008403 + }, + "sent": 244.132624416, + "first": 244.363604333, + "last": 249.061416666, + "chunks": 239 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 54.47267817352149, + "seconds": 4.681245875, + "prefill_seconds": 0.016305875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.2457435, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 238, + "drafted": 20, + "accepted": 17, + "acceptance_rate": 0.85, + "tokens_per_round": 1.0756302521008403 + }, + "sent": 244.133083666, + "first": 244.379900916, + "last": 249.061602625, + "chunks": 239 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.17579718838869, + "seconds": 4.795414709, + "prefill_seconds": 0.01632025, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.1315775, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 238, + "drafted": 20, + "accepted": 17, + "acceptance_rate": 0.85, + "tokens_per_round": 1.0756302521008403 + }, + "sent": 244.13266925, + "first": 244.265597541, + "last": 249.061794458, + "chunks": 239 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.724359588515654, + "seconds": 4.74645025, + "prefill_seconds": 0.016314709, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.180721375, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 238, + "drafted": 20, + "accepted": 17, + "acceptance_rate": 0.85, + "tokens_per_round": 1.0756302521008403 + }, + "sent": 244.132262041, + "first": 244.314685458, + "last": 249.061325375, + "chunks": 239 + } + ] + }, + { + "round": 1, + "arm": "main-plain", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 262.1868024417305, + "ttft_max": 0.019961792000003697, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 262.1737960375258, + "seconds": 0.97263725, + "prefill_seconds": 0.0194825, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.019646792, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 247, + "drafted": 72, + "accepted": 8, + "acceptance_rate": 0.1111111111111111, + "tokens_per_round": 1.0364372469635628 + }, + "sent": 249.082937916, + "first": 249.102899708, + "last": 250.075488708, + "chunks": 248 + } + ] + }, + { + "round": 1, + "arm": "main-plain", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 622.4916929902471, + "ttft_max": 0.07671124999998824, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 157.42673348248232, + "seconds": 1.619801125, + "prefill_seconds": 0.018772542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.03857225, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + }, + "sent": 250.097818916, + "first": 250.136841958, + "last": 251.756618958, + "chunks": 249 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 159.27650569847583, + "seconds": 1.600989417, + "prefill_seconds": 0.01882125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.057384666, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + }, + "sent": 250.09795375, + "first": 250.155666625, + "last": 251.75663625, + "chunks": 249 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 155.6259703736976, + "seconds": 1.638544, + "prefill_seconds": 0.019652541, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.01981975, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + }, + "sent": 250.097881958, + "first": 250.118073, + "last": 251.756569458, + "chunks": 249 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 161.1673614662196, + "seconds": 1.582206209, + "prefill_seconds": 0.018784208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.076146375, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 248, + "drafted": 12, + "accepted": 7, + "acceptance_rate": 0.5833333333333334, + "tokens_per_round": 1.032258064516129 + }, + "sent": 250.097743833, + "first": 250.174455083, + "last": 251.756649083, + "chunks": 249 + } + ] + }, + { + "round": 1, + "arm": "main-plain", + "prompt": "chat", + "streams": 8, + "aggregate_decode_tps": 804.0098677360683, + "ttft_max": 0.152253375000015, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 102.17765117321231, + "seconds": 2.495653375, + "prefill_seconds": 0.018853667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.057652791, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 251.780259666, + "first": 251.838721958, + "last": 254.334342083, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 105.35011616374786, + "seconds": 2.420500416, + "prefill_seconds": 0.018860667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.132804792, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 251.780316375, + "first": 251.913991666, + "last": 254.334616541, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 102.95414641308287, + "seconds": 2.476830792, + "prefill_seconds": 0.018830958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.0764365, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 251.78046625, + "first": 251.857560708, + "last": 254.334366208, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 106.00518504450298, + "seconds": 2.405542709, + "prefill_seconds": 0.018781791, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.1515665, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 30, + "accepted": 3, + "acceptance_rate": 0.1, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 251.7805225, + "first": 251.932775875, + "last": 254.3382855, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 100.65766624393973, + "seconds": 2.533339084, + "prefill_seconds": 0.01976625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.019970708, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 251.7803655, + "first": 251.801003208, + "last": 254.33430275, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 104.53520523380242, + "seconds": 2.439369583, + "prefill_seconds": 0.018769417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.113946458, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 251.780411666, + "first": 251.895127958, + "last": 254.334824375, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 103.73698945476035, + "seconds": 2.458139583, + "prefill_seconds": 0.018799708, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.095197542, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 251.780645541, + "first": 251.87635975, + "last": 254.334739291, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 101.41207868563185, + "seconds": 2.514493375, + "prefill_seconds": 0.018852125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.038842416, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 251.780173291, + "first": 251.81987025, + "last": 254.334317916, + "chunks": 252 + } + ] + }, + { + "round": 1, + "arm": "main-plain", + "prompt": "chat", + "streams": 16, + "aggregate_decode_tps": 797.3471530099499, + "ttft_max": 0.30202095799998574, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.96634395684408, + "seconds": 5.003301791, + "prefill_seconds": 0.018808291, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.131745375, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 254.361831833, + "first": 254.4952495, + "last": 259.498513041, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.5454993116723, + "seconds": 4.852937042, + "prefill_seconds": 0.018816667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.281690666, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 254.3623815, + "first": 254.645720208, + "last": 259.499229166, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.74514911705729, + "seconds": 4.927998167, + "prefill_seconds": 0.018785, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.206731125, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 254.361883375, + "first": 254.570512166, + "last": 259.498974791, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.74997697292843, + "seconds": 4.834125333, + "prefill_seconds": 0.018753667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.300441, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 254.362455208, + "first": 254.664476166, + "last": 259.4988315, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.028713371847566, + "seconds": 5.097072917, + "prefill_seconds": 0.018839959, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.038308708, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 254.361930791, + "first": 254.401221583, + "last": 259.498249, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 49.844636623049375, + "seconds": 5.115896459, + "prefill_seconds": 0.019437292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.019580542, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 254.361976666, + "first": 254.382366291, + "last": 259.498230083, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.5477064118607, + "seconds": 4.946873833, + "prefill_seconds": 0.018806416, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.188014375, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 254.362214791, + "first": 254.551724583, + "last": 259.498775625, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.587413697805275, + "seconds": 5.040779541, + "prefill_seconds": 0.018805667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.094318667, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 254.362335333, + "first": 254.457626083, + "last": 259.499060041, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.141925990882235, + "seconds": 4.890498292, + "prefill_seconds": 0.01876025, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.244148834, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 254.362258791, + "first": 254.608091916, + "last": 259.499184583, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.2140262100971, + "seconds": 5.078262375, + "prefill_seconds": 0.018814833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.056949458, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 254.362019833, + "first": 254.4200285, + "last": 259.498265416, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.40008152954365, + "seconds": 5.059515625, + "prefill_seconds": 0.018778, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.075604375, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 254.362145166, + "first": 254.438818083, + "last": 259.498341875, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.15999607320748, + "seconds": 4.984363166, + "prefill_seconds": 0.018825, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.150512875, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 254.362062166, + "first": 254.514063291, + "last": 259.499334458, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.775720261940045, + "seconds": 5.022085333, + "prefill_seconds": 0.018795583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.113008042, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 254.362104875, + "first": 254.476434333, + "last": 259.499267458, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.343333200012474, + "seconds": 4.871680583, + "prefill_seconds": 0.018804125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.262896292, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 254.362604041, + "first": 254.626896916, + "last": 259.498732166, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.942677792211114, + "seconds": 4.909257875, + "prefill_seconds": 0.018807583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.225384334, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 254.362501541, + "first": 254.589343833, + "last": 259.499018333, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.35307162482252, + "seconds": 4.965623125, + "prefill_seconds": 0.01884425, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.169330792, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 254.361755, + "first": 254.532916458, + "last": 259.498760125, + "chunks": 252 + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 347.10809909124464, + "ttft_max": 0.01903087499999856, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 347.04676560369313, + "seconds": 0.734771291, + "prefill_seconds": 0.0181315, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.0184205, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 2 + }, + "speculative": { + "rounds": 89, + "drafted": 305, + "accepted": 166, + "acceptance_rate": 0.5442622950819672, + "tokens_per_round": 2.8764044943820224 + }, + "sent": 262.414576583, + "first": 262.433607458, + "last": 263.168248916, + "chunks": 90 + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 645.8577525210864, + "ttft_max": 0.07029229199997644, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 167.9839637527971, + "seconds": 1.518002042, + "prefill_seconds": 0.017058292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.0694655, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 222, + "drafted": 41, + "accepted": 33, + "acceptance_rate": 0.8048780487804879, + "tokens_per_round": 1.1531531531531531 + }, + "sent": 263.223389541, + "first": 263.293681833, + "last": 264.811650125, + "chunks": 223 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 165.04391455469687, + "seconds": 1.545043334, + "prefill_seconds": 0.017076334, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.052446625, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 224, + "drafted": 40, + "accepted": 31, + "acceptance_rate": 0.775, + "tokens_per_round": 1.1428571428571428 + }, + "sent": 263.223573125, + "first": 263.276616166, + "last": 264.8216225, + "chunks": 225 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 161.8603572884961, + "seconds": 1.575432084, + "prefill_seconds": 0.017927792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.018190708, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 223, + "drafted": 42, + "accepted": 32, + "acceptance_rate": 0.7619047619047619, + "tokens_per_round": 1.147982062780269 + }, + "sent": 263.223487625, + "first": 263.242327375, + "last": 264.81769975, + "chunks": 224 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 163.64776206179113, + "seconds": 1.558224792, + "prefill_seconds": 0.017220041, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.035387, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 223, + "drafted": 41, + "accepted": 32, + "acceptance_rate": 0.7804878048780488, + "tokens_per_round": 1.147982062780269 + }, + "sent": 263.22323575, + "first": 263.25954125, + "last": 264.817712708, + "chunks": 224 + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 816.154892487187, + "ttft_max": 0.13931495899998936, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 105.6861953253668, + "seconds": 2.412803292, + "prefill_seconds": 0.017041708, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.087976042, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 233, + "drafted": 26, + "accepted": 22, + "acceptance_rate": 0.8461538461538461, + "tokens_per_round": 1.0987124463519313 + }, + "sent": 264.846511833, + "first": 264.935187666, + "last": 267.348084166, + "chunks": 234 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 107.22283690948855, + "seconds": 2.378224708, + "prefill_seconds": 0.016980625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.138756709, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 237, + "drafted": 27, + "accepted": 18, + "acceptance_rate": 0.6666666666666666, + "tokens_per_round": 1.080168776371308 + }, + "sent": 264.846802041, + "first": 264.986117, + "last": 267.364310708, + "chunks": 238 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 104.21882043800122, + "seconds": 2.446774958, + "prefill_seconds": 0.01692975, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.053928416, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 233, + "drafted": 26, + "accepted": 22, + "acceptance_rate": 0.8461538461538461, + "tokens_per_round": 1.0987124463519313 + }, + "sent": 264.846566291, + "first": 264.90109175, + "last": 267.348025083, + "chunks": 234 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 102.69546221437821, + "seconds": 2.483069792, + "prefill_seconds": 0.017370417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.017606417, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 233, + "drafted": 26, + "accepted": 22, + "acceptance_rate": 0.8461538461538461, + "tokens_per_round": 1.0987124463519313 + }, + "sent": 264.846614541, + "first": 264.864785166, + "last": 267.347922041, + "chunks": 234 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 103.50304534879383, + "seconds": 2.463695625, + "prefill_seconds": 0.019371583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.036972417, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 233, + "drafted": 26, + "accepted": 22, + "acceptance_rate": 0.8461538461538461, + "tokens_per_round": 1.0987124463519313 + }, + "sent": 264.846660166, + "first": 264.8841655, + "last": 267.348166375, + "chunks": 234 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 106.43608575659778, + "seconds": 2.395804, + "prefill_seconds": 0.016969625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.104798375, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 233, + "drafted": 26, + "accepted": 22, + "acceptance_rate": 0.8461538461538461, + "tokens_per_round": 1.0987124463519313 + }, + "sent": 264.846848041, + "first": 264.95216475, + "last": 267.348208291, + "chunks": 234 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 106.97931103778156, + "seconds": 2.383638458, + "prefill_seconds": 0.016975375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.121775625, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 264.846755708, + "first": 264.969137041, + "last": 267.352739, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 104.94912645421738, + "seconds": 2.429748666, + "prefill_seconds": 0.017050041, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.070941166, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 233, + "drafted": 26, + "accepted": 22, + "acceptance_rate": 0.8461538461538461, + "tokens_per_round": 1.0987124463519313 + }, + "sent": 264.846436666, + "first": 264.918142291, + "last": 267.348244166, + "chunks": 234 + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 795.3966095266991, + "ttft_max": 0.2762710839999727, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 49.9112888007362, + "seconds": 5.109064625, + "prefill_seconds": 0.019413458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.036750875, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 267.389845166, + "first": 267.427796625, + "last": 272.536823041, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.35051860785666, + "seconds": 4.871011917, + "prefill_seconds": 0.01701525, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.273711042, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 267.389894416, + "first": 267.6661655, + "last": 272.537800791, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.276274000501004, + "seconds": 4.97306025, + "prefill_seconds": 0.017041875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.172224, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 267.390209333, + "first": 267.564042208, + "last": 272.537747166, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.58499563729209, + "seconds": 5.0410205, + "prefill_seconds": 0.017008042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.10443325, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 267.390263458, + "first": 267.4959535, + "last": 272.537241583, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.80861373579655, + "seconds": 4.921961458, + "prefill_seconds": 0.017083208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.223340708, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 267.389942, + "first": 267.615203541, + "last": 272.53764725, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.101519986395736, + "seconds": 4.990066833, + "prefill_seconds": 0.017019792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.155221916, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 267.390074541, + "first": 267.546992875, + "last": 272.537043083, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.168243030579276, + "seconds": 4.888031208, + "prefill_seconds": 0.017024, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.2569055, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 267.389986416, + "first": 267.649160958, + "last": 272.537388916, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 49.7224988993842, + "seconds": 5.128463083, + "prefill_seconds": 0.017152125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.017343333, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 267.390161708, + "first": 267.408377666, + "last": 272.53680225, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.629483441703364, + "seconds": 4.939038375, + "prefill_seconds": 0.017034917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.206259208, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 267.390523833, + "first": 267.598092291, + "last": 272.537894125, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.92779330146235, + "seconds": 5.007089125, + "prefill_seconds": 0.01701475, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.138409708, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 267.390031375, + "first": 267.529982708, + "last": 272.537136375, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.451851445692355, + "seconds": 4.956089875, + "prefill_seconds": 0.017006417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.189219167, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 267.3903485, + "first": 267.581059916, + "last": 272.537550291, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.24548490747553, + "seconds": 5.075082875, + "prefill_seconds": 0.017020375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.07063525, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 267.390606583, + "first": 267.461886208, + "last": 272.537164916, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.75611896995513, + "seconds": 5.024024791, + "prefill_seconds": 0.017000916, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.12143275, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 267.39040525, + "first": 267.512956291, + "last": 272.537279708, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.07825053627812, + "seconds": 5.092030917, + "prefill_seconds": 0.017059167, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.053735833, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 267.390115958, + "first": 267.444854166, + "last": 272.536891, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.41470917348272, + "seconds": 5.058047625, + "prefill_seconds": 0.017059, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.087625167, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 267.390450208, + "first": 267.47895075, + "last": 272.537340916, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.98675938012517, + "seconds": 4.905095125, + "prefill_seconds": 0.0169305, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.240166167, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 4, + "accepted": 3, + "acceptance_rate": 0.75, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 267.38977, + "first": 267.632118166, + "last": 272.537432125, + "chunks": 253 + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 357.09771979100753, + "ttft_max": 0.02024029099999325, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 357.07005135954194, + "seconds": 0.714145583, + "prefill_seconds": 0.019746083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.019916541, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 2 + }, + "speculative": { + "rounds": 91, + "drafted": 283, + "accepted": 164, + "acceptance_rate": 0.5795053003533569, + "tokens_per_round": 2.8131868131868134 + }, + "sent": 272.563330125, + "first": 272.583570416, + "last": 273.297660666, + "chunks": 92 + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 618.4479070805222, + "ttft_max": 0.07851445800002921, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 156.46971991966768, + "seconds": 1.629708292, + "prefill_seconds": 0.01944275, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.039390166, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 249, + "drafted": 11, + "accepted": 6, + "acceptance_rate": 0.5454545454545454, + "tokens_per_round": 1.0281124497991967 + }, + "sent": 273.322900541, + "first": 273.36274, + "last": 274.99240275, + "chunks": 250 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 158.3548155302584, + "seconds": 1.610307834, + "prefill_seconds": 0.019437709, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.05881675, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 249, + "drafted": 11, + "accepted": 6, + "acceptance_rate": 0.5454545454545454, + "tokens_per_round": 1.0281124497991967 + }, + "sent": 273.322958916, + "first": 273.382180416, + "last": 274.992586083, + "chunks": 250 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 160.284594235288, + "seconds": 1.590920208, + "prefill_seconds": 0.019369333, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.078157791, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 249, + "drafted": 17, + "accepted": 6, + "acceptance_rate": 0.35294117647058826, + "tokens_per_round": 1.0281124497991967 + }, + "sent": 273.323042125, + "first": 273.401556583, + "last": 274.992435333, + "chunks": 250 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 154.6267272904501, + "seconds": 1.64913275, + "prefill_seconds": 0.019795417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.019949166, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 249, + "drafted": 11, + "accepted": 6, + "acceptance_rate": 0.5454545454545454, + "tokens_per_round": 1.0281124497991967 + }, + "sent": 273.3228215, + "first": 273.343296, + "last": 274.992382833, + "chunks": 250 + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 8, + "aggregate_decode_tps": 794.1760553413691, + "ttft_max": 0.15879825000001802, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 103.43158275989286, + "seconds": 2.465397833, + "prefill_seconds": 0.019398875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.119088542, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 4, + "accepted": 1, + "acceptance_rate": 0.25, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 275.018285458, + "first": 275.138415416, + "last": 277.604131583, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 104.2490729611327, + "seconds": 2.446064917, + "prefill_seconds": 0.01933225, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.138377875, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 4, + "accepted": 1, + "acceptance_rate": 0.25, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 275.018488125, + "first": 275.157752625, + "last": 277.603875083, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 101.83034555279177, + "seconds": 2.504165125, + "prefill_seconds": 0.019374167, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.080340625, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 4, + "accepted": 1, + "acceptance_rate": 0.25, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 275.018336666, + "first": 275.099585541, + "last": 277.603806583, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 104.91587267057669, + "seconds": 2.430518791, + "prefill_seconds": 0.019425, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.157737583, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 255, + "drafted": 6, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 1.003921568627451 + }, + "sent": 275.018384416, + "first": 275.177182666, + "last": 277.607662291, + "chunks": 256 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 101.0498251393604, + "seconds": 2.523507583, + "prefill_seconds": 0.019428208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.061057208, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 4, + "accepted": 1, + "acceptance_rate": 0.25, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 275.01854475, + "first": 275.080205, + "last": 277.603754291, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 102.62420660546744, + "seconds": 2.484793875, + "prefill_seconds": 0.019429167, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.099766709, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 4, + "accepted": 1, + "acceptance_rate": 0.25, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 275.018601625, + "first": 275.119014791, + "last": 277.603847833, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 100.27760510072491, + "seconds": 2.542940667, + "prefill_seconds": 0.021809916, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.041732958, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 4, + "accepted": 1, + "acceptance_rate": 0.25, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 275.018434791, + "first": 275.060769916, + "last": 277.603710041, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 99.42550415277093, + "seconds": 2.564734292, + "prefill_seconds": 0.019841125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.020004125, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 4, + "accepted": 1, + "acceptance_rate": 0.25, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 275.0182115, + "first": 275.038962333, + "last": 277.603698083, + "chunks": 255 + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 16, + "aggregate_decode_tps": 788.8548028896271, + "ttft_max": 0.31118087500004776, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.06330676926831, + "seconds": 5.093550875, + "prefill_seconds": 0.019405334, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.096908417, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 277.635270458, + "first": 277.733827041, + "last": 282.82798875, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.83831146504552, + "seconds": 4.919141708, + "prefill_seconds": 0.019410667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.270784958, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 277.635318875, + "first": 277.908384208, + "last": 282.82771525, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.249450247420356, + "seconds": 4.880434125, + "prefill_seconds": 0.019322542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.309351459, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 277.635929833, + "first": 277.947110708, + "last": 282.828041416, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.25550159125926, + "seconds": 5.074071334, + "prefill_seconds": 0.019411417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.116295708, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 277.635370625, + "first": 277.753237958, + "last": 282.827383791, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.43216216799861, + "seconds": 4.957987167, + "prefill_seconds": 0.01937825, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.232349833, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 277.635884291, + "first": 277.869507416, + "last": 282.827847833, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.04327510575244, + "seconds": 4.8997685, + "prefill_seconds": 0.019391209, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.290074292, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 277.635418666, + "first": 277.927773666, + "last": 282.827945541, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.83702243367445, + "seconds": 5.016029417, + "prefill_seconds": 0.019409708, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.174288833, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 277.63567375, + "first": 277.811379125, + "last": 282.827585208, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.6411701691176, + "seconds": 5.035428667, + "prefill_seconds": 0.019312167, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.154905541, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 277.635601791, + "first": 277.791968875, + "last": 282.828079333, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.63429242934714, + "seconds": 4.938578375, + "prefill_seconds": 0.019462375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.251585334, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 277.635465666, + "first": 277.888973083, + "last": 282.827899125, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.23187804161023, + "seconds": 4.97736975, + "prefill_seconds": 0.019391084, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.212965209, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 277.636000208, + "first": 277.850126291, + "last": 282.828285458, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 49.68577118614894, + "seconds": 5.132254042, + "prefill_seconds": 0.019380583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.058375583, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 277.635724208, + "first": 277.695039291, + "last": 282.827327833, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 49.499774448454225, + "seconds": 5.151538625, + "prefill_seconds": 0.019430542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.039145041, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 277.635511791, + "first": 277.675665916, + "last": 282.827238041, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.0330804899227, + "seconds": 4.996758917, + "prefill_seconds": 0.019350167, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.193647417, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 277.63577375, + "first": 277.830735541, + "last": 282.827683125, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.447223289623366, + "seconds": 5.054787625, + "prefill_seconds": 0.019404584, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.135652625, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 277.635556958, + "first": 277.772669291, + "last": 282.8278115, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 49.31379809341647, + "seconds": 5.170966542, + "prefill_seconds": 0.01954675, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.01973825, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 277.6358325, + "first": 277.656231, + "last": 282.827159041, + "chunks": 255 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 49.87444922024807, + "seconds": 5.112838417, + "prefill_seconds": 0.019368834, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.077582083, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 254, + "drafted": 2, + "accepted": 1, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0078740157480315 + }, + "sent": 277.635199416, + "first": 277.7144185, + "last": 282.827228416, + "chunks": 255 + } + ] + }, + { + "round": 2, + "arm": "main-plain", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 290.6181341038439, + "ttft_max": 0.016731250000020736, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 290.60274750059926, + "seconds": 0.877486542, + "prefill_seconds": 0.01636925, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.016472375, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 216, + "drafted": 100, + "accepted": 39, + "acceptance_rate": 0.39, + "tokens_per_round": 1.1851851851851851 + }, + "sent": 284.939780666, + "first": 284.956511916, + "last": 285.833952, + "chunks": 217 + } + ] + }, + { + "round": 2, + "arm": "main-plain", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 651.7155973832075, + "ttft_max": 0.0660668329999794, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 163.32064822925992, + "seconds": 1.56134575, + "prefill_seconds": 0.016514084, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.016673583, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 223, + "drafted": 43, + "accepted": 32, + "acceptance_rate": 0.7441860465116279, + "tokens_per_round": 1.147982062780269 + }, + "sent": 285.862577625, + "first": 285.879703041, + "last": 287.441005416, + "chunks": 224 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 168.18952852945154, + "seconds": 1.516146708, + "prefill_seconds": 0.016321125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.065632417, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 224, + "drafted": 46, + "accepted": 31, + "acceptance_rate": 0.6739130434782609, + "tokens_per_round": 1.1428571428571428 + }, + "sent": 285.862633, + "first": 285.928699833, + "last": 287.444802916, + "chunks": 225 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 166.80816283555382, + "seconds": 1.528702167, + "prefill_seconds": 0.016336208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.04929975, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 223, + "drafted": 42, + "accepted": 32, + "acceptance_rate": 0.7619047619047619, + "tokens_per_round": 1.147982062780269 + }, + "sent": 285.86270525, + "first": 285.912371416, + "last": 287.441028291, + "chunks": 224 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 165.0461489711835, + "seconds": 1.545022417, + "prefill_seconds": 0.016329833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.032998208, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 223, + "drafted": 43, + "accepted": 32, + "acceptance_rate": 0.7441860465116279, + "tokens_per_round": 1.147982062780269 + }, + "sent": 285.862509541, + "first": 285.896034666, + "last": 287.441016958, + "chunks": 224 + } + ] + }, + { + "round": 2, + "arm": "main-plain", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 831.0161994582763, + "ttft_max": 0.13210483299997122, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 107.05744545337508, + "seconds": 2.381898792, + "prefill_seconds": 0.016335083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.082185334, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 287.473703583, + "first": 287.556703833, + "last": 289.938711166, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 104.20361821466938, + "seconds": 2.447131917, + "prefill_seconds": 0.016739709, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.01690125, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 287.473754208, + "first": 287.491339583, + "last": 289.938426125, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 108.5428566352171, + "seconds": 2.349302459, + "prefill_seconds": 0.016337209, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.114814833, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 287.473905833, + "first": 287.5893835, + "last": 289.938776916, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 106.33209910232699, + "seconds": 2.398146958, + "prefill_seconds": 0.0163225, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.065811875, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 287.473959458, + "first": 287.540367791, + "last": 289.93848275, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 104.90300174797198, + "seconds": 2.430817, + "prefill_seconds": 0.016336208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.033205166, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 287.473802458, + "first": 287.507666291, + "last": 289.938498916, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 107.79540623645326, + "seconds": 2.365592458, + "prefill_seconds": 0.016332541, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.098416125, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 287.474046916, + "first": 287.57303225, + "last": 289.938839, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 105.61353180011204, + "seconds": 2.414463333, + "prefill_seconds": 0.016364, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.049551, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 287.47384725, + "first": 287.524044458, + "last": 289.938604958, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 108.95265378984647, + "seconds": 2.340466167, + "prefill_seconds": 0.0163505, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.131109708, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 236, + "drafted": 42, + "accepted": 19, + "acceptance_rate": 0.4523809523809524, + "tokens_per_round": 1.0847457627118644 + }, + "sent": 287.473629625, + "first": 287.605734458, + "last": 289.946165375, + "chunks": 237 + } + ] + }, + { + "round": 2, + "arm": "main-plain", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 830.5125280653175, + "ttft_max": 0.26301050000000714, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.97328212515975, + "seconds": 4.813747417, + "prefill_seconds": 0.016346375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.114878917, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 289.974824875, + "first": 290.091412583, + "last": 294.905738875, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.15373023346543, + "seconds": 4.797405542, + "prefill_seconds": 0.016338667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.131151584, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 289.974872666, + "first": 290.10773875, + "last": 294.905436208, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.88709164860404, + "seconds": 4.732116583, + "prefill_seconds": 0.016302834, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.196179958, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 289.9752195, + "first": 290.17302875, + "last": 294.905335416, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.335340783530825, + "seconds": 4.781070042, + "prefill_seconds": 0.016332584, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.147376375, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 289.975468583, + "first": 290.124080791, + "last": 294.905142458, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.26651254634462, + "seconds": 4.878840917, + "prefill_seconds": 0.016349958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.049839875, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 289.974923291, + "first": 290.026090333, + "last": 294.904896666, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 54.26131145645951, + "seconds": 4.699480959, + "prefill_seconds": 0.01632775, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.228727, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 289.974972458, + "first": 290.205723375, + "last": 294.905556041, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 54.44905981674593, + "seconds": 4.683276458, + "prefill_seconds": 0.016322208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.245048459, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 289.975297416, + "first": 290.222047041, + "last": 294.905838458, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.0919816449772, + "seconds": 4.895187166, + "prefill_seconds": 0.016356292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.033592792, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 289.975163875, + "first": 290.009740291, + "last": 294.904886375, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.91876704520609, + "seconds": 4.911518792, + "prefill_seconds": 0.017077209, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.017252375, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 289.975518458, + "first": 289.993361416, + "last": 294.90485675, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.79500372929414, + "seconds": 4.8300025, + "prefill_seconds": 0.016333958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.098574666, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 289.975023208, + "first": 290.075069666, + "last": 294.905689416, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.6172965279471, + "seconds": 4.846315125, + "prefill_seconds": 0.0162985, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.082317875, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 289.975656375, + "first": 290.058719583, + "last": 294.9059905, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.441649342831724, + "seconds": 4.862547292, + "prefill_seconds": 0.016325667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.066120916, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 289.975071166, + "first": 290.042422, + "last": 294.9055095, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 54.64040514687041, + "seconds": 4.666876084, + "prefill_seconds": 0.016307709, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.261180375, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 289.975348291, + "first": 290.238358791, + "last": 294.905790916, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.518162910808236, + "seconds": 4.764737542, + "prefill_seconds": 0.016333209, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.163652334, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 289.97511825, + "first": 290.140429708, + "last": 294.905645291, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.70165697366044, + "seconds": 4.748456833, + "prefill_seconds": 0.016300625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.17993075, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 289.975405166, + "first": 290.15672825, + "last": 294.905898458, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 54.0732705539128, + "seconds": 4.7158235, + "prefill_seconds": 0.0163535, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.212464833, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 19, + "accepted": 16, + "acceptance_rate": 0.8421052631578947, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 289.974755416, + "first": 290.189391083, + "last": 294.905949541, + "chunks": 240 + } + ] + }, + { + "round": 2, + "arm": "main-plain", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 262.3964081747341, + "ttft_max": 0.0202437500000201, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 262.3845845455222, + "seconds": 0.971855875, + "prefill_seconds": 0.019778708, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.01992125, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 247, + "drafted": 72, + "accepted": 8, + "acceptance_rate": 0.1111111111111111, + "tokens_per_round": 1.0364372469635628 + }, + "sent": 294.935785083, + "first": 294.956028833, + "last": 295.927840916, + "chunks": 248 + } + ] + }, + { + "round": 2, + "arm": "main-plain", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 622.917485317143, + "ttft_max": 0.07658683299996483, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 161.28302862963503, + "seconds": 1.5810715, + "prefill_seconds": 0.018786167, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.076180834, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 248, + "drafted": 12, + "accepted": 7, + "acceptance_rate": 0.5833333333333334, + "tokens_per_round": 1.032258064516129 + }, + "sent": 295.956686583, + "first": 296.033273416, + "last": 297.614340125, + "chunks": 249 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 159.38804162765362, + "seconds": 1.599869083, + "prefill_seconds": 0.018792791, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.057388042, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + }, + "sent": 295.956737458, + "first": 296.01448975, + "last": 297.614322583, + "chunks": 249 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 157.53843007753804, + "seconds": 1.618652667, + "prefill_seconds": 0.018812834, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.03856325, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + }, + "sent": 295.956797541, + "first": 295.995684208, + "last": 297.614334, + "chunks": 249 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 155.72805176609543, + "seconds": 1.637469917, + "prefill_seconds": 0.019629625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.01979225, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + }, + "sent": 295.9566095, + "first": 295.976884083, + "last": 297.614306875, + "chunks": 249 + } + ] + }, + { + "round": 2, + "arm": "main-plain", + "prompt": "chat", + "streams": 8, + "aggregate_decode_tps": 804.202708162487, + "ttft_max": 0.15177995799996324, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 100.68020664965388, + "seconds": 2.532771917, + "prefill_seconds": 0.019620959, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.019774875, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 297.64415175, + "first": 297.664605583, + "last": 300.197336166, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 101.43305010971675, + "seconds": 2.5139735, + "prefill_seconds": 0.018797042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.038582458, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 297.644201708, + "first": 297.683404083, + "last": 300.197378416, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 104.55656664589235, + "seconds": 2.438871208, + "prefill_seconds": 0.018806958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.113656292, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 297.644494291, + "first": 297.758623166, + "last": 300.197750125, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 105.36886563708484, + "seconds": 2.420069709, + "prefill_seconds": 0.01880625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.132486167, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 297.64434525, + "first": 297.777433, + "last": 300.197624833, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 106.02447525034125, + "seconds": 2.405105042, + "prefill_seconds": 0.018772417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.151229541, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 30, + "accepted": 3, + "acceptance_rate": 0.1, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 297.644441958, + "first": 297.796221916, + "last": 300.201279458, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 102.19631740884425, + "seconds": 2.495197542, + "prefill_seconds": 0.018787666, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.0573445, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 297.644294041, + "first": 297.702194083, + "last": 300.197352166, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 102.97124018309296, + "seconds": 2.476419625, + "prefill_seconds": 0.018807417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.076114708, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 297.644248791, + "first": 297.720997833, + "last": 300.197710375, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 103.75857642910877, + "seconds": 2.457628167, + "prefill_seconds": 0.018811125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.094907375, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 297.644084541, + "first": 297.739818708, + "last": 300.197828625, + "chunks": 252 + } + ] + }, + { + "round": 2, + "arm": "main-plain", + "prompt": "chat", + "streams": 16, + "aggregate_decode_tps": 800.4553963396994, + "ttft_max": 0.30195220800004563, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.759322934436305, + "seconds": 4.833269, + "prefill_seconds": 0.018797542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.281478292, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 300.231652083, + "first": 300.51546325, + "last": 305.348800291, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.22411279350947, + "seconds": 5.0772425, + "prefill_seconds": 0.018789584, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.03817825, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 300.231700833, + "first": 300.271134916, + "last": 305.348341833, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.039204735749585, + "seconds": 5.09600425, + "prefill_seconds": 0.019426541, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.019594541, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 300.232012, + "first": 300.252341125, + "last": 305.348313, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.95107702209289, + "seconds": 4.908464167, + "prefill_seconds": 0.018742917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.206534083, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 300.232607625, + "first": 300.440241875, + "last": 305.349147916, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.16930216100543, + "seconds": 4.983456667, + "prefill_seconds": 0.0187925, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.1318505, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 300.231747833, + "first": 300.365159166, + "last": 305.349047041, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.15083463192746, + "seconds": 4.889662875, + "prefill_seconds": 0.018801, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.225291708, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 300.23179275, + "first": 300.459042875, + "last": 305.348887208, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.353013017002034, + "seconds": 4.870779833, + "prefill_seconds": 0.018853834, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.244087125, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 300.232240375, + "first": 300.477911166, + "last": 305.349439625, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.55742295669859, + "seconds": 4.945941542, + "prefill_seconds": 0.018782875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.169209, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 300.232067208, + "first": 300.402732291, + "last": 305.349306333, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.36257397740955, + "seconds": 4.964704458, + "prefill_seconds": 0.018785167, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.150635792, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 300.232163083, + "first": 300.38394475, + "last": 305.349396666, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.787357994357095, + "seconds": 5.020934541, + "prefill_seconds": 0.01880575, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.094437167, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 300.23196425, + "first": 300.327546375, + "last": 305.348466166, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.41054903674682, + "seconds": 5.058465041, + "prefill_seconds": 0.018778792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.05694, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 300.231836916, + "first": 300.289925708, + "last": 305.348357458, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.97766116066705, + "seconds": 5.002191042, + "prefill_seconds": 0.018801375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.113120959, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 300.231921875, + "first": 300.34635125, + "last": 305.348693541, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.59845446028725, + "seconds": 5.039679625, + "prefill_seconds": 0.018817625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.075707583, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 300.232111583, + "first": 300.308746625, + "last": 305.348481875, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.55521388941559, + "seconds": 4.85204, + "prefill_seconds": 0.018757833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.262793375, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 300.231878, + "first": 300.49667725, + "last": 305.349356333, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.96454363918808, + "seconds": 4.814541625, + "prefill_seconds": 0.018778625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.3002615, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 300.232298208, + "first": 300.534250416, + "last": 305.348984708, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.754304328772875, + "seconds": 4.927126416, + "prefill_seconds": 0.018756542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.187937709, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 300.231581625, + "first": 300.421495166, + "last": 305.3491995, + "chunks": 252 + } + ] + }, + { + "round": 2, + "arm": "candidate-plain", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 290.11219414456707, + "ttft_max": 0.016669167000031848, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 290.0973561038901, + "seconds": 0.87901525, + "prefill_seconds": 0.016310084, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.016434416, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 216, + "drafted": 100, + "accepted": 39, + "acceptance_rate": 0.39, + "tokens_per_round": 1.1851851851851851 + }, + "sent": 308.221080041, + "first": 308.237749208, + "last": 309.1167195, + "chunks": 217 + } + ] + }, + { + "round": 2, + "arm": "candidate-plain", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 651.566298308141, + "ttft_max": 0.0668896669999981, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 165.01353390530363, + "seconds": 1.545327792, + "prefill_seconds": 0.016315083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.033670666, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 223, + "drafted": 43, + "accepted": 32, + "acceptance_rate": 0.7441860465116279, + "tokens_per_round": 1.147982062780269 + }, + "sent": 309.150666166, + "first": 309.184809333, + "last": 310.730107083, + "chunks": 224 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 163.28918201965183, + "seconds": 1.561646625, + "prefill_seconds": 0.017215208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.017364709, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 223, + "drafted": 43, + "accepted": 32, + "acceptance_rate": 0.7441860465116279, + "tokens_per_round": 1.147982062780269 + }, + "sent": 309.150722833, + "first": 309.168494708, + "last": 310.730092875, + "chunks": 224 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 166.7782701067951, + "seconds": 1.528976166, + "prefill_seconds": 0.01635025, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.049985708, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 223, + "drafted": 42, + "accepted": 32, + "acceptance_rate": 0.7619047619047619, + "tokens_per_round": 1.147982062780269 + }, + "sent": 309.150815208, + "first": 309.201149708, + "last": 310.730115875, + "chunks": 224 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 168.14815632964886, + "seconds": 1.51651975, + "prefill_seconds": 0.0163105, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.066292083, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 224, + "drafted": 46, + "accepted": 31, + "acceptance_rate": 0.6739130434782609, + "tokens_per_round": 1.1428571428571428 + }, + "sent": 309.150580458, + "first": 309.217470125, + "last": 310.733953208, + "chunks": 225 + } + ] + }, + { + "round": 2, + "arm": "candidate-plain", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 830.9803317342685, + "ttft_max": 0.1320373749999817, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 104.89801571795324, + "seconds": 2.430932542, + "prefill_seconds": 0.016297417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.033722542, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 310.766304708, + "first": 310.8007125, + "last": 313.231627958, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 107.78214943631772, + "seconds": 2.365883417, + "prefill_seconds": 0.016259416, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.098723833, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 310.76635525, + "first": 310.865833791, + "last": 313.232120041, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 106.31822267581995, + "seconds": 2.398459959, + "prefill_seconds": 0.016258, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.066167792, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 310.766503166, + "first": 310.833245791, + "last": 313.232074625, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 107.043218122123, + "seconds": 2.382215375, + "prefill_seconds": 0.016325084, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.082435417, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 310.76645175, + "first": 310.849569666, + "last": 313.231988041, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 108.93092430449789, + "seconds": 2.340933042, + "prefill_seconds": 0.016338959, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.131257625, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 236, + "drafted": 42, + "accepted": 19, + "acceptance_rate": 0.4523809523809524, + "tokens_per_round": 1.0847457627118644 + }, + "sent": 310.766405625, + "first": 310.898443, + "last": 313.239348458, + "chunks": 237 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 108.52330130464998, + "seconds": 2.349725791, + "prefill_seconds": 0.016277667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.114925291, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 310.766615375, + "first": 310.882109, + "last": 313.23203375, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 104.19956955362132, + "seconds": 2.447227, + "prefill_seconds": 0.017238375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.017401583, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 310.76654975, + "first": 310.784416708, + "last": 313.231598375, + "chunks": 235 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 105.6030839121993, + "seconds": 2.414702209, + "prefill_seconds": 0.016262292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.049953875, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 234, + "drafted": 25, + "accepted": 21, + "acceptance_rate": 0.84, + "tokens_per_round": 1.0940170940170941 + }, + "sent": 310.766232208, + "first": 310.816983458, + "last": 313.231833333, + "chunks": 235 + } + ] + }, + { + "round": 2, + "arm": "candidate-plain", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 831.0121583537492, + "ttft_max": 0.2618807499999889, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.95027045081604, + "seconds": 4.908540375, + "prefill_seconds": 0.016921542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.017110583, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 18, + "accepted": 16, + "acceptance_rate": 0.8888888888888888, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 313.272857333, + "first": 313.291140416, + "last": 318.199646541, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.12312565623399, + "seconds": 4.89226225, + "prefill_seconds": 0.016278375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.033401167, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 18, + "accepted": 16, + "acceptance_rate": 0.8888888888888888, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 313.272908541, + "first": 313.307429333, + "last": 318.199663041, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 54.66534460310047, + "seconds": 4.664746959, + "prefill_seconds": 0.016233084, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.260031625, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 18, + "accepted": 16, + "acceptance_rate": 0.8888888888888888, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 313.273462375, + "first": 313.535343125, + "last": 318.200459708, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.730871070835924, + "seconds": 4.745875042, + "prefill_seconds": 0.016297833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.179193292, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 18, + "accepted": 16, + "acceptance_rate": 0.8888888888888888, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 313.272953875, + "first": 313.45405375, + "last": 318.200762041, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 54.47625756790819, + "seconds": 4.680938291, + "prefill_seconds": 0.016249583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.243827667, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 18, + "accepted": 16, + "acceptance_rate": 0.8888888888888888, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 313.273226625, + "first": 313.519107333, + "last": 318.200277708, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.184599992366444, + "seconds": 4.794621, + "prefill_seconds": 0.01626375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.130646833, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 18, + "accepted": 16, + "acceptance_rate": 0.8888888888888888, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 313.273403583, + "first": 313.405226166, + "last": 318.200390541, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.36413897051653, + "seconds": 4.778489917, + "prefill_seconds": 0.016247625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.146778292, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 18, + "accepted": 16, + "acceptance_rate": 0.8888888888888888, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 313.27350225, + "first": 313.421475666, + "last": 318.200350208, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.47296465536849, + "seconds": 4.859645375, + "prefill_seconds": 0.016370417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.065873542, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 18, + "accepted": 16, + "acceptance_rate": 0.8888888888888888, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 313.273087, + "first": 313.340081291, + "last": 318.199738166, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.00491396626079, + "seconds": 4.810874708, + "prefill_seconds": 0.016333583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.114492333, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 18, + "accepted": 16, + "acceptance_rate": 0.8888888888888888, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 313.272999583, + "first": 313.388962916, + "last": 318.200301416, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.64890537069318, + "seconds": 4.843405541, + "prefill_seconds": 0.016271167, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.082010875, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 18, + "accepted": 16, + "acceptance_rate": 0.8888888888888888, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 313.273128125, + "first": 313.35635825, + "last": 318.200141375, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.915120953167545, + "seconds": 4.729656458, + "prefill_seconds": 0.016279875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.19541025, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 18, + "accepted": 16, + "acceptance_rate": 0.8888888888888888, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 313.27316975, + "first": 313.470335083, + "last": 318.200551041, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.82363565611979, + "seconds": 4.8273845, + "prefill_seconds": 0.016271416, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.098151458, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 18, + "accepted": 16, + "acceptance_rate": 0.8888888888888888, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 313.273042916, + "first": 313.372629041, + "last": 318.200660833, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 54.10040781099566, + "seconds": 4.713458, + "prefill_seconds": 0.016259792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.211572583, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 18, + "accepted": 16, + "acceptance_rate": 0.8888888888888888, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 313.273303208, + "first": 313.486597958, + "last": 318.200508166, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.54590708472722, + "seconds": 4.76226875, + "prefill_seconds": 0.016263292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.163035166, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 18, + "accepted": 16, + "acceptance_rate": 0.8888888888888888, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 313.273360625, + "first": 313.4377435, + "last": 318.20042525, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 54.28749844480688, + "seconds": 4.697214042, + "prefill_seconds": 0.016251375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.2276415, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 18, + "accepted": 16, + "acceptance_rate": 0.8888888888888888, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 313.2736095, + "first": 313.502852125, + "last": 318.200815875, + "chunks": 240 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.296831998913866, + "seconds": 4.876012375, + "prefill_seconds": 0.016277333, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.049526583, + "sampling": "greedy", + "drafts": true, + "token_sha": "dcc30ceea29d", + "token_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc", + "min_rows": 1 + }, + "speculative": { + "rounds": 239, + "drafted": 18, + "accepted": 16, + "acceptance_rate": 0.8888888888888888, + "tokens_per_round": 1.0711297071129706 + }, + "sent": 313.272782291, + "first": 313.323708, + "last": 318.19969325, + "chunks": 240 + } + ] + }, + { + "round": 2, + "arm": "candidate-plain", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 262.21991974165917, + "ttft_max": 0.019898124999997435, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 262.2083813625824, + "seconds": 0.972508959, + "prefill_seconds": 0.019450166, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.019591042, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 247, + "drafted": 72, + "accepted": 8, + "acceptance_rate": 0.1111111111111111, + "tokens_per_round": 1.0364372469635628 + }, + "sent": 318.232904875, + "first": 318.252803, + "last": 319.225269166, + "chunks": 248 + } + ] + }, + { + "round": 2, + "arm": "candidate-plain", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 622.948506893192, + "ttft_max": 0.07660004199999548, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 157.54933154086103, + "seconds": 1.618540666, + "prefill_seconds": 0.018821792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.038648084, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + }, + "sent": 319.259709291, + "first": 319.298847208, + "last": 320.9173675, + "chunks": 249 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 161.28251001913515, + "seconds": 1.581076584, + "prefill_seconds": 0.018785584, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.076145417, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 248, + "drafted": 12, + "accepted": 7, + "acceptance_rate": 0.5833333333333334, + "tokens_per_round": 1.032258064516129 + }, + "sent": 319.259760166, + "first": 319.336360208, + "last": 320.917396375, + "chunks": 249 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 155.73814530505885, + "seconds": 1.637363791, + "prefill_seconds": 0.019692083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.019849542, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + }, + "sent": 319.259818458, + "first": 319.280021875, + "last": 320.917341833, + "chunks": 249 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 159.39187730854874, + "seconds": 1.599830583, + "prefill_seconds": 0.018723917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.057366708, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 248, + "drafted": 18, + "accepted": 7, + "acceptance_rate": 0.3888888888888889, + "tokens_per_round": 1.032258064516129 + }, + "sent": 319.259639291, + "first": 319.317571416, + "last": 320.917359416, + "chunks": 249 + } + ] + }, + { + "round": 2, + "arm": "candidate-plain", + "prompt": "chat", + "streams": 8, + "aggregate_decode_tps": 804.3912665584089, + "ttft_max": 0.1521238329999619, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 102.9992905473494, + "seconds": 2.475745208, + "prefill_seconds": 0.018804291, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.076227416, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 320.951588666, + "first": 321.028649208, + "last": 323.504430833, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 106.0499026134387, + "seconds": 2.404528375, + "prefill_seconds": 0.018741125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.150904375, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 252, + "drafted": 30, + "accepted": 3, + "acceptance_rate": 0.1, + "tokens_per_round": 1.0158730158730158 + }, + "sent": 320.951646583, + "first": 321.103770416, + "last": 323.508267916, + "chunks": 253 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 103.78672128658181, + "seconds": 2.456961708, + "prefill_seconds": 0.018798958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.094958666, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 320.951808583, + "first": 321.04744825, + "last": 323.504369916, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 102.22434666240264, + "seconds": 2.494513375, + "prefill_seconds": 0.01880375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.057425166, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 320.951908666, + "first": 321.009842, + "last": 323.504324, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 104.58395403722076, + "seconds": 2.438232541, + "prefill_seconds": 0.018791667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.1136865, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 320.951854583, + "first": 321.066244125, + "last": 323.504529625, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 101.45915505061564, + "seconds": 2.513326667, + "prefill_seconds": 0.018847541, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.03865225, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 320.951702833, + "first": 320.991028, + "last": 323.504351708, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 100.7042236486789, + "seconds": 2.532167875, + "prefill_seconds": 0.019715959, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.019900375, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 320.9517515, + "first": 320.972188666, + "last": 323.504311291, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 105.39538069612306, + "seconds": 2.419460875, + "prefill_seconds": 0.018780292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.132466625, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 9, + "accepted": 4, + "acceptance_rate": 0.4444444444444444, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 320.951514833, + "first": 321.085027083, + "last": 323.50478975, + "chunks": 252 + } + ] + }, + { + "round": 2, + "arm": "candidate-plain", + "prompt": "chat", + "streams": 16, + "aggregate_decode_tps": 801.2450260525325, + "ttft_max": 0.3025994579999747, + "requests": [ + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.08908039467293, + "seconds": 5.090929959, + "prefill_seconds": 0.019595375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.019816333, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 323.542316958, + "first": 323.563395791, + "last": 328.654280083, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.27460697129549, + "seconds": 5.072143083, + "prefill_seconds": 0.0188055, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.038602, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 323.542368708, + "first": 323.582208666, + "last": 328.654311625, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.648919005256076, + "seconds": 5.034658291, + "prefill_seconds": 0.018770458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.075997792, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 323.54271425, + "first": 323.619719958, + "last": 328.654323666, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.809909127089995, + "seconds": 4.82863925, + "prefill_seconds": 0.018726458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.281503583, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 323.542420375, + "first": 323.826029125, + "last": 328.655109291, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.40358953285565, + "seconds": 4.866078875, + "prefill_seconds": 0.018754083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.244145875, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 323.542962541, + "first": 323.788552166, + "last": 328.654811041, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.837700640704554, + "seconds": 5.0159625, + "prefill_seconds": 0.018772625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.094714792, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 323.542471, + "first": 323.638498125, + "last": 328.654887041, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.605931519919196, + "seconds": 4.847362125, + "prefill_seconds": 0.018756417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.262877833, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 323.542769458, + "first": 323.80731225, + "last": 328.654937416, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.20315081329055, + "seconds": 4.884762625, + "prefill_seconds": 0.018713334, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.225421875, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 323.542519916, + "first": 323.769792833, + "last": 328.654603416, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 50.46094673811927, + "seconds": 5.053412916, + "prefill_seconds": 0.018739, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.057202375, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 323.543029958, + "first": 323.600936041, + "last": 328.655159375, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 52.00380915298769, + "seconds": 4.903486959, + "prefill_seconds": 0.018685375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.206730875, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 323.542568375, + "first": 323.751071833, + "last": 328.654986166, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.41379407069505, + "seconds": 4.959758458, + "prefill_seconds": 0.018779375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.15092825, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 323.542815041, + "first": 323.69485825, + "last": 328.655296208, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.60879976933287, + "seconds": 4.941017833, + "prefill_seconds": 0.018750542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.169453166, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 323.542615541, + "first": 323.713606541, + "last": 328.655200291, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.805692932797264, + "seconds": 4.922238958, + "prefill_seconds": 0.01877275, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.188166209, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 323.542900833, + "first": 323.732385958, + "last": 328.654701541, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.2207366147053, + "seconds": 4.978452417, + "prefill_seconds": 0.018818875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.13210325, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 323.542660416, + "first": 323.676052791, + "last": 328.655355208, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 51.02788546370878, + "seconds": 4.997267625, + "prefill_seconds": 0.018750125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.113471917, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 323.542856125, + "first": 323.657248791, + "last": 328.654717333, + "chunks": 252 + }, + { + "usage": { + "prompt_tokens": 18, + "completion_tokens": 256, + "total_tokens": 274, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 53.01680247008635, + "seconds": 4.809795916, + "prefill_seconds": 0.018807708, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.300145291, + "sampling": "greedy", + "drafts": true, + "token_sha": "756f2e358866", + "token_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c", + "min_rows": 1 + }, + "speculative": { + "rounds": 251, + "drafted": 8, + "accepted": 4, + "acceptance_rate": 0.5, + "tokens_per_round": 1.0199203187250996 + }, + "sent": 323.542239333, + "first": 323.844838791, + "last": 328.655471083, + "chunks": 252 + } + ] + } + ], + "performance-qwen": [ + { + "round": 0, + "arm": "main-drafted", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 211.92194615871247, + "ttft_max": 0.03147329199998694, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 211.73019910553273, + "seconds": 1.204362916, + "prefill_seconds": 0.030946833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.031178042, + "sampling": "greedy", + "drafts": true, + "token_sha": "a10c9b848ed9", + "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130", + "min_rows": 16 + }, + "speculative": { + "rounds": 43, + "drafted": 658, + "accepted": 213, + "acceptance_rate": 0.32370820668693007, + "tokens_per_round": 5.953488372093023 + }, + "sent": 220.468364583, + "first": 220.499837875, + "last": 221.703111083, + "chunks": 44 + } + ] + }, + { + "round": 0, + "arm": "main-drafted", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 379.28424450667757, + "ttft_max": 0.15881983299999547, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 97.82656232856375, + "seconds": 2.606654, + "prefill_seconds": 0.117248833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.117549, + "sampling": "greedy", + "drafts": true, + "token_sha": "a10c9b848ed9", + "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130", + "min_rows": 4 + }, + "speculative": { + "rounds": 67, + "drafted": 254, + "accepted": 188, + "acceptance_rate": 0.7401574803149606, + "tokens_per_round": 3.8208955223880596 + }, + "sent": 221.705125833, + "first": 221.823199625, + "last": 224.429784583, + "chunks": 68 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 99.3859749337853, + "seconds": 2.565754375, + "prefill_seconds": 0.123503708, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.158455209, + "sampling": "greedy", + "drafts": true, + "token_sha": "a10c9b848ed9", + "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130", + "min_rows": 4 + }, + "speculative": { + "rounds": 67, + "drafted": 254, + "accepted": 188, + "acceptance_rate": 0.7401574803149606, + "tokens_per_round": 3.8208955223880596 + }, + "sent": 221.705284458, + "first": 221.864104291, + "last": 224.429873916, + "chunks": 68 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 96.30150148406732, + "seconds": 2.647933792, + "prefill_seconds": 0.075959041, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.076291, + "sampling": "greedy", + "drafts": true, + "token_sha": "a10c9b848ed9", + "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130", + "min_rows": 4 + }, + "speculative": { + "rounds": 67, + "drafted": 254, + "accepted": 188, + "acceptance_rate": 0.7401574803149606, + "tokens_per_round": 3.8208955223880596 + }, + "sent": 221.705184541, + "first": 221.78189875, + "last": 224.429773291, + "chunks": 68 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 94.82365861929128, + "seconds": 2.689202291, + "prefill_seconds": 0.034662459, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.034964041, + "sampling": "greedy", + "drafts": true, + "token_sha": "a10c9b848ed9", + "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130", + "min_rows": 4 + }, + "speculative": { + "rounds": 67, + "drafted": 254, + "accepted": 188, + "acceptance_rate": 0.7401574803149606, + "tokens_per_round": 3.8208955223880596 + }, + "sent": 221.705046708, + "first": 221.740597958, + "last": 224.429743875, + "chunks": 68 + } + ] + }, + { + "round": 0, + "arm": "main-drafted", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 140.9551661001172, + "ttft_max": 0.03612595900000315, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 140.8800278113749, + "seconds": 1.81005075, + "prefill_seconds": 0.035580708, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.03579525, + "sampling": "greedy", + "drafts": true, + "token_sha": "e2c694d862fd", + "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff", + "min_rows": 16 + }, + "speculative": { + "rounds": 65, + "drafted": 975, + "accepted": 191, + "acceptance_rate": 0.19589743589743588, + "tokens_per_round": 3.9384615384615387 + }, + "sent": 224.430958291, + "first": 224.46708425, + "last": 226.276170125, + "chunks": 66 + } + ] + }, + { + "round": 0, + "arm": "main-drafted", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 314.3405067152877, + "ttft_max": 0.15906683299999713, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 80.5539987986341, + "seconds": 3.165578417, + "prefill_seconds": 0.079156209, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.119072042, + "sampling": "greedy", + "drafts": true, + "token_sha": "e2c694d862fd", + "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff", + "min_rows": 3 + }, + "speculative": { + "rounds": 89, + "drafted": 278, + "accepted": 166, + "acceptance_rate": 0.5971223021582733, + "tokens_per_round": 2.8764044943820224 + }, + "sent": 226.278272291, + "first": 226.397800083, + "last": 229.563340541, + "chunks": 90 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 81.57684958648021, + "seconds": 3.125886833, + "prefill_seconds": 0.118852042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.158741084, + "sampling": "greedy", + "drafts": true, + "token_sha": "e2c694d862fd", + "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff", + "min_rows": 3 + }, + "speculative": { + "rounds": 89, + "drafted": 278, + "accepted": 166, + "acceptance_rate": 0.5971223021582733, + "tokens_per_round": 2.8764044943820224 + }, + "sent": 226.278422708, + "first": 226.437489541, + "last": 229.563531708, + "chunks": 90 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 78.59021380751824, + "seconds": 3.244678792, + "prefill_seconds": 0.039655625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.03991425, + "sampling": "greedy", + "drafts": true, + "token_sha": "e2c694d862fd", + "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff", + "min_rows": 3 + }, + "speculative": { + "rounds": 89, + "drafted": 278, + "accepted": 166, + "acceptance_rate": 0.5971223021582733, + "tokens_per_round": 2.8764044943820224 + }, + "sent": 226.278334458, + "first": 226.318642875, + "last": 229.563267041, + "chunks": 90 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 79.5624005567496, + "seconds": 3.2050315, + "prefill_seconds": 0.079313625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.079552834, + "sampling": "greedy", + "drafts": true, + "token_sha": "e2c694d862fd", + "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff", + "min_rows": 3 + }, + "speculative": { + "rounds": 89, + "drafted": 278, + "accepted": 166, + "acceptance_rate": 0.5971223021582733, + "tokens_per_round": 2.8764044943820224 + }, + "sent": 226.278194875, + "first": 226.358296083, + "last": 229.563297333, + "chunks": 90 + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 212.45570139283868, + "ttft_max": 0.03133441599999287, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 212.2655129411728, + "seconds": 1.201325625, + "prefill_seconds": 0.030853583, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.03105075, + "sampling": "greedy", + "drafts": true, + "token_sha": "a10c9b848ed9", + "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130", + "min_rows": 16 + }, + "speculative": { + "rounds": 43, + "drafted": 658, + "accepted": 213, + "acceptance_rate": 0.32370820668693007, + "tokens_per_round": 5.953488372093023 + }, + "sent": 240.36544375, + "first": 240.396778166, + "last": 241.597028375, + "chunks": 44 + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 377.1968654072174, + "ttft_max": 0.15561083300002565, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 97.27497352907643, + "seconds": 2.621434792, + "prefill_seconds": 0.082676, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.113711334, + "sampling": "greedy", + "drafts": true, + "token_sha": "a10c9b848ed9", + "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130", + "min_rows": 4 + }, + "speculative": { + "rounds": 67, + "drafted": 255, + "accepted": 188, + "acceptance_rate": 0.7372549019607844, + "tokens_per_round": 3.8208955223880596 + }, + "sent": 241.599939541, + "first": 241.714227666, + "last": 244.335596875, + "chunks": 68 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 94.30269209545193, + "seconds": 2.704058541, + "prefill_seconds": 0.030840083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.031085166, + "sampling": "greedy", + "drafts": true, + "token_sha": "a10c9b848ed9", + "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130", + "min_rows": 1 + }, + "speculative": { + "rounds": 67, + "drafted": 251, + "accepted": 188, + "acceptance_rate": 0.749003984063745, + "tokens_per_round": 3.8208955223880596 + }, + "sent": 241.60006575, + "first": 241.631526791, + "last": 244.335539708, + "chunks": 68 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 98.83527473590149, + "seconds": 2.5800505, + "prefill_seconds": 0.12406575, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.155116875, + "sampling": "greedy", + "drafts": true, + "token_sha": "a10c9b848ed9", + "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130", + "min_rows": 4 + }, + "speculative": { + "rounds": 67, + "drafted": 255, + "accepted": 188, + "acceptance_rate": 0.7372549019607844, + "tokens_per_round": 3.8208955223880596 + }, + "sent": 241.599997125, + "first": 241.755607958, + "last": 244.335685, + "chunks": 68 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 95.77020109254082, + "seconds": 2.662623625, + "prefill_seconds": 0.041446, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.07251225, + "sampling": "greedy", + "drafts": true, + "token_sha": "a10c9b848ed9", + "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130", + "min_rows": 4 + }, + "speculative": { + "rounds": 67, + "drafted": 255, + "accepted": 188, + "acceptance_rate": 0.7372549019607844, + "tokens_per_round": 3.8208955223880596 + }, + "sent": 241.599864458, + "first": 241.672987208, + "last": 244.335552958, + "chunks": 68 + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 141.19484385798825, + "ttft_max": 0.036089042000014615, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 141.11870149437556, + "seconds": 1.806989416, + "prefill_seconds": 0.035520292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.0357655, + "sampling": "greedy", + "drafts": true, + "token_sha": "e2c694d862fd", + "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff", + "min_rows": 16 + }, + "speculative": { + "rounds": 65, + "drafted": 975, + "accepted": 191, + "acceptance_rate": 0.19589743589743588, + "tokens_per_round": 3.9384615384615387 + }, + "sent": 244.337321541, + "first": 244.373410583, + "last": 246.179425541, + "chunks": 66 + } + ] + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 316.415283652713, + "ttft_max": 0.15867833299998324, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 79.10471251890549, + "seconds": 3.223575333, + "prefill_seconds": 0.039642875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.039891958, + "sampling": "greedy", + "drafts": true, + "token_sha": "e2c694d862fd", + "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff", + "min_rows": 3 + }, + "speculative": { + "rounds": 88, + "drafted": 276, + "accepted": 167, + "acceptance_rate": 0.605072463768116, + "tokens_per_round": 2.909090909090909 + }, + "sent": 246.182093458, + "first": 246.222425833, + "last": 249.445940416, + "chunks": 89 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 82.12139861437048, + "seconds": 3.105159, + "prefill_seconds": 0.118494625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.15834425, + "sampling": "greedy", + "drafts": true, + "token_sha": "e2c694d862fd", + "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff", + "min_rows": 3 + }, + "speculative": { + "rounds": 88, + "drafted": 276, + "accepted": 167, + "acceptance_rate": 0.605072463768116, + "tokens_per_round": 2.909090909090909 + }, + "sent": 246.182233208, + "first": 246.340911541, + "last": 249.4460375, + "chunks": 89 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 81.08921362016, + "seconds": 3.144684584, + "prefill_seconds": 0.118604958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.118837791, + "sampling": "greedy", + "drafts": true, + "token_sha": "e2c694d862fd", + "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff", + "min_rows": 3 + }, + "speculative": { + "rounds": 88, + "drafted": 276, + "accepted": 167, + "acceptance_rate": 0.605072463768116, + "tokens_per_round": 2.909090909090909 + }, + "sent": 246.182148166, + "first": 246.301384375, + "last": 249.446024291, + "chunks": 89 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 80.08463427060869, + "seconds": 3.184131417, + "prefill_seconds": 0.079125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.07937475, + "sampling": "greedy", + "drafts": true, + "token_sha": "e2c694d862fd", + "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff", + "min_rows": 3 + }, + "speculative": { + "rounds": 88, + "drafted": 276, + "accepted": 167, + "acceptance_rate": 0.605072463768116, + "tokens_per_round": 2.909090909090909 + }, + "sent": 246.18201175, + "first": 246.261895833, + "last": 249.445976166, + "chunks": 89 + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 211.59088653158616, + "ttft_max": 0.0313943330000086, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 211.40970570965607, + "seconds": 1.206188709, + "prefill_seconds": 0.030824792, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.031058833, + "sampling": "greedy", + "drafts": true, + "token_sha": "a10c9b848ed9", + "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130", + "min_rows": 16 + }, + "speculative": { + "rounds": 43, + "drafted": 658, + "accepted": 213, + "acceptance_rate": 0.32370820668693007, + "tokens_per_round": 5.953488372093023 + }, + "sent": 254.012542333, + "first": 254.043936666, + "last": 255.249092541, + "chunks": 44 + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 377.49507004207, + "ttft_max": 0.16005191600001467, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 95.84680597070513, + "seconds": 2.660495542, + "prefill_seconds": 0.076353833, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.076651, + "sampling": "greedy", + "drafts": true, + "token_sha": "a10c9b848ed9", + "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130", + "min_rows": 4 + }, + "speculative": { + "rounds": 67, + "drafted": 254, + "accepted": 188, + "acceptance_rate": 0.7401574803149606, + "tokens_per_round": 3.8208955223880596 + }, + "sent": 255.252503833, + "first": 255.329673833, + "last": 257.9901095, + "chunks": 68 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 97.3572455490473, + "seconds": 2.619219541, + "prefill_seconds": 0.11765925, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.117936625, + "sampling": "greedy", + "drafts": true, + "token_sha": "a10c9b848ed9", + "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130", + "min_rows": 4 + }, + "speculative": { + "rounds": 67, + "drafted": 254, + "accepted": 188, + "acceptance_rate": 0.7401574803149606, + "tokens_per_round": 3.8208955223880596 + }, + "sent": 255.252631541, + "first": 255.370974916, + "last": 257.990136916, + "chunks": 68 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 94.37434527797964, + "seconds": 2.7020055, + "prefill_seconds": 0.034829917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.035106667, + "sampling": "greedy", + "drafts": true, + "token_sha": "a10c9b848ed9", + "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130", + "min_rows": 4 + }, + "speculative": { + "rounds": 67, + "drafted": 254, + "accepted": 188, + "acceptance_rate": 0.7401574803149606, + "tokens_per_round": 3.8208955223880596 + }, + "sent": 255.252561583, + "first": 255.288130666, + "last": 257.990094125, + "chunks": 68 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 98.92393069777353, + "seconds": 2.57773825, + "prefill_seconds": 0.12433075, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.159474, + "sampling": "greedy", + "drafts": true, + "token_sha": "a10c9b848ed9", + "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130", + "min_rows": 4 + }, + "speculative": { + "rounds": 67, + "drafted": 254, + "accepted": 188, + "acceptance_rate": 0.7401574803149606, + "tokens_per_round": 3.8208955223880596 + }, + "sent": 255.252426125, + "first": 255.412478041, + "last": 257.990152708, + "chunks": 68 + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 140.8142774005683, + "ttft_max": 0.03615183299996261, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 140.73994300998172, + "seconds": 1.811852375, + "prefill_seconds": 0.035574334, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.035819583, + "sampling": "greedy", + "drafts": true, + "token_sha": "e2c694d862fd", + "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff", + "min_rows": 16 + }, + "speculative": { + "rounds": 65, + "drafted": 975, + "accepted": 191, + "acceptance_rate": 0.19589743589743588, + "tokens_per_round": 3.9384615384615387 + }, + "sent": 257.992488125, + "first": 258.028639958, + "last": 259.839535875, + "chunks": 66 + } + ] + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 315.9810922490545, + "ttft_max": 0.15894150000002583, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 80.98834255321776, + "seconds": 3.148601292, + "prefill_seconds": 0.079248459, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.119086042, + "sampling": "greedy", + "drafts": true, + "token_sha": "e2c694d862fd", + "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff", + "min_rows": 3 + }, + "speculative": { + "rounds": 88, + "drafted": 276, + "accepted": 167, + "acceptance_rate": 0.605072463768116, + "tokens_per_round": 2.909090909090909 + }, + "sent": 259.842767708, + "first": 259.962342875, + "last": 263.11114075, + "chunks": 89 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 79.98105127825862, + "seconds": 3.188255167, + "prefill_seconds": 0.079208459, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.079422916, + "sampling": "greedy", + "drafts": true, + "token_sha": "e2c694d862fd", + "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff", + "min_rows": 3 + }, + "speculative": { + "rounds": 88, + "drafted": 276, + "accepted": 167, + "acceptance_rate": 0.605072463768116, + "tokens_per_round": 2.909090909090909 + }, + "sent": 259.842903583, + "first": 259.922682583, + "last": 263.110874791, + "chunks": 89 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 82.01407486572037, + "seconds": 3.109222416, + "prefill_seconds": 0.118653292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.158542541, + "sampling": "greedy", + "drafts": true, + "token_sha": "e2c694d862fd", + "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff", + "min_rows": 3 + }, + "speculative": { + "rounds": 88, + "drafted": 276, + "accepted": 167, + "acceptance_rate": 0.605072463768116, + "tokens_per_round": 2.909090909090909 + }, + "sent": 259.842822541, + "first": 260.001764041, + "last": 263.111057041, + "chunks": 89 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 79.00105489643892, + "seconds": 3.227805, + "prefill_seconds": 0.039626042, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.039891209, + "sampling": "greedy", + "drafts": true, + "token_sha": "e2c694d862fd", + "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff", + "min_rows": 3 + }, + "speculative": { + "rounds": 88, + "drafted": 276, + "accepted": 167, + "acceptance_rate": 0.605072463768116, + "tokens_per_round": 2.909090909090909 + }, + "sent": 259.842679041, + "first": 259.8830995, + "last": 263.110842166, + "chunks": 89 + } + ] + }, + { + "round": 1, + "arm": "main-drafted", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 212.29582007221308, + "ttft_max": 0.03140174999998635, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 212.119392935892, + "seconds": 1.202153167, + "prefill_seconds": 0.030904875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.031109208, + "sampling": "greedy", + "drafts": true, + "token_sha": "a10c9b848ed9", + "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130", + "min_rows": 16 + }, + "speculative": { + "rounds": 43, + "drafted": 658, + "accepted": 213, + "acceptance_rate": 0.32370820668693007, + "tokens_per_round": 5.953488372093023 + }, + "sent": 267.675353583, + "first": 267.706755333, + "last": 268.907909458, + "chunks": 44 + } + ] + }, + { + "round": 1, + "arm": "main-drafted", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 379.89684590178143, + "ttft_max": 0.16034129099995198, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 99.56640459004593, + "seconds": 2.561104833, + "prefill_seconds": 0.159479333, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.159775416, + "sampling": "greedy", + "drafts": true, + "token_sha": "a10c9b848ed9", + "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130", + "min_rows": 4 + }, + "speculative": { + "rounds": 67, + "drafted": 254, + "accepted": 188, + "acceptance_rate": 0.7401574803149606, + "tokens_per_round": 3.8208955223880596 + }, + "sent": 268.911875375, + "first": 269.072216666, + "last": 271.633347125, + "chunks": 68 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 97.98995228321633, + "seconds": 2.602307625, + "prefill_seconds": 0.118263, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.118519333, + "sampling": "greedy", + "drafts": true, + "token_sha": "a10c9b848ed9", + "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130", + "min_rows": 4 + }, + "speculative": { + "rounds": 67, + "drafted": 254, + "accepted": 188, + "acceptance_rate": 0.7401574803149606, + "tokens_per_round": 3.8208955223880596 + }, + "sent": 268.911995541, + "first": 269.031000583, + "last": 271.633265916, + "chunks": 68 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 94.97800686107192, + "seconds": 2.684832083, + "prefill_seconds": 0.035676625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.035969, + "sampling": "greedy", + "drafts": true, + "token_sha": "a10c9b848ed9", + "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130", + "min_rows": 4 + }, + "speculative": { + "rounds": 67, + "drafted": 254, + "accepted": 188, + "acceptance_rate": 0.7401574803149606, + "tokens_per_round": 3.8208955223880596 + }, + "sent": 268.911934375, + "first": 268.94840775, + "last": 271.633184708, + "chunks": 68 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 96.45839907288453, + "seconds": 2.643626708, + "prefill_seconds": 0.076915875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.077211458, + "sampling": "greedy", + "drafts": true, + "token_sha": "a10c9b848ed9", + "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130", + "min_rows": 4 + }, + "speculative": { + "rounds": 67, + "drafted": 254, + "accepted": 188, + "acceptance_rate": 0.7401574803149606, + "tokens_per_round": 3.8208955223880596 + }, + "sent": 268.91179475, + "first": 268.989663083, + "last": 271.633212541, + "chunks": 68 + } + ] + }, + { + "round": 1, + "arm": "main-drafted", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 141.11723064905576, + "ttft_max": 0.03655066699997178, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 141.04172122135094, + "seconds": 1.807975667, + "prefill_seconds": 0.036057083, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.036274583, + "sampling": "greedy", + "drafts": true, + "token_sha": "e2c694d862fd", + "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff", + "min_rows": 16 + }, + "speculative": { + "rounds": 65, + "drafted": 975, + "accepted": 191, + "acceptance_rate": 0.19589743589743588, + "tokens_per_round": 3.9384615384615387 + }, + "sent": 271.636266583, + "first": 271.67281725, + "last": 273.4798255, + "chunks": 66 + } + ] + }, + { + "round": 1, + "arm": "main-drafted", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 314.2222738472332, + "ttft_max": 0.1598498339999992, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 80.52107452016692, + "seconds": 3.166872791, + "prefill_seconds": 0.119563166, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.119831125, + "sampling": "greedy", + "drafts": true, + "token_sha": "e2c694d862fd", + "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff", + "min_rows": 3 + }, + "speculative": { + "rounds": 89, + "drafted": 278, + "accepted": 166, + "acceptance_rate": 0.5971223021582733, + "tokens_per_round": 2.8764044943820224 + }, + "sent": 273.483816583, + "first": 273.604093333, + "last": 276.770985458, + "chunks": 90 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 81.54333064693361, + "seconds": 3.12717175, + "prefill_seconds": 0.118842625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.159505875, + "sampling": "greedy", + "drafts": true, + "token_sha": "e2c694d862fd", + "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff", + "min_rows": 3 + }, + "speculative": { + "rounds": 89, + "drafted": 278, + "accepted": 166, + "acceptance_rate": 0.5971223021582733, + "tokens_per_round": 2.8764044943820224 + }, + "sent": 273.483945416, + "first": 273.64379525, + "last": 276.771061, + "chunks": 90 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 78.55952839607261, + "seconds": 3.245946166, + "prefill_seconds": 0.040431458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.040645833, + "sampling": "greedy", + "drafts": true, + "token_sha": "e2c694d862fd", + "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff", + "min_rows": 3 + }, + "speculative": { + "rounds": 89, + "drafted": 278, + "accepted": 166, + "acceptance_rate": 0.5971223021582733, + "tokens_per_round": 2.8764044943820224 + }, + "sent": 273.483870916, + "first": 273.524951208, + "last": 276.770851416, + "chunks": 90 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 79.52676274383238, + "seconds": 3.20646775, + "prefill_seconds": 0.079922917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.080200708, + "sampling": "greedy", + "drafts": true, + "token_sha": "e2c694d862fd", + "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff", + "min_rows": 3 + }, + "speculative": { + "rounds": 89, + "drafted": 278, + "accepted": 166, + "acceptance_rate": 0.5971223021582733, + "tokens_per_round": 2.8764044943820224 + }, + "sent": 273.483737416, + "first": 273.564455, + "last": 276.770873291, + "chunks": 90 + } + ] + }, + { + "round": 2, + "arm": "main-drafted", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 212.0302205426082, + "ttft_max": 0.03127620800000841, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 211.84478174379822, + "seconds": 1.2037115, + "prefill_seconds": 0.03071525, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.030954333, + "sampling": "greedy", + "drafts": true, + "token_sha": "a10c9b848ed9", + "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130", + "min_rows": 16 + }, + "speculative": { + "rounds": 43, + "drafted": 658, + "accepted": 213, + "acceptance_rate": 0.32370820668693007, + "tokens_per_round": 5.953488372093023 + }, + "sent": 281.280602625, + "first": 281.311878833, + "last": 282.514537583, + "chunks": 44 + } + ] + }, + { + "round": 2, + "arm": "main-drafted", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 379.1363966258468, + "ttft_max": 0.16037083300000177, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 99.36423934604692, + "seconds": 2.566315625, + "prefill_seconds": 0.159496, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.159762459, + "sampling": "greedy", + "drafts": true, + "token_sha": "a10c9b848ed9", + "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130", + "min_rows": 4 + }, + "speculative": { + "rounds": 67, + "drafted": 254, + "accepted": 188, + "acceptance_rate": 0.7401574803149606, + "tokens_per_round": 3.8208955223880596 + }, + "sent": 282.519196958, + "first": 282.679567791, + "last": 285.24584725, + "chunks": 68 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 97.78525189785174, + "seconds": 2.607755209, + "prefill_seconds": 0.11804275, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.118341166, + "sampling": "greedy", + "drafts": true, + "token_sha": "a10c9b848ed9", + "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130", + "min_rows": 4 + }, + "speculative": { + "rounds": 67, + "drafted": 254, + "accepted": 188, + "acceptance_rate": 0.7401574803149606, + "tokens_per_round": 3.8208955223880596 + }, + "sent": 282.519314875, + "first": 282.638117083, + "last": 285.245836083, + "chunks": 68 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 94.78495060310176, + "seconds": 2.6903005, + "prefill_seconds": 0.035455125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.0357835, + "sampling": "greedy", + "drafts": true, + "token_sha": "a10c9b848ed9", + "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130", + "min_rows": 4 + }, + "speculative": { + "rounds": 67, + "drafted": 254, + "accepted": 188, + "acceptance_rate": 0.7401574803149606, + "tokens_per_round": 3.8208955223880596 + }, + "sent": 282.519259916, + "first": 282.555522583, + "last": 285.245777125, + "chunks": 68 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 96.25564895196716, + "seconds": 2.649195167, + "prefill_seconds": 0.076599458, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.0768925, + "sampling": "greedy", + "drafts": true, + "token_sha": "a10c9b848ed9", + "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130", + "min_rows": 4 + }, + "speculative": { + "rounds": 67, + "drafted": 254, + "accepted": 188, + "acceptance_rate": 0.7401574803149606, + "tokens_per_round": 3.8208955223880596 + }, + "sent": 282.51910575, + "first": 282.596677875, + "last": 285.245823958, + "chunks": 68 + } + ] + }, + { + "round": 2, + "arm": "main-drafted", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 141.19561917656972, + "ttft_max": 0.03684129200001962, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 141.1207905160334, + "seconds": 1.806962667, + "prefill_seconds": 0.036193416, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.03647625, + "sampling": "greedy", + "drafts": true, + "token_sha": "e2c694d862fd", + "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff", + "min_rows": 16 + }, + "speculative": { + "rounds": 65, + "drafted": 975, + "accepted": 191, + "acceptance_rate": 0.19589743589743588, + "tokens_per_round": 3.9384615384615387 + }, + "sent": 285.249543708, + "first": 285.286385, + "last": 287.092390041, + "chunks": 66 + } + ] + }, + { + "round": 2, + "arm": "main-drafted", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 314.095888200934, + "ttft_max": 0.1594045840000149, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 81.49345545633638, + "seconds": 3.129085625, + "prefill_seconds": 0.158699542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.158901208, + "sampling": "greedy", + "drafts": true, + "token_sha": "e2c694d862fd", + "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff", + "min_rows": 3 + }, + "speculative": { + "rounds": 89, + "drafted": 278, + "accepted": 166, + "acceptance_rate": 0.5971223021582733, + "tokens_per_round": 2.8764044943820224 + }, + "sent": 287.097095166, + "first": 287.25649975, + "last": 290.385539375, + "chunks": 90 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 79.4908501939986, + "seconds": 3.207916375, + "prefill_seconds": 0.079819542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.080064833, + "sampling": "greedy", + "drafts": true, + "token_sha": "e2c694d862fd", + "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff", + "min_rows": 3 + }, + "speculative": { + "rounds": 89, + "drafted": 278, + "accepted": 166, + "acceptance_rate": 0.5971223021582733, + "tokens_per_round": 2.8764044943820224 + }, + "sent": 287.09728275, + "first": 287.177629958, + "last": 290.385496, + "chunks": 90 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 78.52426526343883, + "seconds": 3.247403833, + "prefill_seconds": 0.040318292, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.040579875, + "sampling": "greedy", + "drafts": true, + "token_sha": "e2c694d862fd", + "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff", + "min_rows": 3 + }, + "speculative": { + "rounds": 89, + "drafted": 278, + "accepted": 166, + "acceptance_rate": 0.5971223021582733, + "tokens_per_round": 2.8764044943820224 + }, + "sent": 287.097213666, + "first": 287.138123416, + "last": 290.385478208, + "chunks": 90 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 80.47927459453535, + "seconds": 3.168517625, + "prefill_seconds": 0.119257542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.119520917, + "sampling": "greedy", + "drafts": true, + "token_sha": "e2c694d862fd", + "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff", + "min_rows": 3 + }, + "speculative": { + "rounds": 89, + "drafted": 278, + "accepted": 166, + "acceptance_rate": 0.5971223021582733, + "tokens_per_round": 2.8764044943820224 + }, + "sent": 287.097006458, + "first": 287.217065125, + "last": 290.385527291, + "chunks": 90 + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 212.46021523124688, + "ttft_max": 0.031284874999983, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 212.2716090156747, + "seconds": 1.201291125, + "prefill_seconds": 0.03071125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.03097, + "sampling": "greedy", + "drafts": true, + "token_sha": "a10c9b848ed9", + "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130", + "min_rows": 16 + }, + "speculative": { + "rounds": 43, + "drafted": 658, + "accepted": 213, + "acceptance_rate": 0.32370820668693007, + "tokens_per_round": 5.953488372093023 + }, + "sent": 294.944889416, + "first": 294.976174291, + "last": 296.176399, + "chunks": 44 + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 379.1938092247709, + "ttft_max": 0.1560188750000293, + "requests": [ + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 94.80090021819542, + "seconds": 2.689847875, + "prefill_seconds": 0.031562542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.031844084, + "sampling": "greedy", + "drafts": true, + "token_sha": "a10c9b848ed9", + "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130", + "min_rows": 1 + }, + "speculative": { + "rounds": 67, + "drafted": 251, + "accepted": 188, + "acceptance_rate": 0.749003984063745, + "tokens_per_round": 3.8208955223880596 + }, + "sent": 296.181794916, + "first": 296.214057333, + "last": 298.903864, + "chunks": 68 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 97.7997297917729, + "seconds": 2.607369167, + "prefill_seconds": 0.082535875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.114337917, + "sampling": "greedy", + "drafts": true, + "token_sha": "a10c9b848ed9", + "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130", + "min_rows": 4 + }, + "speculative": { + "rounds": 67, + "drafted": 255, + "accepted": 188, + "acceptance_rate": 0.7372549019607844, + "tokens_per_round": 3.8208955223880596 + }, + "sent": 296.1819095, + "first": 296.296626625, + "last": 298.903974666, + "chunks": 68 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 99.37142863907908, + "seconds": 2.566129958, + "prefill_seconds": 0.123787125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.155574042, + "sampling": "greedy", + "drafts": true, + "token_sha": "a10c9b848ed9", + "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130", + "min_rows": 4 + }, + "speculative": { + "rounds": 67, + "drafted": 255, + "accepted": 188, + "acceptance_rate": 0.7372549019607844, + "tokens_per_round": 3.8208955223880596 + }, + "sent": 296.181857875, + "first": 296.33787675, + "last": 298.903949958, + "chunks": 68 + }, + { + "usage": { + "prompt_tokens": 14, + "completion_tokens": 256, + "total_tokens": 270, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 96.2784902404724, + "seconds": 2.648566667, + "prefill_seconds": 0.041288375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.073060333, + "sampling": "greedy", + "drafts": true, + "token_sha": "a10c9b848ed9", + "token_sha256": "a10c9b848ed99a1439e2054311a64dd9d16933c10009920e35a3eea177bb4130", + "min_rows": 4 + }, + "speculative": { + "rounds": 67, + "drafted": 255, + "accepted": 188, + "acceptance_rate": 0.7372549019607844, + "tokens_per_round": 3.8208955223880596 + }, + "sent": 296.181708541, + "first": 296.255363, + "last": 298.90387875, + "chunks": 68 + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 1, + "aggregate_decode_tps": 141.31967120959874, + "ttft_max": 0.03666970799997671, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 141.24636394193516, + "seconds": 1.805356208, + "prefill_seconds": 0.036061958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.036302625, + "sampling": "greedy", + "drafts": true, + "token_sha": "e2c694d862fd", + "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff", + "min_rows": 16 + }, + "speculative": { + "rounds": 65, + "drafted": 975, + "accepted": 191, + "acceptance_rate": 0.19589743589743588, + "tokens_per_round": 3.9384615384615387 + }, + "sent": 298.908591208, + "first": 298.945260916, + "last": 300.749680625, + "chunks": 66 + } + ] + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt": "chat", + "streams": 4, + "aggregate_decode_tps": 316.7578775374469, + "ttft_max": 0.15931695800003354, + "requests": [ + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 82.21321081821357, + "seconds": 3.101691291, + "prefill_seconds": 0.118398959, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.158899334, + "sampling": "greedy", + "drafts": true, + "token_sha": "e2c694d862fd", + "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff", + "min_rows": 3 + }, + "speculative": { + "rounds": 88, + "drafted": 276, + "accepted": 167, + "acceptance_rate": 0.605072463768116, + "tokens_per_round": 2.909090909090909 + }, + "sent": 300.755190833, + "first": 300.914507791, + "last": 304.016232083, + "chunks": 89 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 80.17450267878608, + "seconds": 3.180562292, + "prefill_seconds": 0.079673916, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.079946708, + "sampling": "greedy", + "drafts": true, + "token_sha": "e2c694d862fd", + "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff", + "min_rows": 3 + }, + "speculative": { + "rounds": 88, + "drafted": 276, + "accepted": 167, + "acceptance_rate": 0.605072463768116, + "tokens_per_round": 2.909090909090909 + }, + "sent": 300.755306, + "first": 300.835610666, + "last": 304.016114666, + "chunks": 89 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 81.17991883253512, + "seconds": 3.141170916, + "prefill_seconds": 0.119087542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.119339375, + "sampling": "greedy", + "drafts": true, + "token_sha": "e2c694d862fd", + "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff", + "min_rows": 3 + }, + "speculative": { + "rounds": 88, + "drafted": 276, + "accepted": 167, + "acceptance_rate": 0.605072463768116, + "tokens_per_round": 2.909090909090909 + }, + "sent": 300.755252541, + "first": 300.875013291, + "last": 304.016133875, + "chunks": 89 + }, + { + "usage": { + "prompt_tokens": 19, + "completion_tokens": 256, + "total_tokens": 275, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "qwen27", + "tokens_per_second": 79.19185080228227, + "seconds": 3.220028291, + "prefill_seconds": 0.040182916, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.040450042, + "sampling": "greedy", + "drafts": true, + "token_sha": "e2c694d862fd", + "token_sha256": "e2c694d862fde073686f73fae065bee5159342b63d54635815db5ece5646efff", + "min_rows": 3 + }, + "speculative": { + "rounds": 88, + "drafted": 276, + "accepted": 167, + "acceptance_rate": 0.605072463768116, + "tokens_per_round": 2.909090909090909 + }, + "sent": 300.755108875, + "first": 300.796106958, + "last": 304.016091958, + "chunks": 89 + } + ] + } + ], + "cold-prefill": [ + { + "round": 0, + "arm": "main-plain", + "prompt_tokens": 2048, + "usage": { + "prompt_tokens": 2048, + "completion_tokens": 1, + "total_tokens": 2049, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 9.475e-05, + "prefill_seconds": 0.152224375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.152786125, + "sampling": "greedy", + "drafts": true, + "token_sha": "f0cbba2470c7", + "token_sha256": "f0cbba2470c7c8706fe77e8d88e947ed8c33100409ffb51ae1af99af7d3077a8", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 0.15443183299999852 + }, + { + "round": 0, + "arm": "main-plain", + "prompt_tokens": 8192, + "usage": { + "prompt_tokens": 8192, + "completion_tokens": 1, + "total_tokens": 8193, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 3.8958e-05, + "prefill_seconds": 0.602027959, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.60268975, + "sampling": "greedy", + "drafts": true, + "token_sha": "94c47d91f1be", + "token_sha256": "94c47d91f1bebd7b85fb5b4210cb24a8125beaf843e6e4a7a80698817c79ae64", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 0.6041695000000011 + }, + { + "round": 0, + "arm": "main-plain", + "prompt_tokens": 32768, + "usage": { + "prompt_tokens": 32768, + "completion_tokens": 1, + "total_tokens": 32769, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 3.6042e-05, + "prefill_seconds": 2.92371825, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 2.924125542, + "sampling": "greedy", + "drafts": true, + "token_sha": "f0cbba2470c7", + "token_sha256": "f0cbba2470c7c8706fe77e8d88e947ed8c33100409ffb51ae1af99af7d3077a8", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 2.9251302499999987 + }, + { + "round": 0, + "arm": "main-plain", + "prompt_tokens": 65536, + "usage": { + "prompt_tokens": 65536, + "completion_tokens": 1, + "total_tokens": 65537, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 3.6083e-05, + "prefill_seconds": 7.367716917, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 7.368493834, + "sampling": "greedy", + "drafts": true, + "token_sha": "3635a91e3da8", + "token_sha256": "3635a91e3da857f7847f68185a116a5260d2593f3913f6b1b66cc2d75b0d6ec0", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 7.370146667 + }, + { + "round": 0, + "arm": "candidate-plain", + "prompt_tokens": 2048, + "usage": { + "prompt_tokens": 2048, + "completion_tokens": 1, + "total_tokens": 2049, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 3.9416e-05, + "prefill_seconds": 0.142128125, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.142321375, + "sampling": "greedy", + "drafts": true, + "token_sha": "f0cbba2470c7", + "token_sha256": "f0cbba2470c7c8706fe77e8d88e947ed8c33100409ffb51ae1af99af7d3077a8", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 0.1428019169999999 + }, + { + "round": 0, + "arm": "candidate-plain", + "prompt_tokens": 8192, + "usage": { + "prompt_tokens": 8192, + "completion_tokens": 1, + "total_tokens": 8193, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 3.4333e-05, + "prefill_seconds": 0.596952333, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.597156042, + "sampling": "greedy", + "drafts": true, + "token_sha": "94c47d91f1be", + "token_sha256": "94c47d91f1bebd7b85fb5b4210cb24a8125beaf843e6e4a7a80698817c79ae64", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 0.5976716660000001 + }, + { + "round": 0, + "arm": "candidate-plain", + "prompt_tokens": 32768, + "usage": { + "prompt_tokens": 32768, + "completion_tokens": 1, + "total_tokens": 32769, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 3.4583e-05, + "prefill_seconds": 2.92241675, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 2.922817417, + "sampling": "greedy", + "drafts": true, + "token_sha": "f0cbba2470c7", + "token_sha256": "f0cbba2470c7c8706fe77e8d88e947ed8c33100409ffb51ae1af99af7d3077a8", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 2.9238302500000017 + }, + { + "round": 0, + "arm": "candidate-plain", + "prompt_tokens": 65536, + "usage": { + "prompt_tokens": 65536, + "completion_tokens": 1, + "total_tokens": 65537, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 3.5083e-05, + "prefill_seconds": 7.322124959, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 7.322879042, + "sampling": "greedy", + "drafts": true, + "token_sha": "3635a91e3da8", + "token_sha256": "3635a91e3da857f7847f68185a116a5260d2593f3913f6b1b66cc2d75b0d6ec0", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 7.324568999999997 + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt_tokens": 2048, + "usage": { + "prompt_tokens": 2048, + "completion_tokens": 1, + "total_tokens": 2049, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 5.0375e-05, + "prefill_seconds": 0.162896375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.16343125, + "sampling": "greedy", + "drafts": true, + "token_sha": "f0cbba2470c7", + "token_sha256": "f0cbba2470c7c8706fe77e8d88e947ed8c33100409ffb51ae1af99af7d3077a8", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 0.16452287500000295 + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt_tokens": 8192, + "usage": { + "prompt_tokens": 8192, + "completion_tokens": 1, + "total_tokens": 8193, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 3.7708e-05, + "prefill_seconds": 0.632174167, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.632618458, + "sampling": "greedy", + "drafts": true, + "token_sha": "94c47d91f1be", + "token_sha256": "94c47d91f1bebd7b85fb5b4210cb24a8125beaf843e6e4a7a80698817c79ae64", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 0.6336222919999983 + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt_tokens": 32768, + "usage": { + "prompt_tokens": 32768, + "completion_tokens": 1, + "total_tokens": 32769, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 3.8084e-05, + "prefill_seconds": 3.046362584, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 3.046803042, + "sampling": "greedy", + "drafts": true, + "token_sha": "f0cbba2470c7", + "token_sha256": "f0cbba2470c7c8706fe77e8d88e947ed8c33100409ffb51ae1af99af7d3077a8", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 3.0479570839999965 + }, + { + "round": 0, + "arm": "candidate-drafted", + "prompt_tokens": 65536, + "usage": { + "prompt_tokens": 65536, + "completion_tokens": 1, + "total_tokens": 65537, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 3.6958e-05, + "prefill_seconds": 7.564455625, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 7.565224041, + "sampling": "greedy", + "drafts": true, + "token_sha": "3635a91e3da8", + "token_sha256": "3635a91e3da857f7847f68185a116a5260d2593f3913f6b1b66cc2d75b0d6ec0", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 7.566995167000002 + }, + { + "round": 1, + "arm": "candidate-plain", + "prompt_tokens": 2048, + "usage": { + "prompt_tokens": 2048, + "completion_tokens": 1, + "total_tokens": 2049, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 3.5834e-05, + "prefill_seconds": 0.142068208, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.142249625, + "sampling": "greedy", + "drafts": true, + "token_sha": "f0cbba2470c7", + "token_sha256": "f0cbba2470c7c8706fe77e8d88e947ed8c33100409ffb51ae1af99af7d3077a8", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 0.14270199999999988 + }, + { + "round": 1, + "arm": "candidate-plain", + "prompt_tokens": 8192, + "usage": { + "prompt_tokens": 8192, + "completion_tokens": 1, + "total_tokens": 8193, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 3.5542e-05, + "prefill_seconds": 0.596721958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.596929459, + "sampling": "greedy", + "drafts": true, + "token_sha": "94c47d91f1be", + "token_sha256": "94c47d91f1bebd7b85fb5b4210cb24a8125beaf843e6e4a7a80698817c79ae64", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 0.5973893749999988 + }, + { + "round": 1, + "arm": "candidate-plain", + "prompt_tokens": 32768, + "usage": { + "prompt_tokens": 32768, + "completion_tokens": 1, + "total_tokens": 32769, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 3.4916e-05, + "prefill_seconds": 2.922063542, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 2.922471709, + "sampling": "greedy", + "drafts": true, + "token_sha": "f0cbba2470c7", + "token_sha256": "f0cbba2470c7c8706fe77e8d88e947ed8c33100409ffb51ae1af99af7d3077a8", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 2.923471584000005 + }, + { + "round": 1, + "arm": "candidate-plain", + "prompt_tokens": 65536, + "usage": { + "prompt_tokens": 65536, + "completion_tokens": 1, + "total_tokens": 65537, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 3.1833e-05, + "prefill_seconds": 7.322574708, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 7.323297667, + "sampling": "greedy", + "drafts": true, + "token_sha": "3635a91e3da8", + "token_sha256": "3635a91e3da857f7847f68185a116a5260d2593f3913f6b1b66cc2d75b0d6ec0", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 7.325058874999996 + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt_tokens": 2048, + "usage": { + "prompt_tokens": 2048, + "completion_tokens": 1, + "total_tokens": 2049, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 4.8791e-05, + "prefill_seconds": 0.16274975, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.163254084, + "sampling": "greedy", + "drafts": true, + "token_sha": "f0cbba2470c7", + "token_sha256": "f0cbba2470c7c8706fe77e8d88e947ed8c33100409ffb51ae1af99af7d3077a8", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 0.16436254200000633 + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt_tokens": 8192, + "usage": { + "prompt_tokens": 8192, + "completion_tokens": 1, + "total_tokens": 8193, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 2.5375e-05, + "prefill_seconds": 0.632193375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.632682667, + "sampling": "greedy", + "drafts": true, + "token_sha": "94c47d91f1be", + "token_sha256": "94c47d91f1bebd7b85fb5b4210cb24a8125beaf843e6e4a7a80698817c79ae64", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 0.6336696670000066 + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt_tokens": 32768, + "usage": { + "prompt_tokens": 32768, + "completion_tokens": 1, + "total_tokens": 32769, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 3.8917e-05, + "prefill_seconds": 3.045708417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 3.046133125, + "sampling": "greedy", + "drafts": true, + "token_sha": "f0cbba2470c7", + "token_sha256": "f0cbba2470c7c8706fe77e8d88e947ed8c33100409ffb51ae1af99af7d3077a8", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 3.0472295000000003 + }, + { + "round": 1, + "arm": "candidate-drafted", + "prompt_tokens": 65536, + "usage": { + "prompt_tokens": 65536, + "completion_tokens": 1, + "total_tokens": 65537, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 4.0959e-05, + "prefill_seconds": 7.564470417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 7.565208167, + "sampling": "greedy", + "drafts": true, + "token_sha": "3635a91e3da8", + "token_sha256": "3635a91e3da857f7847f68185a116a5260d2593f3913f6b1b66cc2d75b0d6ec0", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 7.566989333999999 + }, + { + "round": 1, + "arm": "main-plain", + "prompt_tokens": 2048, + "usage": { + "prompt_tokens": 2048, + "completion_tokens": 1, + "total_tokens": 2049, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 6.125e-05, + "prefill_seconds": 0.152161958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.152673334, + "sampling": "greedy", + "drafts": true, + "token_sha": "f0cbba2470c7", + "token_sha256": "f0cbba2470c7c8706fe77e8d88e947ed8c33100409ffb51ae1af99af7d3077a8", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 0.15385920900000372 + }, + { + "round": 1, + "arm": "main-plain", + "prompt_tokens": 8192, + "usage": { + "prompt_tokens": 8192, + "completion_tokens": 1, + "total_tokens": 8193, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 3.5958e-05, + "prefill_seconds": 0.60266275, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.603419666, + "sampling": "greedy", + "drafts": true, + "token_sha": "94c47d91f1be", + "token_sha256": "94c47d91f1bebd7b85fb5b4210cb24a8125beaf843e6e4a7a80698817c79ae64", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 0.6049087920000034 + }, + { + "round": 1, + "arm": "main-plain", + "prompt_tokens": 32768, + "usage": { + "prompt_tokens": 32768, + "completion_tokens": 1, + "total_tokens": 32769, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 3.7e-05, + "prefill_seconds": 2.924098958, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 2.924542125, + "sampling": "greedy", + "drafts": true, + "token_sha": "f0cbba2470c7", + "token_sha256": "f0cbba2470c7c8706fe77e8d88e947ed8c33100409ffb51ae1af99af7d3077a8", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 2.9256166250000035 + }, + { + "round": 1, + "arm": "main-plain", + "prompt_tokens": 65536, + "usage": { + "prompt_tokens": 65536, + "completion_tokens": 1, + "total_tokens": 65537, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 3.725e-05, + "prefill_seconds": 7.326498708, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 7.32727875, + "sampling": "greedy", + "drafts": true, + "token_sha": "3635a91e3da8", + "token_sha256": "3635a91e3da857f7847f68185a116a5260d2593f3913f6b1b66cc2d75b0d6ec0", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 7.329111124999997 + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt_tokens": 2048, + "usage": { + "prompt_tokens": 2048, + "completion_tokens": 1, + "total_tokens": 2049, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 6.3167e-05, + "prefill_seconds": 0.16316275, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.163665083, + "sampling": "greedy", + "drafts": true, + "token_sha": "f0cbba2470c7", + "token_sha256": "f0cbba2470c7c8706fe77e8d88e947ed8c33100409ffb51ae1af99af7d3077a8", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 0.16489629199999456 + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt_tokens": 8192, + "usage": { + "prompt_tokens": 8192, + "completion_tokens": 1, + "total_tokens": 8193, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 4.0083e-05, + "prefill_seconds": 0.632195333, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.632729583, + "sampling": "greedy", + "drafts": true, + "token_sha": "94c47d91f1be", + "token_sha256": "94c47d91f1bebd7b85fb5b4210cb24a8125beaf843e6e4a7a80698817c79ae64", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 0.633867084000002 + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt_tokens": 32768, + "usage": { + "prompt_tokens": 32768, + "completion_tokens": 1, + "total_tokens": 32769, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 4.2125e-05, + "prefill_seconds": 3.046123, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 3.04655725, + "sampling": "greedy", + "drafts": true, + "token_sha": "f0cbba2470c7", + "token_sha256": "f0cbba2470c7c8706fe77e8d88e947ed8c33100409ffb51ae1af99af7d3077a8", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 3.0476708340000016 + }, + { + "round": 2, + "arm": "candidate-drafted", + "prompt_tokens": 65536, + "usage": { + "prompt_tokens": 65536, + "completion_tokens": 1, + "total_tokens": 65537, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 4.1291e-05, + "prefill_seconds": 7.564734041, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 7.565487958, + "sampling": "greedy", + "drafts": true, + "token_sha": "3635a91e3da8", + "token_sha256": "3635a91e3da857f7847f68185a116a5260d2593f3913f6b1b66cc2d75b0d6ec0", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 7.567298958999999 + }, + { + "round": 2, + "arm": "main-plain", + "prompt_tokens": 2048, + "usage": { + "prompt_tokens": 2048, + "completion_tokens": 1, + "total_tokens": 2049, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 3.9333e-05, + "prefill_seconds": 0.142121708, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.142318875, + "sampling": "greedy", + "drafts": true, + "token_sha": "f0cbba2470c7", + "token_sha256": "f0cbba2470c7c8706fe77e8d88e947ed8c33100409ffb51ae1af99af7d3077a8", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 0.14281366699999865 + }, + { + "round": 2, + "arm": "main-plain", + "prompt_tokens": 8192, + "usage": { + "prompt_tokens": 8192, + "completion_tokens": 1, + "total_tokens": 8193, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 3.575e-05, + "prefill_seconds": 0.597101375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.597292042, + "sampling": "greedy", + "drafts": true, + "token_sha": "94c47d91f1be", + "token_sha256": "94c47d91f1bebd7b85fb5b4210cb24a8125beaf843e6e4a7a80698817c79ae64", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 0.5977911669999969 + }, + { + "round": 2, + "arm": "main-plain", + "prompt_tokens": 32768, + "usage": { + "prompt_tokens": 32768, + "completion_tokens": 1, + "total_tokens": 32769, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 3.6833e-05, + "prefill_seconds": 2.923957417, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 2.924390292, + "sampling": "greedy", + "drafts": true, + "token_sha": "f0cbba2470c7", + "token_sha256": "f0cbba2470c7c8706fe77e8d88e947ed8c33100409ffb51ae1af99af7d3077a8", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 2.9254025830000074 + }, + { + "round": 2, + "arm": "main-plain", + "prompt_tokens": 65536, + "usage": { + "prompt_tokens": 65536, + "completion_tokens": 1, + "total_tokens": 65537, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 3.6583e-05, + "prefill_seconds": 7.32684075, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 7.327646125, + "sampling": "greedy", + "drafts": true, + "token_sha": "3635a91e3da8", + "token_sha256": "3635a91e3da857f7847f68185a116a5260d2593f3913f6b1b66cc2d75b0d6ec0", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 7.329436958000002 + }, + { + "round": 2, + "arm": "candidate-plain", + "prompt_tokens": 2048, + "usage": { + "prompt_tokens": 2048, + "completion_tokens": 1, + "total_tokens": 2049, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 3.7417e-05, + "prefill_seconds": 0.142097667, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.14230825, + "sampling": "greedy", + "drafts": true, + "token_sha": "f0cbba2470c7", + "token_sha256": "f0cbba2470c7c8706fe77e8d88e947ed8c33100409ffb51ae1af99af7d3077a8", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 0.1427672080000093 + }, + { + "round": 2, + "arm": "candidate-plain", + "prompt_tokens": 8192, + "usage": { + "prompt_tokens": 8192, + "completion_tokens": 1, + "total_tokens": 8193, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 3.4916e-05, + "prefill_seconds": 0.596914375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.597105708, + "sampling": "greedy", + "drafts": true, + "token_sha": "94c47d91f1be", + "token_sha256": "94c47d91f1bebd7b85fb5b4210cb24a8125beaf843e6e4a7a80698817c79ae64", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 0.5975776660000065 + }, + { + "round": 2, + "arm": "candidate-plain", + "prompt_tokens": 32768, + "usage": { + "prompt_tokens": 32768, + "completion_tokens": 1, + "total_tokens": 32769, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 3.4459e-05, + "prefill_seconds": 2.922614041, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 2.923045667, + "sampling": "greedy", + "drafts": true, + "token_sha": "f0cbba2470c7", + "token_sha256": "f0cbba2470c7c8706fe77e8d88e947ed8c33100409ffb51ae1af99af7d3077a8", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 2.9240610420000053 + }, + { + "round": 2, + "arm": "candidate-plain", + "prompt_tokens": 65536, + "usage": { + "prompt_tokens": 65536, + "completion_tokens": 1, + "total_tokens": 65537, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "runtime": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lanes", + "tokens_per_second": 0.0, + "seconds": 2.925e-05, + "prefill_seconds": 7.347452875, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 7.348175667, + "sampling": "greedy", + "drafts": true, + "token_sha": "3635a91e3da8", + "token_sha256": "3635a91e3da857f7847f68185a116a5260d2593f3913f6b1b66cc2d75b0d6ec0", + "min_rows": 0 + }, + "speculative": { + "rounds": 0, + "drafted": 0, + "accepted": 0, + "acceptance_rate": 0.0, + "tokens_per_round": 0.0 + }, + "wall": 7.349954542000006 + } + ] + } + } +} diff --git a/docs/receipts/lfm2-dspark-review/qualification.py b/docs/receipts/lfm2-dspark-review/qualification.py new file mode 100644 index 0000000000..7ca0c238cc --- /dev/null +++ b/docs/receipts/lfm2-dspark-review/qualification.py @@ -0,0 +1,220 @@ +import concurrent.futures as cf +import fcntl +import json +import os +from pathlib import Path +import random +import signal +import statistics +import subprocess +import sys +import threading +import time +import urllib.request + +ROOT = Path(__file__).resolve().parents[3] +OUT = Path(os.environ.get('TF_QUAL_OUT', '/tmp/tf1394-evidence')) +OUT.mkdir(exist_ok=True) +HUB = Path.home()/'.cache/huggingface/hub' +MODEL = Path(os.environ.get('TF_QUAL_MODEL', HUB/'models--LiquidAI--LFM2.5-24B-A2B-MLX-4bit/snapshots/667408d494bde191c2cb9dd4f4549827816b9275')) +DRAFT = Path(os.environ.get('TF_QUAL_DRAFTER', Path.home()/'models/LiquidAI--LFM2.5-24B-A2B-DSpark')) +CAND = Path(os.environ.get('TF_QUAL_CANDIDATE', ROOT/'zig-out/native/bin/tensorfold-native')) +MAIN = Path(os.environ.get('TF_QUAL_MAIN', '/tmp/tf1394-main-base/zig-out/native/bin/tensorfold-native')) +BASE = 'http://127.0.0.1:18194' +PROMPTS = json.loads((ROOT/'tools/bench_prompts.json').read_text())[:8] + +def emit(kind, **data): + print(json.dumps({'event':kind, **data}), flush=True) + +def save(name, data): + (OUT/(name+'.json')).write_text(json.dumps(data, indent=2)+'\n') + +def post(body, stream=False, cancel=None): + body = {'model':'review', 'max_tokens':128, 'temperature':0, 'seed':1234, + 'ignore_eos':True, 'return_token_ids':True, **body, 'stream':stream} + if stream: body['stream_options']={'include_usage':True} + chat='messages' in body + if chat: body['chat_template_kwargs']={'enable_thinking':False} + endpoint='/v1/chat/completions' if chat else '/v1/completions' + req = urllib.request.Request(BASE+endpoint, data=json.dumps(body).encode(), headers={'Content-Type':'application/json'}) + sent = time.perf_counter() + with urllib.request.urlopen(req, timeout=3600) as r: + if not stream: + x=json.load(r) + content=x['choices'][0].get('message',{}).get('content',x['choices'][0].get('text','')) + return {'usage':x['usage'], 'runtime':x['tensorfold'], 'speculative':x.get('speculative',{}), 'content':content, 'wall':time.perf_counter()-sent} + usage={}; runtime={}; spec={}; first=None; last=None; chunks=0; done=False + for raw in r: + line=raw.decode().strip() + if line=='data: [DONE]': done=True; break + if not line.startswith('data:'): continue + x=json.loads(line[5:]) + if 'error' in x: raise RuntimeError(str(x['error'])) + usage=x.get('usage') or usage; runtime=x.get('tensorfold') or runtime; spec=x.get('speculative') or spec + if any(c.get('text') or c.get('delta',{}).get('content') for c in x.get('choices',[])): + now=time.perf_counter(); first=first or now; last=now; chunks+=1 + if cancel and chunks>=cancel: return {'cancelled':True} + if not done or not runtime.get('token_sha256'): raise RuntimeError('incomplete stream') + return {'usage':usage, 'runtime':runtime, 'speculative':spec, 'sent':sent, 'first':first, 'last':last, 'chunks':chunks} + +class Server: + def __init__(self, label, binary=CAND, mode='bf16', context=16384, parallel=16, cache=0, validation=False, model=MODEL, draft=DRAFT): + self.label=label; self.binary=binary; self.mode=mode; self.context=context; self.parallel=parallel + self.cache=cache; self.validation=validation; self.model=model; self.draft=draft + def __enter__(self): + processes=subprocess.check_output(['ps','-Ao','pid=,comm='],text=True) + busy=[line for line in processes.splitlines() if line.split() and Path(line.split()[-1]).name in ('tensorfold-native','tf-lfm2-exact','tf-lfm2-dspark')] + if busy: raise RuntimeError('another model process is already running') + args=[str(self.binary),'serve',str(self.model),'--name','review','--host','127.0.0.1','--port','18194','--parallel',str(self.parallel),'--context',str(self.context),'--prompt-cache-gib',str(self.cache),'--keep-warm','900'] + if self.mode!='plain': args+=['--drafter',str(self.draft)] + if self.mode=='8': args+=['--drafter-bits','8'] + env=dict(os.environ) + if self.validation: env['MTL_DEBUG_LAYER']='1' + self.log=open(OUT/(self.label+'.server.log'),'w') + self.proc=subprocess.Popen(args, stdout=self.log, stderr=subprocess.STDOUT, env=env, start_new_session=True) + start=time.monotonic() + while time.monotonic()-start<600: + if self.proc.poll() is not None: raise RuntimeError(self.label+' server exited') + try: + with urllib.request.urlopen(BASE+'/v1/models',timeout=1) as r: json.load(r) + emit('ready', label=self.label, seconds=time.monotonic()-start) + return self + except Exception: time.sleep(.5) + self.__exit__(None,None,None) + raise TimeoutError('load timeout') + def __exit__(self,*args): + started=time.monotonic(); forced=False + if self.proc.poll() is None: + self.proc.send_signal(signal.SIGTERM) + try: self.proc.wait(timeout=90) + except subprocess.TimeoutExpired: + forced=True; os.killpg(self.proc.pid,signal.SIGKILL); self.proc.wait() + self.log.close() + save(self.label+'.shutdown',{'forced_kill':forced,'seconds':time.monotonic()-started,'returncode':self.proc.returncode}) + if forced: raise RuntimeError(self.label+' required forced shutdown') + +def digest(r): return r['runtime']['token_sha256'] + +def functional(): + results={} + first_messages=[{'role':'user','content':'Public cache fixture.\n'+('alpha beta gamma delta\n'*700)+'\nExplain matrix multiplication in one paragraph.'}] + with Server('functional-cache',cache=1,validation=True): + first=post({'messages':first_messages,'max_tokens':64}) + messages=first_messages+[{'role':'assistant','content':first['content']},{'role':'user','content':'Give a small numerical example and explain each step.'}] + resumed=post({'messages':messages,'max_tokens':64}) + if resumed['usage'].get('prompt_tokens_details',{}).get('cached_tokens',resumed['usage'].get('cached_tokens',0))<=0: + raise AssertionError('resume did not report reused tokens: '+str(resumed['usage'])) + results['resumed']=resumed; results['resume_request']={'messages':messages,'max_tokens':64} + refs=[post({'prompt':p['prompt'],'max_tokens':64}) for p in PROMPTS] + results['stress_reference_hashes']=[{'prompt':p['name'],'sha256':digest(r)} for p,r in zip(PROMPTS,refs)] + results['stress']=[] + for slots,seconds in [(6,60),(16,90)]: + end=time.monotonic()+seconds + def worker(i): + rng=random.Random(4242+i); counts={'completed':0,'cancelled':0,'wrong':0,'errors':[]} + while time.monotonic()0 + sampled=[] + for n in [1,4,8,16]: + bodies=[{'prompt':PROMPTS[i%len(PROMPTS)]['prompt'],'temperature':.7,'top_p':.95,'seed':4242+i,'max_tokens':64} for i in range(n)] + plain=[post({**b,'draft':False}) for b in bodies] + solo=[post(b) for b in bodies] + gate=threading.Barrier(n) + def run(b): gate.wait(); return post(b) + with cf.ThreadPoolExecutor(n) as ex: together=list(ex.map(run,bodies)) + row={'streams':n,'hashes':[[digest(p),digest(s),digest(t)] for p,s,t in zip(plain,solo,together)]} + sampled.append(row); emit('sampled',streams=n,equal=sum(len(set(x))==1 for x in row['hashes'])) + assert all(len(set(x))==1 for x in row['hashes']) + results['sampled']=sampled; save('functional',results) + with Server('functional-fresh'): + fresh=post({'messages':messages,'max_tokens':64}) + assert digest(fresh)==digest(results['resumed']) + results['fresh']=fresh; results['resume_equal']=True; save('functional',results); emit('resume',equal=True) + +def long_context(): + results=[] + with Server('long-context',parallel=1,context=131072): + tail=[825,264,15352,4829,9944,13,271,279,854,22373] + for length in [4096,16384,65536,130816]: + ids=(tail*((length+9)//10))[:length] + plain=post({'prompt':ids,'max_tokens':64,'draft':False}) + drafted=post({'prompt':ids,'max_tokens':64}) + row={'length':length,'plain':plain,'drafted':drafted,'equal':digest(plain)==digest(drafted)} + results.append(row); save('long-context',results); emit('long-context',length=length,equal=row['equal']) + assert row['equal'] + for length in [65536,130816]: + emit('long-reference-start',length=length) + with open(OUT/('dspark-reference-'+str(length)+'.log'),'w') as log: + r=subprocess.run([str(ROOT/'zig-out/bin/tf-lfm2-dspark'),str(MODEL),str(DRAFT),str(length)],stdout=log,stderr=subprocess.STDOUT,timeout=3600) + emit('long-reference-done',length=length,exit=r.returncode) + assert r.returncode==0 + +def performance(family='lfm2'): + model=MODEL; draft=DRAFT; widths=[1,4,8,16] + if family=='qwen': + model=HUB/'models--TensorFold--Qwen3.8-27B-MLX-4bit/snapshots/22d8d538154e0e5b6f8dcb1fcc74a60a0798104c' + draft=HUB/'models--z-lab--Qwen3.8-27B-DFlash2/snapshots/50307d4c4cde6860d4eee73e2547cd786fe8e8a4' + widths=[1,4] + prompts=[{'name':'code','prompt':'Write a short Python function that computes the Fibonacci sequence and explain it.'},{'name':'chat','prompt':'Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.'}] + arms={'main-plain':(MAIN,'plain'),'candidate-plain':(CAND,'plain'),'candidate-drafted':(CAND,'bf16')} + if family=='qwen': arms={'main-drafted':(MAIN,'bf16'),'candidate-drafted':(CAND,'bf16')} + results=[]; reference={} + order=list(arms) + for round_ in range(3): + rotated=order[round_%len(order):]+order[:round_%len(order)] + for arm in rotated: + binary,mode=arms[arm] + with Server(f'perf-{family}-{round_}-{arm}',binary,mode,4096,max(widths),0,False,model,draft): + for p in prompts: post({'prompt':p['prompt'],'max_tokens':32}) + for p in prompts: + for width in widths: + gate=threading.Barrier(width) + def run(i): gate.wait(); return post({'prompt':p['prompt'],'max_tokens':256},True) + with cf.ThreadPoolExecutor(width) as ex: rows=list(ex.map(run,range(width))) + key=p['name']; hashes=[digest(x) for x in rows] + if key not in reference: reference[key]=hashes[0] + assert all(h==reference[key] for h in hashes),(family,arm,key,'token mismatch') + first=min(x['first'] for x in rows); last=max(x['last'] for x in rows) + row={'round':round_,'arm':arm,'prompt':key,'streams':width,'aggregate_decode_tps':width*255/(last-first),'ttft_max':max(x['first']-x['sent'] for x in rows),'requests':rows} + results.append(row); save('performance-'+family,results) + emit('performance',family=family,round=round_,arm=arm,prompt=key,streams=width,tps=round(row['aggregate_decode_tps'],2)) + +def cold_prefill(): + results=[] + arms={'main-plain':(MAIN,'plain'),'candidate-plain':(CAND,'plain'),'candidate-drafted':(CAND,'bf16')} + order=list(arms); cycle=[825,264,15352,4829,9944,13,271,279,854,22373] + for round_ in range(3): + for arm in order[round_:]+order[:round_]: + binary,mode=arms[arm] + with Server(f'cold-{round_}-{arm}',binary,mode,131072,1): + for length in [2048,8192,32768,65536]: + ids=(cycle*((length+9)//10))[:length] + r=post({'prompt':ids,'max_tokens':1}) + assert r['usage']['prompt_tokens']==length + assert r['usage']['prompt_tokens_details']['cached_tokens']==0 + row={'round':round_,'arm':arm,'prompt_tokens':length,**r} + results.append(row); save('cold-prefill',results) + emit('cold-prefill',round=round_,arm=arm,length=length,ttft=r['runtime']['time_to_first_token']) + +if __name__=='__main__': + phase=sys.argv[1] + with open('/tmp/tensorfold-model.lock','a') as lock: + emit('waiting-for-model-lock',phase=phase) + fcntl.flock(lock,fcntl.LOCK_EX) + emit('starting',phase=phase) + if phase=='functional': functional() + elif phase=='long': long_context() + elif phase=='performance': performance(sys.argv[2] if len(sys.argv)>2 else 'lfm2') + elif phase=='cold': cold_prefill() From 57354b5489a849f3341ef48c23219e2c7869e96e Mon Sep 17 00:00:00 2001 From: BlivionIaG <15323295+BlivionIaG@users.noreply.github.com> Date: Wed, 28 Oct 2026 15:27:23 +0100 Subject: [PATCH 37/39] style(cli): keep merged CLI within the line limit --- zig/src/server/cli.zig | 1 - 1 file changed, 1 deletion(-) diff --git a/zig/src/server/cli.zig b/zig/src/server/cli.zig index 010160f588..6597ddd6f4 100644 --- a/zig/src/server/cli.zig +++ b/zig/src/server/cli.zig @@ -153,7 +153,6 @@ pub const flags = [_]Flag{ // the native server's own: Cortheon's reasoning program and server-run web research for every chat request .{ .name = "--cortheon", .kind = .store_true, .native = true }, }; - /// The variables this binary honours as the Python engine does, then the CUDA build's. pub const env = [_][]const u8{ "TENSORFOLD_API_KEY", "TENSORFOLD_NO_LIVE", "TENSORFOLD_SEED_SALT", "TENSORFOLD_REQUEST_LOG", "TENSORFOLD_NO_UPDATE_CHECK", "HF_HOME", "HF_HUB_CACHE", "HF_HUB_OFFLINE", "BRAVE_SEARCH_API_KEY" } ++ (if (cuda_build) [_][]const u8{ "TF_CUDA_DEVICE", "TF_CUDA_SEGMENTS", "TENSORFOLD_CUDA_KERNELS", "TENSORFOLD_MEMORY_RESERVE_GIB", "TENSORFOLD_CUDA_MEMORY_LIMIT_GB", "TENSORFOLD_P2P_DIAGNOSTICS", "TENSORFOLD_P2P_PROBE" } else [_][]const u8{}); From b4deee27775d81d27c626bd2c16eeb1a88b0b5f5 Mon Sep 17 00:00:00 2001 From: BlivionIaG <15323295+BlivionIaG@users.noreply.github.com> Date: Wed, 28 Oct 2026 13:10:30 +0100 Subject: [PATCH 38/39] perf(lfm2): tile shared bf16 holds and price measured depths --- docs/recipes/lfm2.5-24b-a2b.md | 12 +++++--- zig/src/core/lanes/hold_price.zig | 36 +++++++++++++++++++--- zig/src/families/lfm2_moe/dspark.zig | 11 +++---- zig/src/families/lfm2_moe/dspark_block.zig | 9 ++++-- zig/tests/contract/metal_lfm2_bf16.zig | 4 ++- 5 files changed, 53 insertions(+), 19 deletions(-) diff --git a/docs/recipes/lfm2.5-24b-a2b.md b/docs/recipes/lfm2.5-24b-a2b.md index 9f13a18e37..13992e4a91 100644 --- a/docs/recipes/lfm2.5-24b-a2b.md +++ b/docs/recipes/lfm2.5-24b-a2b.md @@ -96,7 +96,8 @@ without the head verifies by the core's depth rule, from each stream's measured A stream with a copy ahead verifies the copy and asks no hold, and a drafting stream prices its copy windows by its own copies. The lane core also weighs the hold pass itself: the drafter times holds of 1 to 16 -blocks at load and corrects those prices toward each settled hold's GPU time. +blocks at every admissible depth at load and corrects those prices toward each settled hold's GPU time. +Prices interpolate neighboring measured depths, preserving the cost changes at kernel transitions. `tf-lfm2-dspark` also prints the confidence head against its fp32 reference and times shared holds of 2 to 16 streams. @@ -108,7 +109,10 @@ admits. At load the drafter logs its weights, scratch and a stream's bytes. `tf- allocations over twenty served drafter rounds, which must be none. A hold runs on one concurrent encoder with a barrier only where a step reads an earlier step's writes: q, k and v, -and gate and up, share one sums pass, and the blocks of a shared hold attend side by side. A projection takes the -scalar twin up to 8 rows, the window kernel past them where the GPU has no tensor units, and the tensor units from -9 rows (the head from 5) where it has them. When one lane's stage of 128 verified rows fills, its absorb pass takes +and gate and up, share one sums pass, and the blocks of a shared hold attend side by side. An 8-bit projection takes +the scalar twin up to 8 rows, the window kernel past them where the GPU has no tensor units, and the tensor units +from 9 rows (the head from 5) where it has them. Held blocks use one bf16 16-row group, then the existing +64x64 tiles past 16 rows, avoiding a second weight pass for shared holds of 17 to 32 rows. Context ingestion +keeps the original row kernel through 32 rows, preserving the persistent context projections. +When one lane's stage of 128 verified rows fills, its absorb pass takes every lane's staged rows, so the lanes' stages fill together. diff --git a/zig/src/core/lanes/hold_price.zig b/zig/src/core/lanes/hold_price.zig index 807531a6cd..c3eeb0208a 100644 --- a/zig/src/core/lanes/hold_price.zig +++ b/zig/src/core/lanes/hold_price.zig @@ -30,23 +30,49 @@ pub fn holdMs(costs: []const HoldCost, streams: usize, depth: i64) f64 { return at_lo + t * (atDepth(costs, hi, depth) - at_lo); } -/// One stream count's hold ms at `depth`: linear through its two smallest timed depths (one alone: as timed). +/// Interpolate neighboring depths, or extrapolate the nearest end segment (one alone: as timed). fn atDepth(costs: []const HoldCost, streams: u32, depth: i64) f64 { var a: ?HoldCost = null; var b: ?HoldCost = null; for (costs) |c| { if (c.streams != streams) continue; - if (a == null or c.depth < a.?.depth) { - b = a; + if (c.depth == depth) return c.ms; + if (c.depth < depth and (a == null or c.depth > a.?.depth)) a = c; + if (c.depth > depth and (b == null or c.depth < b.?.depth)) b = c; + } + if (a == null) { + a = b.?; + b = null; + for (costs) |c| if (c.streams == streams and c.depth > a.?.depth and (b == null or c.depth < b.?.depth)) { + b = c; + }; + if (b == null) return a.?.ms; + } else if (b == null) { + b = a.?; + a = null; + for (costs) |c| if (c.streams == streams and c.depth < b.?.depth and (a == null or c.depth > a.?.depth)) { a = c; - } else if (b == null or c.depth < b.?.depth) b = c; + }; + if (a == null) return b.?.ms; } const x = a.?; - const y = b orelse return x.ms; + const y = b.?; const t = (@as(f64, @floatFromInt(depth)) - @as(f64, @floatFromInt(x.depth))) / @as(f64, @floatFromInt(y.depth - x.depth)); return x.ms + t * (y.ms - x.ms); } +test "hold prices retain measured kernel transitions and use the adjacent depth segment" { + const t = [_]HoldCost{ + .{ .streams = 8, .depth = 4, .ms = 18 }, + .{ .streams = 8, .depth = 1, .ms = 10 }, + .{ .streams = 8, .depth = 2, .ms = 17 }, + }; + try std.testing.expectEqual(@as(f64, 17), holdMs(&t, 8, 2)); + try std.testing.expectEqual(@as(f64, 17.5), holdMs(&t, 8, 3)); + try std.testing.expectEqual(@as(f64, 18.5), holdMs(&t, 8, 5)); + try std.testing.expectEqual(@as(f64, 3), holdMs(&t, 8, 0)); +} + test "a hold's ms between and past the timed passes" { const t = [_]HoldCost{ .{ .streams = 1, .depth = 1, .ms = 1.0 }, .{ .streams = 1, .depth = 4, .ms = 2.5 }, .{ .streams = 4, .depth = 1, .ms = 3.0 }, .{ .streams = 4, .depth = 4, .ms = 4.5 }, .{ .streams = 8, .depth = 1, .ms = 4.0 } }; try std.testing.expectApproxEqAbs(@as(f64, 1.0), holdMs(&t, 1, 1), 1e-12); diff --git a/zig/src/families/lfm2_moe/dspark.zig b/zig/src/families/lfm2_moe/dspark.zig index 6993f95fcc..df55a38e9a 100644 --- a/zig/src/families/lfm2_moe/dspark.zig +++ b/zig/src/families/lfm2_moe/dspark.zig @@ -20,7 +20,7 @@ pub const pass_rows = 32; pub const absorb_rows = 2048; /// Verified rows a lane keeps on the host until its next hold absorbs them (a round's rows for no GPU work). pub const stage_rows = 128; -/// The blocks a hold is timed with at load, each at one draft and at four (two at 16 blocks: one 32-row pass). +/// The block counts timed at load, at every depth that fits one pass and the checkpoint's block. const hold_grid = [_]u32{ 1, 2, 4, 8, 16 }; const kw = c.kv_heads * c.head_dim; @@ -86,7 +86,7 @@ pub const Drafter = struct { /// One stream's hold wall ms at full depth, and that over the block (the depth rule's cost a draft). hold_ms: f64 = 0, step_ms: f64 = 0, - holds: [hold_grid.len * 2]lanes.hold_price.HoldCost = undefined, // hold passes timed at load, then by settled holds + holds: [2 * pass_rows]lanes.hold_price.HoldCost = undefined, // the power-of-two block counts sum to fewer than 2 * pass_rows depths holds_len: usize = 0, event: mtl.Event = undefined, signalled: u64 = 0, @@ -540,9 +540,8 @@ pub const Drafter = struct { var blocks: [hold_grid[hold_grid.len - 1]]block.Block = undefined; for (made, &blocks) |lane, *b| b.* = .{ .lane = lane, .pending = 1, .anchor = 0, .depth = 1 }; d.holds_len = 0; - for (hold_grid) |streams| for ([_]u32{ 1, @min(4, pass_rows / streams) }, 0..) |depth, j| { - if (j == 1 and depth == 1) continue; - for (blocks[0..streams]) |*b| b.depth = depth; + for (hold_grid) |streams| for (1..@min(d.config.block, pass_rows / streams) + 1) |depth| { + for (blocks[0..streams]) |*b| b.depth = @intCast(depth); var ms_best = std.math.floatMax(f64); for (0..4) |i| { const t0 = std.Io.Clock.awake.now(d.io); @@ -550,7 +549,7 @@ pub const Drafter = struct { const ms = @as(f64, @floatFromInt(t0.durationTo(std.Io.Clock.awake.now(d.io)).toNanoseconds())) / 1e6; if (i > 0) ms_best = @min(ms_best, ms); } - d.holds[d.holds_len] = .{ .streams = streams, .depth = depth, .ms = ms_best }; + d.holds[d.holds_len] = .{ .streams = streams, .depth = @intCast(depth), .ms = ms_best }; d.holds_len += 1; }; } diff --git a/zig/src/families/lfm2_moe/dspark_block.zig b/zig/src/families/lfm2_moe/dspark_block.zig index 0dd20d8ef5..a737582c5e 100644 --- a/zig/src/families/lfm2_moe/dspark_block.zig +++ b/zig/src/families/lfm2_moe/dspark_block.zig @@ -49,6 +49,7 @@ pub const Block = struct { lane: *Lane, pending: u32, anchor: u64, depth: u32 }; const Enc = struct { d: *Drafter, e: mtl.ComputeEncoder, + bf16_rows_max: usize = 16, conc: bool = false, // a concurrent encoder: each step waits for the writes before it at `sync` /// Later dispatches see the writes before this point (a no-op on a serial encoder). @@ -98,10 +99,10 @@ const Enc = struct { for (ws, outs) |w, out| try tile.encode(x.e, t, w, in.buf, in.off, x.d.s.sums, 0, out.buf, out.off, rows); } - /// y[rows] = x[rows] W^T over the released bf16 matrix: 16 rows a pass of W up to 32 rows, else 64x64 tiles. + /// y[rows] = x[rows] W^T over the released bf16 matrix: one 16-row pass of W, else 64x64 tiles. fn bf16(x: Enc, m: dw.Bf16, in: Ref, out: Ref, rows: usize) void { const args = Bf16Args{ .rows = @intCast(rows), .n = @intCast(m.n), .k = @intCast(m.k) }; - const few = rows <= 32; + const few = rows <= x.bf16_rows_max; if (few) x.pipe("lfm2_bf16_rows") else x.pipe("lfm2_bf16_mma"); x.e.setBuffer(in.buf, in.off, 0); x.e.setBuffer(m.w, 0, 1); @@ -182,7 +183,9 @@ pub fn absorbPass(d: *Drafter, e: mtl.ComputeEncoder, pieces: []const Piece) !vo return absorbOn(.{ .d = d, .e = e }, pieces); } -fn absorbOn(x: Enc, pieces: []const Piece) !void { +fn absorbOn(encoder: Enc, pieces: []const Piece) !void { + var x = encoder; + x.bf16_rows_max = 32; // preserve the context cache's original projection arithmetic const d = x.d; const s = &d.s; var rows: usize = 0; diff --git a/zig/tests/contract/metal_lfm2_bf16.zig b/zig/tests/contract/metal_lfm2_bf16.zig index 0a191a48fa..fcbe718243 100644 --- a/zig/tests/contract/metal_lfm2_bf16.zig +++ b/zig/tests/contract/metal_lfm2_bf16.zig @@ -17,8 +17,10 @@ const rows_served = [_]c.Shape{ .{ .rows = 1, .n = 2048, .k = 2048 }, .{ .rows = 9, .n = 512, .k = 2048 }, .{ .rows = 16, .n = 6144, .k = 2048 }, .{ .rows = 32, .n = 2048, .k = 6144 }, .{ .rows = 4, .n = 2048, .k = 10240 }, }; -/// A cold prompt's absorb: the taps' projection (2,048 from 10,240) and k and v, up to 2,048 rows a pass. +/// Shared holds past 16 rows, and a cold prompt's absorb up to 2,048 rows a pass. const tile_served = [_]c.Shape{ + .{ .rows = 17, .n = 2048, .k = 2048 }, .{ .rows = 18, .n = 512, .k = 2048 }, + .{ .rows = 24, .n = 6144, .k = 2048 }, .{ .rows = 32, .n = 2048, .k = 6144 }, .{ .rows = 33, .n = 2048, .k = 2048 }, .{ .rows = 128, .n = 512, .k = 2048 }, .{ .rows = 2048, .n = 2048, .k = 10240 }, .{ .rows = 70, .n = 6144, .k = 2048 }, }; From 82dfaec0a17f9411364bebfb322bc4c7a9fc0afe Mon Sep 17 00:00:00 2001 From: BlivionIaG <15323295+BlivionIaG@users.noreply.github.com> Date: Wed, 28 Oct 2026 15:34:27 +0100 Subject: [PATCH 39/39] docs: record LFM2 regression investigation and M4/M5 qualification --- docs/receipts/lfm2-dspark-review/README.md | 49 + .../performance-investigation.json | 1517 +++++++++++++++++ .../lfm2-dspark-review/qualification.py | 9 +- 3 files changed, 1573 insertions(+), 2 deletions(-) create mode 100644 docs/receipts/lfm2-dspark-review/performance-investigation.json diff --git a/docs/receipts/lfm2-dspark-review/README.md b/docs/receipts/lfm2-dspark-review/README.md index 17dc78407e..010c84717b 100644 --- a/docs/receipts/lfm2-dspark-review/README.md +++ b/docs/receipts/lfm2-dspark-review/README.md @@ -133,3 +133,52 @@ validation; the rejected shared/lone guards have not been restored. M1, M2, M3, CUDA and HIP were not rerun for this revision. Focused Metal checks are not a full cross-family kernel-contract receipt. All nine GitHub checks passed on `2dbbc0e8f` before this receipt-only update. + +### Performance follow-up (`b4deee277`, M4 and M5) + +Candidate source `b4deee277` keeps context ingestion on the original bf16 row +kernel through 32 rows, and uses the existing tiled kernel for held blocks past +16 rows. It measures each admissible held depth and interpolates neighboring +measured prices. The hold-price unit test covers a nonlinear timing curve. +Qwen/DFlash2 dispatch and scheduling are unchanged; only LFM2 supplies this hold +price table. + +`performance-investigation.json` retains all 64 M4 comparison cells, their output +hashes, source hashes and actual arm order. Four-round median changes against +the published PR are +1.06%, +0.55%, effectively flat and +0.57% for code at +1/4/8/16 streams; prose changes are +0.27%, +0.75%, +0.41% and +0.42%. +The 43.99 tok/s lone-code outlier is retained. The final arm order repeats the +previous order, so this is not a perfectly balanced four-round experiment. +All reply hashes agree. Independent fp32 drafter picks agree in all nine cases. +Focused DSpark, shared-lane equality and native-option checks pass: 49 build +steps and 56 tests. Formatting, whitespace and harness syntax checks pass. +These small gains do not establish a no-loss result against main. + +An earlier candidate changed context projections as well as held blocks and +regressed M4 16-stream prose by 8.38%. Preserving context projections removes +that regression in this comparison, supporting context arithmetic as a cause. +A separate acceptance-cap scheduler experiment lost roughly 2–3% in some M5 +cells and is excluded. Faster individual holds alone did not guarantee faster +serving. The exact retained candidate also completed M5 validation, with source hashes +matching this tree and tested executable hashes retained in the JSON receipt. + + +M5 completed 72 cells in three Latin rotations of main without the external +DSpark drafter, the published PR and the candidate. Four-round M4 and three-round +M5 results use the same 256-token requests and aggregate decode metric. The main +arm still permits suffix copying; it is not the literal `"draft": false` oracle. +Candidate versus published median changes at 1/4/8/16 streams are -0.26%, +0.04%, ++0.31%, +0.58% for code and -0.04%, -0.02%, +0.06%, +0.72% for prose. +Against main, code is +21.87%, -0.90%, -2.22%, -3.51%; prose is +36.18%, -0.55%, +-1.19%, -0.36%. Wider concurrency therefore still does not clear a no-loss gate. + +Final M5 LFM2 contracts: 11 pass, zero failures, all 17 planted controls red, +zero limit problems. Independent FP32 drafter picks agree in all nine cases, +with zero host allocations/resizes in 20 steady rounds. Resumed/fresh equality, +29 seeded sampled cases, eight 8-bit cells across 1/4/8/16 streams, and long +contexts of 65,536 and 130,816 tokens (drafted/undrafted and four concurrent/solo) +all pass. Cancellation stress completes 423 requests with 234 cancellations at +six slots and 827 with 445 cancellations at sixteen slots, zero wrong replies +or request errors. All servers shut down gracefully. Hashes and counters are in +`performance-investigation.json`. This extends, rather than replaces, the +historical full forward/cache and qualification receipts above. diff --git a/docs/receipts/lfm2-dspark-review/performance-investigation.json b/docs/receipts/lfm2-dspark-review/performance-investigation.json new file mode 100644 index 0000000000..6bc04e9b0a --- /dev/null +++ b/docs/receipts/lfm2-dspark-review/performance-investigation.json @@ -0,0 +1,1517 @@ +{ + "status": "M4 and M5 comparison complete; no no-loss pass against main", + "published_source": "8bd05874d98d9449ad2f8ddb8796bf26e229874b", + "device": "Apple M4 Pro, 20 GPU cores, 48 GiB", + "protocol": { + "generated_tokens": 256, + "warmup_tokens": 32, + "context": 4096, + "parallel_limit": 16, + "cache": 0, + "sampling": "greedy, seed 1234", + "aggregate_tps": "streams * 255 / (latest last token - earliest first token)", + "arm_order_by_round": [ + [ + "published", + "candidate" + ], + [ + "candidate", + "published" + ], + [ + "published", + "candidate" + ], + [ + "published", + "candidate" + ] + ] + }, + "source_sha256": { + "zig/src/families/lfm2_moe/dspark_block.zig": "d2dd077581904759967446ebe2ad195c083ae73c004862de09907ffb4671ac41", + "zig/src/families/lfm2_moe/dspark.zig": "06b5dda907dcb6f44804edfd0c66466bb7422365516613478c777a6e55f46302", + "zig/src/core/lanes/hold_price.zig": "2e6cbe4c477ae19c9edc527a74471748fc8af012725f402ee4c81ed3bf021dba" + }, + "cells": [ + { + "round": 0, + "arm": "published", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 135.60663824374626, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 0, + "arm": "published", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 237.5475197924822, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 0, + "arm": "published", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 270.9110025112889, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 0, + "arm": "published", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 325.10385752712824, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 0, + "arm": "published", + "prompt": "prose", + "streams": 1, + "aggregate_decode_tps": 148.10711121092461, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 0, + "arm": "published", + "prompt": "prose", + "streams": 4, + "aggregate_decode_tps": 270.0303059152453, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 0, + "arm": "published", + "prompt": "prose", + "streams": 8, + "aggregate_decode_tps": 295.6105956217913, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 0, + "arm": "published", + "prompt": "prose", + "streams": 16, + "aggregate_decode_tps": 322.43096510198524, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 0, + "arm": "candidate", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 43.99498661294338, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 0, + "arm": "candidate", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 258.03903047977514, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 0, + "arm": "candidate", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 287.71439774595365, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 0, + "arm": "candidate", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 329.26775089681786, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 0, + "arm": "candidate", + "prompt": "prose", + "streams": 1, + "aggregate_decode_tps": 155.17129240785718, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 0, + "arm": "candidate", + "prompt": "prose", + "streams": 4, + "aggregate_decode_tps": 274.42316268438344, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 0, + "arm": "candidate", + "prompt": "prose", + "streams": 8, + "aggregate_decode_tps": 293.0455742669573, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 0, + "arm": "candidate", + "prompt": "prose", + "streams": 16, + "aggregate_decode_tps": 324.3134795211021, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 1, + "arm": "candidate", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 137.1292625523315, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 1, + "arm": "candidate", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 252.08277989639137, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 1, + "arm": "candidate", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 284.07929135629126, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 1, + "arm": "candidate", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 327.5277632360025, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 1, + "arm": "candidate", + "prompt": "prose", + "streams": 1, + "aggregate_decode_tps": 150.82092426693282, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 1, + "arm": "candidate", + "prompt": "prose", + "streams": 4, + "aggregate_decode_tps": 268.80377082796355, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 1, + "arm": "candidate", + "prompt": "prose", + "streams": 8, + "aggregate_decode_tps": 294.66891749695225, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 1, + "arm": "candidate", + "prompt": "prose", + "streams": 16, + "aggregate_decode_tps": 323.9697195614469, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 1, + "arm": "published", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 136.14276531242143, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 1, + "arm": "published", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 252.69384755422968, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 1, + "arm": "published", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 284.64110269593544, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 1, + "arm": "published", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 325.6922837054757, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 1, + "arm": "published", + "prompt": "prose", + "streams": 1, + "aggregate_decode_tps": 150.98369206524345, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 1, + "arm": "published", + "prompt": "prose", + "streams": 4, + "aggregate_decode_tps": 268.60609628493586, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 1, + "arm": "published", + "prompt": "prose", + "streams": 8, + "aggregate_decode_tps": 293.288959146046, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 1, + "arm": "published", + "prompt": "prose", + "streams": 16, + "aggregate_decode_tps": 323.24421841141884, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 2, + "arm": "published", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 136.28826488314226, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 2, + "arm": "published", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 251.47134877024175, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 2, + "arm": "published", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 284.5448538421833, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 2, + "arm": "published", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 325.79926741762347, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 2, + "arm": "published", + "prompt": "prose", + "streams": 1, + "aggregate_decode_tps": 150.90184937588137, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 2, + "arm": "published", + "prompt": "prose", + "streams": 4, + "aggregate_decode_tps": 234.90340221222957, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 2, + "arm": "published", + "prompt": "prose", + "streams": 8, + "aggregate_decode_tps": 293.26972222916555, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 2, + "arm": "published", + "prompt": "prose", + "streams": 16, + "aggregate_decode_tps": 322.47025493738624, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 2, + "arm": "candidate", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 138.34750502292397, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 2, + "arm": "candidate", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 254.00120377530746, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 2, + "arm": "candidate", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 284.5375280962798, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 2, + "arm": "candidate", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 327.2831823875495, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 2, + "arm": "candidate", + "prompt": "prose", + "streams": 1, + "aggregate_decode_tps": 151.00505713344813, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 2, + "arm": "candidate", + "prompt": "prose", + "streams": 4, + "aggregate_decode_tps": 271.8840625705605, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 2, + "arm": "candidate", + "prompt": "prose", + "streams": 8, + "aggregate_decode_tps": 298.09945887703077, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 2, + "arm": "candidate", + "prompt": "prose", + "streams": 16, + "aggregate_decode_tps": 324.2089086579436, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 3, + "arm": "published", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 136.32707890068104, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 3, + "arm": "published", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 251.8205867417788, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 3, + "arm": "published", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 284.349538989573, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 3, + "arm": "published", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 325.9626710889029, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 3, + "arm": "published", + "prompt": "prose", + "streams": 1, + "aggregate_decode_tps": 151.20030809404938, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 3, + "arm": "published", + "prompt": "prose", + "streams": 4, + "aggregate_decode_tps": 268.07908129520496, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 3, + "arm": "published", + "prompt": "prose", + "streams": 8, + "aggregate_decode_tps": 293.3722674786694, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 3, + "arm": "published", + "prompt": "prose", + "streams": 16, + "aggregate_decode_tps": 323.0746966258725, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 3, + "arm": "candidate", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 138.18860967197506, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 3, + "arm": "candidate", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 251.84917011393054, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 3, + "arm": "candidate", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 284.3401360103994, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 3, + "arm": "candidate", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 327.6480036147952, + "reply_sha256": "f66e3fd7e5093c8bd530e7b6782d587fd8df18f17a2b4cbbe2e22ee71038890f" + }, + { + "round": 3, + "arm": "candidate", + "prompt": "prose", + "streams": 1, + "aggregate_decode_tps": 151.6843393961539, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 3, + "arm": "candidate", + "prompt": "prose", + "streams": 4, + "aggregate_decode_tps": 268.8005329779927, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 3, + "arm": "candidate", + "prompt": "prose", + "streams": 8, + "aggregate_decode_tps": 294.4169012094223, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + }, + { + "round": 3, + "arm": "candidate", + "prompt": "prose", + "streams": 16, + "aggregate_decode_tps": 324.05536974614046, + "reply_sha256": "5a6a547bd3c4834ce019f947db389a30e48e297485f20f2b721d340a5432d784" + } + ], + "limitations": [ + "One candidate lone-code cell was 43.99 tok/s; retained in the four-round median. No competing model process was observed when subsequently checked.", + "Four rounds were not perfectly balanced: the final order repeated published then candidate.", + "Earlier kernel/context candidate regressed M4 16-stream prose by 8.38%; the retained version preserves original context-ingestion arithmetic.", + "An acceptance-cap scheduler experiment lost 2\u20133% on some M5 workloads and is excluded." + ], + "candidate_source": "b4deee277", + "m5": { + "device": "Apple M5 Ultra, 256 GiB; macOS 27.0 (26A428); Zig 0.17.0; Metal 32023.921.6", + "source_sha256": { + "zig/src/core/lanes/windows.zig": "88eedd02202e42e91fcdef4dd046d969038cc391cac282604893a417af3ad13a", + "zig/src/core/lanes/stream.zig": "7a129645b82569a0f9b680088fe35f5e9bd1defb9149e14a8fa85f777ebf48d0", + "zig/src/families/lfm2_moe/dspark_block.zig": "d2dd077581904759967446ebe2ad195c083ae73c004862de09907ffb4671ac41", + "zig/tests/contract/metal_lfm2_bf16.zig": "06f08a81d48a17371bdaed3b1c036aee085af7d7c0e0eed258b845e51a85c774" + }, + "cells": [ + { + "round": 0, + "arm": "main-no-external-drafter", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 289.99017871497716, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "main-no-external-drafter", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 651.5857918743773, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "main-no-external-drafter", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 831.2316832063025, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "main-no-external-drafter", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 831.1221850446695, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "main-no-external-drafter", + "prompt": "prose", + "streams": 1, + "aggregate_decode_tps": 261.653824038573, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "main-no-external-drafter", + "prompt": "prose", + "streams": 4, + "aggregate_decode_tps": 622.0862892391041, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "main-no-external-drafter", + "prompt": "prose", + "streams": 8, + "aggregate_decode_tps": 804.3648359431719, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "main-no-external-drafter", + "prompt": "prose", + "streams": 16, + "aggregate_decode_tps": 799.608159390429, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "published", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 354.34974736947413, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "published", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 645.481121931619, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "published", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 810.5269440440464, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "published", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 797.0694737428038, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "published", + "prompt": "prose", + "streams": 1, + "aggregate_decode_tps": 356.3219816253091, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "published", + "prompt": "prose", + "streams": 4, + "aggregate_decode_tps": 618.0402465118209, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "published", + "prompt": "prose", + "streams": 8, + "aggregate_decode_tps": 789.0806197623405, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "published", + "prompt": "prose", + "streams": 16, + "aggregate_decode_tps": 791.0577258755956, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "candidate", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 354.7364418761545, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "candidate", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 646.0307516882782, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "candidate", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 815.7156281542954, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "candidate", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 798.470003561415, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 0, + "arm": "candidate", + "prompt": "prose", + "streams": 1, + "aggregate_decode_tps": 356.5780465065517, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "candidate", + "prompt": "prose", + "streams": 4, + "aggregate_decode_tps": 618.6789173179606, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "candidate", + "prompt": "prose", + "streams": 8, + "aggregate_decode_tps": 796.0023513285163, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 0, + "arm": "candidate", + "prompt": "prose", + "streams": 16, + "aggregate_decode_tps": 792.391417523257, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "published", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 356.9031926500828, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "published", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 645.8077958032181, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "published", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 813.5162265998285, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "published", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 796.4582658126303, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "published", + "prompt": "prose", + "streams": 1, + "aggregate_decode_tps": 356.61902076187044, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "published", + "prompt": "prose", + "streams": 4, + "aggregate_decode_tps": 618.9625704407207, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "published", + "prompt": "prose", + "streams": 8, + "aggregate_decode_tps": 794.9411091902847, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "published", + "prompt": "prose", + "streams": 16, + "aggregate_decode_tps": 790.0706195599455, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "candidate", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 351.7006307459781, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "candidate", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 644.9662400483767, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "candidate", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 813.0724108239679, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "candidate", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 801.6591921437237, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "candidate", + "prompt": "prose", + "streams": 1, + "aggregate_decode_tps": 355.0560737481366, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "candidate", + "prompt": "prose", + "streams": 4, + "aggregate_decode_tps": 618.0832683633394, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "candidate", + "prompt": "prose", + "streams": 8, + "aggregate_decode_tps": 795.426957848166, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "candidate", + "prompt": "prose", + "streams": 16, + "aggregate_decode_tps": 796.7172061115025, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "main-no-external-drafter", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 289.0384163078719, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "main-no-external-drafter", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 652.2329226077759, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "main-no-external-drafter", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 831.6711661442548, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "main-no-external-drafter", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 830.8481776811126, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 1, + "arm": "main-no-external-drafter", + "prompt": "prose", + "streams": 1, + "aggregate_decode_tps": 262.70151064092926, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "main-no-external-drafter", + "prompt": "prose", + "streams": 4, + "aggregate_decode_tps": 622.7723581572978, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "main-no-external-drafter", + "prompt": "prose", + "streams": 8, + "aggregate_decode_tps": 805.1425054893175, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 1, + "arm": "main-no-external-drafter", + "prompt": "prose", + "streams": 16, + "aggregate_decode_tps": 800.4781222863546, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "candidate", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 353.41984243522757, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "candidate", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 645.7325520628694, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "candidate", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 810.1592109631267, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "candidate", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 804.1400599123751, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "candidate", + "prompt": "prose", + "streams": 1, + "aggregate_decode_tps": 356.47342949874684, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "candidate", + "prompt": "prose", + "streams": 4, + "aggregate_decode_tps": 618.7608129248248, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "candidate", + "prompt": "prose", + "streams": 8, + "aggregate_decode_tps": 789.0114172609233, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "candidate", + "prompt": "prose", + "streams": 16, + "aggregate_decode_tps": 798.1338794206057, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "main-no-external-drafter", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 290.3072664135369, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "main-no-external-drafter", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 651.6165425845753, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "main-no-external-drafter", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 831.5210047708392, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "main-no-external-drafter", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 829.3651009986223, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "main-no-external-drafter", + "prompt": "prose", + "streams": 1, + "aggregate_decode_tps": 261.76873963295424, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "main-no-external-drafter", + "prompt": "prose", + "streams": 4, + "aggregate_decode_tps": 622.0101018709207, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "main-no-external-drafter", + "prompt": "prose", + "streams": 8, + "aggregate_decode_tps": 804.9815598809657, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "main-no-external-drafter", + "prompt": "prose", + "streams": 16, + "aggregate_decode_tps": 798.5681100696453, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "published", + "prompt": "code", + "streams": 1, + "aggregate_decode_tps": 353.57965911849703, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "published", + "prompt": "code", + "streams": 4, + "aggregate_decode_tps": 645.0730230346089, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "published", + "prompt": "code", + "streams": 8, + "aggregate_decode_tps": 809.4871362230531, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "published", + "prompt": "code", + "streams": 16, + "aggregate_decode_tps": 797.7172187774961, + "reply_sha256": "dcc30ceea29dc7510fd59ee9a4c2554b780db9dcafe7a843c29afdfd57231edc" + }, + { + "round": 2, + "arm": "published", + "prompt": "prose", + "streams": 1, + "aggregate_decode_tps": 356.74670092367637, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "published", + "prompt": "prose", + "streams": 4, + "aggregate_decode_tps": 618.772261173814, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "published", + "prompt": "prose", + "streams": 8, + "aggregate_decode_tps": 797.1883738155475, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + }, + { + "round": 2, + "arm": "published", + "prompt": "prose", + "streams": 16, + "aggregate_decode_tps": 791.4675338150536, + "reply_sha256": "756f2e35886612286b0797e4b20f7a65f5434e1c0997b033a224a1b9830c452c" + } + ], + "binary_sha256": { + "zig-out/native/bin/tensorfold-native": "d5c4358dc174587536ab242aa438fb9fea9557df7a719bef83c35dd9397127c9", + "zig-out/bin/tf-lfm2-dspark": "29519d0208e77721fab62fbf63d982ec8db82a302b603999e185b462b19eb058", + "zig-out/bin/tf-kernel-contract": "4fd08e7265bcec265b92a682f3fe927c7580acf6a1585b69daccfde0936ef6ad" + }, + "protocol": { + "rounds": 3, + "arm_order_by_round": [ + [ + "main", + "published", + "candidate" + ], + [ + "published", + "candidate", + "main" + ], + [ + "candidate", + "main", + "published" + ] + ], + "same_requests_as_m4": true, + "all_72_cells_match": true + }, + "qualification": { + "source": "b4deee277", + "stress": [ + { + "slots": 6, + "seconds": 60, + "completed": 423, + "cancelled": 234, + "wrong": 0, + "errors": [] + }, + { + "slots": 16, + "seconds": 90, + "completed": 827, + "cancelled": 445, + "wrong": 0, + "errors": [] + } + ], + "sampled": [ + { + "streams": 1, + "hashes": [ + [ + "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d", + "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d", + "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d" + ] + ] + }, + { + "streams": 4, + "hashes": [ + [ + "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d", + "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d", + "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d" + ], + [ + "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5", + "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5", + "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5" + ], + [ + "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770", + "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770", + "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770" + ], + [ + "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133", + "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133", + "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133" + ] + ] + }, + { + "streams": 8, + "hashes": [ + [ + "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d", + "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d", + "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d" + ], + [ + "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5", + "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5", + "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5" + ], + [ + "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770", + "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770", + "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770" + ], + [ + "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133", + "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133", + "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133" + ], + [ + "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366", + "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366", + "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366" + ], + [ + "ba44af1f43a9f23b0bbadc69ea88414df1e7625f0aa2ade7acd73666362ec4ff", + "ba44af1f43a9f23b0bbadc69ea88414df1e7625f0aa2ade7acd73666362ec4ff", + "ba44af1f43a9f23b0bbadc69ea88414df1e7625f0aa2ade7acd73666362ec4ff" + ], + [ + "36b7dc4d8daac1d982e69cb8a13b97aa3d76403555565c13e19ff839f6a50c31", + "36b7dc4d8daac1d982e69cb8a13b97aa3d76403555565c13e19ff839f6a50c31", + "36b7dc4d8daac1d982e69cb8a13b97aa3d76403555565c13e19ff839f6a50c31" + ], + [ + "4e295b6a9b36a30ee500d0c93fb4d23083bf86a26a0d2dbdc720516e39c3b2cf", + "4e295b6a9b36a30ee500d0c93fb4d23083bf86a26a0d2dbdc720516e39c3b2cf", + "4e295b6a9b36a30ee500d0c93fb4d23083bf86a26a0d2dbdc720516e39c3b2cf" + ] + ] + }, + { + "streams": 16, + "hashes": [ + [ + "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d", + "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d", + "30c10286f6d8d5c1e874fe7698627e604bcc3a286b3da84ad487c184c2a8840d" + ], + [ + "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5", + "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5", + "c99ab72197e17fde3d3663c329d67152ae5641c49b11e8194c7db40eccf711d5" + ], + [ + "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770", + "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770", + "c10b43f97e6e3a72f57980bafcf598931d5c4e57fe313580303911fc40830770" + ], + [ + "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133", + "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133", + "51b0d3cc4c23679db3b995d9decd5b4c8e04f1543c5da88c0076dc550f87c133" + ], + [ + "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366", + "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366", + "1330f02fdc900dbab934f2c0097e46a2608c35e620be479bf9819f68c1ad2366" + ], + [ + "ba44af1f43a9f23b0bbadc69ea88414df1e7625f0aa2ade7acd73666362ec4ff", + "ba44af1f43a9f23b0bbadc69ea88414df1e7625f0aa2ade7acd73666362ec4ff", + "ba44af1f43a9f23b0bbadc69ea88414df1e7625f0aa2ade7acd73666362ec4ff" + ], + [ + "36b7dc4d8daac1d982e69cb8a13b97aa3d76403555565c13e19ff839f6a50c31", + "36b7dc4d8daac1d982e69cb8a13b97aa3d76403555565c13e19ff839f6a50c31", + "36b7dc4d8daac1d982e69cb8a13b97aa3d76403555565c13e19ff839f6a50c31" + ], + [ + "4e295b6a9b36a30ee500d0c93fb4d23083bf86a26a0d2dbdc720516e39c3b2cf", + "4e295b6a9b36a30ee500d0c93fb4d23083bf86a26a0d2dbdc720516e39c3b2cf", + "4e295b6a9b36a30ee500d0c93fb4d23083bf86a26a0d2dbdc720516e39c3b2cf" + ], + [ + "16e209d6f6ca13b44c6c2ac86018d6117f48739fe679c8b10f219711ff9cb986", + "16e209d6f6ca13b44c6c2ac86018d6117f48739fe679c8b10f219711ff9cb986", + "16e209d6f6ca13b44c6c2ac86018d6117f48739fe679c8b10f219711ff9cb986" + ], + [ + "45cab8cc38d33c168e96169e6cd61ecb339df2e61855515953cdeb5314d7e351", + "45cab8cc38d33c168e96169e6cd61ecb339df2e61855515953cdeb5314d7e351", + "45cab8cc38d33c168e96169e6cd61ecb339df2e61855515953cdeb5314d7e351" + ], + [ + "0bfbd2b2a91cab827879cb2bd2aa696bf6bd49516c2aa5439c83ecb386ea7369", + "0bfbd2b2a91cab827879cb2bd2aa696bf6bd49516c2aa5439c83ecb386ea7369", + "0bfbd2b2a91cab827879cb2bd2aa696bf6bd49516c2aa5439c83ecb386ea7369" + ], + [ + "1e81d9e26277be61b38974b207ca6f31e045350d3e939c4ec700e40603553f0a", + "1e81d9e26277be61b38974b207ca6f31e045350d3e939c4ec700e40603553f0a", + "1e81d9e26277be61b38974b207ca6f31e045350d3e939c4ec700e40603553f0a" + ], + [ + "97ba371a7ee308d84fd24f6cf3e574b817a1fe7a56728a122cfe9ab4aae75363", + "97ba371a7ee308d84fd24f6cf3e574b817a1fe7a56728a122cfe9ab4aae75363", + "97ba371a7ee308d84fd24f6cf3e574b817a1fe7a56728a122cfe9ab4aae75363" + ], + [ + "6884f675a0b28c8d331d9d245919814741dcefeaf12900e783d1b525d4490398", + "6884f675a0b28c8d331d9d245919814741dcefeaf12900e783d1b525d4490398", + "6884f675a0b28c8d331d9d245919814741dcefeaf12900e783d1b525d4490398" + ], + [ + "7b81161c5817e10474e598a19f3fc2008bdece4a9e4c68e1dc52024e4d5d32db", + "7b81161c5817e10474e598a19f3fc2008bdece4a9e4c68e1dc52024e4d5d32db", + "7b81161c5817e10474e598a19f3fc2008bdece4a9e4c68e1dc52024e4d5d32db" + ], + [ + "e5997fd3392b57373c76457f8c69bd05899ad99c92abf281efc274e148dd74fd", + "e5997fd3392b57373c76457f8c69bd05899ad99c92abf281efc274e148dd74fd", + "e5997fd3392b57373c76457f8c69bd05899ad99c92abf281efc274e148dd74fd" + ] + ] + } + ], + "resume": { + "equal": true, + "cached_tokens": 3517, + "resumed_sha256": "2fed3f850c53a2a46620eb8b0facca0a7db4890e9462c4db2a39a5597f462a91", + "fresh_sha256": "2fed3f850c53a2a46620eb8b0facca0a7db4890e9462c4db2a39a5597f462a91" + }, + "8bit": [ + { + "streams": 1, + "plain_sha256": "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "solo_sha256": "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "concurrent_sha256": [ + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58" + ] + }, + { + "streams": 4, + "plain_sha256": "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "solo_sha256": "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "concurrent_sha256": [ + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58" + ] + }, + { + "streams": 8, + "plain_sha256": "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "solo_sha256": "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "concurrent_sha256": [ + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58" + ] + }, + { + "streams": 16, + "plain_sha256": "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "solo_sha256": "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "concurrent_sha256": [ + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58", + "4168f12f73c60b83168f60b474ea92b8f63c7833ecf3295c3fb2d5331156ed58" + ] + }, + { + "streams": 1, + "plain_sha256": "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "solo_sha256": "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "concurrent_sha256": [ + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b" + ] + }, + { + "streams": 4, + "plain_sha256": "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "solo_sha256": "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "concurrent_sha256": [ + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b" + ] + }, + { + "streams": 8, + "plain_sha256": "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "solo_sha256": "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "concurrent_sha256": [ + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b" + ] + }, + { + "streams": 16, + "plain_sha256": "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "solo_sha256": "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "concurrent_sha256": [ + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b", + "728482497427a444e71dbb503dd96423f4ffa65df139bf2fab5c08d9be44548b" + ] + } + ], + "long_context": [ + { + "length": 65536, + "plain_sha256": "c2ba0e4604c6beb356eefe87cdec6c938f5f4a484521bfd73e8f9e6b57bc22b6", + "solo_sha256": "c2ba0e4604c6beb356eefe87cdec6c938f5f4a484521bfd73e8f9e6b57bc22b6", + "concurrent_sha256": [ + "c2ba0e4604c6beb356eefe87cdec6c938f5f4a484521bfd73e8f9e6b57bc22b6", + "c2ba0e4604c6beb356eefe87cdec6c938f5f4a484521bfd73e8f9e6b57bc22b6", + "c2ba0e4604c6beb356eefe87cdec6c938f5f4a484521bfd73e8f9e6b57bc22b6", + "c2ba0e4604c6beb356eefe87cdec6c938f5f4a484521bfd73e8f9e6b57bc22b6" + ] + }, + { + "length": 130816, + "plain_sha256": "3cc9886102163ddcfa01f3a3f20f723929d6cb01e51cb4bd3e960f05ca0569c3", + "solo_sha256": "3cc9886102163ddcfa01f3a3f20f723929d6cb01e51cb4bd3e960f05ca0569c3", + "concurrent_sha256": [ + "3cc9886102163ddcfa01f3a3f20f723929d6cb01e51cb4bd3e960f05ca0569c3", + "3cc9886102163ddcfa01f3a3f20f723929d6cb01e51cb4bd3e960f05ca0569c3", + "3cc9886102163ddcfa01f3a3f20f723929d6cb01e51cb4bd3e960f05ca0569c3", + "3cc9886102163ddcfa01f3a3f20f723929d6cb01e51cb4bd3e960f05ca0569c3" + ] + } + ], + "kernel_contract": "11 pass, 0 fail; 17 controls red, 0 missed; 0 limit problems", + "drafter_reference": [ + "step 6: confidence reference 0.1449, GPU 0.1465", + "step 7: reference 496 (margin 1.722, second 55658), GPU 496 equal", + "step 7: confidence reference 0.1030, GPU 0.1027", + "step 8: reference 49 (margin 0.100, second 28358), GPU 49 equal", + "step 8: confidence reference 0.0880, GPU 0.0873", + "the reference's largest margin over a different GPU pick: 0.000", + "the confidences' worst difference from the fp32 reference: 0.00158", + "9 of 9 drafts equal the fp32 reference's picks on the same chain (255-token context)" + ] + } + } +} diff --git a/docs/receipts/lfm2-dspark-review/qualification.py b/docs/receipts/lfm2-dspark-review/qualification.py index 7ca0c238cc..e0987af059 100644 --- a/docs/receipts/lfm2-dspark-review/qualification.py +++ b/docs/receipts/lfm2-dspark-review/qualification.py @@ -161,7 +161,7 @@ def long_context(): emit('long-reference-done',length=length,exit=r.returncode) assert r.returncode==0 -def performance(family='lfm2'): +def performance(family='lfm2', compare=False): model=MODEL; draft=DRAFT; widths=[1,4,8,16] if family=='qwen': model=HUB/'models--TensorFold--Qwen3.8-27B-MLX-4bit/snapshots/22d8d538154e0e5b6f8dcb1fcc74a60a0798104c' @@ -170,6 +170,10 @@ def performance(family='lfm2'): prompts=[{'name':'code','prompt':'Write a short Python function that computes the Fibonacci sequence and explain it.'},{'name':'chat','prompt':'Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.'}] arms={'main-plain':(MAIN,'plain'),'candidate-plain':(CAND,'plain'),'candidate-drafted':(CAND,'bf16')} if family=='qwen': arms={'main-drafted':(MAIN,'bf16'),'candidate-drafted':(CAND,'bf16')} + if compare: + published=os.environ.get('TF_QUAL_PUBLISHED') + if not published: raise SystemExit('compare requires TF_QUAL_PUBLISHED pointing to the earlier PR binary') + arms={'main':(MAIN,'plain'),'published':(Path(published),'bf16'),'candidate':(CAND,'bf16')} results=[]; reference={} order=list(arms) for round_ in range(3): @@ -188,7 +192,7 @@ def run(i): gate.wait(); return post({'prompt':p['prompt'],'max_tokens':256},Tru assert all(h==reference[key] for h in hashes),(family,arm,key,'token mismatch') first=min(x['first'] for x in rows); last=max(x['last'] for x in rows) row={'round':round_,'arm':arm,'prompt':key,'streams':width,'aggregate_decode_tps':width*255/(last-first),'ttft_max':max(x['first']-x['sent'] for x in rows),'requests':rows} - results.append(row); save('performance-'+family,results) + results.append(row); save(('comparison-' if compare else 'performance-')+family,results) emit('performance',family=family,round=round_,arm=arm,prompt=key,streams=width,tps=round(row['aggregate_decode_tps'],2)) def cold_prefill(): @@ -217,4 +221,5 @@ def cold_prefill(): if phase=='functional': functional() elif phase=='long': long_context() elif phase=='performance': performance(sys.argv[2] if len(sys.argv)>2 else 'lfm2') + elif phase=='compare': performance(compare=True) elif phase=='cold': cold_prefill()