From a126372a20165f4840bcba141c9459ad21f6104b Mon Sep 17 00:00:00 2001 From: JinYan Su <751080330@qq.com> Date: Wed, 7 Oct 2026 15:33:39 +0000 Subject: [PATCH 1/6] manifest: a manifest cuts into pipeline stages at call indices `kern cut --at --module pp_mailbox.cubin --out ` splits a verified manifest into S stage manifests. Liveness across each cut point finds the boundary workspaces; they go through a send/recv op pair over a mailbox on the downstream GPU (tools/kernels-src/pp_mailbox.cu: full/empty counters, system-scope release/acquire, a device-side spin that traps after 60 s). Each stage carries `cut {id, stage, stages}` and a `clock` output fill the mailbox kernels stamp per item with globaltimer. A stage's group `pp.` joins two different manifests that agree only on the peer's `of` buffer; import_peers already matches by name and size, so the runtime is unchanged. Protocol requires a token fill on stage 0 only. Gate: a property test (400 random manifests, symbolic interpreter) shows every cut that is made computes the whole; qwen3-4b cut at l18 into two stages, run in sequence (examples/pp_chain), gives bitwise identical logits and tokens to the whole manifest on 7 prompts, chunk 512 and 64. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/kern-manifest/src/constants.rs | 2 + crates/kern-manifest/src/cut.rs | 485 ++++++++++++++++++++++++++ crates/kern-manifest/src/lib.rs | 1 + crates/kern-manifest/src/protocol.rs | 18 +- crates/kern-manifest/src/types.rs | 20 +- crates/kern-manifest/src/verify.rs | 11 +- crates/kern-manifest/tests/cut.rs | 294 ++++++++++++++++ crates/kern-run/examples/pp_chain.rs | 233 +++++++++++++ crates/kern-run/src/bin/kern.rs | 48 ++- crates/kern-run/src/lib.rs | 82 +++-- crates/kern-serve/src/tray.rs | 2 +- docs/manifest.md | 41 ++- schema/manifest-v5.schema.json | 66 +++- tools/kernels-src/pp_mailbox.cu | 150 ++++++++ 14 files changed, 1420 insertions(+), 33 deletions(-) create mode 100644 crates/kern-manifest/src/cut.rs create mode 100644 crates/kern-manifest/tests/cut.rs create mode 100644 crates/kern-run/examples/pp_chain.rs create mode 100644 tools/kernels-src/pp_mailbox.cu diff --git a/crates/kern-manifest/src/constants.rs b/crates/kern-manifest/src/constants.rs index 0838f6d1..a77b9dec 100644 --- a/crates/kern-manifest/src/constants.rs +++ b/crates/kern-manifest/src/constants.rs @@ -18,6 +18,8 @@ struct LiteralManifest { #[serde(default)] topology: Option, #[serde(default)] + cut: Option, + #[serde(default)] vars: BTreeMap, #[serde(default)] states: BTreeMap, diff --git a/crates/kern-manifest/src/cut.rs b/crates/kern-manifest/src/cut.rs new file mode 100644 index 00000000..1023c6fe --- /dev/null +++ b/crates/kern-manifest/src/cut.rs @@ -0,0 +1,485 @@ +//! Cutting a manifest into a pipeline: stage `s` runs the calls between +//! two cut points of every program a serving loop drives, and the +//! activations live across a cut travel through a mailbox. +//! +//! A forward is a pure function of its inputs and its states, so a cut +//! changes no arithmetic: the stages run the same calls in the same order +//! on the same values, and the pipeline's outputs are the unsplit +//! manifest's byte for byte. What a cut adds is transport. At a cut point +//! the buffers live across it are the ones the calls after it read before +//! they write them whole (an `out` at offset 0 is taken to write the whole +//! buffer, as every generator writes a view at its base) and that the +//! calls before it wrote; each must be a workspace, since an output is read +//! from the last stage and a carry would cross programs. The upstream stage +//! ends every program by putting them into a box on the downstream GPU +//! (`tools/kernels-src/pp_mailbox.cu`): wait until the box is empty, put, +//! post. The downstream stage starts by waiting for the post, taking them +//! out and freeing the box. One box per edge, laid out per program at the +//! bounds of its vars; the box sits on the downstream GPU and the upstream +//! reaches it through a `peer` buffer of a two-member topology group +//! `pp.`, its own copy of the box unused. +//! +//! A state never travels. Every region of a state (a state and a call's +//! offset into it) belongs to the one stage whose calls touch it, which is +//! what makes the stages' page tables interchangeable: a sequence's pages +//! hold the first stage's layers on one GPU and the next stage's on +//! another, under the same page ids. A region two stages touch is refused. +//! A program run once after load is sliced per stage to the calls the +//! stage's own calls need, so every stage derives its own tables. +//! +//! The prompt reaches the step program's first token the way it does +//! unsplit; a chunk program that hands a token back itself is refused, so +//! a pipeline's first token always comes from a step. Everything a stage +//! does not use is dropped, and every stage verifies. + +use std::collections::{BTreeMap, BTreeSet}; + +use serde_json::json; + +use crate::types::*; +use crate::verify::{verify, Verified}; + +/// How long a mailbox wait spins before the stage gives up and traps. +pub const TIMEOUT_NS: i64 = 60_000_000_000; +/// Bytes of a box before its data: the two counters, a cache line each. +const HEADER: u64 = 256; +/// Each carried buffer starts at a multiple of this in the box. +const ALIGN: u64 = 256; +/// Entries of a stage's clock. +const RING: u64 = 256; +/// The module name the pipeline kernels go under in every stage. +const MODULE: &str = "pp"; +/// The stage's clock (fill `clock`). +const CLOCK: &str = "pp.clock"; + +/// Every reason a manifest cannot be cut where asked, reported together. +#[derive(Debug)] +pub struct CutErrors(pub Vec); + +impl std::fmt::Display for CutErrors { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("manifest cannot be cut:")?; + for e in &self.0 { + write!(f, "\n - {e}")?; + } + Ok(()) + } +} + +impl std::error::Error for CutErrors {} + +/// A buffer an edge carries in one program: where in the box, and how +/// many bytes per call. +#[derive(Debug, Clone)] +struct Carried { + buffer: String, + dtype: DType, + offset: u64, + bytes: Expr, +} + +/// Cut `m` before each call index in `at` of every program with a +/// `batch` (all of them must be longer than the last point), wiring in +/// the mailbox kernels of `pp`. Returns the stages in order, each +/// verified. +pub fn cut(m: &Verified, at: &[usize], pp: &Module) -> Result, CutErrors> { + let mut errs = Vec::new(); + if m.topology.is_some() { + errs.push("the manifest has a topology; a cut takes a single-rank manifest".to_string()); + } + if m.cut.is_some() { + errs.push("the manifest is already a stage of a cut".to_string()); + } + if at.is_empty() || at[0] == 0 || at.windows(2).any(|w| w[0] >= w[1]) { + errs.push(format!("cut points {at:?}: at least one, increasing, none at 0")); + } + let driven: Vec<(&String, &Program)> = m.programs.iter().filter(|(_, p)| p.batch.is_some()).collect(); + if driven.is_empty() { + errs.push("no program declares a `batch`: nothing to cut".to_string()); + } + for (name, p) in &driven { + if let Some(&last) = at.last() { + if last >= p.calls.len() { + errs.push(format!("program `{name}` has {} calls; it cannot be cut before call {last}", p.calls.len())); + } + } + let chunk = p.batch.as_ref().is_some_and(|b| matches!(b.rows, Dim::Var(_))); + if chunk && writes(m, &p.calls).iter().any(|b| m.buffers[*b].fill == Some(Fill::Tokens)) { + errs.push(format!( + "program `{name}` takes a chunk and hands a token back; a pipeline takes its first token from a step" + )); + } + } + if !errs.is_empty() { + return Err(CutErrors(errs)); + } + + let stages = at.len() + 1; + let bounds: Vec = std::iter::once(0).chain(at.iter().copied()).collect(); + let span = |p: &Program, s: usize| bounds[s]..bounds.get(s + 1).copied().unwrap_or(p.calls.len()); + let max: BTreeMap = m.vars.iter().map(|(k, v)| (k.clone(), v.max)).collect(); + + // What each edge carries in each program, and each box's size. + let mut carried: BTreeMap<(&str, usize), Vec> = BTreeMap::new(); + for (name, p) in &driven { + for (e, &a) in at.iter().enumerate() { + let before = writes(m, &p.calls[..a]); + let mut offset = 0; + let mut list = Vec::new(); + for b in live_in(m, &p.calls[a..]).into_iter().filter(|b| before.contains(b)) { + let buf = &m.buffers[b]; + if buf.kind != BufferKind::Workspace { + errs.push(format!( + "program `{name}`: {} buffer `{b}` is live across the cut before call {a}; only a workspace travels", + buf.kind + )); + continue; + } + match bytes_of(buf) { + Some(bytes) => { + let at_max = bytes.eval(&max).unwrap_or(u64::MAX); + list.push(Carried { buffer: b.to_string(), dtype: buf.dtype, offset, bytes }); + offset += at_max.div_ceil(ALIGN) * ALIGN; + } + None => errs.push(format!( + "program `{name}`: buffer `{b}` is shaped {:?}; a carried buffer has at most one var dim", + buf.shape + )), + } + } + carried.insert((name.as_str(), e), list); + } + for (i, c) in p.calls.iter().enumerate().take(bounds[stages - 1]) { + for (b, _, dir) in accesses(m, c) { + if dir != Dir::In && matches!(m.buffers[b].kind, BufferKind::Output | BufferKind::Carry) { + errs.push(format!( + "program `{name}` call {i} writes {} buffer `{b}` before the last stage; a pipeline's outputs come from its last stage", + m.buffers[b].kind + )); + } + } + } + } + let mut owner: BTreeMap<(String, u64), usize> = BTreeMap::new(); + for (name, p) in &driven { + for s in 0..stages { + for c in &p.calls[span(p, s)] { + for (st, off) in state_args(c) { + match owner.insert((st.to_string(), off), s) { + Some(o) if o != s => errs.push(format!( + "program `{name}`: state `{st}`+{off} is touched by stages {o} and {s}; a state region belongs to one stage" + )), + _ => {} + } + } + } + } + } + if !errs.is_empty() { + return Err(CutErrors(errs)); + } + let boxes: Vec = (0..at.len()) + .map(|e| { + let data = carried + .iter() + .filter(|((_, ee), _)| *ee == e) + .map(|(_, list)| list.last().map_or(0, |c| c.offset + c.bytes.eval(&max).unwrap_or(0))) + .max() + .unwrap_or(0); + HEADER + data.max(1).div_ceil(ALIGN) * ALIGN + }) + .collect(); + + let id = id_of(m, at); + let built: Vec = (0..stages) + .map(|s| { + let programs: BTreeMap = driven + .iter() + .map(|(name, p)| { + let into = (s > 0).then(|| &carried[&(name.as_str(), s - 1)][..]); + let out = (s + 1 < stages).then(|| &carried[&(name.as_str(), s)][..]); + let calls = + receive(s, into).into_iter().chain(p.calls[span(p, s)].iter().cloned()).chain(send(s, out)); + ((*name).clone(), Program { calls: calls.collect(), ..(*p).clone() }) + }) + .collect(); + stage(m, programs, s, stages, &boxes, &id, pp) + }) + .collect(); + built + .into_iter() + .enumerate() + .map(|(s, st)| verify(st).map_err(|e| CutErrors(e.0.into_iter().map(|d| format!("stage {s}: {d}")).collect()))) + .collect() +} + +fn box_name(e: usize) -> String { + format!("pp.{e}.box") +} + +fn peer_name(e: usize) -> String { + format!("pp.{e}.peer") +} + +fn call(op: &str, args: serde_json::Value) -> Call { + serde_json::from_value(json!({"label": op, "op": op, "args": args})).expect("a well-formed call") +} + +/// The calls that start stage `s`: take what the edge into it carries, or +/// stamp the start of a first stage. +fn receive(s: usize, into: Option<&[Carried]>) -> Vec { + let Some(list) = into else { + return vec![call("pp.stamp_first", json!([{"buf": peer_name(s)}, {"buf": CLOCK}]))]; + }; + let b = box_name(s - 1); + std::iter::once(call("pp.wait_full", json!([{"buf": b}, {"buf": CLOCK}]))) + .chain(list.iter().map(|c| { + call( + &format!("pp.take.{}", c.dtype), + json!([{"buf": b}, {"buf": c.buffer}, {"expr": c.bytes}, {"i64": c.offset}]), + ) + })) + .chain(std::iter::once(call("pp.post_empty", json!([{"buf": b}])))) + .collect() +} + +/// The calls that end stage `s`: put what the edge out of it carries, or +/// stamp the end of a last stage. +fn send(s: usize, out: Option<&[Carried]>) -> Vec { + let Some(list) = out else { + return vec![call("pp.stamp_last", json!([{"buf": box_name(s - 1)}, {"buf": CLOCK}]))]; + }; + let p = peer_name(s); + std::iter::once(call("pp.wait_empty", json!([{"buf": p}, {"buf": CLOCK}]))) + .chain(list.iter().map(|c| { + call( + &format!("pp.put.{}", c.dtype), + json!([{"buf": p}, {"buf": c.buffer}, {"expr": c.bytes}, {"i64": c.offset}]), + ) + })) + .chain(std::iter::once(call("pp.post_full", json!([{"buf": p}])))) + .collect() +} + +/// The pipeline ops a stage's calls name: `pp.put.` and +/// `pp.take.` per carried dtype, the waits, posts and stamps. +fn pp_op(name: &str) -> Op { + let single = |entry: &str, params: serde_json::Value| { + json!({"params": params, "impl": {"launches": [ + {"module": MODULE, "entry": entry, "block": [1, 1, 1], "grid": [1, 1, 1]}]}}) + }; + let wait = |entry: &str, first: &str| { + json!({"params": [first, "out buffer"], "impl": {"launches": [ + {"module": MODULE, "entry": entry, "block": [1, 1, 1], "grid": [1, 1, 1], + "params": [first, "out buffer", "i64"], + "args": [{"param": 0}, {"param": 1}, {"i64": TIMEOUT_NS}]}]}}) + }; + let copy = |entry: &str, params: serde_json::Value| { + json!({"params": params, "impl": {"launches": [ + {"module": MODULE, "entry": entry, "block": [256, 1, 1], "grid": [264, 1, 1]}]}}) + }; + let v = match name.split('.').collect::>().as_slice() { + ["pp", "wait_empty"] => wait("kern_pp_wait_empty", "in buffer"), + ["pp", "wait_full"] => wait("kern_pp_wait_full", "in buffer"), + ["pp", "post_full"] => single("kern_pp_post_full", json!(["in buffer"])), + ["pp", "post_empty"] => single("kern_pp_post_empty", json!(["inout buffer"])), + ["pp", "stamp_first"] => single("kern_pp_stamp_first", json!(["in buffer", "out buffer"])), + ["pp", "stamp_last"] => single("kern_pp_stamp_last", json!(["in buffer", "out buffer"])), + ["pp", "put", dt] => copy("kern_pp_put", json!(["in buffer", format!("in buffer<{dt}>"), "i64", "i64"])), + ["pp", "take", dt] => copy("kern_pp_take", json!(["in buffer", format!("out buffer<{dt}>"), "i64", "i64"])), + _ => unreachable!("the cut names only the pipeline ops above"), + }; + serde_json::from_value(v).expect("a well-formed pipeline op") +} + +/// Stage `s` of `stages`: `programs` (already cut and wired), the once +/// programs sliced to what they need, and every declaration they use. +fn stage( + m: &Manifest, + mut programs: BTreeMap, + s: usize, + stages: usize, + boxes: &[u64], + id: &str, + pp: &Module, +) -> Manifest { + let mut needed: BTreeSet = + programs.values().flat_map(|p| &p.calls).flat_map(|c| reads(m, c)).map(str::to_string).collect(); + for (name, p) in m.programs.iter().filter(|(_, p)| p.once) { + let mut kept = Vec::new(); + for c in p.calls.iter().rev() { + if accesses(m, c).iter().any(|(b, _, d)| *d != Dir::In && needed.contains(*b)) { + needed.extend(reads(m, c).map(str::to_string)); + kept.push(c.clone()); + } + } + if !kept.is_empty() { + kept.reverse(); + programs.insert(name.clone(), Program { calls: kept, ..p.clone() }); + } + } + + let mut buffers: BTreeMap = BTreeMap::new(); + let u8_box = |bytes: u64, kind: &str| -> Buffer { + serde_json::from_value(json!({"dtype": "u8", "shape": [bytes], "kind": kind, "export": true})).expect("a box") + }; + buffers.insert( + CLOCK.to_string(), + serde_json::from_value(json!({"dtype": "i64", "shape": [RING * 5], "kind": "output", "fill": "clock"})) + .expect("a clock"), + ); + if s > 0 { + buffers.insert(box_name(s - 1), u8_box(boxes[s - 1], "carry")); + } + let mut topology = None; + if s + 1 < stages { + buffers.insert(box_name(s), u8_box(boxes[s], "workspace")); + buffers.insert( + peer_name(s), + serde_json::from_value( + json!({"dtype": "u64", "shape": [2], "kind": "peer", "of": box_name(s), "group": format!("pp.{s}")}), + ) + .expect("a peer"), + ); + topology = Some(Topology { groups: BTreeMap::from([(format!("pp.{s}"), 2)]) }); + } + let calls: Vec<&Call> = programs.values().flat_map(|p| &p.calls).collect(); + for c in &calls { + for a in &c.args { + if let Arg::Buf { buf, .. } = a { + if let Some(b) = m.buffers.get(buf) { + buffers.insert(buf.clone(), b.clone()); + } + } + } + } + let states: BTreeMap = + calls.iter().flat_map(|c| state_args(c)).map(|(st, _)| (st.to_string(), m.states[st].clone())).collect(); + let names: BTreeSet = buffers.keys().chain(states.keys()).cloned().collect(); + for b in buffers.values_mut() { + if b.domain.as_ref().and_then(|d| d.index_into.as_ref()).is_some_and(|t| !names.contains(t)) { + b.domain = None; + } + } + let ops: BTreeMap = + calls.iter().map(|c| (c.op.clone(), m.ops.get(&c.op).cloned().unwrap_or_else(|| pp_op(&c.op)))).collect(); + let mut modules: BTreeMap = ops + .values() + .flat_map(|o| &o.imp.launches) + .filter_map(Launch::module) + .filter(|n| *n != MODULE) + .map(|n| (n.to_string(), m.modules[n].clone())) + .collect(); + modules.insert(MODULE.to_string(), pp.clone()); + // A var stays when some kept declaration names it: shapes, grids, + // call args, batches and domains all spell a var as a bare string. + let mut strings = BTreeSet::new(); + for v in [serde_json::to_value(&buffers), serde_json::to_value(&ops), serde_json::to_value(&programs)] { + collect_strings(&v.expect("serializable"), &mut strings); + } + let vars = m.vars.iter().filter(|(k, _)| strings.contains(*k)).map(|(k, v)| (k.clone(), v.clone())).collect(); + Manifest { + schema_version: m.schema_version, + model: m.model.clone(), + topology, + cut: Some(Cut { id: id.to_string(), stage: s as u64, stages: stages as u64 }), + vars, + states, + buffers, + modules, + ops, + programs, + } +} + +fn collect_strings(v: &serde_json::Value, out: &mut BTreeSet) { + match v { + serde_json::Value::String(s) => { + out.insert(s.clone()); + } + serde_json::Value::Array(a) => a.iter().for_each(|x| collect_strings(x, out)), + serde_json::Value::Object(o) => o.values().for_each(|x| collect_strings(x, out)), + _ => {} + } +} + +/// Bytes one call moves of `b`: its size at the call's vars, when at most +/// one dim is a var. +fn bytes_of(b: &Buffer) -> Option { + let constant: u64 = b.shape.iter().filter_map(|d| if let Dim::Const(c) = d { Some(*c) } else { None }).product(); + let vars: Vec<&String> = b.shape.iter().filter_map(|d| if let Dim::Var(v) = d { Some(v) } else { None }).collect(); + let c = constant * b.dtype.bytes(); + match vars.as_slice() { + [] => Some(Expr::Const(c)), + [v] => Some(Expr::Mul { mul: (Box::new(Expr::Var((*v).clone())), c) }), + _ => None, + } +} + +/// Every buffer argument of `c`: name, offset and the direction its op +/// declares. +fn accesses<'a>(m: &'a Manifest, c: &'a Call) -> Vec<(&'a str, u64, Dir)> { + let params = m.ops.get(&c.op).map(|o| o.params.as_slice()).unwrap_or(&[]); + c.args + .iter() + .zip(params) + .filter_map(|(a, p)| match a { + Arg::Buf { buf, offset } => Some((buf.as_str(), *offset, p.dir().unwrap_or(Dir::In))), + _ => None, + }) + .collect() +} + +/// The buffers `c` reads, by name; a pipeline call reads its buffers too. +fn reads<'a>(m: &'a Manifest, c: &'a Call) -> impl Iterator { + let known = m.ops.contains_key(&c.op); + let acc = accesses(m, c); + let pipeline: Vec<&str> = if known { + Vec::new() + } else { + c.args.iter().filter_map(|a| if let Arg::Buf { buf, .. } = a { Some(buf.as_str()) } else { None }).collect() + }; + acc.into_iter().filter(|(_, _, d)| *d != Dir::Out).map(|(b, _, _)| b).chain(pipeline) +} + +/// The buffers `calls` write. +fn writes<'a>(m: &'a Manifest, calls: &'a [Call]) -> BTreeSet<&'a str> { + calls.iter().flat_map(|c| accesses(m, c)).filter(|(_, _, d)| *d != Dir::In).map(|(b, _, _)| b).collect() +} + +/// The buffers `calls` read before writing them whole: what must already +/// hold its value when they start. +fn live_in<'a>(m: &'a Manifest, calls: &'a [Call]) -> BTreeSet<&'a str> { + let mut killed = BTreeSet::new(); + let mut live = BTreeSet::new(); + for c in calls { + let acc = accesses(m, c); + for &(b, _, d) in &acc { + if d != Dir::Out && !killed.contains(b) { + live.insert(b); + } + } + for &(b, off, d) in &acc { + if d == Dir::Out && off == 0 { + killed.insert(b); + } + } + } + live +} + +/// Every state region `c` touches. +fn state_args(c: &Call) -> impl Iterator { + c.args.iter().filter_map(|a| match a { + Arg::State { state, offset } => Some((state.as_str(), *offset)), + _ => None, + }) +} + +/// The cut's name: FNV-1a over the manifest and the cut points, the same +/// bytes for the same cut. +fn id_of(m: &Manifest, at: &[usize]) -> String { + let bytes = m.to_json().into_bytes().into_iter().chain(at.iter().flat_map(|a| (*a as u64).to_le_bytes())); + let h = bytes.fold(0xcbf2_9ce4_8422_2325u64, |h, b| (h ^ b as u64).wrapping_mul(0x0000_0100_0000_01b3)); + format!("{h:016x}") +} diff --git a/crates/kern-manifest/src/lib.rs b/crates/kern-manifest/src/lib.rs index d5a684b5..1093c58a 100644 --- a/crates/kern-manifest/src/lib.rs +++ b/crates/kern-manifest/src/lib.rs @@ -17,6 +17,7 @@ mod constants; mod host; pub use constants::json_schema; +pub mod cut; pub mod protocol; pub mod types; pub mod values; diff --git a/crates/kern-manifest/src/protocol.rs b/crates/kern-manifest/src/protocol.rs index ca792aa7..5f28a060 100644 --- a/crates/kern-manifest/src/protocol.rs +++ b/crates/kern-manifest/src/protocol.rs @@ -322,6 +322,7 @@ impl Protocol { Fill::Tokens => axis == Axis::Groups || (axis == Axis::Tray && width == 1), Fill::SpanAt | Fill::Error => axis == Axis::Fixed(1) && b.dtype == DType::I32, Fill::Blocks => matches!(axis, Axis::Fixed(n) if n >= 2) && b.dtype == DType::I32, + Fill::Clock => matches!(axis, Axis::Fixed(n) if n > 0 && n % 5 == 0) && b.dtype == DType::I64, }; if !ok { errs.push(format!( @@ -335,6 +336,7 @@ impl Protocol { Fill::Tokens => "expected [groups], [groups, w] or [tray]", Fill::SpanAt | Fill::Error => "expected i32 [1]", Fill::Blocks => "expected i32 [members + 1]", + Fill::Clock => "expected i64 [ring × 5]", } )); continue; @@ -347,14 +349,24 @@ impl Protocol { errs.push(format!("fill `{fill}` over {axis:?} is on {n} buffers, expected one")); } } - if !fills.iter().any(|f| f.fill == Fill::Token && f.axis != Axis::Groups) { + // A stage past the first of a cut takes activations, not tokens. + let fed = m.cut.as_ref().is_none_or(|c| c.stage == 0); + if fed && !fills.iter().any(|f| f.fill == Fill::Token && f.axis != Axis::Groups) { errs.push("no input has fill `token` over the rows: nothing carries the tokens a call feeds".into()); } if fills.iter().filter(|f| f.fill == Fill::Token && f.axis == Axis::Groups).count() > 1 { errs.push("fill `token` over the sequences is on more than one buffer".into()); } - for fill in [Fill::Position, Fill::Valid, Fill::CuSeqlens, Fill::SpanAt, Fill::Blocks, Fill::Count, Fill::Error] - { + for fill in [ + Fill::Position, + Fill::Valid, + Fill::CuSeqlens, + Fill::SpanAt, + Fill::Blocks, + Fill::Count, + Fill::Error, + Fill::Clock, + ] { if one(fill).is_none() && m.buffers.values().any(|b| b.fill == Some(fill)) { errs.push(format!("fill `{fill}` is on more than one buffer")); } diff --git a/crates/kern-manifest/src/types.rs b/crates/kern-manifest/src/types.rs index 0800af6e..af89c7ce 100644 --- a/crates/kern-manifest/src/types.rs +++ b/crates/kern-manifest/src/types.rs @@ -71,9 +71,12 @@ pub struct Manifest { pub schema_version: u32, /// Free-form model label, e.g. `"qwen3-4b"`. pub model: String, - /// Rank groups a multi-GPU manifest is SPMD over, e.g. `{"groups": {"ep": 4}}`; every rank loads the same manifest. + /// Rank groups of a multi-GPU manifest, e.g. `{"groups": {"ep": 4}}`: every rank of an SPMD group loads the same manifest; the two stages a cut's group joins load their own, agreeing only on the peer's `of` buffer. #[serde(default, skip_serializing_if = "Option::is_none")] pub topology: Option, + /// The pipeline stage this manifest is, when it is one piece of a cut manifest, e.g. `{"id": "9f2c…", "stage": 1, "stages": 2}`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cut: Option, /// Per-call scalars the caller supplies, e.g. `{"tokens": {"max": 2048}}`. #[serde(default, deserialize_with = "unique_map", skip_serializing_if = "BTreeMap::is_empty")] pub vars: BTreeMap, @@ -197,6 +200,8 @@ pub enum Fill { Count, /// Output: a one-word error flag of the collectives, read after every call; nonzero is a failed step. Error, + /// Output: a pipeline stage's clock, `i64 [ring × 5]`: entry `item % ring` is {item + 1, receive wait begin, end, send wait begin, end} in globaltimer ns, 0 where the stage has no such edge. A cut declares it; the stage reads it when it pleases. + Clock, } impl fmt::Display for Fill { @@ -213,6 +218,7 @@ impl fmt::Display for Fill { Fill::Tokens => "tokens", Fill::Count => "count", Fill::Error => "error", + Fill::Clock => "clock", }) } } @@ -226,6 +232,18 @@ pub struct Topology { pub groups: BTreeMap, } +/// One stage of a manifest cut into a pipeline (`kern cut`): every stage of one cut carries the same `id`; stage `stage` of `stages` receives its predecessor's activations and sends its own to the next. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct Cut { + /// Names the cut: a hash of the manifest it was cut from and where. + pub id: String, + /// This stage's place in the pipeline, from 0. + pub stage: u64, + /// Stages in the pipeline, at least 2. + pub stages: u64, +} + /// A per-call scalar the caller supplies, bounded `1..=max`; the only kind of number that may size a shape or a grid. #[derive(Debug, Clone, Serialize, Deserialize, schemars::JsonSchema)] #[serde(deny_unknown_fields)] diff --git a/crates/kern-manifest/src/verify.rs b/crates/kern-manifest/src/verify.rs index f00f0fe3..3aa8bd64 100644 --- a/crates/kern-manifest/src/verify.rs +++ b/crates/kern-manifest/src/verify.rs @@ -37,6 +37,7 @@ //! `of` and `group` only where they mean something; `{"rank": g}` //! binds only to i32/i64 params; an op with an extern launch never //! receives a peer buffer (runtime built-ins may not touch peer memory) +//! 11. cut: at least two stages, the stage below their count, an id //! //! What this deliberately cannot check: kernel *behavior*, and the //! *semantics* of interface params (that a replacement implementation @@ -195,6 +196,14 @@ fn diagnostics(m: &Manifest) -> Vec { } } } + if let Some(c) = &m.cut { + if c.id.is_empty() || c.stages < 2 || c.stage >= c.stages { + errs.push(format!( + "cut: stage {} of {} with id `{}`; a cut has at least 2 stages, a stage below their count and an id", + c.stage, c.stages, c.id + )); + } + } let group_ctx = |g: &str, errs: &mut Vec, used_groups: &mut BTreeSet, ctx: &str| -> Option { match m.group_size(g) { Some(sz) => { @@ -241,7 +250,7 @@ fn diagnostics(m: &Manifest) -> Vec { let ctx = format!("buffer `{name}`"); // 9b. fill if let Some(fill) = b.fill { - let output = matches!(fill, Fill::Tokens | Fill::Count | Fill::Error); + let output = matches!(fill, Fill::Tokens | Fill::Count | Fill::Error | Fill::Clock); match (b.kind, output) { (BufferKind::Input, false) | (BufferKind::Output, true) => {} (BufferKind::Input | BufferKind::Output, _) => errs.push(format!( diff --git a/crates/kern-manifest/tests/cut.rs b/crates/kern-manifest/tests/cut.rs new file mode 100644 index 00000000..748f1135 --- /dev/null +++ b/crates/kern-manifest/tests/cut.rs @@ -0,0 +1,294 @@ +//! A cut changes transport, never arithmetic: run symbolically, the stages +//! of any cut that verifies compute what the whole manifest computes. + +use std::collections::BTreeMap; + +use kern_manifest::cut::cut; +use kern_manifest::types::{Arg, Dir, Manifest, Module, ParamType}; +use kern_manifest::{verify, Protocol, Verified}; +use serde_json::{json, Value}; + +const QWEN3: &str = include_str!("../../../examples/qwen3-4b.json"); + +fn pp() -> Module { + Module { source: "pp_mailbox.cubin".into(), sha256: "ab".repeat(32) } +} + +struct Rand(u64); + +impl Rand { + fn next(&mut self, n: usize) -> usize { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 7; + self.0 ^= self.0 << 17; + (self.0 >> 33) as usize % n + } +} + +fn hash(parts: &[u64]) -> u64 { + parts + .iter() + .flat_map(|p| p.to_le_bytes()) + .fold(0xcbf2_9ce4_8422_2325, |h, b| (h ^ b as u64).wrapping_mul(0x100_0000_01b3)) +} + +fn name_hash(s: &str) -> u64 { + hash(&s.bytes().map(u64::from).collect::>()) +} + +/// A random straight-line program over four workspaces, an input, a carry +/// the once program derives and one paged state in per-layer regions, its +/// calls reading only what is written. Ops are opaque functions of what +/// they read. +fn random_manifest(r: &mut Rand) -> Manifest { + let ops = json!({ + "f1": ["in buffer", "out buffer"], + "f2": ["in buffer", "in buffer", "out buffer"], + "g": ["inout buffer"], + "sw": ["in buffer", "inout state"], + "sr": ["in state", "in buffer", "out buffer"], + "z": ["out buffer"] + }); + let ws = ["w0", "w1", "w2", "w3"]; + let mut readable = vec!["x", "c"]; + let mut calls: Vec = vec![json!({"op": "f1", "args": [{"buf": "x"}, {"buf": ws[r.next(4)]}]})]; + let pick = |r: &mut Rand, from: &[&'static str]| from[r.next(from.len())]; + readable.push(match &calls[0]["args"][1]["buf"] { + Value::String(s) => ws.into_iter().find(|w| w == s).unwrap(), + _ => unreachable!(), + }); + for i in 1..4 + r.next(9) { + // A layer is three calls and owns one region of the state. + let region = json!({"state": "s", "offset": 4 * (i / 3)}); + let out = ws[r.next(4)]; + let c = match r.next(5) { + 0 => json!({"op": "f1", "args": [{"buf": pick(r, &readable)}, {"buf": out}]}), + 1 => json!({"op": "f2", "args": [{"buf": pick(r, &readable)}, {"buf": pick(r, &readable)}, {"buf": out}]}), + 2 => { + let b = pick(r, &readable[2..]); + json!({"op": "g", "args": [{"buf": b}]}) + } + 3 => json!({"op": "sw", "args": [{"buf": pick(r, &readable)}, region]}), + _ => json!({"op": "sr", "args": [region, {"buf": pick(r, &readable)}, {"buf": out}]}), + }; + if c["op"] != "g" && c["op"] != "sw" && !readable.contains(&out) { + readable.push(out); + } + calls.push(c); + } + calls.push(json!({"op": "f1", "args": [{"buf": pick(r, &readable)}, {"buf": "y"}]})); + for (i, c) in calls.iter_mut().enumerate() { + c["label"] = json!(format!("c{i}")); + } + let used: Vec = calls.iter().map(|c| c["op"].as_str().unwrap().to_string()).chain(["z".into()]).collect(); + let buf = |kind: &str, shape: Value| json!({"dtype": "bf16", "shape": shape, "kind": kind}); + let mut buffers = json!({ + "x": buf("input", json!(["tokens", 8])), "y": buf("output", json!(["tokens", 8])), + "c": buf("carry", json!([8])), "d": buf("carry", json!([8])) + }); + let named = serde_json::to_string(&calls).unwrap(); + for w in ws.iter().filter(|w| named.contains(&format!("\"{w}\""))) { + buffers[*w] = buf("workspace", json!(["tokens", 8])); + } + let ops: serde_json::Map = ops + .as_object() + .unwrap() + .iter() + .filter(|(k, _)| used.contains(k)) + .map(|(k, p)| (k.clone(), json!({"params": p, "impl": {"launches": [{"entry": "extern:x"}]}}))) + .collect(); + let uses_state = named.contains("\"state\""); + Manifest::from_json( + &json!({ + "schema_version": 5, "model": "t", "vars": {"tokens": {"max": 4}}, + "states": if uses_state { json!({"s": {"bytes_per_token": 16}}) } else { json!({}) }, + "buffers": buffers, "modules": {}, "ops": ops, + "programs": { + "init": {"once": true, "calls": [ + {"label": "i0", "op": "z", "args": [{"buf": "c"}]}, + {"label": "i1", "op": "z", "args": [{"buf": "d"}]}]}, + "p": {"batch": {"groups": 1, "rows": 1}, "calls": calls} + } + }) + .to_string(), + ) + .unwrap() +} + +/// What one stage (or the whole manifest) leaves behind: its buffers and +/// the state regions it wrote. +#[derive(Default)] +struct Machine { + bufs: BTreeMap, + states: BTreeMap<(String, u64), u64>, +} + +/// The edge a pipeline buffer belongs to: `pp..box` / `pp..peer`. +fn edge(name: &str) -> u64 { + name.split('.').nth(1).unwrap().parse().unwrap() +} + +impl Machine { + /// Run `program` of `m`: every op a hash of what it reads into what it + /// writes; the pipeline ops move values through `mail`. + fn run(&mut self, m: &Manifest, program: &str, mail: &mut BTreeMap<(u64, i64), u64>) { + for c in &m.programs[program].calls { + let label = c.label.clone().unwrap(); + let buf = |a: &Arg| match a { + Arg::Buf { buf, .. } => buf.clone(), + _ => unreachable!(), + }; + let off = |a: &Arg| match a { + Arg::I64 { i64: v } => *v, + _ => unreachable!(), + }; + if c.op.starts_with("pp.put.") { + mail.insert((edge(&buf(&c.args[0])), off(&c.args[3])), self.bufs[&buf(&c.args[1])]); + continue; + } + if c.op.starts_with("pp.take.") { + self.bufs.insert(buf(&c.args[1]), mail[&(edge(&buf(&c.args[0])), off(&c.args[3]))]); + continue; + } + if c.op.starts_with("pp.") { + continue; + } + let params = &m.ops[&c.op].params; + let mut read = vec![name_hash(&c.op), name_hash(&label)]; + for (a, p) in c.args.iter().zip(params) { + match (a, p) { + (Arg::Buf { buf, .. }, ParamType::Buf { dir: Dir::In | Dir::InOut, .. }) => { + read.push(*self.bufs.get(buf).unwrap_or_else(|| panic!("{label} reads unset `{buf}`"))) + } + (Arg::State { state, offset }, ParamType::State { dir: Dir::In | Dir::InOut }) => { + read.push(*self.states.get(&(state.clone(), *offset)).unwrap_or(&0)) + } + _ => {} + } + } + let h = hash(&read); + for (j, (a, p)) in c.args.iter().zip(params).enumerate() { + let v = hash(&[h, j as u64]); + match (a, p) { + (Arg::Buf { buf, .. }, ParamType::Buf { dir: Dir::Out | Dir::InOut, .. }) => { + self.bufs.insert(buf.clone(), v); + } + (Arg::State { state, offset }, ParamType::State { dir: Dir::Out | Dir::InOut }) => { + self.states.insert((state.clone(), *offset), v); + } + _ => {} + } + } + } + } + + fn start(m: &Manifest) -> Machine { + let mut me = Machine::default(); + me.bufs.insert("x".into(), name_hash("x")); + if m.programs.contains_key("init") { + me.run(m, "init", &mut BTreeMap::new()); + } + me + } +} + +#[test] +fn every_cut_that_verifies_computes_the_whole() { + let (mut cuts, mut refused) = (0, 0); + for seed in 1..=400u64 { + let mut r = Rand(seed.wrapping_mul(0x9e37_79b9_7f4a_7c15)); + let m = verify(random_manifest(&mut r)).unwrap_or_else(|e| panic!("seed {seed}: {e}")); + let n = m.programs["p"].calls.len(); + let a = 1 + r.next(n - 1); + let at = if a + 1 < n && r.next(2) == 0 { vec![a, a + 1 + r.next(n - a - 1)] } else { vec![a] }; + + let mut whole = Machine::start(&m); + whole.run(&m, "p", &mut BTreeMap::new()); + + let stages = match cut(&m, &at, &pp()) { + Ok(s) => s, + Err(e) => { + assert!(e.0.iter().all(|d| d.contains("touched by stages")), "seed {seed} at {at:?}: {e}"); + refused += 1; + continue; + } + }; + cuts += 1; + assert_eq!(stages.len(), at.len() + 1); + let mut mail = BTreeMap::new(); + let mut states = BTreeMap::new(); + let mut y = None; + for (s, st) in stages.iter().enumerate() { + assert_eq!(st.cut.as_ref().map(|c| (c.stage, c.stages)), Some((s as u64, stages.len() as u64))); + let mut me = Machine::start(st); + me.run(st, "p", &mut mail); + for (k, v) in me.states { + assert!(states.insert(k, v).is_none(), "seed {seed}: a state region written by two stages"); + } + y = me.bufs.get("y").copied(); + } + assert_eq!((y, states), (whole.bufs.get("y").copied(), whole.states), "seed {seed} at {at:?}"); + } + assert!(cuts > 200 && refused > 0, "{cuts} cuts, {refused} refused"); +} + +fn qwen3() -> Verified { + verify(Manifest::from_json(QWEN3).unwrap()).unwrap() +} + +/// The buffers stage `s`'s program puts into its outgoing box. +fn carried(st: &Manifest, program: &str) -> Vec { + st.programs[program] + .calls + .iter() + .filter(|c| c.op.starts_with("pp.put.")) + .map(|c| match &c.args[1] { + Arg::Buf { buf, .. } => buf.clone(), + _ => unreachable!(), + }) + .collect() +} + +#[test] +fn qwen3_cuts_at_a_layer_into_stages_that_each_serve() { + // l18.qkv_proj: embed, l0 norm, 18 layers of 12 calls. + let at = 2 + 18 * 12; + assert_eq!(qwen3().programs["prefill"].calls[at].label.as_deref(), Some("l18.qkv_proj")); + let stages = cut(&qwen3(), &[at], &pp()).unwrap_or_else(|e| panic!("{e}")); + let (head, tail) = (&stages[0], &stages[1]); + // The residual stream and the normed input are all that crosses, in + // every program. + for p in ["prefill", "decode", "decode_batch"] { + assert_eq!(carried(head, p), ["residual", "x"]); + } + // Each stage holds its own weights and its own calls of the KV. + assert!( + head.buffers.contains_key("model.embed_tokens.weight") + && !tail.buffers.contains_key("model.embed_tokens.weight") + ); + assert!(tail.buffers.contains_key("lm_head.weight") && !head.buffers.contains_key("lm_head.weight")); + assert!(!head.buffers.contains_key("model.layers.18.self_attn.qkv_proj.weight")); + // The head stages tokens and hands nothing back; the tail is fed + // activations and samples. + let ph = Protocol::check_unsampled(head).unwrap_or_else(|e| panic!("{e}")); + assert!(ph.forwards.iter().all(|f| f.emits.is_none())); + let pt = Protocol::check(tail).unwrap_or_else(|e| panic!("{e}")); + assert_eq!(pt.forward(1, kern_manifest::protocol::Rows::Const(1)).map(|f| f.emits.is_some()), Some(true)); + assert_eq!(head.cut.as_ref().map(|c| &c.id), tail.cut.as_ref().map(|c| &c.id)); + // The same cut is the same bytes. + let again = cut(&qwen3(), &[at], &pp()).unwrap(); + assert_eq!(again[1].to_json(), tail.to_json()); + + let three = cut(&qwen3(), &[2 + 12 * 12, 2 + 24 * 12], &pp()).unwrap_or_else(|e| panic!("{e}")); + assert_eq!(three.len(), 3); + Protocol::check_unsampled(&three[1]).unwrap_or_else(|e| panic!("{e}")); + assert_ne!(three[0].cut.as_ref().map(|c| &c.id), head.cut.as_ref().map(|c| &c.id)); +} + +#[test] +fn a_cut_says_why_it_cannot_be_made() { + let n = qwen3().programs["prefill"].calls.len(); + for at in [vec![], vec![0], vec![5, 5], vec![n]] { + assert!(cut(&qwen3(), &at, &pp()).is_err(), "{at:?}"); + } +} diff --git a/crates/kern-run/examples/pp_chain.rs b/crates/kern-run/examples/pp_chain.rs new file mode 100644 index 00000000..d9f87e43 --- /dev/null +++ b/crates/kern-run/examples/pp_chain.rs @@ -0,0 +1,233 @@ +//! The cut gate: the stage manifests of `kern cut`, run in sequence in +//! one process, compute what the unsplit manifest computes, bit for bit. +//! +//! pp_chain --manifest whole.json --stages s0.json,s1.json --kernels dir +//! --weights dir --gpus 0,1,2 --prompts ids.txt [--chunk 512] +//! [--steps 8] [--capacity 65536] +//! +//! The first GPU runs the whole manifest, the rest one stage each, the +//! stages wired through their mailboxes as a pipeline is (here with +//! in-process handles). Each line of `--prompts` is one prompt's token +//! ids. A prompt but its last token goes through the chunk program in +//! `--chunk` pieces, then `--steps` one-row steps feed back what they +//! pick. After every step the input of the call writing the tokens (the +//! logits) and the tokens themselves must be equal on the whole and on the +//! last stage. All stages are issued before any is waited on: a stage's +//! device waits for its predecessor's mailbox on its own. + +use std::collections::BTreeMap; +use std::path::PathBuf; + +use anyhow::{bail, ensure, Context, Result}; +use kern_manifest::protocol::{Forward, Rows}; +use kern_manifest::types::Arg; +use kern_manifest::{Protocol, Verified}; +use kern_pool::Lease; +use kern_run::{fills, run_once, write_named, Vars}; +use kern_runtime::{Capacity, GroupRank, PeerHandle, Runtime, Topology}; + +struct Opts { + manifest: PathBuf, + stages: Vec, + kernels: PathBuf, + weights: Vec, + gpus: Vec, + prompts: PathBuf, + chunk: usize, + steps: usize, + capacity: u64, +} + +fn opts() -> Result { + let mut o = Opts { + manifest: PathBuf::new(), + stages: Vec::new(), + kernels: PathBuf::new(), + weights: Vec::new(), + gpus: Vec::new(), + prompts: PathBuf::new(), + chunk: 512, + steps: 8, + capacity: 65536, + }; + let mut args = std::env::args().skip(1); + while let Some(a) = args.next() { + let v = args.next().with_context(|| format!("{a}: a value"))?; + match a.as_str() { + "--manifest" => o.manifest = v.into(), + "--stages" => o.stages = v.split(',').map(PathBuf::from).collect(), + "--kernels" => o.kernels = v.into(), + "--weights" => o.weights.push(v), + "--gpus" => o.gpus = v.split(',').map(str::parse).collect::>()?, + "--prompts" => o.prompts = v.into(), + "--chunk" => o.chunk = v.parse()?, + "--steps" => o.steps = v.parse()?, + "--capacity" => o.capacity = v.parse()?, + _ => bail!("unknown argument {a}"), + } + } + ensure!(o.gpus.len() == o.stages.len() + 1, "one GPU for the whole manifest and one per stage"); + Ok(o) +} + +fn load(path: &PathBuf, o: &Opts, gpu: usize, weights: &kern_run::Weights) -> Result<(Runtime, Protocol)> { + let m = Verified::from_json(&std::fs::read_to_string(path)?).with_context(|| format!("{}", path.display()))?; + let topology = m.topology.as_ref().map(|t| Topology { + groups: t.groups.iter().map(|(g, &size)| (g.clone(), GroupRank { index: 0, size })).collect(), + }); + let capacity = Capacity { tokens: Some(o.capacity), seqs: 2 }; + let mut rt = Runtime::load(&m, Some(&o.kernels), gpu, Some(capacity), topology.as_ref())?; + weights.bind(&mut rt)?; + let p = Protocol::check_unsampled(&rt.manifest)?; + Ok((rt, p)) +} + +/// The buffer the call writing `tokens` reads: the logits it picks from. +fn logits(m: &Verified, program: &str, tokens: &str) -> Result { + let call = m.programs[program] + .calls + .iter() + .rev() + .find(|c| c.args.iter().any(|a| matches!(a, Arg::Buf { buf, .. } if buf == tokens))) + .with_context(|| format!("`{program}` writes no `{tokens}`"))?; + call.args + .iter() + .find_map(|a| match a { + Arg::Buf { buf, .. } if buf != tokens => Some(buf.clone()), + _ => None, + }) + .with_context(|| format!("the call writing `{tokens}` reads no buffer")) +} + +/// Stage one call of `ids` at `pos` into `rt` from the values `p` and +/// `lease` give. +fn item(p: &Protocol, lease: &Lease, pos: usize, ids: &[i64]) -> Result<(Vars, BTreeMap>)> { + let (vars, values) = fills(p, std::slice::from_ref(lease), &[pos], ids.len(), ids); + let mut rows: BTreeMap> = values.into_iter().map(|(f, v)| (f.name.clone(), v)).collect(); + for t in &p.page_tables { + let mut v = Vec::new(); + lease.extend_row(&t.name, &mut v)?; + rows.insert(t.name.clone(), v.into_iter().map(i64::from).collect()); + } + Ok((vars, rows)) +} + +fn issue(rt: &mut Runtime, p: &Protocol, program: &str, vars: &Vars, rows: &BTreeMap>) -> Result<()> { + let vars: Vars = + vars.iter().filter(|(k, _)| rt.manifest.vars.contains_key(*k)).map(|(k, v)| (k.clone(), *v)).collect(); + for (name, v) in rows { + write_named(rt, p, name, v, &vars)?; + } + Ok(rt.issue(program, &vars)?) +} + +fn forwards(p: &Protocol) -> Result<(Forward, Forward)> { + Ok(( + p.chunk().cloned().context("no chunk program")?, + p.forward(1, Rows::Const(1)).cloned().context("no one-row program")?, + )) +} + +fn main() -> Result<()> { + tracing_subscriber::fmt().with_env_filter(tracing_subscriber::EnvFilter::from_default_env()).init(); + let o = opts()?; + let weights = kern_run::Weights::parse(&o.weights)?; + let (mut whole, pw) = load(&o.manifest, &o, o.gpus[0], &weights)?; + let mut stages: Vec<(Runtime, Protocol)> = + o.stages.iter().zip(&o.gpus[1..]).map(|(s, &g)| load(s, &o, g, &weights)).collect::>()?; + let exports: Vec> = + stages.iter().map(|(rt, _)| rt.export_handles()).collect::>()?; + for (s, (rt, _)) in stages.iter_mut().enumerate() { + let groups: Vec = rt.manifest.topology.iter().flat_map(|t| t.groups.keys().cloned()).collect(); + for g in groups { + rt.import_peers(&g, &[exports[s].clone(), exports[s + 1].clone()])?; + } + } + run_once(&whole, &pw)?; + for (rt, p) in &stages { + run_once(rt, p)?; + } + + let (wchunk, wstep) = forwards(&pw)?; + let (_, sstep) = forwards(&stages[0].1)?; + let tokens = &pw.fills[wstep.emits.context("the whole manifest's step hands back no tokens")?]; + let (last, lp) = stages.last().unwrap(); + let ltokens = lp.fills[lp + .forwards + .iter() + .find(|f| f.name == sstep.name) + .and_then(|f| f.emits) + .context("the last stage's step hands back no tokens")?] + .clone(); + let wlogits = logits(&whole.manifest, &wstep.name, &tokens.name)?; + let llogits = logits(&last.manifest, &sstep.name, <okens.name)?; + ensure!(wlogits == llogits, "logits are `{wlogits}` whole, `{llogits}` on the last stage"); + + let wlease = whole.lease(o.capacity as usize)?; + let slease = stages[0].0.lease(o.capacity as usize)?; + let prompts: Vec> = std::fs::read_to_string(&o.prompts)? + .lines() + .filter(|l| !l.trim().is_empty()) + .map(|l| l.split_whitespace().map(str::parse).collect::>()) + .collect::>()?; + let mut failed = 0; + for (n, prompt) in prompts.iter().enumerate() { + let mut pos = 0; + let mut calls: Vec<(String, usize, Vec)> = Vec::new(); + while pos < prompt.len() - 1 { + let c = (prompt.len() - 1 - pos).min(o.chunk); + calls.push((wchunk.name.clone(), pos, prompt[pos..pos + c].to_vec())); + pos += c; + } + for (program, pos, ids) in &calls { + let (vars, rows) = item(&pw, &wlease, *pos, ids)?; + issue(&mut whole, &pw, program, &vars, &rows)?; + let (vars, rows) = item(&stages[0].1, &slease, *pos, ids)?; + for (rt, p) in &mut stages { + issue(rt, p, program, &vars, &rows)?; + } + } + let mut tok = *prompt.last().unwrap(); + let mut picked = Vec::new(); + let mut same = true; + for step in 0..o.steps { + let at = prompt.len() - 1 + step; + let (vars, rows) = item(&pw, &wlease, at, &[tok])?; + issue(&mut whole, &pw, &wstep.name, &vars, &rows)?; + let (vars, rows) = item(&stages[0].1, &slease, at, &[tok])?; + for (rt, p) in &mut stages { + issue(rt, p, &sstep.name, &vars, &rows)?; + } + for (rt, _) in &stages { + rt.synchronize()?; + } + let (last, _) = stages.last().unwrap(); + let wt = tokens.decode(&whole.read_output(&tokens.name)?)[0]; + let st = ltokens.decode(&last.read_output(<okens.name)?)[0]; + let wl = whole.read_buffer(&wlogits)?; + let sl = last.read_buffer(&llogits)?; + if wt != st || wl != sl { + let differ = wl.iter().zip(&sl).filter(|(a, b)| a != b).count(); + println!( + "prompt {n} step {step}: token {wt} whole, {st} staged; logits bytes differing {differ}/{}", + wl.len() + ); + same = false; + } + picked.push(wt); + tok = wt; + } + println!( + "prompt {n}: {} tokens, {} chunk calls, {} steps: {} {:?}", + prompt.len(), + calls.len(), + o.steps, + if same { "IDENTICAL" } else { "DIFFERENT" }, + picked + ); + failed += usize::from(!same); + } + ensure!(failed == 0, "{failed} of {} prompts differ", prompts.len()); + println!("PASS: {} prompts bitwise identical", prompts.len()); + Ok(()) +} diff --git a/crates/kern-run/src/bin/kern.rs b/crates/kern-run/src/bin/kern.rs index 771159e5..d98e1ee0 100644 --- a/crates/kern-run/src/bin/kern.rs +++ b/crates/kern-run/src/bin/kern.rs @@ -71,6 +71,23 @@ enum Cmd { /// Manifest JSON to verify (does not read kern.toml) manifest: PathBuf, }, + /// Cut a manifest into pipeline stages before the given call indices + /// of every program a serving loop drives; writes `stage.json` + /// into `--out`, each verified. No GPU + Cut { + /// Manifest JSON to cut (does not read kern.toml) + manifest: PathBuf, + /// Call indices to cut before, increasing + #[arg(long, value_delimiter = ',', required = true)] + at: Vec, + /// The mailbox kernels' cubin (`tools/kernels-src/pp_mailbox.cu`), + /// pinned by its sha256; it goes into each stage's kernels dir + #[arg(long)] + module: PathBuf, + /// Directory the stages are written to + #[arg(long)] + out: PathBuf, + }, } fn main() -> Result<()> { @@ -85,7 +102,7 @@ fn main() -> Result<()> { .init(); let cli = Cli::parse(); let cfg = match &cli.cmd { - Cmd::Verify { .. } => None, + Cmd::Verify { .. } | Cmd::Cut { .. } => None, _ => Config::find(cli.config.as_deref())?, }; match cli.cmd { @@ -118,9 +135,38 @@ fn main() -> Result<()> { } Ok(()) } + Cmd::Cut { manifest, at, module, out } => cut(&manifest, &at, &module, &out), } } +/// `kern cut`: the stages of a manifest cut before `at`, written as +/// `stage.json`. +fn cut(manifest: &Path, at: &[usize], module: &Path, out: &Path) -> Result<()> { + use sha2::Digest; + let json = std::fs::read_to_string(manifest).with_context(|| format!("reading {}", manifest.display()))?; + let m = kern_manifest::Verified::from_json(&json)?; + let bytes = std::fs::read(module).with_context(|| format!("reading {}", module.display()))?; + let pp = kern_manifest::types::Module { + source: module.file_name().context("--module names a file")?.to_string_lossy().into_owned(), + sha256: hex::encode(sha2::Sha256::digest(&bytes)), + }; + let stages = kern_manifest::cut::cut(&m, at, &pp)?; + std::fs::create_dir_all(out).with_context(|| format!("creating {}", out.display()))?; + for (s, st) in stages.iter().enumerate() { + let path = out.join(format!("stage{s}.json")); + std::fs::write(&path, st.to_json()).with_context(|| format!("writing {}", path.display()))?; + let calls: Vec = st.programs.iter().map(|(n, p)| format!("{n} {}", p.calls.len())).collect(); + tracing::info!( + "stage {s}: {} ({} weights; calls: {})", + path.display(), + st.buffers.values().filter(|b| b.kind == kern_manifest::types::BufferKind::Weight).count(), + calls.join(", ") + ); + } + tracing::info!("cut `{}` into {} stages", stages[0].cut.as_ref().map_or("", |c| c.id.as_str()), stages.len()); + Ok(()) +} + /// The target a command runs on: the named one, else the only one, else /// none at all — every command takes its inputs from flags when there is /// no `kern.toml` to look in. diff --git a/crates/kern-run/src/lib.rs b/crates/kern-run/src/lib.rs index 4f63c713..a451b617 100644 --- a/crates/kern-run/src/lib.rs +++ b/crates/kern-run/src/lib.rs @@ -26,7 +26,7 @@ use std::sync::LazyLock; use std::time::Duration; use anyhow::{bail, ensure, Context, Result}; -use kern_manifest::protocol::{Axis, Forward, LineTable, PageTable, Rows}; +use kern_manifest::protocol::{Axis, Filled, Forward, LineTable, PageTable, Rows}; use kern_manifest::types::Fill; use kern_manifest::{Protocol, Verified}; use kern_pool::Lease; @@ -94,8 +94,8 @@ pub fn le_bytes_i32(v: &[i32]) -> Vec { v.iter().flat_map(|x| x.to_le_bytes()).collect() } -/// The var vars of one call. -pub(crate) type Vars = BTreeMap; +/// The var values of one call. +pub type Vars = BTreeMap; /// The safetensors a `--weights` entry stands for: the file itself, or /// every `*.safetensors` under a directory (a checkpoint's shards) in @@ -134,11 +134,60 @@ fn map_file(f: &std::path::Path) -> Result { unsafe { memmap2::Mmap::map(&file) }.with_context(|| format!("mapping weights {}", f.display())) } -/// Stage one call: `rows` rows for each lease, at that sequence's -/// position, `ids` in row order. Returns the call's vars. +/// What one call writes into each fill: `rows` rows for each lease, at +/// that sequence's position, `ids` in row order; and the call's vars. /// -/// Every fill a caller of one rank produces is written; `blocks` is the +/// Every fill a caller of one rank produces is there; `blocks` is the /// tray batch's and only a caller that spans a rank group can fill it. +pub fn fills<'p>( + p: &'p Protocol, + leases: &[Lease], + positions: &[usize], + rows: usize, + ids: &[i64], +) -> (Vars, Vec<(&'p Filled, Vec)>) { + let b = leases.len(); + let longest = positions.iter().map(|&pos| pos + rows).max().unwrap_or(rows); + let vars = p.vars(b as u64, rows as u64, (b * rows) as u64, longest as u64); + let values = p + .fills + .iter() + .filter_map(|f| { + let v: Vec = match (f.fill, f.axis) { + // Each sequence's first token: the anchor a drafting program + // splices its own rows from. + (Fill::Token, Axis::Groups) => ids.chunks(rows).map(|x| x[0]).collect(), + (Fill::Token, _) => ids.to_vec(), + (Fill::Valid, _) => vec![1; b * rows], + (Fill::Position, _) => positions.iter().flat_map(|&pos| (pos..pos + rows).map(|v| v as i64)).collect(), + (Fill::Slot, _) => { + leases.iter().zip(positions).flat_map(|(l, &pos)| l.slots(pos..pos + rows)).collect() + } + (Fill::SeqLen, _) => positions.iter().map(|&pos| (pos + rows) as i64).collect(), + (Fill::CuSeqlens, _) => (0..=b).map(|i| (i * rows) as i64).collect(), + (Fill::SpanAt, _) => vec![0], + (Fill::Blocks | Fill::Tokens | Fill::Count | Fill::Error | Fill::Clock, _) => return None, + }; + Some((f, v)) + }) + .collect(); + (vars, values) +} + +/// Write `values` into input `name` of `rt` the way `p` reads it: a fill +/// in its dtype, a page table as i32. `false` when the manifest has no +/// such input (a pipeline stage that never reads it). +pub fn write_named(rt: &mut Runtime, p: &Protocol, name: &str, values: &[i64], vars: &Vars) -> Result { + let bytes = match (p.fills.iter().find(|f| f.name == name), p.page_tables.iter().any(|t| t.name == name)) { + (Some(f), _) => f.encode(values), + (None, true) => values.iter().flat_map(|&v| (v as i32).to_le_bytes()).collect(), + (None, false) => return Ok(false), + }; + rt.write_input_at(name, &bytes, vars)?; + Ok(true) +} + +/// Stage one call (see [`fills`]). Returns the call's vars. fn stage( rt: &mut Runtime, p: &Protocol, @@ -147,23 +196,8 @@ fn stage( rows: usize, ids: &[i64], ) -> Result { - let b = leases.len(); - let longest = positions.iter().map(|&pos| pos + rows).max().unwrap_or(rows); - let vars = p.vars(b as u64, rows as u64, (b * rows) as u64, longest as u64); - for f in &p.fills { - let v: Vec = match (f.fill, f.axis) { - // Each sequence's first token: the anchor a drafting program - // splices its own rows from. - (Fill::Token, Axis::Groups) => ids.chunks(rows).map(|x| x[0]).collect(), - (Fill::Token, _) => ids.to_vec(), - (Fill::Valid, _) => vec![1; b * rows], - (Fill::Position, _) => positions.iter().flat_map(|&pos| (pos..pos + rows).map(|v| v as i64)).collect(), - (Fill::Slot, _) => leases.iter().zip(positions).flat_map(|(l, &pos)| l.slots(pos..pos + rows)).collect(), - (Fill::SeqLen, _) => positions.iter().map(|&pos| (pos + rows) as i64).collect(), - (Fill::CuSeqlens, _) => (0..=b).map(|i| (i * rows) as i64).collect(), - (Fill::SpanAt, _) => vec![0], - (Fill::Blocks | Fill::Tokens | Fill::Count | Fill::Error, _) => continue, - }; + let (vars, values) = fills(p, leases, positions, rows, ids); + for (f, v) in values { rt.write_input_at(&f.name, &f.encode(&v), &vars)?; } Ok(vars) @@ -197,7 +231,7 @@ fn line_rows(t: &LineTable, leases: &[Lease], cols: usize) -> Result> { /// What the manifest runs once after load (the derived tables a weight /// prep program computes), with every var at 1: a once program takes no /// call shape. -fn run_once(rt: &Runtime, p: &Protocol) -> Result<()> { +pub fn run_once(rt: &Runtime, p: &Protocol) -> Result<()> { let vars: Vars = rt.manifest.vars.keys().map(|v| (v.clone(), 1)).collect(); p.once.iter().try_for_each(|name| rt.run(name, &vars).with_context(|| format!("`{name}`"))) } diff --git a/crates/kern-serve/src/tray.rs b/crates/kern-serve/src/tray.rs index f20e6a74..d80fab00 100644 --- a/crates/kern-serve/src/tray.rs +++ b/crates/kern-serve/src/tray.rs @@ -822,7 +822,7 @@ impl Tray { (Fill::CuSeqlens, _) => (0..=b as i64).map(|i| i * per as i64).collect(), (Fill::SpanAt, _) => vec![span_at], (Fill::Blocks, _) => l.offsets(&groups, q).into_iter().map(|o| (o * per) as i64).collect(), - (Fill::Tokens | Fill::Count | Fill::Error, _) => continue, + (Fill::Tokens | Fill::Count | Fill::Error | Fill::Clock, _) => continue, (Fill::Token, Axis::Fixed(_)) => unreachable!("the protocol checks fill shapes"), }; writes.push((f, v)); diff --git a/docs/manifest.md b/docs/manifest.md index a5d1e616..fb438dab 100644 --- a/docs/manifest.md +++ b/docs/manifest.md @@ -16,6 +16,7 @@ vars. var caller 每次调用供应的标量，有 states. state 不透明持久内存，runtime 按字节供应 buffers. buffer 有类型的张量：input / output / weight / workspace / carry / peer topology.groups. group 多卡 SPMD 的 rank 组：只有名字和大小 +cut cut 流水线的一段：id、第几段、共几段 ``` `call` 调 `op`，`op` 由若干 `launch` 实现，`launch` 起 `module` 里的 @@ -68,6 +69,10 @@ topology.groups. group 多卡 SPMD 的 rank 组：只有名字和 / TRT-LLM / DeepEP intranode 都是这个形状）。 - `{"rank": "ep"}`：call 或 launch 的标量实参来源，本 rank 在组里的 下标，load 时烧成常量，只能接 `i32`/`i64` 参。 +- `cut`（可选，`kern cut` 写的）：`{"id": "6a23…", "stage": 1, "stages": + 2}`——这份 manifest 是一份整 manifest 在某些 call 下标处切开后的第 + `stage` 段，同一刀切出的各段 `id` 相同（整 manifest 的 JSON 与切点的 + hash）。见「流水线切分」。 - `modules`：manifest 的依赖清单（必填）——每个 kernel launch 钉住的代码工件： `source`（本地文件名 `argmax.cubin`，或 registry ref `hf://[@revision]`）+ `sha256`。**身份是 sha256， @@ -385,6 +390,40 @@ kern-serve 只拿 `Protocol` 驱动 runtime，不读 JSON（CI 用 grep 保证 段代码，`--rows` 选每组行数（缺省取 manifest 声明的最宽）。设计与取舍见 [v4-design.md](v4-design.md)。 +## 流水线切分：`kern cut` + +`kern cut --at [,…] --module pp_mailbox.cubin --out +` 把一份单卡 manifest 在 call 下标处切成 S 段（`tools/kernels-src/pp_mailbox.cu` +是邮箱核，与模型无关；`kern_manifest::cut::cut` 是纯函数，不碰 GPU）： + +- 每个 batch program 在同一下标处切开（下标须小于每个 batch program 的 + 长度）；once program 在每段按"本段读到的东西"往回切片。活跃性分析找出 + 切点两侧都活着的 buffer：只许是 workspace、至多一个 var 维（字节数 + `var × 常数`）；output / carry 只许在末段写；一个 (state, offset) 区域 + 只许一段碰（按层切 KV 时各段各持自己那些层）。做不到的切点报错并说明 + 哪条不满足，不会静默切出错的东西。 +- 边界处插一对 send / recv：上游段末尾 `pp.wait_empty → pp.put.… + → pp.post_full`，下游段开头 `pp.wait_full → pp.take.… → + pp.post_empty`；邮箱 `pp..box`（`u8[256 + 最大载荷]`）在下游 GPU 上， + `full` / `empty` 两个计数在头里，上游经 `pp..peer` 远程写。等待在设备 + 上自旋，超时（60 s）`__trap`——进程死于 launch failure，不会挂住。 +- 每段声明 `pp.clock`（`fill: clock`，`i64 [256 × 5]` output）：第 k 项 = + {k+1, 收等开始, 收等结束, 发等开始, 发等结束}（globaltimer ns），首段的 + "收等结束"是本段开始、末段的"发等开始"是本段结束，于是每段 busy = 发等 + 开始 − 收等结束。item 下标由邮箱计数在设备上导出，不需要输入。 +- 非末段声明拓扑组 `pp.`（大小 2，本段是 0 号）。**这里放宽了 SPMD**： + 组两端装的是不同的 manifest，只在 `of` buffer（同名同字节数的 + `pp..box`，上游那份是不用的 workspace + export）上对上—— + `import_peers` 本来就只按名字和字节数匹配，runtime 不用改。 +- 首段之外不需要 `token` fill（`Protocol` 只对 `cut.stage == 0` 要求它）。 + 一个 chunk program 若自己吐 token（hybrid GDN）则拒切：prefill-only 的 + 流水线约定首个 token 从一行的 step program 出。 + +门禁：`crates/kern-manifest/tests/cut.rs` 用随机 manifest（400 个种子） ++ 符号解释器验证"能切的切点，各段依次跑 == 整份跑"，且确实有切点被拒； +GPU 上 `crates/kern-run/examples/pp_chain.rs` 把各段在同一进程里串起来， +logits 与 token 与整份逐位同（serve.md「流水线」）。 + ## Verifier（`kern-manifest`） `verify()` 收集全部错误一次报告（`VerifyErrors`）： @@ -423,7 +462,7 @@ kern-serve 只拿 `Protocol` 驱动 runtime，不读 JSON（CI 用 grep 保证 `export`；`of`/`group` 只许出现在 peer 上；peer 对 op 只读且视为初始 已写；`{"rank": g}` 只接 `i32`/`i64` 参且 g 已声明；**带 extern launch 的 op 不得收到 peer buffer**——runtime 内置（cublasLt）永远不碰 peer - 内存； + 内存；`cut` 有 id、`stages ≥ 2`、`stage < stages`； 12. pack / tensormap / cluster：`bytes` 只许作 launch 参（不进 extern）、 只接 `{"pack"}` 实参且 `size == n`，字段都在 image 内、互不重叠，引用的 接口参 / scratch / var / group 都存在；引用 `out` 接口参的指针字段算 diff --git a/schema/manifest-v5.schema.json b/schema/manifest-v5.schema.json index 10abf59b..28e7d4a7 100644 --- a/schema/manifest-v5.schema.json +++ b/schema/manifest-v5.schema.json @@ -419,6 +419,54 @@ ], "type": "object" }, + "Cut": { + "additionalProperties": false, + "description": "One stage of a manifest cut into a pipeline (`kern cut`): every stage of one cut carries the same `id`; stage `stage` of `stages` receives its predecessor's activations and sends its own to the next.", + "properties": { + "id": { + "description": "Names the cut: a hash of the manifest it was cut from and where.", + "type": "string" + }, + "stage": { + "anyOf": [ + { + "description": "This stage's place in the pipeline, from 0.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + { + "description": "Name of a numeric constant declared in constants.", + "minLength": 1, + "type": "string" + } + ], + "description": "This stage's place in the pipeline, from 0." + }, + "stages": { + "anyOf": [ + { + "description": "Stages in the pipeline, at least 2.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + { + "description": "Name of a numeric constant declared in constants.", + "minLength": 1, + "type": "string" + } + ], + "description": "Stages in the pipeline, at least 2." + } + }, + "required": [ + "id", + "stage", + "stages" + ], + "type": "object" + }, "DType": { "description": "Element type of a buffer or scratch, e.g. `\"bf16\"`.", "enum": [ @@ -959,6 +1007,11 @@ "const": "error", "description": "Output: a one-word error flag of the collectives, read after every call; nonzero is a failed step.", "type": "string" + }, + { + "const": "clock", + "description": "Output: a pipeline stage's clock, `i64 [ring × 5]`: entry `item % ring` is {item + 1, receive wait begin, end, send wait begin, end} in globaltimer ns, 0 where the stage has no such edge. A cut declares it; the stage reads it when it pleases.", + "type": "string" } ] }, @@ -2154,6 +2207,17 @@ }, "type": "object" }, + "cut": { + "anyOf": [ + { + "$ref": "#/$defs/Cut" + }, + { + "type": "null" + } + ], + "description": "The pipeline stage this manifest is, when it is one piece of a cut manifest, e.g. `{\"id\": \"9f2c…\", \"stage\": 1, \"stages\": 2}`." + }, "model": { "description": "Free-form model label, e.g. `\"qwen3-4b\"`.", "type": "string" @@ -2201,7 +2265,7 @@ "type": "null" } ], - "description": "Rank groups a multi-GPU manifest is SPMD over, e.g. `{\"groups\": {\"ep\": 4}}`; every rank loads the same manifest." + "description": "Rank groups of a multi-GPU manifest, e.g. `{\"groups\": {\"ep\": 4}}`: every rank of an SPMD group loads the same manifest; the two stages a cut's group joins load their own, agreeing only on the peer's `of` buffer." }, "vars": { "additionalProperties": { diff --git a/tools/kernels-src/pp_mailbox.cu b/tools/kernels-src/pp_mailbox.cu new file mode 100644 index 00000000..c3bcb897 --- /dev/null +++ b/tools/kernels-src/pp_mailbox.cu @@ -0,0 +1,150 @@ +// The pipeline edge between two stages of a cut manifest (`kern cut`): one +// mailbox on the downstream GPU, written by the upstream stage through a +// peer pointer. +// +// The box is `u8[256 + data]`. Its header holds two item counters, `full` +// at byte 0 (items the upstream has posted) and `empty` at byte 64 (items +// the downstream has taken out). Both live on the downstream GPU, so the +// downstream spins locally and the upstream reads and writes remote +// memory; neither side keeps a counter of its own. An upstream item: wait +// until `empty == full` (the box is free), put each boundary buffer at its +// offset, post `full + 1`. A downstream item: wait until `full > empty`, +// take each buffer out, post `empty + 1`. Each step is its own launch, so +// stream order puts every copy of an item between its two flags; a post +// fences system-wide before its release store. +// +// A wait gives up after `timeout_ns` of globaltimer and traps: the process +// dies with a launch failure instead of hanging on a dead stage. +// +// Every stage also keeps a clock: `clock` is `i64[RING * 5]`, entry `item % +// RING` = {item + 1, recv wait begin, recv wait end, send wait begin, send +// wait end} in globaltimer ns. The item is read off the counters, never +// passed: a downstream stage's is its box's `empty` before the take, an +// upstream stage's the downstream box's `full` before the post. The waits +// stamp their own spin; a first stage stamps its start as "recv end" and a +// last stage its end as "send begin", so busy = send begin - recv end on +// every stage. One thread writes each field. +// +// kern_pp_wait_empty (in u64 peers[2], out i64 clock[], i64 timeout_ns) +// kern_pp_put (in u64 peers[2], in u8 src[], i64 bytes, i64 offset) +// kern_pp_post_full (in u64 peers[2]) +// kern_pp_wait_full (in u8 box[], out i64 clock[], i64 timeout_ns) +// kern_pp_take (in u8 box[], out u8 dst[], i64 bytes, i64 offset) +// kern_pp_post_empty (inout u8 box[]) +// kern_pp_stamp_first(in u64 peers[2], out i64 clock[]) +// kern_pp_stamp_last (in u8 box[], out i64 clock[]) +// waits, posts, stamps: block [1,1,1], grid [1,1,1]; put / take: any grid. +// peers[1] is the downstream box; the upstream's own box (peers[0]) is unused. + +#define PP_HEADER 256 +#define PP_RING 256 + +typedef unsigned long long u64; + +__device__ __forceinline__ u64 gtimer() { + u64 t; + asm volatile("mov.u64 %0, %%globaltimer;" : "=l"(t)); + return t; +} + +__device__ __forceinline__ u64 ld_acquire_sys(const u64* p) { + u64 v; + asm volatile("ld.acquire.sys.global.u64 %0, [%1];" : "=l"(v) : "l"(p) : "memory"); + return v; +} + +__device__ __forceinline__ void st_release_sys(u64* p, u64 v) { + asm volatile("st.release.sys.global.u64 [%0], %1;" ::"l"(p), "l"(v) : "memory"); +} + +__device__ __forceinline__ u64* full_of(const unsigned char* box) { + return reinterpret_cast(const_cast(box)); +} + +__device__ __forceinline__ u64* empty_of(const unsigned char* box) { + return reinterpret_cast(const_cast(box) + 64); +} + +__device__ __forceinline__ unsigned char* downstream(const u64* peers) { + return reinterpret_cast(peers[1]); +} + +__device__ __forceinline__ long long* entry(long long* clock, u64 item) { + long long* e = clock + (item % PP_RING) * 5; + e[0] = (long long)item + 1; + return e; +} + +__device__ void copy(unsigned char* dst, const unsigned char* src, long long bytes) { + const long long stride = (long long)gridDim.x * blockDim.x; + const long long t = (long long)blockIdx.x * blockDim.x + threadIdx.x; + const long long vecs = bytes / 16; + const uint4* s = reinterpret_cast(src); + uint4* d = reinterpret_cast(dst); + for (long long i = t; i < vecs; i += stride) { + d[i] = s[i]; + } + for (long long i = vecs * 16 + t; i < bytes; i += stride) { + dst[i] = src[i]; + } +} + +extern "C" __global__ void kern_pp_wait_empty(const u64* peers, long long* clock, long long timeout_ns) { + const unsigned char* box = downstream(peers); + const u64 item = ld_acquire_sys(full_of(box)); + long long* e = entry(clock, item); + const u64 t0 = gtimer(); + e[3] = (long long)t0; + while (ld_acquire_sys(empty_of(box)) != item) { + if ((long long)(gtimer() - t0) > timeout_ns) { + printf("kern pp: item %llu waited %lld ns for the downstream stage to take the last one\n", item, + timeout_ns); + __trap(); + } + } + e[4] = (long long)gtimer(); +} + +extern "C" __global__ void kern_pp_put(const u64* peers, const unsigned char* src, long long bytes, + long long offset) { + copy(downstream(peers) + PP_HEADER + offset, src, bytes); +} + +extern "C" __global__ void kern_pp_post_full(const u64* peers) { + const unsigned char* box = downstream(peers); + __threadfence_system(); + st_release_sys(full_of(box), ld_acquire_sys(full_of(box)) + 1); +} + +extern "C" __global__ void kern_pp_wait_full(const unsigned char* box, long long* clock, long long timeout_ns) { + const u64 item = ld_acquire_sys(empty_of(box)); + long long* e = entry(clock, item); + const u64 t0 = gtimer(); + e[1] = (long long)t0; + while (ld_acquire_sys(full_of(box)) <= item) { + if ((long long)(gtimer() - t0) > timeout_ns) { + printf("kern pp: item %llu waited %lld ns for the upstream stage\n", item, timeout_ns); + __trap(); + } + } + __threadfence_system(); + e[2] = (long long)gtimer(); +} + +extern "C" __global__ void kern_pp_take(const unsigned char* box, unsigned char* dst, long long bytes, + long long offset) { + copy(dst, box + PP_HEADER + offset, bytes); +} + +extern "C" __global__ void kern_pp_post_empty(unsigned char* box) { + __threadfence_system(); + st_release_sys(empty_of(box), *empty_of(box) + 1); +} + +extern "C" __global__ void kern_pp_stamp_first(const u64* peers, long long* clock) { + entry(clock, ld_acquire_sys(full_of(downstream(peers))))[2] = (long long)gtimer(); +} + +extern "C" __global__ void kern_pp_stamp_last(const unsigned char* box, long long* clock) { + entry(clock, ld_acquire_sys(empty_of(box)) - 1)[3] = (long long)gtimer(); +} From b2e23399fa4db76164ddcefe2608ed8c3f78f4d6 Mon Sep 17 00:00:00 2001 From: JinYan Su <751080330@qq.com> Date: Wed, 7 Oct 2026 15:33:39 +0000 Subject: [PATCH 2/6] kern-serve: a cut manifest serves prefill as one process per stage Stage 0 (`--pp-listen`) is the head: HTTP, scheduler and the only pool, sized to the fewest pages any stage has. Other stages (`--pp-head`) send Hello {stage, cut, pages, mailbox}, get Table {downstream_mailbox, pages}, then replay the head's Item {program, vars, rows} in order; the last stage answers tokens. Activations go GPU to GPU through the mailbox. Any error exits the process; a closed connection ends the other side. Each stage logs one `pp item` line per item from its clock: rows, busy, recv_wait, send_wait. New dependencies: tokio-util (codec) for length-delimited framing, futures-util (sink/stream) to drive it, bytes for its frames; all three were already in the lock through pegainfer. Gate (tray07, qwen3-4b, 2 stages): first tokens identical to single-GPU kern-serve on 7 prompts; 8193-token prompt TTFT 313 -> 174 ms at conc1. Co-Authored-By: Claude Opus 5.5 (1M context) --- Cargo.lock | 5 + Cargo.toml | 3 + crates/kern-serve/Cargo.toml | 7 +- crates/kern-serve/src/lib.rs | 117 ++++++++--- crates/kern-serve/src/pp/head.rs | 322 ++++++++++++++++++++++++++++++ crates/kern-serve/src/pp/mod.rs | 210 +++++++++++++++++++ crates/kern-serve/src/pp/stage.rs | 143 +++++++++++++ docs/roadmap.md | 7 + docs/serve.md | 42 ++++ 9 files changed, 823 insertions(+), 33 deletions(-) create mode 100644 crates/kern-serve/src/pp/head.rs create mode 100644 crates/kern-serve/src/pp/mod.rs create mode 100644 crates/kern-serve/src/pp/stage.rs diff --git a/Cargo.lock b/Cargo.lock index f7448052..859ca407 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2292,14 +2292,19 @@ name = "kern-serve" version = "0.2.3" dependencies = [ "anyhow", + "bytes", "clap", + "futures-util", + "hex", "kern-manifest", "kern-pool", "kern-run", "kern-runtime", "pegainfer-frontend", + "serde", "serde_json", "tokio", + "tokio-util", "tracing", "tracing-subscriber", "vllm-chat", diff --git a/Cargo.toml b/Cargo.toml index 098d40f7..ab21a18e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,6 +37,7 @@ pegainfer-frontend = { git = "https://github.com/pegainfer-project/pegainfer", r vllm-chat = { git = "https://github.com/vllm-project/vllm.git", rev = "89dbb2644552d6e473a7e97da0ce8f0aa8e32c9d" } anyhow = "1" +bytes = "1" # Recorded references are ragged tables (a row per token position, list # columns for the top-k), which parquet holds and pandas / duckdb read as is. arrow-array = "=60.0.0" @@ -46,6 +47,7 @@ clap = { version = "4", features = ["derive"] } # The CUDA API kern binds is a fact about the binary, not about the build # machine: `cuda-13000` pins it, and no toolkit is needed to build. cudarc = { version = "=0.19.7", features = ["cuda-13000", "f16", "nccl"] } +futures-util = { version = "0.3", default-features = false, features = ["sink"] } half = "=2.7.1" hex = "0.4" libc = "0.2" @@ -58,6 +60,7 @@ sha2 = "0.11" thiserror = "2" tokenizers = "=0.23.2" tokio = { version = "1", features = ["rt-multi-thread", "macros"] } +tokio-util = { version = "0.7", features = ["codec"] } toml = "1.1" tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } diff --git a/crates/kern-serve/Cargo.toml b/crates/kern-serve/Cargo.toml index 13eeae10..a5f5e8c1 100644 --- a/crates/kern-serve/Cargo.toml +++ b/crates/kern-serve/Cargo.toml @@ -12,9 +12,14 @@ kern-run.workspace = true pegainfer-frontend.workspace = true anyhow.workspace = true +bytes.workspace = true clap.workspace = true +futures-util.workspace = true +hex.workspace = true +serde.workspace = true serde_json.workspace = true -tokio.workspace = true +tokio = { workspace = true, features = ["net", "time"] } +tokio-util.workspace = true tracing.workspace = true tracing-subscriber.workspace = true diff --git a/crates/kern-serve/src/lib.rs b/crates/kern-serve/src/lib.rs index 211ee83f..3b3218ec 100644 --- a/crates/kern-serve/src/lib.rs +++ b/crates/kern-serve/src/lib.rs @@ -11,6 +11,7 @@ #![deny(unsafe_code)] pub mod logline; +mod pp; mod scheduler; mod tray; @@ -22,7 +23,7 @@ use clap::Args; use kern_manifest::Verified; use kern_runtime::{Capacity, Runtime, Topology}; use pegainfer_frontend::engine::{ - drive, scheduler_pair, Engine, EngineInfo, KvCapacity, LaunchedEngine, LiveScheduler, + drive, scheduler_pair, Engine, EngineInfo, KvCapacity, LaunchedEngine, LiveScheduler, Scheduler, }; use pegainfer_frontend::vllm; use tracing::info; @@ -90,6 +91,15 @@ pub struct ServeOpts { /// (0: off) #[arg(long, default_value_t = 0.0)] pub host_gib: f64, + + /// Serve stage 0 of a cut manifest (`kern cut`): HTTP, scheduler and + /// pool, waiting at this host:port for every other stage to join + #[arg(long, conflicts_with = "pp_head")] + pub pp_listen: Option, + + /// Serve a later stage of a cut manifest, joining the head at host:port + #[arg(long)] + pub pp_head: Option, } pub fn serve(o: ServeOpts, art: Artifacts) -> Result<()> { @@ -102,6 +112,26 @@ pub fn serve(o: ServeOpts, art: Artifacts) -> Result<()> { "loading" ); let gpus = if o.gpus.is_empty() { vec![0] } else { o.gpus.clone() }; + let manifest_json = std::fs::read_to_string(&art.manifest) + .with_context(|| format!("reading manifest {}", art.manifest.display()))?; + let manifest = + Verified::from_json(&manifest_json).with_context(|| format!("manifest {}", art.manifest.display()))?; + anyhow::ensure!( + manifest.cut.is_none() || (gpus.len() == 1 && (o.pp_listen.is_some() || o.pp_head.is_some())), + "a pipeline stage runs on one GPU, with `--pp-listen` (stage 0) or `--pp-head` (the others)" + ); + if let Some(head) = &o.pp_head { + let stage = pp::stage::Stage { + head: head.clone(), + gpu: gpus[0], + kernels: art.kernels.clone(), + capacity: o.capacity, + max_seqs: o.max_seqs, + eager: o.eager, + }; + return pp::stage::run(manifest, &weights, stage); + } + let model_path = weights.dirs().remove(0); let mut stop_tokens: Vec = kern_run::eos_ids(&model_path).into_iter().map(|x| x as u32).collect(); stop_tokens.extend(&o.stop_tokens); @@ -113,52 +143,75 @@ pub fn serve(o: ServeOpts, art: Artifacts) -> Result<()> { model_path.display() ); info!(ids = ?stop_tokens, "stop tokens"); - - let manifest_json = std::fs::read_to_string(&art.manifest) - .with_context(|| format!("reading manifest {}", art.manifest.display()))?; - let manifest = - Verified::from_json(&manifest_json).with_context(|| format!("manifest {}", art.manifest.display()))?; let served_name = o.served_model_name.clone().unwrap_or_else(|| manifest.model.clone()); + + if let Some(listen) = o.pp_listen.clone() { + let head = pp::head::Head { + listen, + gpu: gpus[0], + kernels: art.kernels.clone(), + capacity: o.capacity, + chunk: o.chunk, + max_seqs: o.max_seqs, + eager: o.eager, + stop_tokens, + }; + let load = move || -> Result<(pp::head::PpHead, scheduler::Facts)> { + let h = pp::head::load(manifest, &weights, head)?; + let facts = h.facts(); + Ok((h, facts)) + }; + return launch(load, served_name, model_path, o.port); + } + // Every sequence of a tray batch group holds a token slot on each of // its `t` ranks, and each rank its pad. let t = manifest.group_size("tp").unwrap_or(1) as usize; let capacity = Capacity { tokens: o.capacity, seqs: ((o.max_seqs + 1) * t) as u64 }; - - // The scheduler thread owns the tray for its whole life: load there, - // report readiness, then drive. - let (handle, backend) = scheduler_pair(); - let (ready_tx, ready_rx) = std::sync::mpsc::channel::>(); let host_bytes = (o.host_gib * (1u64 << 30) as f64) as u64; let policy = Policy { chunk: o.chunk.map(|c| c as usize), max_seqs: o.max_seqs, stop_tokens, rows: o.rows, host_bytes }; + let (kernels, eager) = (art.kernels.clone(), o.eager); + let load = move || -> Result<(KernScheduler, scheduler::Facts)> { + let t0 = Instant::now(); + let bind = |rt: &mut Runtime, _: &Topology| weights.bind(rt); + let tray = Tray::load(&manifest, kernels.as_deref(), &gpus, capacity, &bind, host_bytes, eager)?; + info!(model = %tray.manifest().model, gpus = ?gpus, load_s = logline::secs(t0.elapsed()), "tray loaded"); + let sched = KernScheduler::new(tray, policy)?; + let facts = sched.facts(); + Ok((sched, facts)) + }; + launch(load, served_name, model_path, o.port) +} + +/// Run the scheduler `load` builds on a thread of its own, which owns it +/// (and its GPUs) for its whole life, and serve HTTP once it is ready. +fn launch(load: L, served_name: String, model_path: PathBuf, port: u16) -> Result<()> +where + S: Scheduler + 'static, + L: FnOnce() -> Result<(S, scheduler::Facts)> + Send + 'static, +{ + let (handle, backend) = scheduler_pair(); + let (ready_tx, ready_rx) = std::sync::mpsc::channel::>(); let join = std::thread::Builder::new() .name("kern-scheduler".into()) - .spawn(move || { - let load = || -> Result { - let t0 = Instant::now(); - let bind = |rt: &mut Runtime, _: &Topology| weights.bind(rt); - let tray = Tray::load(&manifest, art.kernels.as_deref(), &gpus, capacity, &bind, host_bytes, o.eager)?; - info!(model = %tray.manifest().model, gpus = ?gpus, load_s = logline::secs(t0.elapsed()), "tray loaded"); - KernScheduler::new(tray, policy) - }; - match load() { - Ok(sched) => { - let _ = ready_tx.send(Ok(sched.facts())); - // A scheduler that panicked would leave the port open - // and every request hanging: the process goes with it. - if std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| drive(sched, backend))).is_err() { - std::process::exit(101); - } - } - Err(e) => { - let _ = ready_tx.send(Err(e)); + .spawn(move || match load() { + Ok((sched, facts)) => { + let _ = ready_tx.send(Ok(facts)); + // A scheduler that panicked would leave the port open + // and every request hanging: the process goes with it. + if std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| drive(sched, backend))).is_err() { + std::process::exit(101); } } + Err(e) => { + let _ = ready_tx.send(Err(e)); + } }) .context("spawning the scheduler thread")?; // The port opens once the engine is up: `serving` follows readiness. - let (model, model_dir, port) = (served_name.clone(), model_path.clone(), o.port); + let (model, model_dir) = (served_name.clone(), model_path.clone()); let engine = async move { let facts = tokio::task::spawn_blocking(move || ready_rx.recv()) .await @@ -179,6 +232,6 @@ pub fn serve(o: ServeOpts, art: Artifacts) -> Result<()> { rt.block_on(async move { // Needs the runtime: it spawns the signal listener. let shutdown = vllm::shutdown_token_from_ctrl_c(); - vllm::serve_with_engine_count(engine, &model_path, vec![served_name], o.port, None, 1, shutdown).await + vllm::serve_with_engine_count(engine, &model_path, vec![served_name], port, None, 1, shutdown).await }) } diff --git a/crates/kern-serve/src/pp/head.rs b/crates/kern-serve/src/pp/head.rs new file mode 100644 index 00000000..0af0399c --- /dev/null +++ b/crates/kern-serve/src/pp/head.rs @@ -0,0 +1,322 @@ +//! The head: stage 0 of a pipeline, the scheduler every stage follows. +//! +//! A request leases its prompt's pages from the head's pool, the one pool +//! of the pipeline, and goes out as items: the prompt but its last token +//! in chunks through the chunk program, then the last token through the +//! one-row step, whose tokens the last stage hands back. Each item is sent +//! to every stage before the head stages and issues it itself, so a stage +//! waiting on its device for an item always has it on the way. The first +//! token finishes the request. Nothing waits on the head's device but the +//! clock read, done when nothing is in flight or the clock ring is half +//! full. + +use std::collections::{BTreeMap, VecDeque}; +use std::sync::mpsc; + +use anyhow::{bail, ensure, Context, Result}; +use kern_manifest::protocol::{Forward, Rows}; +use kern_manifest::{Protocol, Verified}; +use kern_pool::{Denied, Lease}; +use kern_run::Weights; +use kern_runtime::{Capacity, Error, Runtime}; +use pegainfer_frontend::engine::{ + FinishReason, QueuedRequest, RejectReason, RequestId, RequestLedger, Scheduler, SchedulerMetrics, +}; +use tokio::net::TcpListener; +use tracing::{error, info}; + +use super::{connect_downstream, recv, send, topology, wire, Clock, Handles, Hello, Item, Reader, Table, Writer}; +use crate::scheduler::Facts; + +/// How the head loads and what it may run at once. +pub struct Head { + pub listen: String, + pub gpu: usize, + pub kernels: Option, + pub capacity: Option, + pub chunk: Option, + pub max_seqs: usize, + pub eager: bool, + pub stop_tokens: Vec, +} + +/// A request whose items are out and whose token is not back yet. +struct Inflight { + id: RequestId, + _lease: Lease, +} + +pub struct PpHead { + rt: Runtime, + p: Protocol, + prefill: Forward, + step: Forward, + chunk: usize, + max_seqs: usize, + stop_tokens: Vec, + io: tokio::runtime::Runtime, + stages: Vec, + replies: mpsc::Receiver>, + waiting: VecDeque, + inflight: VecDeque, + clock: Clock, +} + +/// Wait for every other stage, load stage 0 at the pages they all have, +/// map stage 1's mailbox and answer each stage with its table. +pub fn load(m: Verified, weights: &Weights, h: Head) -> Result { + let cut = m.cut.clone().context("the head serves a cut manifest")?; + ensure!(cut.stage == 0, "`--pp-listen` serves stage 0; this manifest is stage {} of {}", cut.stage, cut.stages); + let io = super::io()?; + let listener = io.block_on(TcpListener::bind(&h.listen)).with_context(|| format!("binding {}", h.listen))?; + info!(listen = %h.listen, stages = cut.stages, cut = %cut.id, "waiting for the stages"); + let mut hellos: BTreeMap = BTreeMap::new(); + while (hellos.len() as u64) < cut.stages - 1 { + let (s, addr) = io.block_on(listener.accept())?; + let (mut r, w) = wire(s); + let hello: Hello = io.block_on(recv(&mut r))?.with_context(|| format!("{addr} closed before its hello"))?; + ensure!( + hello.cut == cut.id, + "{addr}: stage {} is of cut `{}`, the head of `{}`", + hello.stage, + hello.cut, + cut.id + ); + ensure!( + (1..cut.stages).contains(&hello.stage) && !hellos.contains_key(&hello.stage), + "{addr}: stage {} is not a stage still missing of 1..{}", + hello.stage, + cut.stages + ); + info!(stage = hello.stage, %addr, pages = hello.pages, "stage joined"); + hellos.insert(hello.stage, (hello, r, w)); + } + let unit = kern_pool::page_unit(&m); + let fewest = hellos.values().map(|(x, ..)| x.pages).min().expect("at least one other stage"); + let tokens = h.capacity.map_or(fewest * unit, |c| c.min(fewest * unit)); + let seqs = (h.max_seqs + 1) as u64; + let mut rt = Runtime::load( + &m, + h.kernels.as_deref(), + h.gpu, + Some(Capacity { tokens: Some(tokens), seqs }), + topology(&m).as_ref(), + )?; + rt.set_eager(h.eager); + weights.bind(&mut rt)?; + connect_downstream(&mut rt, &hellos[&1].0.mailbox)?; + let p = Protocol::check_unsampled(&rt.manifest)?; + kern_run::run_once(&rt, &p)?; + ensure!(p.line_tables.is_empty(), "a pipeline serves paged state only; this manifest has per-sequence states"); + let pages = rt.pages_total() as u64; + + let downstream: Vec> = + (1..cut.stages).map(|s| hellos.get(&(s + 1)).map(|(x, ..)| x.mailbox.clone())).collect(); + let (tx, replies) = mpsc::channel(); + let mut stages = Vec::new(); + for ((s, (_, r, mut w)), down) in hellos.into_iter().zip(downstream) { + io.block_on(send(&mut w, &Table { downstream_mailbox: down, pages }))?; + let tx = (s == cut.stages - 1).then(|| tx.clone()); + io.spawn(listen(s, r, tx)); + stages.push(w); + } + + let prefill = + p.chunk().cloned().context("a pipeline prefills through a program taking one sequence's rows as fed")?; + let step = p.forward(1, Rows::Const(1)).cloned().context("no program takes one sequence of one row")?; + let max = p.rows.max as usize; + let chunk = match h.chunk { + None => max, + Some(c) if (1..=max as u64).contains(&c) => c as usize, + Some(c) => bail!("--chunk {c}: the manifest's `tokens` bound is {max}; a chunk can only be smaller"), + }; + let clock = Clock::new(&p)?; + info!(pages, page = rt.page(), chunk, prefill = %prefill.name, step = %step.name, "pipeline head ready"); + Ok(PpHead { + rt, + p, + prefill, + step, + chunk, + max_seqs: h.max_seqs, + stop_tokens: h.stop_tokens, + io, + stages, + replies, + waiting: VecDeque::new(), + inflight: VecDeque::new(), + clock, + }) +} + +/// Read one stage's connection to its end: the last stage's tokens go to +/// the scheduler; anything closing ends the head. +async fn listen(stage: u64, mut r: Reader, tokens: Option>>) { + let why = loop { + match (recv::>(&mut r).await, &tokens) { + (Ok(Some(t)), Some(tx)) => { + if tx.send(t).is_err() { + return; + } + } + (Ok(Some(_)), None) => break "a stage other than the last handed tokens back".to_string(), + (Ok(None), _) => break "stage closed its connection; the pipeline is gone".to_string(), + (Err(e), _) => break format!("stage connection: {e:#}"), + } + }; + error!(stage, "{why}"); + std::process::exit(1); +} + +impl PpHead { + pub fn facts(&self) -> Facts { + Facts { + total_blocks: self.rt.pages_total(), + block_size: self.rt.page() as usize, + max_request_tokens: self.rt.max_seq_tokens(), + } + } + + /// Lease each waiting prompt in order and send its items out; one that + /// cannot be seated stops the scan. + fn admit(&mut self, ledger: &mut RequestLedger) -> Result<()> { + while let Some(q) = self.waiting.front() { + let id = q.id; + if ledger.is_aborted(id) { + ledger.retire(id); + self.waiting.pop_front(); + continue; + } + if self.inflight.len() >= self.max_seqs { + break; + } + let (prompt, max_tokens) = (q.request.prompt_tokens.len(), q.request.max_tokens); + let limit = self.rt.max_seq_tokens(); + let leased = + if prompt == 0 { Err(Error::Denied(Denied::ExceedsRow { limit })) } else { self.rt.lease(prompt) }; + let reject = match leased { + Ok(lease) => { + let q = self.waiting.pop_front().unwrap(); + ledger.admit(id); + ledger.set_cached_tokens(id, 0); + self.send_prompt(&q, &lease)?; + self.inflight.push_back(Inflight { id, _lease: lease }); + continue; + } + Err(Error::Denied(Denied::Busy | Denied::Remapping)) => break, + Err(Error::Denied(Denied::ExceedsRow { limit })) => { + Some(RejectReason::ContextLength { prompt_tokens: prompt, max_tokens, limit }) + } + Err(Error::Denied(Denied::ExceedsPool)) => { + Some(RejectReason::KvBudget { prompt_tokens: prompt, worst_case_tokens: prompt }) + } + Err(e) => return Err(e.into()), + }; + if let Some(r) = reject { + ledger.reject(id, r); + self.waiting.pop_front(); + } + } + Ok(()) + } + + /// The prompt as items: all but its last token in chunks, the last + /// through the step, whose token comes back. + fn send_prompt(&mut self, q: &QueuedRequest, lease: &Lease) -> Result<()> { + let ids: Vec = q.request.prompt_tokens.iter().map(|&t| t as i64).collect(); + let n = ids.len(); + let (prefill, step) = (self.prefill.clone(), self.step.clone()); + let mut pos = 0; + while pos < n - 1 { + let c = (n - 1 - pos).min(self.chunk); + self.send_item(&prefill, lease, pos, &ids[pos..pos + c])?; + pos += c; + } + self.send_item(&step, lease, n - 1, &ids[n - 1..]) + } + + fn send_item(&mut self, f: &Forward, lease: &Lease, pos: usize, ids: &[i64]) -> Result<()> { + let (vars, values) = kern_run::fills(&self.p, std::slice::from_ref(lease), &[pos], ids.len(), ids); + let mut rows: BTreeMap> = values.into_iter().map(|(f, v)| (f.name.clone(), v)).collect(); + for t in &self.p.page_tables { + let mut v = Vec::with_capacity(t.width); + lease.extend_row(&t.name, &mut v)?; + rows.insert(t.name.clone(), v.into_iter().map(i64::from).collect()); + } + let item = Item { program: f.name.clone(), vars, rows }; + for w in &mut self.stages { + self.io.block_on(send(w, &item))?; + } + for (name, v) in &item.rows { + kern_run::write_named(&mut self.rt, &self.p, name, v, &item.vars)?; + } + self.rt.issue(&item.program, &item.vars).with_context(|| format!("`{}`", item.program))?; + self.clock.issued(ids.len() as u64); + Ok(()) + } + + /// Admit, collect, and read the clock whenever this step issued + /// nothing: the read waits for the head's device, which is also how a + /// mailbox wait that gave up on a dead stage surfaces here. + fn advance(&mut self, ledger: &mut RequestLedger) -> Result<()> { + let issued = self.clock.pending(); + self.admit(ledger)?; + self.collect(ledger)?; + if self.clock.pending() == issued || self.clock.half_full() { + self.clock.flush(&self.rt, 0)?; + } + Ok(()) + } + + /// Every token back since the last step finishes its request, oldest + /// first: the last stage answers in issue order. + fn collect(&mut self, ledger: &mut RequestLedger) -> Result<()> { + loop { + let t = match self.replies.try_recv() { + Ok(t) => t, + Err(mpsc::TryRecvError::Empty) => return Ok(()), + Err(mpsc::TryRecvError::Disconnected) => bail!("the last stage's connection is gone"), + }; + let Some(r) = self.inflight.pop_front() else { bail!("a token came back for no request") }; + let Some(&tok) = t.first() else { bail!("the last stage handed back no token") }; + if ledger.is_aborted(r.id) { + ledger.retire(r.id); + continue; + } + let tok = tok as u32; + ledger.push_tokens(r.id, &[tok], &[]); + let why = if self.stop_tokens.contains(&tok) { FinishReason::Stop } else { FinishReason::Length }; + ledger.finish(r.id, why); + } + } +} + +// Built on the scheduler thread and used there only, like the `Tray`. +#[allow(unsafe_code)] +unsafe impl Send for PpHead {} + +impl Scheduler for PpHead { + fn submit(&mut self, request: QueuedRequest) { + self.waiting.push_back(request); + } + + /// A failed step ends the process: the stages see the connection close + /// and follow, and no half-fed pipeline lingers behind an open port. + fn step(&mut self, ledger: &mut RequestLedger) -> Result<()> { + if let Err(e) = self.advance(ledger) { + error!("pipeline head: {e:#}"); + std::process::exit(1); + } + Ok(()) + } + + fn metrics(&self) -> SchedulerMetrics { + SchedulerMetrics { + kv_used_blocks: self.rt.pages_used() as u64, + kv_total_blocks: self.rt.pages_total() as u64, + num_running_reqs: self.inflight.len() as u64, + num_waiting_reqs: self.waiting.len() as u64, + spec_decode: None, + } + } +} diff --git a/crates/kern-serve/src/pp/mod.rs b/crates/kern-serve/src/pp/mod.rs new file mode 100644 index 00000000..0c0eaf4c --- /dev/null +++ b/crates/kern-serve/src/pp/mod.rs @@ -0,0 +1,210 @@ +//! Pipeline serving: one process per stage of a cut manifest (`kern cut`), +//! prefill only — a request gets its first token and finishes. +//! +//! Stage 0 is the head: the HTTP front end, the scheduler and the one +//! [`kern_pool::Pool`] that decides every lease. The other stages are +//! executors that hold no accounting of their own. Each stage's KV pages +//! hold that stage's layers under the head's page ids, which is why the +//! head's pool is sized to the smallest stage: the stages say how many +//! pages they have in their [`Hello`], the head loads with the minimum. +//! +//! One TCP connection per downstream stage, framed by `tokio-util`'s +//! length-delimited codec, each frame one JSON message. A stage connects +//! and sends its [`Hello`]: its place, the cut it belongs to, its pages and +//! the export handles of its allocations (its mailbox among them). Once +//! every stage has said hello the head loads, maps the first stage's +//! mailbox and answers each stage with a [`Table`]: the next stage's +//! handles, to map its own outgoing peer from, and the pages. From then on +//! the connection carries the log: the head sends every stage every +//! [`Item`] in issue order — the program, its vars and the values of each +//! input it staged — and the last stage answers each item whose program +//! hands tokens back with the tokens. TCP keeps the order; there is no +//! sequence number, no stop message and no fault message. A stage that +//! fails exits; a closed connection ends the process on the other side. +//! +//! The activations never touch the connection: they go GPU to GPU through +//! the mailbox the cut wired in, and each stage's device waits for its +//! predecessor on its own. A stage only enqueues; it synchronizes when it +//! has nothing to enqueue (to read its clock) or when it must read tokens. + +pub mod head; +pub mod stage; + +use std::collections::{BTreeMap, VecDeque}; + +use anyhow::{bail, Context, Result}; +use bytes::Bytes; +use futures_util::{SinkExt, StreamExt}; +use kern_manifest::protocol::Axis; +use kern_manifest::types::Fill; +use kern_manifest::Protocol; +use kern_runtime::{GroupRank, PeerHandle, Runtime, Topology}; +use serde::{Deserialize, Serialize}; +use tokio::net::tcp::{OwnedReadHalf, OwnedWriteHalf}; +use tokio::net::TcpStream; +use tokio_util::codec::{FramedRead, FramedWrite, LengthDelimitedCodec}; + +/// A stage introducing itself to the head. +#[derive(Debug, Serialize, Deserialize)] +pub struct Hello { + pub stage: u64, + pub cut: String, + pub pages: u64, + /// Every allocation the stage exports, by name, as hex fabric handles. + pub mailbox: Handles, +} + +/// The head's answer: the next stage's exports (none for the last stage) +/// and the pages every stage addresses. +#[derive(Debug, Serialize, Deserialize)] +pub struct Table { + pub downstream_mailbox: Option, + pub pages: u64, +} + +/// One call, as the head staged it. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Item { + pub program: String, + pub vars: BTreeMap, + /// Each input's values by name: the fills and the page tables. + pub rows: BTreeMap>, +} + +pub type Handles = BTreeMap; + +/// This runtime's exports as wire handles; a stage's box must be mappable +/// from another process, so a local handle is an error. +fn handles(rt: &Runtime) -> Result { + rt.export_handles()? + .into_iter() + .map(|(name, h)| { + let b = h.to_bytes().with_context(|| { + format!("`{name}` has no fabric handle; a pipeline stage maps its neighbour across processes") + })?; + Ok((name, hex::encode(b))) + }) + .collect() +} + +fn peer_handles(h: &Handles) -> Result> { + h.iter() + .map(|(name, x)| { + let b = hex::decode(x).with_context(|| format!("handle of `{name}`"))?; + Ok((name.clone(), PeerHandle::from_bytes(&b).with_context(|| format!("handle of `{name}`"))?)) + }) + .collect() +} + +/// A stage's place in the group of its outgoing edge: always the +/// upstream member, rank 0 of 2. +fn topology(rt_manifest: &kern_manifest::Manifest) -> Option { + rt_manifest.topology.as_ref().map(|t| Topology { + groups: t.groups.iter().map(|(g, &size)| (g.clone(), GroupRank { index: 0, size })).collect(), + }) +} + +/// Map the next stage's box into this stage's outgoing peer buffer. +fn connect_downstream(rt: &mut Runtime, downstream: &Handles) -> Result<()> { + let groups: Vec = rt.manifest.topology.iter().flat_map(|t| t.groups.keys().cloned()).collect(); + let own = rt.export_handles()?; + let theirs = peer_handles(downstream)?; + for g in groups { + rt.import_peers(&g, &[own.clone(), theirs.clone()]) + .with_context(|| format!("mapping the next stage over `{g}`"))?; + } + let pending = rt.pending_peers(); + if !pending.is_empty() { + bail!("peer buffers {pending:?} still unfilled after the next stage was mapped"); + } + Ok(()) +} + +type Reader = FramedRead; +type Writer = FramedWrite; + +fn wire(s: TcpStream) -> (Reader, Writer) { + let _ = s.set_nodelay(true); + let (r, w) = s.into_split(); + (FramedRead::new(r, LengthDelimitedCodec::new()), FramedWrite::new(w, LengthDelimitedCodec::new())) +} + +async fn send(w: &mut Writer, msg: &T) -> Result<()> { + w.send(Bytes::from(serde_json::to_vec(msg)?)).await.context("sending") +} + +/// The next message; `None` when the other side closed. +async fn recv Deserialize<'de>>(r: &mut Reader) -> Result> { + match r.next().await { + None => Ok(None), + Some(frame) => Ok(Some(serde_json::from_slice(&frame.context("receiving")?)?)), + } +} + +/// The IO runtime a process's connections live on, one thread beside the +/// one driving the GPU. +fn io() -> Result { + Ok(tokio::runtime::Builder::new_multi_thread().worker_threads(1).enable_all().build()?) +} + +/// What a stage's device spent per item, read off its clock output: one +/// log line per item, busy and waiting on either side. +struct Clock { + name: String, + ring: u64, + logged: u64, + /// Rows of each item issued and not yet logged, oldest first. + rows: VecDeque, +} + +impl Clock { + fn new(p: &Protocol) -> Result { + let f = p.any(Fill::Clock).context("a cut stage declares a `clock` fill")?; + let Axis::Fixed(n) = f.axis else { bail!("`{}`: a clock is a fixed ring", f.name) }; + Ok(Clock { name: f.name.clone(), ring: n / 5, logged: 0, rows: VecDeque::new() }) + } + + fn issued(&mut self, rows: u64) { + self.rows.push_back(rows); + } + + /// Items issued and not yet logged. + fn pending(&self) -> usize { + self.rows.len() + } + + /// Whether the oldest unread entry is half a ring from being overwritten. + fn half_full(&self) -> bool { + self.rows.len() as u64 >= self.ring / 2 + } + + /// Read the clock (waiting for the stage's stream) and log every item + /// issued since the last read. + fn flush(&mut self, rt: &Runtime, stage: u64) -> Result<()> { + if self.rows.is_empty() { + return Ok(()); + } + let raw = rt.read_output(&self.name)?; + let e: Vec = raw.as_chunks::<8>().0.iter().map(|c| i64::from_le_bytes(*c)).collect(); + let span = |a: i64, b: i64| (a != 0 && b != 0).then_some(b - a); + for rows in self.rows.drain(..) { + let k = self.logged; + self.logged += 1; + let x = &e[(k % self.ring) as usize * 5..][..5]; + if x[0] != k as i64 + 1 { + tracing::warn!(stage, item = k, "pp clock entry overwritten before it was read"); + continue; + } + tracing::info!( + stage, + item = k, + rows, + busy_ns = span(x[2], x[3]), + recv_wait_ns = span(x[1], x[2]), + send_wait_ns = span(x[3], x[4]), + "pp item" + ); + } + Ok(()) + } +} diff --git a/crates/kern-serve/src/pp/stage.rs b/crates/kern-serve/src/pp/stage.rs new file mode 100644 index 00000000..64ddb35f --- /dev/null +++ b/crates/kern-serve/src/pp/stage.rs @@ -0,0 +1,143 @@ +//! A downstream stage: loads its slice of the model, says hello to the +//! head, maps the next stage's mailbox from the table it gets back, then +//! replays the head's items in order. It keeps no accounting: the page +//! ids, positions and slots of every item are the head's. The last stage +//! hands back the tokens of every item whose program emits them. + +use std::sync::mpsc; +use std::time::{Duration, Instant}; + +use anyhow::{bail, ensure, Context, Result}; +use kern_manifest::{Protocol, Verified}; +use kern_run::{Vars, Weights}; +use kern_runtime::{Capacity, Runtime}; +use tokio::net::TcpStream; +use tracing::{error, info}; + +use super::{connect_downstream, handles, recv, send, topology, wire, Clock, Hello, Item, Table}; + +/// How a stage loads and where its head listens. +pub struct Stage { + pub head: String, + pub gpu: usize, + pub kernels: Option, + pub capacity: Option, + pub max_seqs: usize, + pub eager: bool, +} + +/// How long a stage keeps trying to reach a head that is not up yet. +const CONNECT: Duration = Duration::from_secs(600); + +pub fn run(m: Verified, weights: &Weights, o: Stage) -> Result<()> { + let cut = m.cut.clone().context("`--pp-head` serves a stage of a cut manifest")?; + ensure!(cut.stage > 0, "stage 0 is the head; it takes `--pp-listen`"); + let last = cut.stage + 1 == cut.stages; + let t0 = Instant::now(); + let capacity = Capacity { tokens: o.capacity, seqs: (o.max_seqs + 1) as u64 }; + let mut rt = Runtime::load(&m, o.kernels.as_deref(), o.gpu, Some(capacity), topology(&m).as_ref())?; + rt.set_eager(o.eager); + weights.bind(&mut rt)?; + let p = Protocol::check_unsampled(&rt.manifest)?; + info!( + stage = cut.stage, + gpu = o.gpu, + pages = rt.pages_total(), + load_s = t0.elapsed().as_secs_f64(), + "stage loaded" + ); + + let io = super::io()?; + let stream = io.block_on(connect(&o.head))?; + let (mut r, mut w) = wire(stream); + let hello = Hello { stage: cut.stage, cut: cut.id.clone(), pages: rt.pages_total() as u64, mailbox: handles(&rt)? }; + io.block_on(send(&mut w, &hello))?; + let table: Table = io.block_on(recv(&mut r))?.context("the head closed before its table")?; + ensure!( + table.pages <= rt.pages_total() as u64, + "the head addresses {} pages, this stage has {}", + table.pages, + rt.pages_total() + ); + match (&table.downstream_mailbox, last) { + (Some(d), false) => connect_downstream(&mut rt, d)?, + (None, true) => {} + (d, _) => bail!( + "stage {} of {}: the head sent {} next stage's mailbox", + cut.stage, + cut.stages, + if d.is_some() { "a" } else { "no" } + ), + } + kern_run::run_once(&rt, &p)?; + info!(stage = cut.stage, head = %o.head, last, "stage serving"); + + let (tx, items) = mpsc::channel::(); + io.spawn(async move { + loop { + match recv::(&mut r).await { + Ok(Some(item)) => { + if tx.send(item).is_err() { + return; + } + } + Ok(None) => { + info!("the head closed its connection"); + return; + } + Err(e) => { + error!("head connection: {e:#}"); + return; + } + } + } + }); + + let mut clock = Clock::new(&p)?; + loop { + let item = match items.try_recv() { + Ok(item) => item, + Err(mpsc::TryRecvError::Empty) => { + clock.flush(&rt, cut.stage)?; + match items.recv() { + Ok(item) => item, + Err(_) => break, + } + } + Err(mpsc::TryRecvError::Disconnected) => break, + }; + let vars: Vars = item.vars.into_iter().filter(|(k, _)| rt.manifest.vars.contains_key(k)).collect(); + for (name, v) in &item.rows { + kern_run::write_named(&mut rt, &p, name, v, &vars)?; + } + rt.issue(&item.program, &vars).with_context(|| format!("`{}`", item.program))?; + clock.issued(vars.get(&p.rows.var).copied().unwrap_or(0)); + let emits = p.forwards.iter().find(|f| f.name == item.program).and_then(|f| f.emits); + if let Some(i) = emits.filter(|_| last) { + let t = &p.fills[i]; + let mut tokens = t.decode(&rt.read_output(&t.name)?); + tokens.truncate(t.width as usize); + io.block_on(send(&mut w, &tokens))?; + } + if clock.half_full() { + clock.flush(&rt, cut.stage)?; + } + } + clock.flush(&rt, cut.stage)?; + Ok(()) +} + +/// Reach the head, retrying while it is still loading. +async fn connect(head: &str) -> Result { + let t0 = Instant::now(); + loop { + match TcpStream::connect(head).await { + Ok(s) => return Ok(s), + Err(e) if t0.elapsed() < CONNECT => { + tracing::debug!("head {head}: {e}"); + tokio::time::sleep(Duration::from_millis(500)).await; + } + Err(e) => return Err(e).with_context(|| format!("connecting to the head at {head}")), + } + } +} diff --git a/docs/roadmap.md b/docs/roadmap.md index 2f41e89e..872c5e50 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -58,6 +58,13 @@ kern 在内部 RSI 里是 kernel agent 的反馈环境（`kern test` 裁判、`k | R2 ✅ | blob store `Pegainfer/kern-kernels`（private，`blobs/` 平铺）+ `$KERN_INDEX_DIR/.toml` + `tools/kernels/abi/.py` + `check.py`（CI）；三个预编译核（trtllm fmha ctx、DSL mla decode、flash_kda）与手写核迁入，`handwritten.hw()` / `prebuilt()` 收编为 index 查询，`tools/kernels-bin/` 与生成器里的 nvcc 路径删除 | 重生成的 `examples/k3-*.json` 除 module `source` 外逐字节相同；4 层 TP4 prefill c8/c40/c7 与 EP4 c8/c40+free16 全 exact（同 P 行口径）；`check.py --online` 全部命中。**2026-09-19 tray14 过**：10 个 example 重生成 0 行非 source 差异；43 族 49 变体 53 blob 上传，`check.py` 离线/在线 0 问题；k3_golden 用空 `--cubins` 目录跑 TP4 c8/c40/c7、EP4 c8/c40+free16、decode span8/plain 七项，exact 数与 09-18 逐项相同（span8 的 1 wrong 是同一个近平局）；`~/bench_results/2026-09-19-registry/` | | R3 ✅ | 第一个新族 `trtllm_bmm_mxe4m3_mxe2m1_mxe4m3`（flashinfer-cubin 0.6.18 的 trtllm-gen batched GEMM，`siTuGlu` 变体即 K3 的激活）：整族导入、ABI 从 bundle 的 `KernelParams.h` + `flashinferMetaInfo.h` 写、vLLM 容器里 `trtllm_fp4_block_scale_moe` 探针 capture 验证；K3 prefill 的 MoE 换成 allgather(fp8 latent) → CTA 表 → FC1 → FC2 → finalize → reduce-scatter，MegaMoE 留给 decode 形态；`[[pick]]` 由 `kern bench` 对 16k/4 行的 K3 形状扫 tile 变体写入 | 4 层 TP4 prefill 门禁不变；93 层 12.9k 判定点对；16k over 0 < 780 ms（MegaMoE 231 ms 里 GEMM 之外的 86 ms 是靶，新增两次集合通信预算 ~40 ms）。**2026-09-19 tray03 过**：两族 215 变体导入、ABI 移植对 capture 逐字段 0 差异、probe 门禁 `out` 逐位同；`kern bench` 扫 18 对 tile 变体（`k3-prefill-16k-ep4`）写入 pick（tile 128：FC1 0.85 / FC2 0.40 ms 每层，捕获的 tile 64 是 1.23 / 0.58）；4 层 TP4 c40 1/1、c7 6/6，c8 4/5（step 15 是 4 ulp 近平局，逐层对比 MegaMoE 形态：同专家集合的行差 ~5% 即两条路各自的 fp8 中间量化）；93 层 12.9k 单块 2/2、c8 16/17（之前 13/17）；**16k over 0 = 749.5 ms**（之前 780；MoE 本体 231 → ~156，reduce-scatter 多出的 54 ms 是等最慢 rank 的专家负载）；`~/bench_results/2026-09-19-registry/` | +## 流水线线（PP） + +| 级 | 内容 | 门禁 | +|---|---|---| +| PP1 ✅ | `kern cut`（按 call 下标切、活跃性找边界、send/recv 邮箱核、`clock` fill）+ kern-serve 一段一进程的 prefill-only 流水（head 持唯一的池，TCP 发 item，GPU 邮箱传激活） | **2026-10-07 tray07 过**：qwen3-4b 2 段，各段依次跑 logits/token 与整份逐位同（chunk 512/64）；两进程首 token 与单卡同；8193 token conc1 TTFT 313 → 174 ms（serve.md「流水线」） | +| PP2 | 每段只分自己那些层的 KV（现在按整份 `bytes_per_token`）；decode 走流水；per-seq state（GDN）与吐 token 的 chunk program | | + ## 协议线 | 级 | 内容 | 门禁 | diff --git a/docs/serve.md b/docs/serve.md index d9edbccb..02c8e101 100644 --- a/docs/serve.md +++ b/docs/serve.md @@ -567,6 +567,48 @@ toy 抓到的几条： toy 不测 kernel 数值和性能，也没有 tray（EP4 的 toy 要一个走 peer 指针的 collective，还没写）。 +## 流水线（prefill-only PP v1，2026-10-07，tray07 GB300） + +一段一个进程：`kern cut` 切出的 stage 0 是 head（`--pp-listen host:port`：HTTP、scheduler、唯一的 +`Pool`），其余段 `--pp-head host:port`。握手：每段连上 head 发 `Hello { stage, cut, pages, mailbox }` +（export handle 的 hex），head 收齐 S−1 个、按最少的页数装自己的池，回 `Table { downstream_mailbox, +pages }`；之后同一条 TCP（tokio-util `LengthDelimitedCodec` + JSON）上 head 按发射顺序给每段发 +`Item { program, vars, rows }`（fill 与页表的值，页号全流水共用），末段对吐 token 的 item 回 token。 +激活不走 TCP，走 GPU 上的邮箱（manifest.md「流水线切分」），各段设备自己等上游。没有序号、 +stop、fault 消息：任何错误进程退出，连接断了对端退出；设备上的邮箱等待 60 s 超时 trap。 +只做 prefill：prompt 除末 token 按 chunk 走 chunk program，末 token 走一行的 step，首 token 即结束 +（`max_tokens` 视为 1）。每段每个 item 一行 `pp item` 日志：rows、busy_ns、recv_wait_ns、send_wait_ns +（读 `clock` fill，空闲时读，不加协议消息）。 + +门禁（qwen3-4b，切在 `l18.qkv_proj` 前，2 段）： +- 只切不分进程：`pp_chain` 各段依次跑 vs 整份，7 条 prompt（1–2049 token），chunk 512 × 8 步、 + chunk 64 × 16 步（2049 token 32 个 chunk），每步 logits 与 token **逐位同**。 +- 两进程 serve：7 条 gate prompt 首 token 与单卡 kern-serve 全同；杀末段 → head 立即退出；冻住末段 + （SIGSTOP）→ head 61 s 后 trap 退出，恢复的末段 61 s 后同样退出。 + +扫描（chunk 512，`--max-seqs 16`，每格 conc1 4 条 / conc4 16 条不同 prompt；TTFT = max_tokens=1 的 +请求延迟；busy/wait 是每 item 均值）： + +| conc | prompt | chunks | TTFT unsplit ms | TTFT pp ms | speedup | stage | chunk busy ms | step busy ms | recv_wait ms/item | send_wait ms/item | +|---|---:|---:|---:|---:|---:|---|---:|---:|---:|---:| +| c1 | 513 | 1 | 8.1 | 8.5 | 0.95x | 0 | 2.99 | 1.32 | - | 0.00 | +| | | | | | | 1 | 2.90 | 1.46 | 1.48 | - | +| c1 | 2049 | 4 | 38.2 | 25.3 | 1.51x | 0 | 4.27 | 1.46 | - | 0.00 | +| | | | | | | 1 | 4.21 | 1.60 | 1.16 | - | +| c1 | 8193 | 16 | 313.4 | 173.8 | 1.80x | 0 | 9.57 | 1.99 | - | 0.00 | +| | | | | | | 1 | 9.52 | 2.13 | 1.01 | - | +| c4 | 513 | 1 | 29.7 | 17.6 | 1.68x | 0 | 2.96 | 1.32 | - | 0.00 | +| | | | | | | 1 | 2.91 | 1.46 | 0.09 | - | +| c4 | 2049 | 4 | 139.6 | 78.5 | 1.78x | 0 | 4.26 | 1.46 | - | 0.19 | +| | | | | | | 1 | 4.20 | 1.60 | 0.30 | - | +| c4 | 8193 | 16 | 1194.0 | 664.5 | 1.80x | 0 | 9.57 | 1.99 | - | 0.61 | +| | | | | | | 1 | 9.52 | 2.13 | 0.73 | - | + +单请求内 chunk 之间就流水起来（stage 0 算第 k+1 块时 stage 1 算第 k 块），所以长 prompt conc1 也 +接近 2×；1 块的 prompt 没有可重叠的东西，略慢（邮箱 + 两段 launch）。v1 的限制：每段按整份的 +`bytes_per_token` 分 KV（只用自己那些层，浪费一半）；只认纯 paged 的 manifest；chunk program +自己吐 token 的模型拒切；上游那份 box 是不用的占位。 + ## 没做（按需要加） span 长的预算 policy（K5 D2：按稠密 / attention 预算定 c，现在是 `--chunk` 上限）、抢占 / 动态页分配、 From bbc94b8ec94412f05f464991adbd77448a2dbabc Mon Sep 17 00:00:00 2001 From: JinYan Su <751080330@qq.com> Date: Wed, 7 Oct 2026 15:44:10 +0000 Subject: [PATCH 3/6] qwen3: the target kv is one paged state per layer gen_qwen3_decode.py emits `kv.0`..`kv.35` (vLLM's per-layer pool layout, 4096 bytes per token each) instead of one layer-interleaved `kv`; the pool hands every paged state the same page ids, so nothing else changes. The draft's `draft_kv` stays interleaved. The generator had not run since the registry move (dead `handwritten` import, `variant` unimported, launch `label` dropped); it runs again, so the regenerated examples also take the registry builds of argmax, embedding and weight_prep. Gate (tray07): `kern test qwen3-4b` bit-identical at every span; new vs old qwen3-4b.json bit-identical at every span; `kern run` 32 steps text identical; dspark round and --rows 1, 128 steps, text identical to before. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/kern-manifest/tests/qwen3_example.rs | 6 +- examples/qwen3-4b-dspark.json | 2468 +++++++++---------- examples/qwen3-4b-silu-mined.json | 1668 ++++++------- examples/qwen3-4b.json | 1668 ++++++------- tools/gen_qwen3_decode.py | 43 +- 5 files changed, 2789 insertions(+), 3064 deletions(-) diff --git a/crates/kern-manifest/tests/qwen3_example.rs b/crates/kern-manifest/tests/qwen3_example.rs index d51b1589..2510d3d0 100644 --- a/crates/kern-manifest/tests/qwen3_example.rs +++ b/crates/kern-manifest/tests/qwen3_example.rs @@ -24,8 +24,10 @@ fn qwen3_decode_mined_verifies() { // prefill: same forward minus the final_norm/lm_head/sample tail — the // last prompt token goes through `decode` instead. assert_eq!(m.programs["prefill"].calls.len(), 2 + 36 * 12 - 1); - // The runtime's entire knowledge of the KV cache: a byte count. - assert_eq!(m.states["kv"].bytes_per_token, 36 * 2 * 1024 * 2); + // The runtime's entire knowledge of the KV cache: a byte count per + // layer's state, every one paged under the same ids. + assert_eq!(m.states.len(), 36); + assert!(m.states.values().all(|s| s.bytes_per_token == 2 * 1024 * 2)); let again = Manifest::from_json(&m.to_json()).expect("reparse"); assert_eq!(m.to_json(), again.to_json()); diff --git a/examples/qwen3-4b-dspark.json b/examples/qwen3-4b-dspark.json index ed29590d..78cb3ff9 100644 --- a/examples/qwen3-4b-dspark.json +++ b/examples/qwen3-4b-dspark.json @@ -22,7 +22,42 @@ "DRAFT_KV_SCALE_COUNT": 10, "MAX_TOKENS": 2048, "MAX_SEQS": 256, - "KV_BYTES_PER_TOKEN": 147456, + "KV.0_BYTES_PER_TOKEN": 4096, + "KV.1_BYTES_PER_TOKEN": 4096, + "KV.2_BYTES_PER_TOKEN": 4096, + "KV.3_BYTES_PER_TOKEN": 4096, + "KV.4_BYTES_PER_TOKEN": 4096, + "KV.5_BYTES_PER_TOKEN": 4096, + "KV.6_BYTES_PER_TOKEN": 4096, + "KV.7_BYTES_PER_TOKEN": 4096, + "KV.8_BYTES_PER_TOKEN": 4096, + "KV.9_BYTES_PER_TOKEN": 4096, + "KV.10_BYTES_PER_TOKEN": 4096, + "KV.11_BYTES_PER_TOKEN": 4096, + "KV.12_BYTES_PER_TOKEN": 4096, + "KV.13_BYTES_PER_TOKEN": 4096, + "KV.14_BYTES_PER_TOKEN": 4096, + "KV.15_BYTES_PER_TOKEN": 4096, + "KV.16_BYTES_PER_TOKEN": 4096, + "KV.17_BYTES_PER_TOKEN": 4096, + "KV.18_BYTES_PER_TOKEN": 4096, + "KV.19_BYTES_PER_TOKEN": 4096, + "KV.20_BYTES_PER_TOKEN": 4096, + "KV.21_BYTES_PER_TOKEN": 4096, + "KV.22_BYTES_PER_TOKEN": 4096, + "KV.23_BYTES_PER_TOKEN": 4096, + "KV.24_BYTES_PER_TOKEN": 4096, + "KV.25_BYTES_PER_TOKEN": 4096, + "KV.26_BYTES_PER_TOKEN": 4096, + "KV.27_BYTES_PER_TOKEN": 4096, + "KV.28_BYTES_PER_TOKEN": 4096, + "KV.29_BYTES_PER_TOKEN": 4096, + "KV.30_BYTES_PER_TOKEN": 4096, + "KV.31_BYTES_PER_TOKEN": 4096, + "KV.32_BYTES_PER_TOKEN": 4096, + "KV.33_BYTES_PER_TOKEN": 4096, + "KV.34_BYTES_PER_TOKEN": 4096, + "KV.35_BYTES_PER_TOKEN": 4096, "DRAFT_KV_BYTES_PER_TOKEN": 20480, "RMS_NORM_EPS": 9.999999974752427e-07, "NUM_HEADS": 32, @@ -41,8 +76,113 @@ } }, "states": { - "kv": { - "bytes_per_token": "KV_BYTES_PER_TOKEN" + "kv.0": { + "bytes_per_token": "KV.0_BYTES_PER_TOKEN" + }, + "kv.1": { + "bytes_per_token": "KV.1_BYTES_PER_TOKEN" + }, + "kv.2": { + "bytes_per_token": "KV.2_BYTES_PER_TOKEN" + }, + "kv.3": { + "bytes_per_token": "KV.3_BYTES_PER_TOKEN" + }, + "kv.4": { + "bytes_per_token": "KV.4_BYTES_PER_TOKEN" + }, + "kv.5": { + "bytes_per_token": "KV.5_BYTES_PER_TOKEN" + }, + "kv.6": { + "bytes_per_token": "KV.6_BYTES_PER_TOKEN" + }, + "kv.7": { + "bytes_per_token": "KV.7_BYTES_PER_TOKEN" + }, + "kv.8": { + "bytes_per_token": "KV.8_BYTES_PER_TOKEN" + }, + "kv.9": { + "bytes_per_token": "KV.9_BYTES_PER_TOKEN" + }, + "kv.10": { + "bytes_per_token": "KV.10_BYTES_PER_TOKEN" + }, + "kv.11": { + "bytes_per_token": "KV.11_BYTES_PER_TOKEN" + }, + "kv.12": { + "bytes_per_token": "KV.12_BYTES_PER_TOKEN" + }, + "kv.13": { + "bytes_per_token": "KV.13_BYTES_PER_TOKEN" + }, + "kv.14": { + "bytes_per_token": "KV.14_BYTES_PER_TOKEN" + }, + "kv.15": { + "bytes_per_token": "KV.15_BYTES_PER_TOKEN" + }, + "kv.16": { + "bytes_per_token": "KV.16_BYTES_PER_TOKEN" + }, + "kv.17": { + "bytes_per_token": "KV.17_BYTES_PER_TOKEN" + }, + "kv.18": { + "bytes_per_token": "KV.18_BYTES_PER_TOKEN" + }, + "kv.19": { + "bytes_per_token": "KV.19_BYTES_PER_TOKEN" + }, + "kv.20": { + "bytes_per_token": "KV.20_BYTES_PER_TOKEN" + }, + "kv.21": { + "bytes_per_token": "KV.21_BYTES_PER_TOKEN" + }, + "kv.22": { + "bytes_per_token": "KV.22_BYTES_PER_TOKEN" + }, + "kv.23": { + "bytes_per_token": "KV.23_BYTES_PER_TOKEN" + }, + "kv.24": { + "bytes_per_token": "KV.24_BYTES_PER_TOKEN" + }, + "kv.25": { + "bytes_per_token": "KV.25_BYTES_PER_TOKEN" + }, + "kv.26": { + "bytes_per_token": "KV.26_BYTES_PER_TOKEN" + }, + "kv.27": { + "bytes_per_token": "KV.27_BYTES_PER_TOKEN" + }, + "kv.28": { + "bytes_per_token": "KV.28_BYTES_PER_TOKEN" + }, + "kv.29": { + "bytes_per_token": "KV.29_BYTES_PER_TOKEN" + }, + "kv.30": { + "bytes_per_token": "KV.30_BYTES_PER_TOKEN" + }, + "kv.31": { + "bytes_per_token": "KV.31_BYTES_PER_TOKEN" + }, + "kv.32": { + "bytes_per_token": "KV.32_BYTES_PER_TOKEN" + }, + "kv.33": { + "bytes_per_token": "KV.33_BYTES_PER_TOKEN" + }, + "kv.34": { + "bytes_per_token": "KV.34_BYTES_PER_TOKEN" + }, + "kv.35": { + "bytes_per_token": "KV.35_BYTES_PER_TOKEN" }, "draft_kv": { "bytes_per_token": "DRAFT_KV_BYTES_PER_TOKEN" @@ -79,7 +219,7 @@ "kind": "input", "fill": "slot", "domain": { - "index_into": "kv" + "index_into": "kv.0" } }, "block_table": { @@ -90,7 +230,7 @@ ], "kind": "input", "domain": { - "index_into": "kv", + "index_into": "kv.0", "stride": "KV_BLOCK_SIZE" } }, @@ -5051,20 +5191,20 @@ }, "modules": { "argmax": { - "source": "argmax.cubin", - "sha256": "25374575bd35dbd9fe07f90982d0da0c2d4a72cc6ac88df5ec92cdb62a3beb16" + "source": "hf:Pegainfer/kern-kernels/blobs/ebe4ba79544ddfac43ee2dcf6a7f5299aba1124e176c201133d4ef9400e3dc25", + "sha256": "ebe4ba79544ddfac43ee2dcf6a7f5299aba1124e176c201133d4ef9400e3dc25" }, "copy_rows": { - "source": "copy_rows.cubin", - "sha256": "a49dcdf873038dbf4de19e8a324ac420afc669dca2cf122d601e034bcab10edd" + "source": "hf:Pegainfer/kern-kernels/blobs/f301274a5957faac2f0891ea9d6e80e0dabdb56ee4c14c3795e836f6bf31fb54", + "sha256": "f301274a5957faac2f0891ea9d6e80e0dabdb56ee4c14c3795e836f6bf31fb54" }, "embedding": { - "source": "embedding.cubin", - "sha256": "1c3dd3e591d872aed259d7ef59d44649061f77a4247a5df864a7fbf0ccbde92f" + "source": "hf:Pegainfer/kern-kernels/blobs/868e93ae6e46e36a11c56af1fae49aeff9fcdfdc79f64ca99475a309e721e9e7", + "sha256": "868e93ae6e46e36a11c56af1fae49aeff9fcdfdc79f64ca99475a309e721e9e7" }, "markov_rows": { - "source": "markov_rows.cubin", - "sha256": "73e7038bafa6c50023ad2db1a1534b83d59890c80573a23c79e16a77390fe72b" + "source": "hf:Pegainfer/kern-kernels/blobs/307c90a3eb9e8219ffa9045134edfc881d7c6d74e8a32b5ec1e36ef5b8421f87", + "sha256": "307c90a3eb9e8219ffa9045134edfc881d7c6d74e8a32b5ec1e36ef5b8421f87" }, "reduce_segments": { "source": "reduce_segments.cubin", @@ -5075,8 +5215,8 @@ "sha256": "1e5912bed482f8555e91a145a3cff4a1ce9b0c4240e23aa21b184c32d4716f13" }, "spec_round": { - "source": "spec_round.cubin", - "sha256": "83a1cd23bb15864787adb483d4a8b0737fb2c1c4d3a875750306d9a0f1ac0b8a" + "source": "hf:Pegainfer/kern-kernels/blobs/f8b61d2c217c24d55b020825ae4af59e80c6a08165c901e1385d12b8ffefec8d", + "sha256": "f8b61d2c217c24d55b020825ae4af59e80c6a08165c901e1385d12b8ffefec8d" }, "unified_causal": { "source": "unified_causal.cubin", @@ -5103,7 +5243,7 @@ "sha256": "116de812ac5a3a186f76fa239d4c16abbed6b8b0af06d16e1baec0aefe8fe8fa" }, "weight_prep": { - "source": "weight_prep.cubin", + "source": "hf:Pegainfer/kern-kernels/blobs/040f51b01c421b9c69a2090034ff1bc4741e40c02f5ea52bd04a3e62e5c1af4d", "sha256": "040f51b01c421b9c69a2090034ff1bc4741e40c02f5ea52bd04a3e62e5c1af4d" } }, @@ -5683,7 +5823,7 @@ "param": 8 }, { - "i64": 1179648 + "i64": 32768 }, { "i64": 2048 @@ -5692,7 +5832,7 @@ "i64": 256 }, { - "i64": 1179648 + "i64": 32768 }, { "i64": 2048 @@ -5914,7 +6054,7 @@ "param": 8 }, { - "i64": 1179648 + "i64": 32768 }, { "i64": 2048 @@ -5923,7 +6063,7 @@ "i64": 256 }, { - "i64": 1179648 + "i64": 32768 }, { "i64": 2048 @@ -6074,7 +6214,7 @@ "param": 8 }, { - "i64": 1179648 + "i64": 32768 }, { "i64": 2048 @@ -6083,7 +6223,7 @@ "i64": 256 }, { - "i64": 1179648 + "i64": 32768 }, { "i64": 2048 @@ -6664,7 +6804,7 @@ "param": 8 }, { - "i64": 1179648 + "i64": 32768 }, { "i64": 2048 @@ -6673,7 +6813,7 @@ "i64": 256 }, { - "i64": 1179648 + "i64": 32768 }, { "i64": 2048 @@ -7477,10 +7617,10 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv" + "state": "kv.0" }, { - "state": "kv", + "state": "kv.0", "offset": 256 }, { @@ -7497,7 +7637,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -7512,10 +7652,10 @@ "buf": "q_n" }, { - "state": "kv" + "state": "kv.0" }, { - "state": "kv", + "state": "kv.0", "offset": 256 }, { @@ -7802,12 +7942,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 65536 + "state": "kv.1" }, { - "state": "kv", - "offset": 65792 + "state": "kv.1", + "offset": 256 }, { "buf": "slot_mapping" @@ -7824,7 +7963,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -7839,12 +7978,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 65536 + "state": "kv.1" }, { - "state": "kv", - "offset": 65792 + "state": "kv.1", + "offset": 256 }, { "buf": "block_table" @@ -8107,12 +8245,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 131072 + "state": "kv.2" }, { - "state": "kv", - "offset": 131328 + "state": "kv.2", + "offset": 256 }, { "buf": "slot_mapping" @@ -8129,7 +8266,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -8144,12 +8281,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 131072 + "state": "kv.2" }, { - "state": "kv", - "offset": 131328 + "state": "kv.2", + "offset": 256 }, { "buf": "block_table" @@ -8412,12 +8548,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 196608 + "state": "kv.3" }, { - "state": "kv", - "offset": 196864 + "state": "kv.3", + "offset": 256 }, { "buf": "slot_mapping" @@ -8434,7 +8569,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -8449,12 +8584,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 196608 + "state": "kv.3" }, { - "state": "kv", - "offset": 196864 + "state": "kv.3", + "offset": 256 }, { "buf": "block_table" @@ -8717,12 +8851,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 262144 + "state": "kv.4" }, { - "state": "kv", - "offset": 262400 + "state": "kv.4", + "offset": 256 }, { "buf": "slot_mapping" @@ -8739,7 +8872,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -8754,12 +8887,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 262144 + "state": "kv.4" }, { - "state": "kv", - "offset": 262400 + "state": "kv.4", + "offset": 256 }, { "buf": "block_table" @@ -9022,12 +9154,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 327680 + "state": "kv.5" }, { - "state": "kv", - "offset": 327936 + "state": "kv.5", + "offset": 256 }, { "buf": "slot_mapping" @@ -9044,7 +9175,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -9059,12 +9190,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 327680 + "state": "kv.5" }, { - "state": "kv", - "offset": 327936 + "state": "kv.5", + "offset": 256 }, { "buf": "block_table" @@ -9327,12 +9457,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 393216 + "state": "kv.6" }, { - "state": "kv", - "offset": 393472 + "state": "kv.6", + "offset": 256 }, { "buf": "slot_mapping" @@ -9349,7 +9478,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -9364,12 +9493,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 393216 + "state": "kv.6" }, { - "state": "kv", - "offset": 393472 + "state": "kv.6", + "offset": 256 }, { "buf": "block_table" @@ -9632,12 +9760,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 458752 + "state": "kv.7" }, { - "state": "kv", - "offset": 459008 + "state": "kv.7", + "offset": 256 }, { "buf": "slot_mapping" @@ -9654,7 +9781,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -9669,12 +9796,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 458752 + "state": "kv.7" }, { - "state": "kv", - "offset": 459008 + "state": "kv.7", + "offset": 256 }, { "buf": "block_table" @@ -9937,12 +10063,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 524288 + "state": "kv.8" }, { - "state": "kv", - "offset": 524544 + "state": "kv.8", + "offset": 256 }, { "buf": "slot_mapping" @@ -9959,7 +10084,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -9974,12 +10099,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 524288 + "state": "kv.8" }, { - "state": "kv", - "offset": 524544 + "state": "kv.8", + "offset": 256 }, { "buf": "block_table" @@ -10260,12 +10384,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 589824 + "state": "kv.9" }, { - "state": "kv", - "offset": 590080 + "state": "kv.9", + "offset": 256 }, { "buf": "slot_mapping" @@ -10282,7 +10405,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -10297,12 +10420,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 589824 + "state": "kv.9" }, { - "state": "kv", - "offset": 590080 + "state": "kv.9", + "offset": 256 }, { "buf": "block_table" @@ -10565,12 +10687,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 655360 + "state": "kv.10" }, { - "state": "kv", - "offset": 655616 + "state": "kv.10", + "offset": 256 }, { "buf": "slot_mapping" @@ -10587,7 +10708,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -10602,12 +10723,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 655360 + "state": "kv.10" }, { - "state": "kv", - "offset": 655616 + "state": "kv.10", + "offset": 256 }, { "buf": "block_table" @@ -10870,12 +10990,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 720896 + "state": "kv.11" }, { - "state": "kv", - "offset": 721152 + "state": "kv.11", + "offset": 256 }, { "buf": "slot_mapping" @@ -10892,7 +11011,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -10907,12 +11026,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 720896 + "state": "kv.11" }, { - "state": "kv", - "offset": 721152 + "state": "kv.11", + "offset": 256 }, { "buf": "block_table" @@ -11175,12 +11293,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 786432 + "state": "kv.12" }, { - "state": "kv", - "offset": 786688 + "state": "kv.12", + "offset": 256 }, { "buf": "slot_mapping" @@ -11197,7 +11314,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -11212,12 +11329,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 786432 + "state": "kv.12" }, { - "state": "kv", - "offset": 786688 + "state": "kv.12", + "offset": 256 }, { "buf": "block_table" @@ -11480,12 +11596,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 851968 + "state": "kv.13" }, { - "state": "kv", - "offset": 852224 + "state": "kv.13", + "offset": 256 }, { "buf": "slot_mapping" @@ -11502,7 +11617,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -11517,12 +11632,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 851968 + "state": "kv.13" }, { - "state": "kv", - "offset": 852224 + "state": "kv.13", + "offset": 256 }, { "buf": "block_table" @@ -11785,12 +11899,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 917504 + "state": "kv.14" }, { - "state": "kv", - "offset": 917760 + "state": "kv.14", + "offset": 256 }, { "buf": "slot_mapping" @@ -11807,7 +11920,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -11822,12 +11935,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 917504 + "state": "kv.14" }, { - "state": "kv", - "offset": 917760 + "state": "kv.14", + "offset": 256 }, { "buf": "block_table" @@ -12090,12 +12202,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 983040 + "state": "kv.15" }, { - "state": "kv", - "offset": 983296 + "state": "kv.15", + "offset": 256 }, { "buf": "slot_mapping" @@ -12112,7 +12223,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -12127,12 +12238,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 983040 + "state": "kv.15" }, { - "state": "kv", - "offset": 983296 + "state": "kv.15", + "offset": 256 }, { "buf": "block_table" @@ -12395,12 +12505,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1048576 + "state": "kv.16" }, { - "state": "kv", - "offset": 1048832 + "state": "kv.16", + "offset": 256 }, { "buf": "slot_mapping" @@ -12417,7 +12526,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -12432,12 +12541,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1048576 + "state": "kv.16" }, { - "state": "kv", - "offset": 1048832 + "state": "kv.16", + "offset": 256 }, { "buf": "block_table" @@ -12718,12 +12826,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1114112 + "state": "kv.17" }, { - "state": "kv", - "offset": 1114368 + "state": "kv.17", + "offset": 256 }, { "buf": "slot_mapping" @@ -12740,7 +12847,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -12755,12 +12862,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1114112 + "state": "kv.17" }, { - "state": "kv", - "offset": 1114368 + "state": "kv.17", + "offset": 256 }, { "buf": "block_table" @@ -13023,12 +13129,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1179648 + "state": "kv.18" }, { - "state": "kv", - "offset": 1179904 + "state": "kv.18", + "offset": 256 }, { "buf": "slot_mapping" @@ -13045,7 +13150,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -13060,12 +13165,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1179648 + "state": "kv.18" }, { - "state": "kv", - "offset": 1179904 + "state": "kv.18", + "offset": 256 }, { "buf": "block_table" @@ -13328,12 +13432,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1245184 + "state": "kv.19" }, { - "state": "kv", - "offset": 1245440 + "state": "kv.19", + "offset": 256 }, { "buf": "slot_mapping" @@ -13350,7 +13453,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -13365,12 +13468,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1245184 + "state": "kv.19" }, { - "state": "kv", - "offset": 1245440 + "state": "kv.19", + "offset": 256 }, { "buf": "block_table" @@ -13633,12 +13735,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1310720 + "state": "kv.20" }, { - "state": "kv", - "offset": 1310976 + "state": "kv.20", + "offset": 256 }, { "buf": "slot_mapping" @@ -13655,7 +13756,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -13670,12 +13771,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1310720 + "state": "kv.20" }, { - "state": "kv", - "offset": 1310976 + "state": "kv.20", + "offset": 256 }, { "buf": "block_table" @@ -13938,12 +14038,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1376256 + "state": "kv.21" }, { - "state": "kv", - "offset": 1376512 + "state": "kv.21", + "offset": 256 }, { "buf": "slot_mapping" @@ -13960,7 +14059,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -13975,12 +14074,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1376256 + "state": "kv.21" }, { - "state": "kv", - "offset": 1376512 + "state": "kv.21", + "offset": 256 }, { "buf": "block_table" @@ -14243,12 +14341,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1441792 + "state": "kv.22" }, { - "state": "kv", - "offset": 1442048 + "state": "kv.22", + "offset": 256 }, { "buf": "slot_mapping" @@ -14265,7 +14362,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -14280,12 +14377,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1441792 + "state": "kv.22" }, { - "state": "kv", - "offset": 1442048 + "state": "kv.22", + "offset": 256 }, { "buf": "block_table" @@ -14548,12 +14644,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1507328 + "state": "kv.23" }, { - "state": "kv", - "offset": 1507584 + "state": "kv.23", + "offset": 256 }, { "buf": "slot_mapping" @@ -14570,7 +14665,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -14585,12 +14680,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1507328 + "state": "kv.23" }, { - "state": "kv", - "offset": 1507584 + "state": "kv.23", + "offset": 256 }, { "buf": "block_table" @@ -14853,12 +14947,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1572864 + "state": "kv.24" }, { - "state": "kv", - "offset": 1573120 + "state": "kv.24", + "offset": 256 }, { "buf": "slot_mapping" @@ -14875,7 +14968,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -14890,12 +14983,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1572864 + "state": "kv.24" }, { - "state": "kv", - "offset": 1573120 + "state": "kv.24", + "offset": 256 }, { "buf": "block_table" @@ -15176,12 +15268,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1638400 + "state": "kv.25" }, { - "state": "kv", - "offset": 1638656 + "state": "kv.25", + "offset": 256 }, { "buf": "slot_mapping" @@ -15198,7 +15289,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -15213,12 +15304,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1638400 + "state": "kv.25" }, { - "state": "kv", - "offset": 1638656 + "state": "kv.25", + "offset": 256 }, { "buf": "block_table" @@ -15481,12 +15571,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1703936 + "state": "kv.26" }, { - "state": "kv", - "offset": 1704192 + "state": "kv.26", + "offset": 256 }, { "buf": "slot_mapping" @@ -15503,7 +15592,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -15518,12 +15607,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1703936 + "state": "kv.26" }, { - "state": "kv", - "offset": 1704192 + "state": "kv.26", + "offset": 256 }, { "buf": "block_table" @@ -15786,12 +15874,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1769472 + "state": "kv.27" }, { - "state": "kv", - "offset": 1769728 + "state": "kv.27", + "offset": 256 }, { "buf": "slot_mapping" @@ -15808,7 +15895,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -15823,12 +15910,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1769472 + "state": "kv.27" }, { - "state": "kv", - "offset": 1769728 + "state": "kv.27", + "offset": 256 }, { "buf": "block_table" @@ -16091,12 +16177,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1835008 + "state": "kv.28" }, { - "state": "kv", - "offset": 1835264 + "state": "kv.28", + "offset": 256 }, { "buf": "slot_mapping" @@ -16113,7 +16198,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -16128,12 +16213,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1835008 + "state": "kv.28" }, { - "state": "kv", - "offset": 1835264 + "state": "kv.28", + "offset": 256 }, { "buf": "block_table" @@ -16396,12 +16480,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1900544 + "state": "kv.29" }, { - "state": "kv", - "offset": 1900800 + "state": "kv.29", + "offset": 256 }, { "buf": "slot_mapping" @@ -16418,7 +16501,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -16433,12 +16516,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1900544 + "state": "kv.29" }, { - "state": "kv", - "offset": 1900800 + "state": "kv.29", + "offset": 256 }, { "buf": "block_table" @@ -16701,12 +16783,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1966080 + "state": "kv.30" }, { - "state": "kv", - "offset": 1966336 + "state": "kv.30", + "offset": 256 }, { "buf": "slot_mapping" @@ -16723,7 +16804,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -16738,12 +16819,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1966080 + "state": "kv.30" }, { - "state": "kv", - "offset": 1966336 + "state": "kv.30", + "offset": 256 }, { "buf": "block_table" @@ -17006,12 +17086,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 2031616 + "state": "kv.31" }, { - "state": "kv", - "offset": 2031872 + "state": "kv.31", + "offset": 256 }, { "buf": "slot_mapping" @@ -17028,7 +17107,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -17043,12 +17122,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 2031616 + "state": "kv.31" }, { - "state": "kv", - "offset": 2031872 + "state": "kv.31", + "offset": 256 }, { "buf": "block_table" @@ -17311,12 +17389,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 2097152 + "state": "kv.32" }, { - "state": "kv", - "offset": 2097408 + "state": "kv.32", + "offset": 256 }, { "buf": "slot_mapping" @@ -17333,7 +17410,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -17348,12 +17425,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 2097152 + "state": "kv.32" }, { - "state": "kv", - "offset": 2097408 + "state": "kv.32", + "offset": 256 }, { "buf": "block_table" @@ -17634,12 +17710,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 2162688 + "state": "kv.33" }, { - "state": "kv", - "offset": 2162944 + "state": "kv.33", + "offset": 256 }, { "buf": "slot_mapping" @@ -17656,7 +17731,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -17671,12 +17746,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 2162688 + "state": "kv.33" }, { - "state": "kv", - "offset": 2162944 + "state": "kv.33", + "offset": 256 }, { "buf": "block_table" @@ -17939,12 +18013,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 2228224 + "state": "kv.34" }, { - "state": "kv", - "offset": 2228480 + "state": "kv.34", + "offset": 256 }, { "buf": "slot_mapping" @@ -17961,7 +18034,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -17976,12 +18049,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 2228224 + "state": "kv.34" }, { - "state": "kv", - "offset": 2228480 + "state": "kv.34", + "offset": 256 }, { "buf": "block_table" @@ -18244,12 +18316,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 2293760 + "state": "kv.35" }, { - "state": "kv", - "offset": 2294016 + "state": "kv.35", + "offset": 256 }, { "buf": "slot_mapping" @@ -18266,7 +18337,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -18281,12 +18352,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 2293760 + "state": "kv.35" }, { - "state": "kv", - "offset": 2294016 + "state": "kv.35", + "offset": 256 }, { "buf": "block_table" @@ -19159,10 +19229,10 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv" + "state": "kv.0" }, { - "state": "kv", + "state": "kv.0", "offset": 256 }, { @@ -19179,7 +19249,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -19194,10 +19264,10 @@ "buf": "q_n" }, { - "state": "kv" + "state": "kv.0" }, { - "state": "kv", + "state": "kv.0", "offset": 256 }, { @@ -19457,12 +19527,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 65536 + "state": "kv.1" }, { - "state": "kv", - "offset": 65792 + "state": "kv.1", + "offset": 256 }, { "buf": "slot_mapping" @@ -19479,7 +19548,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -19494,12 +19563,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 65536 + "state": "kv.1" }, { - "state": "kv", - "offset": 65792 + "state": "kv.1", + "offset": 256 }, { "buf": "block_table" @@ -19759,12 +19827,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 131072 + "state": "kv.2" }, { - "state": "kv", - "offset": 131328 + "state": "kv.2", + "offset": 256 }, { "buf": "slot_mapping" @@ -19781,7 +19848,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -19796,12 +19863,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 131072 + "state": "kv.2" }, { - "state": "kv", - "offset": 131328 + "state": "kv.2", + "offset": 256 }, { "buf": "block_table" @@ -20061,12 +20127,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 196608 + "state": "kv.3" }, { - "state": "kv", - "offset": 196864 + "state": "kv.3", + "offset": 256 }, { "buf": "slot_mapping" @@ -20083,7 +20148,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -20098,12 +20163,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 196608 + "state": "kv.3" }, { - "state": "kv", - "offset": 196864 + "state": "kv.3", + "offset": 256 }, { "buf": "block_table" @@ -20363,12 +20427,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 262144 + "state": "kv.4" }, { - "state": "kv", - "offset": 262400 + "state": "kv.4", + "offset": 256 }, { "buf": "slot_mapping" @@ -20385,7 +20448,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -20400,12 +20463,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 262144 + "state": "kv.4" }, { - "state": "kv", - "offset": 262400 + "state": "kv.4", + "offset": 256 }, { "buf": "block_table" @@ -20665,12 +20727,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 327680 + "state": "kv.5" }, { - "state": "kv", - "offset": 327936 + "state": "kv.5", + "offset": 256 }, { "buf": "slot_mapping" @@ -20687,7 +20748,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -20702,12 +20763,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 327680 + "state": "kv.5" }, { - "state": "kv", - "offset": 327936 + "state": "kv.5", + "offset": 256 }, { "buf": "block_table" @@ -20967,12 +21027,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 393216 + "state": "kv.6" }, { - "state": "kv", - "offset": 393472 + "state": "kv.6", + "offset": 256 }, { "buf": "slot_mapping" @@ -20989,7 +21048,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -21004,12 +21063,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 393216 + "state": "kv.6" }, { - "state": "kv", - "offset": 393472 + "state": "kv.6", + "offset": 256 }, { "buf": "block_table" @@ -21269,12 +21327,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 458752 + "state": "kv.7" }, { - "state": "kv", - "offset": 459008 + "state": "kv.7", + "offset": 256 }, { "buf": "slot_mapping" @@ -21291,7 +21348,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -21306,12 +21363,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 458752 + "state": "kv.7" }, { - "state": "kv", - "offset": 459008 + "state": "kv.7", + "offset": 256 }, { "buf": "block_table" @@ -21571,12 +21627,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 524288 + "state": "kv.8" }, { - "state": "kv", - "offset": 524544 + "state": "kv.8", + "offset": 256 }, { "buf": "slot_mapping" @@ -21593,7 +21648,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -21608,12 +21663,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 524288 + "state": "kv.8" }, { - "state": "kv", - "offset": 524544 + "state": "kv.8", + "offset": 256 }, { "buf": "block_table" @@ -21873,12 +21927,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 589824 + "state": "kv.9" }, { - "state": "kv", - "offset": 590080 + "state": "kv.9", + "offset": 256 }, { "buf": "slot_mapping" @@ -21895,7 +21948,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -21910,12 +21963,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 589824 + "state": "kv.9" }, { - "state": "kv", - "offset": 590080 + "state": "kv.9", + "offset": 256 }, { "buf": "block_table" @@ -22175,12 +22227,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 655360 + "state": "kv.10" }, { - "state": "kv", - "offset": 655616 + "state": "kv.10", + "offset": 256 }, { "buf": "slot_mapping" @@ -22197,7 +22248,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -22212,12 +22263,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 655360 + "state": "kv.10" }, { - "state": "kv", - "offset": 655616 + "state": "kv.10", + "offset": 256 }, { "buf": "block_table" @@ -22477,12 +22527,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 720896 + "state": "kv.11" }, { - "state": "kv", - "offset": 721152 + "state": "kv.11", + "offset": 256 }, { "buf": "slot_mapping" @@ -22499,7 +22548,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -22514,12 +22563,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 720896 + "state": "kv.11" }, { - "state": "kv", - "offset": 721152 + "state": "kv.11", + "offset": 256 }, { "buf": "block_table" @@ -22779,12 +22827,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 786432 + "state": "kv.12" }, { - "state": "kv", - "offset": 786688 + "state": "kv.12", + "offset": 256 }, { "buf": "slot_mapping" @@ -22801,7 +22848,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -22816,12 +22863,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 786432 + "state": "kv.12" }, { - "state": "kv", - "offset": 786688 + "state": "kv.12", + "offset": 256 }, { "buf": "block_table" @@ -23081,12 +23127,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 851968 + "state": "kv.13" }, { - "state": "kv", - "offset": 852224 + "state": "kv.13", + "offset": 256 }, { "buf": "slot_mapping" @@ -23103,7 +23148,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -23118,12 +23163,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 851968 + "state": "kv.13" }, { - "state": "kv", - "offset": 852224 + "state": "kv.13", + "offset": 256 }, { "buf": "block_table" @@ -23383,12 +23427,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 917504 + "state": "kv.14" }, { - "state": "kv", - "offset": 917760 + "state": "kv.14", + "offset": 256 }, { "buf": "slot_mapping" @@ -23405,7 +23448,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -23420,12 +23463,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 917504 + "state": "kv.14" }, { - "state": "kv", - "offset": 917760 + "state": "kv.14", + "offset": 256 }, { "buf": "block_table" @@ -23685,12 +23727,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 983040 + "state": "kv.15" }, { - "state": "kv", - "offset": 983296 + "state": "kv.15", + "offset": 256 }, { "buf": "slot_mapping" @@ -23707,7 +23748,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -23722,12 +23763,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 983040 + "state": "kv.15" }, { - "state": "kv", - "offset": 983296 + "state": "kv.15", + "offset": 256 }, { "buf": "block_table" @@ -23987,12 +24027,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1048576 + "state": "kv.16" }, { - "state": "kv", - "offset": 1048832 + "state": "kv.16", + "offset": 256 }, { "buf": "slot_mapping" @@ -24009,7 +24048,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -24024,12 +24063,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1048576 + "state": "kv.16" }, { - "state": "kv", - "offset": 1048832 + "state": "kv.16", + "offset": 256 }, { "buf": "block_table" @@ -24289,12 +24327,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1114112 + "state": "kv.17" }, { - "state": "kv", - "offset": 1114368 + "state": "kv.17", + "offset": 256 }, { "buf": "slot_mapping" @@ -24311,7 +24348,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -24326,12 +24363,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1114112 + "state": "kv.17" }, { - "state": "kv", - "offset": 1114368 + "state": "kv.17", + "offset": 256 }, { "buf": "block_table" @@ -24591,12 +24627,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1179648 + "state": "kv.18" }, { - "state": "kv", - "offset": 1179904 + "state": "kv.18", + "offset": 256 }, { "buf": "slot_mapping" @@ -24613,7 +24648,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -24628,12 +24663,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1179648 + "state": "kv.18" }, { - "state": "kv", - "offset": 1179904 + "state": "kv.18", + "offset": 256 }, { "buf": "block_table" @@ -24893,12 +24927,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1245184 + "state": "kv.19" }, { - "state": "kv", - "offset": 1245440 + "state": "kv.19", + "offset": 256 }, { "buf": "slot_mapping" @@ -24915,7 +24948,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -24930,12 +24963,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1245184 + "state": "kv.19" }, { - "state": "kv", - "offset": 1245440 + "state": "kv.19", + "offset": 256 }, { "buf": "block_table" @@ -25195,12 +25227,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1310720 + "state": "kv.20" }, { - "state": "kv", - "offset": 1310976 + "state": "kv.20", + "offset": 256 }, { "buf": "slot_mapping" @@ -25217,7 +25248,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -25232,12 +25263,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1310720 + "state": "kv.20" }, { - "state": "kv", - "offset": 1310976 + "state": "kv.20", + "offset": 256 }, { "buf": "block_table" @@ -25497,12 +25527,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1376256 + "state": "kv.21" }, { - "state": "kv", - "offset": 1376512 + "state": "kv.21", + "offset": 256 }, { "buf": "slot_mapping" @@ -25519,7 +25548,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -25534,12 +25563,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1376256 + "state": "kv.21" }, { - "state": "kv", - "offset": 1376512 + "state": "kv.21", + "offset": 256 }, { "buf": "block_table" @@ -25799,12 +25827,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1441792 + "state": "kv.22" }, { - "state": "kv", - "offset": 1442048 + "state": "kv.22", + "offset": 256 }, { "buf": "slot_mapping" @@ -25821,7 +25848,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -25836,12 +25863,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1441792 + "state": "kv.22" }, { - "state": "kv", - "offset": 1442048 + "state": "kv.22", + "offset": 256 }, { "buf": "block_table" @@ -26101,12 +26127,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1507328 + "state": "kv.23" }, { - "state": "kv", - "offset": 1507584 + "state": "kv.23", + "offset": 256 }, { "buf": "slot_mapping" @@ -26123,7 +26148,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -26138,12 +26163,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1507328 + "state": "kv.23" }, { - "state": "kv", - "offset": 1507584 + "state": "kv.23", + "offset": 256 }, { "buf": "block_table" @@ -26403,12 +26427,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1572864 + "state": "kv.24" }, { - "state": "kv", - "offset": 1573120 + "state": "kv.24", + "offset": 256 }, { "buf": "slot_mapping" @@ -26425,7 +26448,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -26440,12 +26463,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1572864 + "state": "kv.24" }, { - "state": "kv", - "offset": 1573120 + "state": "kv.24", + "offset": 256 }, { "buf": "block_table" @@ -26705,12 +26727,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1638400 + "state": "kv.25" }, { - "state": "kv", - "offset": 1638656 + "state": "kv.25", + "offset": 256 }, { "buf": "slot_mapping" @@ -26727,7 +26748,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -26742,12 +26763,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1638400 + "state": "kv.25" }, { - "state": "kv", - "offset": 1638656 + "state": "kv.25", + "offset": 256 }, { "buf": "block_table" @@ -27007,12 +27027,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1703936 + "state": "kv.26" }, { - "state": "kv", - "offset": 1704192 + "state": "kv.26", + "offset": 256 }, { "buf": "slot_mapping" @@ -27029,7 +27048,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -27044,12 +27063,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1703936 + "state": "kv.26" }, { - "state": "kv", - "offset": 1704192 + "state": "kv.26", + "offset": 256 }, { "buf": "block_table" @@ -27309,12 +27327,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1769472 + "state": "kv.27" }, { - "state": "kv", - "offset": 1769728 + "state": "kv.27", + "offset": 256 }, { "buf": "slot_mapping" @@ -27331,7 +27348,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -27346,12 +27363,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1769472 + "state": "kv.27" }, { - "state": "kv", - "offset": 1769728 + "state": "kv.27", + "offset": 256 }, { "buf": "block_table" @@ -27611,12 +27627,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1835008 + "state": "kv.28" }, { - "state": "kv", - "offset": 1835264 + "state": "kv.28", + "offset": 256 }, { "buf": "slot_mapping" @@ -27633,7 +27648,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -27648,12 +27663,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1835008 + "state": "kv.28" }, { - "state": "kv", - "offset": 1835264 + "state": "kv.28", + "offset": 256 }, { "buf": "block_table" @@ -27913,12 +27927,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1900544 + "state": "kv.29" }, { - "state": "kv", - "offset": 1900800 + "state": "kv.29", + "offset": 256 }, { "buf": "slot_mapping" @@ -27935,7 +27948,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -27950,12 +27963,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1900544 + "state": "kv.29" }, { - "state": "kv", - "offset": 1900800 + "state": "kv.29", + "offset": 256 }, { "buf": "block_table" @@ -28215,12 +28227,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1966080 + "state": "kv.30" }, { - "state": "kv", - "offset": 1966336 + "state": "kv.30", + "offset": 256 }, { "buf": "slot_mapping" @@ -28237,7 +28248,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -28252,12 +28263,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1966080 + "state": "kv.30" }, { - "state": "kv", - "offset": 1966336 + "state": "kv.30", + "offset": 256 }, { "buf": "block_table" @@ -28517,12 +28527,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 2031616 + "state": "kv.31" }, { - "state": "kv", - "offset": 2031872 + "state": "kv.31", + "offset": 256 }, { "buf": "slot_mapping" @@ -28539,7 +28548,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -28554,12 +28563,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 2031616 + "state": "kv.31" }, { - "state": "kv", - "offset": 2031872 + "state": "kv.31", + "offset": 256 }, { "buf": "block_table" @@ -28819,12 +28827,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 2097152 + "state": "kv.32" }, { - "state": "kv", - "offset": 2097408 + "state": "kv.32", + "offset": 256 }, { "buf": "slot_mapping" @@ -28841,7 +28848,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -28856,12 +28863,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 2097152 + "state": "kv.32" }, { - "state": "kv", - "offset": 2097408 + "state": "kv.32", + "offset": 256 }, { "buf": "block_table" @@ -29121,12 +29127,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 2162688 + "state": "kv.33" }, { - "state": "kv", - "offset": 2162944 + "state": "kv.33", + "offset": 256 }, { "buf": "slot_mapping" @@ -29143,7 +29148,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -29158,12 +29163,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 2162688 + "state": "kv.33" }, { - "state": "kv", - "offset": 2162944 + "state": "kv.33", + "offset": 256 }, { "buf": "block_table" @@ -29423,12 +29427,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 2228224 + "state": "kv.34" }, { - "state": "kv", - "offset": 2228480 + "state": "kv.34", + "offset": 256 }, { "buf": "slot_mapping" @@ -29445,7 +29448,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -29460,12 +29463,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 2228224 + "state": "kv.34" }, { - "state": "kv", - "offset": 2228480 + "state": "kv.34", + "offset": 256 }, { "buf": "block_table" @@ -29725,12 +29727,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 2293760 + "state": "kv.35" }, { - "state": "kv", - "offset": 2294016 + "state": "kv.35", + "offset": 256 }, { "buf": "slot_mapping" @@ -29747,7 +29748,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -29762,12 +29763,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 2293760 + "state": "kv.35" }, { - "state": "kv", - "offset": 2294016 + "state": "kv.35", + "offset": 256 }, { "buf": "block_table" @@ -30108,10 +30108,10 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv" + "state": "kv.0" }, { - "state": "kv", + "state": "kv.0", "offset": 256 }, { @@ -30128,7 +30128,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -30143,10 +30143,10 @@ "buf": "q_n" }, { - "state": "kv" + "state": "kv.0" }, { - "state": "kv", + "state": "kv.0", "offset": 256 }, { @@ -30409,12 +30409,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 65536 + "state": "kv.1" }, { - "state": "kv", - "offset": 65792 + "state": "kv.1", + "offset": 256 }, { "buf": "slot_mapping" @@ -30431,7 +30430,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -30446,12 +30445,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 65536 + "state": "kv.1" }, { - "state": "kv", - "offset": 65792 + "state": "kv.1", + "offset": 256 }, { "buf": "block_table" @@ -30714,12 +30712,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 131072 + "state": "kv.2" }, { - "state": "kv", - "offset": 131328 + "state": "kv.2", + "offset": 256 }, { "buf": "slot_mapping" @@ -30736,7 +30733,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -30751,12 +30748,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 131072 + "state": "kv.2" }, { - "state": "kv", - "offset": 131328 + "state": "kv.2", + "offset": 256 }, { "buf": "block_table" @@ -31019,12 +31015,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 196608 + "state": "kv.3" }, { - "state": "kv", - "offset": 196864 + "state": "kv.3", + "offset": 256 }, { "buf": "slot_mapping" @@ -31041,7 +31036,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -31056,12 +31051,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 196608 + "state": "kv.3" }, { - "state": "kv", - "offset": 196864 + "state": "kv.3", + "offset": 256 }, { "buf": "block_table" @@ -31324,12 +31318,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 262144 + "state": "kv.4" }, { - "state": "kv", - "offset": 262400 + "state": "kv.4", + "offset": 256 }, { "buf": "slot_mapping" @@ -31346,7 +31339,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -31361,12 +31354,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 262144 + "state": "kv.4" }, { - "state": "kv", - "offset": 262400 + "state": "kv.4", + "offset": 256 }, { "buf": "block_table" @@ -31629,12 +31621,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 327680 + "state": "kv.5" }, { - "state": "kv", - "offset": 327936 + "state": "kv.5", + "offset": 256 }, { "buf": "slot_mapping" @@ -31651,7 +31642,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -31666,12 +31657,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 327680 + "state": "kv.5" }, { - "state": "kv", - "offset": 327936 + "state": "kv.5", + "offset": 256 }, { "buf": "block_table" @@ -31934,12 +31924,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 393216 + "state": "kv.6" }, { - "state": "kv", - "offset": 393472 + "state": "kv.6", + "offset": 256 }, { "buf": "slot_mapping" @@ -31956,7 +31945,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -31971,12 +31960,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 393216 + "state": "kv.6" }, { - "state": "kv", - "offset": 393472 + "state": "kv.6", + "offset": 256 }, { "buf": "block_table" @@ -32239,12 +32227,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 458752 + "state": "kv.7" }, { - "state": "kv", - "offset": 459008 + "state": "kv.7", + "offset": 256 }, { "buf": "slot_mapping" @@ -32261,7 +32248,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -32276,12 +32263,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 458752 + "state": "kv.7" }, { - "state": "kv", - "offset": 459008 + "state": "kv.7", + "offset": 256 }, { "buf": "block_table" @@ -32544,12 +32530,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 524288 + "state": "kv.8" }, { - "state": "kv", - "offset": 524544 + "state": "kv.8", + "offset": 256 }, { "buf": "slot_mapping" @@ -32566,7 +32551,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -32581,12 +32566,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 524288 + "state": "kv.8" }, { - "state": "kv", - "offset": 524544 + "state": "kv.8", + "offset": 256 }, { "buf": "block_table" @@ -32849,12 +32833,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 589824 + "state": "kv.9" }, { - "state": "kv", - "offset": 590080 + "state": "kv.9", + "offset": 256 }, { "buf": "slot_mapping" @@ -32871,7 +32854,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -32886,12 +32869,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 589824 + "state": "kv.9" }, { - "state": "kv", - "offset": 590080 + "state": "kv.9", + "offset": 256 }, { "buf": "block_table" @@ -33154,12 +33136,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 655360 + "state": "kv.10" }, { - "state": "kv", - "offset": 655616 + "state": "kv.10", + "offset": 256 }, { "buf": "slot_mapping" @@ -33176,7 +33157,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -33191,12 +33172,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 655360 + "state": "kv.10" }, { - "state": "kv", - "offset": 655616 + "state": "kv.10", + "offset": 256 }, { "buf": "block_table" @@ -33459,12 +33439,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 720896 + "state": "kv.11" }, { - "state": "kv", - "offset": 721152 + "state": "kv.11", + "offset": 256 }, { "buf": "slot_mapping" @@ -33481,7 +33460,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -33496,12 +33475,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 720896 + "state": "kv.11" }, { - "state": "kv", - "offset": 721152 + "state": "kv.11", + "offset": 256 }, { "buf": "block_table" @@ -33764,12 +33742,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 786432 + "state": "kv.12" }, { - "state": "kv", - "offset": 786688 + "state": "kv.12", + "offset": 256 }, { "buf": "slot_mapping" @@ -33786,7 +33763,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -33801,12 +33778,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 786432 + "state": "kv.12" }, { - "state": "kv", - "offset": 786688 + "state": "kv.12", + "offset": 256 }, { "buf": "block_table" @@ -34069,12 +34045,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 851968 + "state": "kv.13" }, { - "state": "kv", - "offset": 852224 + "state": "kv.13", + "offset": 256 }, { "buf": "slot_mapping" @@ -34091,7 +34066,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -34106,12 +34081,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 851968 + "state": "kv.13" }, { - "state": "kv", - "offset": 852224 + "state": "kv.13", + "offset": 256 }, { "buf": "block_table" @@ -34374,12 +34348,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 917504 + "state": "kv.14" }, { - "state": "kv", - "offset": 917760 + "state": "kv.14", + "offset": 256 }, { "buf": "slot_mapping" @@ -34396,7 +34369,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -34411,12 +34384,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 917504 + "state": "kv.14" }, { - "state": "kv", - "offset": 917760 + "state": "kv.14", + "offset": 256 }, { "buf": "block_table" @@ -34679,12 +34651,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 983040 + "state": "kv.15" }, { - "state": "kv", - "offset": 983296 + "state": "kv.15", + "offset": 256 }, { "buf": "slot_mapping" @@ -34701,7 +34672,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -34716,12 +34687,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 983040 + "state": "kv.15" }, { - "state": "kv", - "offset": 983296 + "state": "kv.15", + "offset": 256 }, { "buf": "block_table" @@ -34984,12 +34954,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1048576 + "state": "kv.16" }, { - "state": "kv", - "offset": 1048832 + "state": "kv.16", + "offset": 256 }, { "buf": "slot_mapping" @@ -35006,7 +34975,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -35021,12 +34990,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1048576 + "state": "kv.16" }, { - "state": "kv", - "offset": 1048832 + "state": "kv.16", + "offset": 256 }, { "buf": "block_table" @@ -35289,12 +35257,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1114112 + "state": "kv.17" }, { - "state": "kv", - "offset": 1114368 + "state": "kv.17", + "offset": 256 }, { "buf": "slot_mapping" @@ -35311,7 +35278,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -35326,12 +35293,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1114112 + "state": "kv.17" }, { - "state": "kv", - "offset": 1114368 + "state": "kv.17", + "offset": 256 }, { "buf": "block_table" @@ -35594,12 +35560,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1179648 + "state": "kv.18" }, { - "state": "kv", - "offset": 1179904 + "state": "kv.18", + "offset": 256 }, { "buf": "slot_mapping" @@ -35616,7 +35581,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -35631,12 +35596,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1179648 + "state": "kv.18" }, { - "state": "kv", - "offset": 1179904 + "state": "kv.18", + "offset": 256 }, { "buf": "block_table" @@ -35899,12 +35863,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1245184 + "state": "kv.19" }, { - "state": "kv", - "offset": 1245440 + "state": "kv.19", + "offset": 256 }, { "buf": "slot_mapping" @@ -35921,7 +35884,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -35936,12 +35899,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1245184 + "state": "kv.19" }, { - "state": "kv", - "offset": 1245440 + "state": "kv.19", + "offset": 256 }, { "buf": "block_table" @@ -36204,12 +36166,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1310720 + "state": "kv.20" }, { - "state": "kv", - "offset": 1310976 + "state": "kv.20", + "offset": 256 }, { "buf": "slot_mapping" @@ -36226,7 +36187,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -36241,12 +36202,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1310720 + "state": "kv.20" }, { - "state": "kv", - "offset": 1310976 + "state": "kv.20", + "offset": 256 }, { "buf": "block_table" @@ -36509,12 +36469,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1376256 + "state": "kv.21" }, { - "state": "kv", - "offset": 1376512 + "state": "kv.21", + "offset": 256 }, { "buf": "slot_mapping" @@ -36531,7 +36490,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -36546,12 +36505,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1376256 + "state": "kv.21" }, { - "state": "kv", - "offset": 1376512 + "state": "kv.21", + "offset": 256 }, { "buf": "block_table" @@ -36814,12 +36772,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1441792 + "state": "kv.22" }, { - "state": "kv", - "offset": 1442048 + "state": "kv.22", + "offset": 256 }, { "buf": "slot_mapping" @@ -36836,7 +36793,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -36851,12 +36808,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1441792 + "state": "kv.22" }, { - "state": "kv", - "offset": 1442048 + "state": "kv.22", + "offset": 256 }, { "buf": "block_table" @@ -37119,12 +37075,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1507328 + "state": "kv.23" }, { - "state": "kv", - "offset": 1507584 + "state": "kv.23", + "offset": 256 }, { "buf": "slot_mapping" @@ -37141,7 +37096,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -37156,12 +37111,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1507328 + "state": "kv.23" }, { - "state": "kv", - "offset": 1507584 + "state": "kv.23", + "offset": 256 }, { "buf": "block_table" @@ -37424,12 +37378,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1572864 + "state": "kv.24" }, { - "state": "kv", - "offset": 1573120 + "state": "kv.24", + "offset": 256 }, { "buf": "slot_mapping" @@ -37446,7 +37399,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -37461,12 +37414,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1572864 + "state": "kv.24" }, { - "state": "kv", - "offset": 1573120 + "state": "kv.24", + "offset": 256 }, { "buf": "block_table" @@ -37729,12 +37681,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1638400 + "state": "kv.25" }, { - "state": "kv", - "offset": 1638656 + "state": "kv.25", + "offset": 256 }, { "buf": "slot_mapping" @@ -37751,7 +37702,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -37766,12 +37717,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1638400 + "state": "kv.25" }, { - "state": "kv", - "offset": 1638656 + "state": "kv.25", + "offset": 256 }, { "buf": "block_table" @@ -38034,12 +37984,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1703936 + "state": "kv.26" }, { - "state": "kv", - "offset": 1704192 + "state": "kv.26", + "offset": 256 }, { "buf": "slot_mapping" @@ -38056,7 +38005,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -38071,12 +38020,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1703936 + "state": "kv.26" }, { - "state": "kv", - "offset": 1704192 + "state": "kv.26", + "offset": 256 }, { "buf": "block_table" @@ -38339,12 +38287,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1769472 + "state": "kv.27" }, { - "state": "kv", - "offset": 1769728 + "state": "kv.27", + "offset": 256 }, { "buf": "slot_mapping" @@ -38361,7 +38308,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -38376,12 +38323,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1769472 + "state": "kv.27" }, { - "state": "kv", - "offset": 1769728 + "state": "kv.27", + "offset": 256 }, { "buf": "block_table" @@ -38644,12 +38590,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1835008 + "state": "kv.28" }, { - "state": "kv", - "offset": 1835264 + "state": "kv.28", + "offset": 256 }, { "buf": "slot_mapping" @@ -38666,7 +38611,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -38681,12 +38626,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1835008 + "state": "kv.28" }, { - "state": "kv", - "offset": 1835264 + "state": "kv.28", + "offset": 256 }, { "buf": "block_table" @@ -38949,12 +38893,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1900544 + "state": "kv.29" }, { - "state": "kv", - "offset": 1900800 + "state": "kv.29", + "offset": 256 }, { "buf": "slot_mapping" @@ -38971,7 +38914,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -38986,12 +38929,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1900544 + "state": "kv.29" }, { - "state": "kv", - "offset": 1900800 + "state": "kv.29", + "offset": 256 }, { "buf": "block_table" @@ -39254,12 +39196,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1966080 + "state": "kv.30" }, { - "state": "kv", - "offset": 1966336 + "state": "kv.30", + "offset": 256 }, { "buf": "slot_mapping" @@ -39276,7 +39217,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -39291,12 +39232,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1966080 + "state": "kv.30" }, { - "state": "kv", - "offset": 1966336 + "state": "kv.30", + "offset": 256 }, { "buf": "block_table" @@ -39559,12 +39499,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 2031616 + "state": "kv.31" }, { - "state": "kv", - "offset": 2031872 + "state": "kv.31", + "offset": 256 }, { "buf": "slot_mapping" @@ -39581,7 +39520,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -39596,12 +39535,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 2031616 + "state": "kv.31" }, { - "state": "kv", - "offset": 2031872 + "state": "kv.31", + "offset": 256 }, { "buf": "block_table" @@ -39864,12 +39802,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 2097152 + "state": "kv.32" }, { - "state": "kv", - "offset": 2097408 + "state": "kv.32", + "offset": 256 }, { "buf": "slot_mapping" @@ -39886,7 +39823,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -39901,12 +39838,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 2097152 + "state": "kv.32" }, { - "state": "kv", - "offset": 2097408 + "state": "kv.32", + "offset": 256 }, { "buf": "block_table" @@ -40169,12 +40105,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 2162688 + "state": "kv.33" }, { - "state": "kv", - "offset": 2162944 + "state": "kv.33", + "offset": 256 }, { "buf": "slot_mapping" @@ -40191,7 +40126,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -40206,12 +40141,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 2162688 + "state": "kv.33" }, { - "state": "kv", - "offset": 2162944 + "state": "kv.33", + "offset": 256 }, { "buf": "block_table" @@ -40474,12 +40408,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 2228224 + "state": "kv.34" }, { - "state": "kv", - "offset": 2228480 + "state": "kv.34", + "offset": 256 }, { "buf": "slot_mapping" @@ -40496,7 +40429,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -40511,12 +40444,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 2228224 + "state": "kv.34" }, { - "state": "kv", - "offset": 2228480 + "state": "kv.34", + "offset": 256 }, { "buf": "block_table" @@ -40779,12 +40711,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 2293760 + "state": "kv.35" }, { - "state": "kv", - "offset": 2294016 + "state": "kv.35", + "offset": 256 }, { "buf": "slot_mapping" @@ -40801,7 +40732,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -40816,12 +40747,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 2293760 + "state": "kv.35" }, { - "state": "kv", - "offset": 2294016 + "state": "kv.35", + "offset": 256 }, { "buf": "block_table" @@ -43132,10 +43062,10 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv" + "state": "kv.0" }, { - "state": "kv", + "state": "kv.0", "offset": 256 }, { @@ -43152,7 +43082,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -43167,10 +43097,10 @@ "buf": "q_n" }, { - "state": "kv" + "state": "kv.0" }, { - "state": "kv", + "state": "kv.0", "offset": 256 }, { @@ -43457,12 +43387,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 65536 + "state": "kv.1" }, { - "state": "kv", - "offset": 65792 + "state": "kv.1", + "offset": 256 }, { "buf": "slot_mapping" @@ -43479,7 +43408,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -43494,12 +43423,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 65536 + "state": "kv.1" }, { - "state": "kv", - "offset": 65792 + "state": "kv.1", + "offset": 256 }, { "buf": "block_table" @@ -43762,12 +43690,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 131072 + "state": "kv.2" }, { - "state": "kv", - "offset": 131328 + "state": "kv.2", + "offset": 256 }, { "buf": "slot_mapping" @@ -43784,7 +43711,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -43799,12 +43726,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 131072 + "state": "kv.2" }, { - "state": "kv", - "offset": 131328 + "state": "kv.2", + "offset": 256 }, { "buf": "block_table" @@ -44067,12 +43993,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 196608 + "state": "kv.3" }, { - "state": "kv", - "offset": 196864 + "state": "kv.3", + "offset": 256 }, { "buf": "slot_mapping" @@ -44089,7 +44014,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -44104,12 +44029,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 196608 + "state": "kv.3" }, { - "state": "kv", - "offset": 196864 + "state": "kv.3", + "offset": 256 }, { "buf": "block_table" @@ -44372,12 +44296,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 262144 + "state": "kv.4" }, { - "state": "kv", - "offset": 262400 + "state": "kv.4", + "offset": 256 }, { "buf": "slot_mapping" @@ -44394,7 +44317,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -44409,12 +44332,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 262144 + "state": "kv.4" }, { - "state": "kv", - "offset": 262400 + "state": "kv.4", + "offset": 256 }, { "buf": "block_table" @@ -44677,12 +44599,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 327680 + "state": "kv.5" }, { - "state": "kv", - "offset": 327936 + "state": "kv.5", + "offset": 256 }, { "buf": "slot_mapping" @@ -44699,7 +44620,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -44714,12 +44635,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 327680 + "state": "kv.5" }, { - "state": "kv", - "offset": 327936 + "state": "kv.5", + "offset": 256 }, { "buf": "block_table" @@ -44982,12 +44902,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 393216 + "state": "kv.6" }, { - "state": "kv", - "offset": 393472 + "state": "kv.6", + "offset": 256 }, { "buf": "slot_mapping" @@ -45004,7 +44923,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -45019,12 +44938,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 393216 + "state": "kv.6" }, { - "state": "kv", - "offset": 393472 + "state": "kv.6", + "offset": 256 }, { "buf": "block_table" @@ -45287,12 +45205,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 458752 + "state": "kv.7" }, { - "state": "kv", - "offset": 459008 + "state": "kv.7", + "offset": 256 }, { "buf": "slot_mapping" @@ -45309,7 +45226,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -45324,12 +45241,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 458752 + "state": "kv.7" }, { - "state": "kv", - "offset": 459008 + "state": "kv.7", + "offset": 256 }, { "buf": "block_table" @@ -45592,12 +45508,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 524288 + "state": "kv.8" }, { - "state": "kv", - "offset": 524544 + "state": "kv.8", + "offset": 256 }, { "buf": "slot_mapping" @@ -45614,7 +45529,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -45629,12 +45544,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 524288 + "state": "kv.8" }, { - "state": "kv", - "offset": 524544 + "state": "kv.8", + "offset": 256 }, { "buf": "block_table" @@ -45915,12 +45829,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 589824 + "state": "kv.9" }, { - "state": "kv", - "offset": 590080 + "state": "kv.9", + "offset": 256 }, { "buf": "slot_mapping" @@ -45937,7 +45850,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -45952,12 +45865,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 589824 + "state": "kv.9" }, { - "state": "kv", - "offset": 590080 + "state": "kv.9", + "offset": 256 }, { "buf": "block_table" @@ -46220,12 +46132,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 655360 + "state": "kv.10" }, { - "state": "kv", - "offset": 655616 + "state": "kv.10", + "offset": 256 }, { "buf": "slot_mapping" @@ -46242,7 +46153,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -46257,12 +46168,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 655360 + "state": "kv.10" }, { - "state": "kv", - "offset": 655616 + "state": "kv.10", + "offset": 256 }, { "buf": "block_table" @@ -46525,12 +46435,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 720896 + "state": "kv.11" }, { - "state": "kv", - "offset": 721152 + "state": "kv.11", + "offset": 256 }, { "buf": "slot_mapping" @@ -46547,7 +46456,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -46562,12 +46471,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 720896 + "state": "kv.11" }, { - "state": "kv", - "offset": 721152 + "state": "kv.11", + "offset": 256 }, { "buf": "block_table" @@ -46830,12 +46738,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 786432 + "state": "kv.12" }, { - "state": "kv", - "offset": 786688 + "state": "kv.12", + "offset": 256 }, { "buf": "slot_mapping" @@ -46852,7 +46759,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -46867,12 +46774,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 786432 + "state": "kv.12" }, { - "state": "kv", - "offset": 786688 + "state": "kv.12", + "offset": 256 }, { "buf": "block_table" @@ -47135,12 +47041,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 851968 + "state": "kv.13" }, { - "state": "kv", - "offset": 852224 + "state": "kv.13", + "offset": 256 }, { "buf": "slot_mapping" @@ -47157,7 +47062,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -47172,12 +47077,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 851968 + "state": "kv.13" }, { - "state": "kv", - "offset": 852224 + "state": "kv.13", + "offset": 256 }, { "buf": "block_table" @@ -47440,12 +47344,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 917504 + "state": "kv.14" }, { - "state": "kv", - "offset": 917760 + "state": "kv.14", + "offset": 256 }, { "buf": "slot_mapping" @@ -47462,7 +47365,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -47477,12 +47380,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 917504 + "state": "kv.14" }, { - "state": "kv", - "offset": 917760 + "state": "kv.14", + "offset": 256 }, { "buf": "block_table" @@ -47745,12 +47647,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 983040 + "state": "kv.15" }, { - "state": "kv", - "offset": 983296 + "state": "kv.15", + "offset": 256 }, { "buf": "slot_mapping" @@ -47767,7 +47668,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -47782,12 +47683,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 983040 + "state": "kv.15" }, { - "state": "kv", - "offset": 983296 + "state": "kv.15", + "offset": 256 }, { "buf": "block_table" @@ -48050,12 +47950,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1048576 + "state": "kv.16" }, { - "state": "kv", - "offset": 1048832 + "state": "kv.16", + "offset": 256 }, { "buf": "slot_mapping" @@ -48072,7 +47971,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -48087,12 +47986,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1048576 + "state": "kv.16" }, { - "state": "kv", - "offset": 1048832 + "state": "kv.16", + "offset": 256 }, { "buf": "block_table" @@ -48373,12 +48271,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1114112 + "state": "kv.17" }, { - "state": "kv", - "offset": 1114368 + "state": "kv.17", + "offset": 256 }, { "buf": "slot_mapping" @@ -48395,7 +48292,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -48410,12 +48307,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1114112 + "state": "kv.17" }, { - "state": "kv", - "offset": 1114368 + "state": "kv.17", + "offset": 256 }, { "buf": "block_table" @@ -48678,12 +48574,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1179648 + "state": "kv.18" }, { - "state": "kv", - "offset": 1179904 + "state": "kv.18", + "offset": 256 }, { "buf": "slot_mapping" @@ -48700,7 +48595,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -48715,12 +48610,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1179648 + "state": "kv.18" }, { - "state": "kv", - "offset": 1179904 + "state": "kv.18", + "offset": 256 }, { "buf": "block_table" @@ -48983,12 +48877,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1245184 + "state": "kv.19" }, { - "state": "kv", - "offset": 1245440 + "state": "kv.19", + "offset": 256 }, { "buf": "slot_mapping" @@ -49005,7 +48898,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -49020,12 +48913,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1245184 + "state": "kv.19" }, { - "state": "kv", - "offset": 1245440 + "state": "kv.19", + "offset": 256 }, { "buf": "block_table" @@ -49288,12 +49180,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1310720 + "state": "kv.20" }, { - "state": "kv", - "offset": 1310976 + "state": "kv.20", + "offset": 256 }, { "buf": "slot_mapping" @@ -49310,7 +49201,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -49325,12 +49216,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1310720 + "state": "kv.20" }, { - "state": "kv", - "offset": 1310976 + "state": "kv.20", + "offset": 256 }, { "buf": "block_table" @@ -49593,12 +49483,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1376256 + "state": "kv.21" }, { - "state": "kv", - "offset": 1376512 + "state": "kv.21", + "offset": 256 }, { "buf": "slot_mapping" @@ -49615,7 +49504,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -49630,12 +49519,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1376256 + "state": "kv.21" }, { - "state": "kv", - "offset": 1376512 + "state": "kv.21", + "offset": 256 }, { "buf": "block_table" @@ -49898,12 +49786,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1441792 + "state": "kv.22" }, { - "state": "kv", - "offset": 1442048 + "state": "kv.22", + "offset": 256 }, { "buf": "slot_mapping" @@ -49920,7 +49807,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -49935,12 +49822,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1441792 + "state": "kv.22" }, { - "state": "kv", - "offset": 1442048 + "state": "kv.22", + "offset": 256 }, { "buf": "block_table" @@ -50203,12 +50089,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1507328 + "state": "kv.23" }, { - "state": "kv", - "offset": 1507584 + "state": "kv.23", + "offset": 256 }, { "buf": "slot_mapping" @@ -50225,7 +50110,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -50240,12 +50125,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1507328 + "state": "kv.23" }, { - "state": "kv", - "offset": 1507584 + "state": "kv.23", + "offset": 256 }, { "buf": "block_table" @@ -50508,12 +50392,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1572864 + "state": "kv.24" }, { - "state": "kv", - "offset": 1573120 + "state": "kv.24", + "offset": 256 }, { "buf": "slot_mapping" @@ -50530,7 +50413,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -50545,12 +50428,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1572864 + "state": "kv.24" }, { - "state": "kv", - "offset": 1573120 + "state": "kv.24", + "offset": 256 }, { "buf": "block_table" @@ -50831,12 +50713,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1638400 + "state": "kv.25" }, { - "state": "kv", - "offset": 1638656 + "state": "kv.25", + "offset": 256 }, { "buf": "slot_mapping" @@ -50853,7 +50734,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -50868,12 +50749,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1638400 + "state": "kv.25" }, { - "state": "kv", - "offset": 1638656 + "state": "kv.25", + "offset": 256 }, { "buf": "block_table" @@ -51136,12 +51016,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1703936 + "state": "kv.26" }, { - "state": "kv", - "offset": 1704192 + "state": "kv.26", + "offset": 256 }, { "buf": "slot_mapping" @@ -51158,7 +51037,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -51173,12 +51052,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1703936 + "state": "kv.26" }, { - "state": "kv", - "offset": 1704192 + "state": "kv.26", + "offset": 256 }, { "buf": "block_table" @@ -51441,12 +51319,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1769472 + "state": "kv.27" }, { - "state": "kv", - "offset": 1769728 + "state": "kv.27", + "offset": 256 }, { "buf": "slot_mapping" @@ -51463,7 +51340,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -51478,12 +51355,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1769472 + "state": "kv.27" }, { - "state": "kv", - "offset": 1769728 + "state": "kv.27", + "offset": 256 }, { "buf": "block_table" @@ -51746,12 +51622,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1835008 + "state": "kv.28" }, { - "state": "kv", - "offset": 1835264 + "state": "kv.28", + "offset": 256 }, { "buf": "slot_mapping" @@ -51768,7 +51643,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -51783,12 +51658,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1835008 + "state": "kv.28" }, { - "state": "kv", - "offset": 1835264 + "state": "kv.28", + "offset": 256 }, { "buf": "block_table" @@ -52051,12 +51925,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1900544 + "state": "kv.29" }, { - "state": "kv", - "offset": 1900800 + "state": "kv.29", + "offset": 256 }, { "buf": "slot_mapping" @@ -52073,7 +51946,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -52088,12 +51961,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1900544 + "state": "kv.29" }, { - "state": "kv", - "offset": 1900800 + "state": "kv.29", + "offset": 256 }, { "buf": "block_table" @@ -52356,12 +52228,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1966080 + "state": "kv.30" }, { - "state": "kv", - "offset": 1966336 + "state": "kv.30", + "offset": 256 }, { "buf": "slot_mapping" @@ -52378,7 +52249,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -52393,12 +52264,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1966080 + "state": "kv.30" }, { - "state": "kv", - "offset": 1966336 + "state": "kv.30", + "offset": 256 }, { "buf": "block_table" @@ -52661,12 +52531,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 2031616 + "state": "kv.31" }, { - "state": "kv", - "offset": 2031872 + "state": "kv.31", + "offset": 256 }, { "buf": "slot_mapping" @@ -52683,7 +52552,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -52698,12 +52567,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 2031616 + "state": "kv.31" }, { - "state": "kv", - "offset": 2031872 + "state": "kv.31", + "offset": 256 }, { "buf": "block_table" @@ -52966,12 +52834,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 2097152 + "state": "kv.32" }, { - "state": "kv", - "offset": 2097408 + "state": "kv.32", + "offset": 256 }, { "buf": "slot_mapping" @@ -52988,7 +52855,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -53003,12 +52870,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 2097152 + "state": "kv.32" }, { - "state": "kv", - "offset": 2097408 + "state": "kv.32", + "offset": 256 }, { "buf": "block_table" @@ -53289,12 +53155,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 2162688 + "state": "kv.33" }, { - "state": "kv", - "offset": 2162944 + "state": "kv.33", + "offset": 256 }, { "buf": "slot_mapping" @@ -53311,7 +53176,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -53326,12 +53191,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 2162688 + "state": "kv.33" }, { - "state": "kv", - "offset": 2162944 + "state": "kv.33", + "offset": 256 }, { "buf": "block_table" @@ -53594,12 +53458,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 2228224 + "state": "kv.34" }, { - "state": "kv", - "offset": 2228480 + "state": "kv.34", + "offset": 256 }, { "buf": "slot_mapping" @@ -53616,7 +53479,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -53631,12 +53494,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 2228224 + "state": "kv.34" }, { - "state": "kv", - "offset": 2228480 + "state": "kv.34", + "offset": 256 }, { "buf": "block_table" @@ -53899,12 +53761,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 2293760 + "state": "kv.35" }, { - "state": "kv", - "offset": 2294016 + "state": "kv.35", + "offset": 256 }, { "buf": "slot_mapping" @@ -53921,7 +53782,7 @@ "i64": 6144 }, { - "i64": 1179648 + "i64": 32768 } ] }, @@ -53936,12 +53797,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 2293760 + "state": "kv.35" }, { - "state": "kv", - "offset": 2294016 + "state": "kv.35", + "offset": 256 }, { "buf": "block_table" diff --git a/examples/qwen3-4b-silu-mined.json b/examples/qwen3-4b-silu-mined.json index a607f8e9..48086007 100644 --- a/examples/qwen3-4b-silu-mined.json +++ b/examples/qwen3-4b-silu-mined.json @@ -17,7 +17,42 @@ "KV_SCALE_COUNT": 72, "MAX_TOKENS": 2048, "MAX_SEQS": 256, - "KV_BYTES_PER_TOKEN": 147456, + "KV.0_BYTES_PER_TOKEN": 4096, + "KV.1_BYTES_PER_TOKEN": 4096, + "KV.2_BYTES_PER_TOKEN": 4096, + "KV.3_BYTES_PER_TOKEN": 4096, + "KV.4_BYTES_PER_TOKEN": 4096, + "KV.5_BYTES_PER_TOKEN": 4096, + "KV.6_BYTES_PER_TOKEN": 4096, + "KV.7_BYTES_PER_TOKEN": 4096, + "KV.8_BYTES_PER_TOKEN": 4096, + "KV.9_BYTES_PER_TOKEN": 4096, + "KV.10_BYTES_PER_TOKEN": 4096, + "KV.11_BYTES_PER_TOKEN": 4096, + "KV.12_BYTES_PER_TOKEN": 4096, + "KV.13_BYTES_PER_TOKEN": 4096, + "KV.14_BYTES_PER_TOKEN": 4096, + "KV.15_BYTES_PER_TOKEN": 4096, + "KV.16_BYTES_PER_TOKEN": 4096, + "KV.17_BYTES_PER_TOKEN": 4096, + "KV.18_BYTES_PER_TOKEN": 4096, + "KV.19_BYTES_PER_TOKEN": 4096, + "KV.20_BYTES_PER_TOKEN": 4096, + "KV.21_BYTES_PER_TOKEN": 4096, + "KV.22_BYTES_PER_TOKEN": 4096, + "KV.23_BYTES_PER_TOKEN": 4096, + "KV.24_BYTES_PER_TOKEN": 4096, + "KV.25_BYTES_PER_TOKEN": 4096, + "KV.26_BYTES_PER_TOKEN": 4096, + "KV.27_BYTES_PER_TOKEN": 4096, + "KV.28_BYTES_PER_TOKEN": 4096, + "KV.29_BYTES_PER_TOKEN": 4096, + "KV.30_BYTES_PER_TOKEN": 4096, + "KV.31_BYTES_PER_TOKEN": 4096, + "KV.32_BYTES_PER_TOKEN": 4096, + "KV.33_BYTES_PER_TOKEN": 4096, + "KV.34_BYTES_PER_TOKEN": 4096, + "KV.35_BYTES_PER_TOKEN": 4096, "RMS_NORM_EPS": 9.999999974752427e-07, "NUM_HEADS": 32, "NUM_KV_HEADS": 8, @@ -34,8 +69,113 @@ } }, "states": { - "kv": { - "bytes_per_token": "KV_BYTES_PER_TOKEN" + "kv.0": { + "bytes_per_token": "KV.0_BYTES_PER_TOKEN" + }, + "kv.1": { + "bytes_per_token": "KV.1_BYTES_PER_TOKEN" + }, + "kv.2": { + "bytes_per_token": "KV.2_BYTES_PER_TOKEN" + }, + "kv.3": { + "bytes_per_token": "KV.3_BYTES_PER_TOKEN" + }, + "kv.4": { + "bytes_per_token": "KV.4_BYTES_PER_TOKEN" + }, + "kv.5": { + "bytes_per_token": "KV.5_BYTES_PER_TOKEN" + }, + "kv.6": { + "bytes_per_token": "KV.6_BYTES_PER_TOKEN" + }, + "kv.7": { + "bytes_per_token": "KV.7_BYTES_PER_TOKEN" + }, + "kv.8": { + "bytes_per_token": "KV.8_BYTES_PER_TOKEN" + }, + "kv.9": { + "bytes_per_token": "KV.9_BYTES_PER_TOKEN" + }, + "kv.10": { + "bytes_per_token": "KV.10_BYTES_PER_TOKEN" + }, + "kv.11": { + "bytes_per_token": "KV.11_BYTES_PER_TOKEN" + }, + "kv.12": { + "bytes_per_token": "KV.12_BYTES_PER_TOKEN" + }, + "kv.13": { + "bytes_per_token": "KV.13_BYTES_PER_TOKEN" + }, + "kv.14": { + "bytes_per_token": "KV.14_BYTES_PER_TOKEN" + }, + "kv.15": { + "bytes_per_token": "KV.15_BYTES_PER_TOKEN" + }, + "kv.16": { + "bytes_per_token": "KV.16_BYTES_PER_TOKEN" + }, + "kv.17": { + "bytes_per_token": "KV.17_BYTES_PER_TOKEN" + }, + "kv.18": { + "bytes_per_token": "KV.18_BYTES_PER_TOKEN" + }, + "kv.19": { + "bytes_per_token": "KV.19_BYTES_PER_TOKEN" + }, + "kv.20": { + "bytes_per_token": "KV.20_BYTES_PER_TOKEN" + }, + "kv.21": { + "bytes_per_token": "KV.21_BYTES_PER_TOKEN" + }, + "kv.22": { + "bytes_per_token": "KV.22_BYTES_PER_TOKEN" + }, + "kv.23": { + "bytes_per_token": "KV.23_BYTES_PER_TOKEN" + }, + "kv.24": { + "bytes_per_token": "KV.24_BYTES_PER_TOKEN" + }, + "kv.25": { + "bytes_per_token": "KV.25_BYTES_PER_TOKEN" + }, + "kv.26": { + "bytes_per_token": "KV.26_BYTES_PER_TOKEN" + }, + "kv.27": { + "bytes_per_token": "KV.27_BYTES_PER_TOKEN" + }, + "kv.28": { + "bytes_per_token": "KV.28_BYTES_PER_TOKEN" + }, + "kv.29": { + "bytes_per_token": "KV.29_BYTES_PER_TOKEN" + }, + "kv.30": { + "bytes_per_token": "KV.30_BYTES_PER_TOKEN" + }, + "kv.31": { + "bytes_per_token": "KV.31_BYTES_PER_TOKEN" + }, + "kv.32": { + "bytes_per_token": "KV.32_BYTES_PER_TOKEN" + }, + "kv.33": { + "bytes_per_token": "KV.33_BYTES_PER_TOKEN" + }, + "kv.34": { + "bytes_per_token": "KV.34_BYTES_PER_TOKEN" + }, + "kv.35": { + "bytes_per_token": "KV.35_BYTES_PER_TOKEN" } }, "buffers": { @@ -69,7 +209,7 @@ "kind": "input", "fill": "slot", "domain": { - "index_into": "kv" + "index_into": "kv.0" } }, "block_table": { @@ -80,7 +220,7 @@ ], "kind": "input", "domain": { - "index_into": "kv", + "index_into": "kv.0", "stride": "KV_BLOCK_SIZE" } }, @@ -4182,12 +4322,12 @@ }, "modules": { "argmax": { - "source": "argmax.cubin", - "sha256": "25374575bd35dbd9fe07f90982d0da0c2d4a72cc6ac88df5ec92cdb62a3beb16" + "source": "hf:Pegainfer/kern-kernels/blobs/ebe4ba79544ddfac43ee2dcf6a7f5299aba1124e176c201133d4ef9400e3dc25", + "sha256": "ebe4ba79544ddfac43ee2dcf6a7f5299aba1124e176c201133d4ef9400e3dc25" }, "embedding": { - "source": "embedding.cubin", - "sha256": "1c3dd3e591d872aed259d7ef59d44649061f77a4247a5df864a7fbf0ccbde92f" + "source": "hf:Pegainfer/kern-kernels/blobs/868e93ae6e46e36a11c56af1fae49aeff9fcdfdc79f64ca99475a309e721e9e7", + "sha256": "868e93ae6e46e36a11c56af1fae49aeff9fcdfdc79f64ca99475a309e721e9e7" }, "reduce_segments": { "source": "reduce_segments.cubin", @@ -4218,7 +4358,7 @@ "sha256": "116de812ac5a3a186f76fa239d4c16abbed6b8b0af06d16e1baec0aefe8fe8fa" }, "weight_prep": { - "source": "weight_prep.cubin", + "source": "hf:Pegainfer/kern-kernels/blobs/040f51b01c421b9c69a2090034ff1bc4741e40c02f5ea52bd04a3e62e5c1af4d", "sha256": "040f51b01c421b9c69a2090034ff1bc4741e40c02f5ea52bd04a3e62e5c1af4d" } }, @@ -4621,7 +4761,7 @@ "i64": "QKV_DIM" }, { - "i64": 1179648 + "i64": 32768 }, { "i64": 256 @@ -4791,7 +4931,7 @@ "param": 8 }, { - "i64": 1179648 + "i64": 32768 }, { "i64": 2048 @@ -4800,7 +4940,7 @@ "i64": 256 }, { - "i64": 1179648 + "i64": 32768 }, { "i64": 2048 @@ -5022,7 +5162,7 @@ "param": 8 }, { - "i64": 1179648 + "i64": 32768 }, { "i64": 2048 @@ -5031,7 +5171,7 @@ "i64": 256 }, { - "i64": 1179648 + "i64": 32768 }, { "i64": 2048 @@ -5182,7 +5322,7 @@ "param": 8 }, { - "i64": 1179648 + "i64": 32768 }, { "i64": 2048 @@ -5191,7 +5331,7 @@ "i64": 256 }, { - "i64": 1179648 + "i64": 32768 }, { "i64": 2048 @@ -5727,10 +5867,10 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv" + "state": "kv.0" }, { - "state": "kv", + "state": "kv.0", "offset": 256 }, { @@ -5756,10 +5896,10 @@ "buf": "q_n" }, { - "state": "kv" + "state": "kv.0" }, { - "state": "kv", + "state": "kv.0", "offset": 256 }, { @@ -6007,12 +6147,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 65536 + "state": "kv.1" }, { - "state": "kv", - "offset": 65792 + "state": "kv.1", + "offset": 256 }, { "buf": "slot_mapping" @@ -6038,12 +6177,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 65536 + "state": "kv.1" }, { - "state": "kv", - "offset": 65792 + "state": "kv.1", + "offset": 256 }, { "buf": "block_table" @@ -6291,12 +6429,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 131072 + "state": "kv.2" }, { - "state": "kv", - "offset": 131328 + "state": "kv.2", + "offset": 256 }, { "buf": "slot_mapping" @@ -6322,12 +6459,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 131072 + "state": "kv.2" }, { - "state": "kv", - "offset": 131328 + "state": "kv.2", + "offset": 256 }, { "buf": "block_table" @@ -6575,12 +6711,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 196608 + "state": "kv.3" }, { - "state": "kv", - "offset": 196864 + "state": "kv.3", + "offset": 256 }, { "buf": "slot_mapping" @@ -6606,12 +6741,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 196608 + "state": "kv.3" }, { - "state": "kv", - "offset": 196864 + "state": "kv.3", + "offset": 256 }, { "buf": "block_table" @@ -6859,12 +6993,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 262144 + "state": "kv.4" }, { - "state": "kv", - "offset": 262400 + "state": "kv.4", + "offset": 256 }, { "buf": "slot_mapping" @@ -6890,12 +7023,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 262144 + "state": "kv.4" }, { - "state": "kv", - "offset": 262400 + "state": "kv.4", + "offset": 256 }, { "buf": "block_table" @@ -7143,12 +7275,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 327680 + "state": "kv.5" }, { - "state": "kv", - "offset": 327936 + "state": "kv.5", + "offset": 256 }, { "buf": "slot_mapping" @@ -7174,12 +7305,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 327680 + "state": "kv.5" }, { - "state": "kv", - "offset": 327936 + "state": "kv.5", + "offset": 256 }, { "buf": "block_table" @@ -7427,12 +7557,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 393216 + "state": "kv.6" }, { - "state": "kv", - "offset": 393472 + "state": "kv.6", + "offset": 256 }, { "buf": "slot_mapping" @@ -7458,12 +7587,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 393216 + "state": "kv.6" }, { - "state": "kv", - "offset": 393472 + "state": "kv.6", + "offset": 256 }, { "buf": "block_table" @@ -7711,12 +7839,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 458752 + "state": "kv.7" }, { - "state": "kv", - "offset": 459008 + "state": "kv.7", + "offset": 256 }, { "buf": "slot_mapping" @@ -7742,12 +7869,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 458752 + "state": "kv.7" }, { - "state": "kv", - "offset": 459008 + "state": "kv.7", + "offset": 256 }, { "buf": "block_table" @@ -7995,12 +8121,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 524288 + "state": "kv.8" }, { - "state": "kv", - "offset": 524544 + "state": "kv.8", + "offset": 256 }, { "buf": "slot_mapping" @@ -8026,12 +8151,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 524288 + "state": "kv.8" }, { - "state": "kv", - "offset": 524544 + "state": "kv.8", + "offset": 256 }, { "buf": "block_table" @@ -8279,12 +8403,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 589824 + "state": "kv.9" }, { - "state": "kv", - "offset": 590080 + "state": "kv.9", + "offset": 256 }, { "buf": "slot_mapping" @@ -8310,12 +8433,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 589824 + "state": "kv.9" }, { - "state": "kv", - "offset": 590080 + "state": "kv.9", + "offset": 256 }, { "buf": "block_table" @@ -8563,12 +8685,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 655360 + "state": "kv.10" }, { - "state": "kv", - "offset": 655616 + "state": "kv.10", + "offset": 256 }, { "buf": "slot_mapping" @@ -8594,12 +8715,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 655360 + "state": "kv.10" }, { - "state": "kv", - "offset": 655616 + "state": "kv.10", + "offset": 256 }, { "buf": "block_table" @@ -8847,12 +8967,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 720896 + "state": "kv.11" }, { - "state": "kv", - "offset": 721152 + "state": "kv.11", + "offset": 256 }, { "buf": "slot_mapping" @@ -8878,12 +8997,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 720896 + "state": "kv.11" }, { - "state": "kv", - "offset": 721152 + "state": "kv.11", + "offset": 256 }, { "buf": "block_table" @@ -9131,12 +9249,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 786432 + "state": "kv.12" }, { - "state": "kv", - "offset": 786688 + "state": "kv.12", + "offset": 256 }, { "buf": "slot_mapping" @@ -9162,12 +9279,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 786432 + "state": "kv.12" }, { - "state": "kv", - "offset": 786688 + "state": "kv.12", + "offset": 256 }, { "buf": "block_table" @@ -9415,12 +9531,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 851968 + "state": "kv.13" }, { - "state": "kv", - "offset": 852224 + "state": "kv.13", + "offset": 256 }, { "buf": "slot_mapping" @@ -9446,12 +9561,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 851968 + "state": "kv.13" }, { - "state": "kv", - "offset": 852224 + "state": "kv.13", + "offset": 256 }, { "buf": "block_table" @@ -9699,12 +9813,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 917504 + "state": "kv.14" }, { - "state": "kv", - "offset": 917760 + "state": "kv.14", + "offset": 256 }, { "buf": "slot_mapping" @@ -9730,12 +9843,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 917504 + "state": "kv.14" }, { - "state": "kv", - "offset": 917760 + "state": "kv.14", + "offset": 256 }, { "buf": "block_table" @@ -9983,12 +10095,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 983040 + "state": "kv.15" }, { - "state": "kv", - "offset": 983296 + "state": "kv.15", + "offset": 256 }, { "buf": "slot_mapping" @@ -10014,12 +10125,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 983040 + "state": "kv.15" }, { - "state": "kv", - "offset": 983296 + "state": "kv.15", + "offset": 256 }, { "buf": "block_table" @@ -10267,12 +10377,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1048576 + "state": "kv.16" }, { - "state": "kv", - "offset": 1048832 + "state": "kv.16", + "offset": 256 }, { "buf": "slot_mapping" @@ -10298,12 +10407,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1048576 + "state": "kv.16" }, { - "state": "kv", - "offset": 1048832 + "state": "kv.16", + "offset": 256 }, { "buf": "block_table" @@ -10551,12 +10659,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1114112 + "state": "kv.17" }, { - "state": "kv", - "offset": 1114368 + "state": "kv.17", + "offset": 256 }, { "buf": "slot_mapping" @@ -10582,12 +10689,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1114112 + "state": "kv.17" }, { - "state": "kv", - "offset": 1114368 + "state": "kv.17", + "offset": 256 }, { "buf": "block_table" @@ -10835,12 +10941,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1179648 + "state": "kv.18" }, { - "state": "kv", - "offset": 1179904 + "state": "kv.18", + "offset": 256 }, { "buf": "slot_mapping" @@ -10866,12 +10971,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1179648 + "state": "kv.18" }, { - "state": "kv", - "offset": 1179904 + "state": "kv.18", + "offset": 256 }, { "buf": "block_table" @@ -11119,12 +11223,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1245184 + "state": "kv.19" }, { - "state": "kv", - "offset": 1245440 + "state": "kv.19", + "offset": 256 }, { "buf": "slot_mapping" @@ -11150,12 +11253,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1245184 + "state": "kv.19" }, { - "state": "kv", - "offset": 1245440 + "state": "kv.19", + "offset": 256 }, { "buf": "block_table" @@ -11403,12 +11505,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1310720 + "state": "kv.20" }, { - "state": "kv", - "offset": 1310976 + "state": "kv.20", + "offset": 256 }, { "buf": "slot_mapping" @@ -11434,12 +11535,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1310720 + "state": "kv.20" }, { - "state": "kv", - "offset": 1310976 + "state": "kv.20", + "offset": 256 }, { "buf": "block_table" @@ -11687,12 +11787,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1376256 + "state": "kv.21" }, { - "state": "kv", - "offset": 1376512 + "state": "kv.21", + "offset": 256 }, { "buf": "slot_mapping" @@ -11718,12 +11817,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1376256 + "state": "kv.21" }, { - "state": "kv", - "offset": 1376512 + "state": "kv.21", + "offset": 256 }, { "buf": "block_table" @@ -11971,12 +12069,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1441792 + "state": "kv.22" }, { - "state": "kv", - "offset": 1442048 + "state": "kv.22", + "offset": 256 }, { "buf": "slot_mapping" @@ -12002,12 +12099,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1441792 + "state": "kv.22" }, { - "state": "kv", - "offset": 1442048 + "state": "kv.22", + "offset": 256 }, { "buf": "block_table" @@ -12255,12 +12351,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1507328 + "state": "kv.23" }, { - "state": "kv", - "offset": 1507584 + "state": "kv.23", + "offset": 256 }, { "buf": "slot_mapping" @@ -12286,12 +12381,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1507328 + "state": "kv.23" }, { - "state": "kv", - "offset": 1507584 + "state": "kv.23", + "offset": 256 }, { "buf": "block_table" @@ -12539,12 +12633,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1572864 + "state": "kv.24" }, { - "state": "kv", - "offset": 1573120 + "state": "kv.24", + "offset": 256 }, { "buf": "slot_mapping" @@ -12570,12 +12663,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1572864 + "state": "kv.24" }, { - "state": "kv", - "offset": 1573120 + "state": "kv.24", + "offset": 256 }, { "buf": "block_table" @@ -12823,12 +12915,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1638400 + "state": "kv.25" }, { - "state": "kv", - "offset": 1638656 + "state": "kv.25", + "offset": 256 }, { "buf": "slot_mapping" @@ -12854,12 +12945,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1638400 + "state": "kv.25" }, { - "state": "kv", - "offset": 1638656 + "state": "kv.25", + "offset": 256 }, { "buf": "block_table" @@ -13107,12 +13197,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1703936 + "state": "kv.26" }, { - "state": "kv", - "offset": 1704192 + "state": "kv.26", + "offset": 256 }, { "buf": "slot_mapping" @@ -13138,12 +13227,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1703936 + "state": "kv.26" }, { - "state": "kv", - "offset": 1704192 + "state": "kv.26", + "offset": 256 }, { "buf": "block_table" @@ -13391,12 +13479,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1769472 + "state": "kv.27" }, { - "state": "kv", - "offset": 1769728 + "state": "kv.27", + "offset": 256 }, { "buf": "slot_mapping" @@ -13422,12 +13509,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1769472 + "state": "kv.27" }, { - "state": "kv", - "offset": 1769728 + "state": "kv.27", + "offset": 256 }, { "buf": "block_table" @@ -13675,12 +13761,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1835008 + "state": "kv.28" }, { - "state": "kv", - "offset": 1835264 + "state": "kv.28", + "offset": 256 }, { "buf": "slot_mapping" @@ -13706,12 +13791,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1835008 + "state": "kv.28" }, { - "state": "kv", - "offset": 1835264 + "state": "kv.28", + "offset": 256 }, { "buf": "block_table" @@ -13959,12 +14043,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1900544 + "state": "kv.29" }, { - "state": "kv", - "offset": 1900800 + "state": "kv.29", + "offset": 256 }, { "buf": "slot_mapping" @@ -13990,12 +14073,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1900544 + "state": "kv.29" }, { - "state": "kv", - "offset": 1900800 + "state": "kv.29", + "offset": 256 }, { "buf": "block_table" @@ -14243,12 +14325,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1966080 + "state": "kv.30" }, { - "state": "kv", - "offset": 1966336 + "state": "kv.30", + "offset": 256 }, { "buf": "slot_mapping" @@ -14274,12 +14355,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1966080 + "state": "kv.30" }, { - "state": "kv", - "offset": 1966336 + "state": "kv.30", + "offset": 256 }, { "buf": "block_table" @@ -14527,12 +14607,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 2031616 + "state": "kv.31" }, { - "state": "kv", - "offset": 2031872 + "state": "kv.31", + "offset": 256 }, { "buf": "slot_mapping" @@ -14558,12 +14637,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 2031616 + "state": "kv.31" }, { - "state": "kv", - "offset": 2031872 + "state": "kv.31", + "offset": 256 }, { "buf": "block_table" @@ -14811,12 +14889,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 2097152 + "state": "kv.32" }, { - "state": "kv", - "offset": 2097408 + "state": "kv.32", + "offset": 256 }, { "buf": "slot_mapping" @@ -14842,12 +14919,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 2097152 + "state": "kv.32" }, { - "state": "kv", - "offset": 2097408 + "state": "kv.32", + "offset": 256 }, { "buf": "block_table" @@ -15095,12 +15171,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 2162688 + "state": "kv.33" }, { - "state": "kv", - "offset": 2162944 + "state": "kv.33", + "offset": 256 }, { "buf": "slot_mapping" @@ -15126,12 +15201,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 2162688 + "state": "kv.33" }, { - "state": "kv", - "offset": 2162944 + "state": "kv.33", + "offset": 256 }, { "buf": "block_table" @@ -15379,12 +15453,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 2228224 + "state": "kv.34" }, { - "state": "kv", - "offset": 2228480 + "state": "kv.34", + "offset": 256 }, { "buf": "slot_mapping" @@ -15410,12 +15483,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 2228224 + "state": "kv.34" }, { - "state": "kv", - "offset": 2228480 + "state": "kv.34", + "offset": 256 }, { "buf": "block_table" @@ -15663,12 +15735,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 2293760 + "state": "kv.35" }, { - "state": "kv", - "offset": 2294016 + "state": "kv.35", + "offset": 256 }, { "buf": "slot_mapping" @@ -15694,12 +15765,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 2293760 + "state": "kv.35" }, { - "state": "kv", - "offset": 2294016 + "state": "kv.35", + "offset": 256 }, { "buf": "block_table" @@ -15974,10 +16044,10 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv" + "state": "kv.0" }, { - "state": "kv", + "state": "kv.0", "offset": 256 }, { @@ -16003,10 +16073,10 @@ "buf": "q_n" }, { - "state": "kv" + "state": "kv.0" }, { - "state": "kv", + "state": "kv.0", "offset": 256 }, { @@ -16251,12 +16321,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 65536 + "state": "kv.1" }, { - "state": "kv", - "offset": 65792 + "state": "kv.1", + "offset": 256 }, { "buf": "slot_mapping" @@ -16282,12 +16351,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 65536 + "state": "kv.1" }, { - "state": "kv", - "offset": 65792 + "state": "kv.1", + "offset": 256 }, { "buf": "block_table" @@ -16532,12 +16600,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 131072 + "state": "kv.2" }, { - "state": "kv", - "offset": 131328 + "state": "kv.2", + "offset": 256 }, { "buf": "slot_mapping" @@ -16563,12 +16630,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 131072 + "state": "kv.2" }, { - "state": "kv", - "offset": 131328 + "state": "kv.2", + "offset": 256 }, { "buf": "block_table" @@ -16813,12 +16879,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 196608 + "state": "kv.3" }, { - "state": "kv", - "offset": 196864 + "state": "kv.3", + "offset": 256 }, { "buf": "slot_mapping" @@ -16844,12 +16909,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 196608 + "state": "kv.3" }, { - "state": "kv", - "offset": 196864 + "state": "kv.3", + "offset": 256 }, { "buf": "block_table" @@ -17094,12 +17158,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 262144 + "state": "kv.4" }, { - "state": "kv", - "offset": 262400 + "state": "kv.4", + "offset": 256 }, { "buf": "slot_mapping" @@ -17125,12 +17188,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 262144 + "state": "kv.4" }, { - "state": "kv", - "offset": 262400 + "state": "kv.4", + "offset": 256 }, { "buf": "block_table" @@ -17375,12 +17437,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 327680 + "state": "kv.5" }, { - "state": "kv", - "offset": 327936 + "state": "kv.5", + "offset": 256 }, { "buf": "slot_mapping" @@ -17406,12 +17467,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 327680 + "state": "kv.5" }, { - "state": "kv", - "offset": 327936 + "state": "kv.5", + "offset": 256 }, { "buf": "block_table" @@ -17656,12 +17716,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 393216 + "state": "kv.6" }, { - "state": "kv", - "offset": 393472 + "state": "kv.6", + "offset": 256 }, { "buf": "slot_mapping" @@ -17687,12 +17746,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 393216 + "state": "kv.6" }, { - "state": "kv", - "offset": 393472 + "state": "kv.6", + "offset": 256 }, { "buf": "block_table" @@ -17937,12 +17995,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 458752 + "state": "kv.7" }, { - "state": "kv", - "offset": 459008 + "state": "kv.7", + "offset": 256 }, { "buf": "slot_mapping" @@ -17968,12 +18025,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 458752 + "state": "kv.7" }, { - "state": "kv", - "offset": 459008 + "state": "kv.7", + "offset": 256 }, { "buf": "block_table" @@ -18218,12 +18274,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 524288 + "state": "kv.8" }, { - "state": "kv", - "offset": 524544 + "state": "kv.8", + "offset": 256 }, { "buf": "slot_mapping" @@ -18249,12 +18304,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 524288 + "state": "kv.8" }, { - "state": "kv", - "offset": 524544 + "state": "kv.8", + "offset": 256 }, { "buf": "block_table" @@ -18499,12 +18553,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 589824 + "state": "kv.9" }, { - "state": "kv", - "offset": 590080 + "state": "kv.9", + "offset": 256 }, { "buf": "slot_mapping" @@ -18530,12 +18583,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 589824 + "state": "kv.9" }, { - "state": "kv", - "offset": 590080 + "state": "kv.9", + "offset": 256 }, { "buf": "block_table" @@ -18780,12 +18832,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 655360 + "state": "kv.10" }, { - "state": "kv", - "offset": 655616 + "state": "kv.10", + "offset": 256 }, { "buf": "slot_mapping" @@ -18811,12 +18862,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 655360 + "state": "kv.10" }, { - "state": "kv", - "offset": 655616 + "state": "kv.10", + "offset": 256 }, { "buf": "block_table" @@ -19061,12 +19111,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 720896 + "state": "kv.11" }, { - "state": "kv", - "offset": 721152 + "state": "kv.11", + "offset": 256 }, { "buf": "slot_mapping" @@ -19092,12 +19141,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 720896 + "state": "kv.11" }, { - "state": "kv", - "offset": 721152 + "state": "kv.11", + "offset": 256 }, { "buf": "block_table" @@ -19342,12 +19390,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 786432 + "state": "kv.12" }, { - "state": "kv", - "offset": 786688 + "state": "kv.12", + "offset": 256 }, { "buf": "slot_mapping" @@ -19373,12 +19420,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 786432 + "state": "kv.12" }, { - "state": "kv", - "offset": 786688 + "state": "kv.12", + "offset": 256 }, { "buf": "block_table" @@ -19623,12 +19669,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 851968 + "state": "kv.13" }, { - "state": "kv", - "offset": 852224 + "state": "kv.13", + "offset": 256 }, { "buf": "slot_mapping" @@ -19654,12 +19699,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 851968 + "state": "kv.13" }, { - "state": "kv", - "offset": 852224 + "state": "kv.13", + "offset": 256 }, { "buf": "block_table" @@ -19904,12 +19948,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 917504 + "state": "kv.14" }, { - "state": "kv", - "offset": 917760 + "state": "kv.14", + "offset": 256 }, { "buf": "slot_mapping" @@ -19935,12 +19978,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 917504 + "state": "kv.14" }, { - "state": "kv", - "offset": 917760 + "state": "kv.14", + "offset": 256 }, { "buf": "block_table" @@ -20185,12 +20227,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 983040 + "state": "kv.15" }, { - "state": "kv", - "offset": 983296 + "state": "kv.15", + "offset": 256 }, { "buf": "slot_mapping" @@ -20216,12 +20257,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 983040 + "state": "kv.15" }, { - "state": "kv", - "offset": 983296 + "state": "kv.15", + "offset": 256 }, { "buf": "block_table" @@ -20466,12 +20506,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1048576 + "state": "kv.16" }, { - "state": "kv", - "offset": 1048832 + "state": "kv.16", + "offset": 256 }, { "buf": "slot_mapping" @@ -20497,12 +20536,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1048576 + "state": "kv.16" }, { - "state": "kv", - "offset": 1048832 + "state": "kv.16", + "offset": 256 }, { "buf": "block_table" @@ -20747,12 +20785,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1114112 + "state": "kv.17" }, { - "state": "kv", - "offset": 1114368 + "state": "kv.17", + "offset": 256 }, { "buf": "slot_mapping" @@ -20778,12 +20815,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1114112 + "state": "kv.17" }, { - "state": "kv", - "offset": 1114368 + "state": "kv.17", + "offset": 256 }, { "buf": "block_table" @@ -21028,12 +21064,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1179648 + "state": "kv.18" }, { - "state": "kv", - "offset": 1179904 + "state": "kv.18", + "offset": 256 }, { "buf": "slot_mapping" @@ -21059,12 +21094,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1179648 + "state": "kv.18" }, { - "state": "kv", - "offset": 1179904 + "state": "kv.18", + "offset": 256 }, { "buf": "block_table" @@ -21309,12 +21343,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1245184 + "state": "kv.19" }, { - "state": "kv", - "offset": 1245440 + "state": "kv.19", + "offset": 256 }, { "buf": "slot_mapping" @@ -21340,12 +21373,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1245184 + "state": "kv.19" }, { - "state": "kv", - "offset": 1245440 + "state": "kv.19", + "offset": 256 }, { "buf": "block_table" @@ -21590,12 +21622,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1310720 + "state": "kv.20" }, { - "state": "kv", - "offset": 1310976 + "state": "kv.20", + "offset": 256 }, { "buf": "slot_mapping" @@ -21621,12 +21652,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1310720 + "state": "kv.20" }, { - "state": "kv", - "offset": 1310976 + "state": "kv.20", + "offset": 256 }, { "buf": "block_table" @@ -21871,12 +21901,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1376256 + "state": "kv.21" }, { - "state": "kv", - "offset": 1376512 + "state": "kv.21", + "offset": 256 }, { "buf": "slot_mapping" @@ -21902,12 +21931,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1376256 + "state": "kv.21" }, { - "state": "kv", - "offset": 1376512 + "state": "kv.21", + "offset": 256 }, { "buf": "block_table" @@ -22152,12 +22180,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1441792 + "state": "kv.22" }, { - "state": "kv", - "offset": 1442048 + "state": "kv.22", + "offset": 256 }, { "buf": "slot_mapping" @@ -22183,12 +22210,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1441792 + "state": "kv.22" }, { - "state": "kv", - "offset": 1442048 + "state": "kv.22", + "offset": 256 }, { "buf": "block_table" @@ -22433,12 +22459,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1507328 + "state": "kv.23" }, { - "state": "kv", - "offset": 1507584 + "state": "kv.23", + "offset": 256 }, { "buf": "slot_mapping" @@ -22464,12 +22489,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1507328 + "state": "kv.23" }, { - "state": "kv", - "offset": 1507584 + "state": "kv.23", + "offset": 256 }, { "buf": "block_table" @@ -22714,12 +22738,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1572864 + "state": "kv.24" }, { - "state": "kv", - "offset": 1573120 + "state": "kv.24", + "offset": 256 }, { "buf": "slot_mapping" @@ -22745,12 +22768,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1572864 + "state": "kv.24" }, { - "state": "kv", - "offset": 1573120 + "state": "kv.24", + "offset": 256 }, { "buf": "block_table" @@ -22995,12 +23017,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1638400 + "state": "kv.25" }, { - "state": "kv", - "offset": 1638656 + "state": "kv.25", + "offset": 256 }, { "buf": "slot_mapping" @@ -23026,12 +23047,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1638400 + "state": "kv.25" }, { - "state": "kv", - "offset": 1638656 + "state": "kv.25", + "offset": 256 }, { "buf": "block_table" @@ -23276,12 +23296,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1703936 + "state": "kv.26" }, { - "state": "kv", - "offset": 1704192 + "state": "kv.26", + "offset": 256 }, { "buf": "slot_mapping" @@ -23307,12 +23326,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1703936 + "state": "kv.26" }, { - "state": "kv", - "offset": 1704192 + "state": "kv.26", + "offset": 256 }, { "buf": "block_table" @@ -23557,12 +23575,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1769472 + "state": "kv.27" }, { - "state": "kv", - "offset": 1769728 + "state": "kv.27", + "offset": 256 }, { "buf": "slot_mapping" @@ -23588,12 +23605,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1769472 + "state": "kv.27" }, { - "state": "kv", - "offset": 1769728 + "state": "kv.27", + "offset": 256 }, { "buf": "block_table" @@ -23838,12 +23854,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1835008 + "state": "kv.28" }, { - "state": "kv", - "offset": 1835264 + "state": "kv.28", + "offset": 256 }, { "buf": "slot_mapping" @@ -23869,12 +23884,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1835008 + "state": "kv.28" }, { - "state": "kv", - "offset": 1835264 + "state": "kv.28", + "offset": 256 }, { "buf": "block_table" @@ -24119,12 +24133,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1900544 + "state": "kv.29" }, { - "state": "kv", - "offset": 1900800 + "state": "kv.29", + "offset": 256 }, { "buf": "slot_mapping" @@ -24150,12 +24163,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1900544 + "state": "kv.29" }, { - "state": "kv", - "offset": 1900800 + "state": "kv.29", + "offset": 256 }, { "buf": "block_table" @@ -24400,12 +24412,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1966080 + "state": "kv.30" }, { - "state": "kv", - "offset": 1966336 + "state": "kv.30", + "offset": 256 }, { "buf": "slot_mapping" @@ -24431,12 +24442,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1966080 + "state": "kv.30" }, { - "state": "kv", - "offset": 1966336 + "state": "kv.30", + "offset": 256 }, { "buf": "block_table" @@ -24681,12 +24691,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 2031616 + "state": "kv.31" }, { - "state": "kv", - "offset": 2031872 + "state": "kv.31", + "offset": 256 }, { "buf": "slot_mapping" @@ -24712,12 +24721,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 2031616 + "state": "kv.31" }, { - "state": "kv", - "offset": 2031872 + "state": "kv.31", + "offset": 256 }, { "buf": "block_table" @@ -24962,12 +24970,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 2097152 + "state": "kv.32" }, { - "state": "kv", - "offset": 2097408 + "state": "kv.32", + "offset": 256 }, { "buf": "slot_mapping" @@ -24993,12 +25000,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 2097152 + "state": "kv.32" }, { - "state": "kv", - "offset": 2097408 + "state": "kv.32", + "offset": 256 }, { "buf": "block_table" @@ -25243,12 +25249,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 2162688 + "state": "kv.33" }, { - "state": "kv", - "offset": 2162944 + "state": "kv.33", + "offset": 256 }, { "buf": "slot_mapping" @@ -25274,12 +25279,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 2162688 + "state": "kv.33" }, { - "state": "kv", - "offset": 2162944 + "state": "kv.33", + "offset": 256 }, { "buf": "block_table" @@ -25524,12 +25528,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 2228224 + "state": "kv.34" }, { - "state": "kv", - "offset": 2228480 + "state": "kv.34", + "offset": 256 }, { "buf": "slot_mapping" @@ -25555,12 +25558,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 2228224 + "state": "kv.34" }, { - "state": "kv", - "offset": 2228480 + "state": "kv.34", + "offset": 256 }, { "buf": "block_table" @@ -25805,12 +25807,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 2293760 + "state": "kv.35" }, { - "state": "kv", - "offset": 2294016 + "state": "kv.35", + "offset": 256 }, { "buf": "slot_mapping" @@ -25836,12 +25837,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 2293760 + "state": "kv.35" }, { - "state": "kv", - "offset": 2294016 + "state": "kv.35", + "offset": 256 }, { "buf": "block_table" @@ -26167,10 +26167,10 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv" + "state": "kv.0" }, { - "state": "kv", + "state": "kv.0", "offset": 256 }, { @@ -26196,10 +26196,10 @@ "buf": "q_n" }, { - "state": "kv" + "state": "kv.0" }, { - "state": "kv", + "state": "kv.0", "offset": 256 }, { @@ -26447,12 +26447,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 65536 + "state": "kv.1" }, { - "state": "kv", - "offset": 65792 + "state": "kv.1", + "offset": 256 }, { "buf": "slot_mapping" @@ -26478,12 +26477,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 65536 + "state": "kv.1" }, { - "state": "kv", - "offset": 65792 + "state": "kv.1", + "offset": 256 }, { "buf": "block_table" @@ -26731,12 +26729,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 131072 + "state": "kv.2" }, { - "state": "kv", - "offset": 131328 + "state": "kv.2", + "offset": 256 }, { "buf": "slot_mapping" @@ -26762,12 +26759,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 131072 + "state": "kv.2" }, { - "state": "kv", - "offset": 131328 + "state": "kv.2", + "offset": 256 }, { "buf": "block_table" @@ -27015,12 +27011,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 196608 + "state": "kv.3" }, { - "state": "kv", - "offset": 196864 + "state": "kv.3", + "offset": 256 }, { "buf": "slot_mapping" @@ -27046,12 +27041,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 196608 + "state": "kv.3" }, { - "state": "kv", - "offset": 196864 + "state": "kv.3", + "offset": 256 }, { "buf": "block_table" @@ -27299,12 +27293,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 262144 + "state": "kv.4" }, { - "state": "kv", - "offset": 262400 + "state": "kv.4", + "offset": 256 }, { "buf": "slot_mapping" @@ -27330,12 +27323,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 262144 + "state": "kv.4" }, { - "state": "kv", - "offset": 262400 + "state": "kv.4", + "offset": 256 }, { "buf": "block_table" @@ -27583,12 +27575,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 327680 + "state": "kv.5" }, { - "state": "kv", - "offset": 327936 + "state": "kv.5", + "offset": 256 }, { "buf": "slot_mapping" @@ -27614,12 +27605,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 327680 + "state": "kv.5" }, { - "state": "kv", - "offset": 327936 + "state": "kv.5", + "offset": 256 }, { "buf": "block_table" @@ -27867,12 +27857,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 393216 + "state": "kv.6" }, { - "state": "kv", - "offset": 393472 + "state": "kv.6", + "offset": 256 }, { "buf": "slot_mapping" @@ -27898,12 +27887,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 393216 + "state": "kv.6" }, { - "state": "kv", - "offset": 393472 + "state": "kv.6", + "offset": 256 }, { "buf": "block_table" @@ -28151,12 +28139,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 458752 + "state": "kv.7" }, { - "state": "kv", - "offset": 459008 + "state": "kv.7", + "offset": 256 }, { "buf": "slot_mapping" @@ -28182,12 +28169,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 458752 + "state": "kv.7" }, { - "state": "kv", - "offset": 459008 + "state": "kv.7", + "offset": 256 }, { "buf": "block_table" @@ -28435,12 +28421,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 524288 + "state": "kv.8" }, { - "state": "kv", - "offset": 524544 + "state": "kv.8", + "offset": 256 }, { "buf": "slot_mapping" @@ -28466,12 +28451,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 524288 + "state": "kv.8" }, { - "state": "kv", - "offset": 524544 + "state": "kv.8", + "offset": 256 }, { "buf": "block_table" @@ -28719,12 +28703,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 589824 + "state": "kv.9" }, { - "state": "kv", - "offset": 590080 + "state": "kv.9", + "offset": 256 }, { "buf": "slot_mapping" @@ -28750,12 +28733,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 589824 + "state": "kv.9" }, { - "state": "kv", - "offset": 590080 + "state": "kv.9", + "offset": 256 }, { "buf": "block_table" @@ -29003,12 +28985,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 655360 + "state": "kv.10" }, { - "state": "kv", - "offset": 655616 + "state": "kv.10", + "offset": 256 }, { "buf": "slot_mapping" @@ -29034,12 +29015,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 655360 + "state": "kv.10" }, { - "state": "kv", - "offset": 655616 + "state": "kv.10", + "offset": 256 }, { "buf": "block_table" @@ -29287,12 +29267,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 720896 + "state": "kv.11" }, { - "state": "kv", - "offset": 721152 + "state": "kv.11", + "offset": 256 }, { "buf": "slot_mapping" @@ -29318,12 +29297,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 720896 + "state": "kv.11" }, { - "state": "kv", - "offset": 721152 + "state": "kv.11", + "offset": 256 }, { "buf": "block_table" @@ -29571,12 +29549,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 786432 + "state": "kv.12" }, { - "state": "kv", - "offset": 786688 + "state": "kv.12", + "offset": 256 }, { "buf": "slot_mapping" @@ -29602,12 +29579,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 786432 + "state": "kv.12" }, { - "state": "kv", - "offset": 786688 + "state": "kv.12", + "offset": 256 }, { "buf": "block_table" @@ -29855,12 +29831,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 851968 + "state": "kv.13" }, { - "state": "kv", - "offset": 852224 + "state": "kv.13", + "offset": 256 }, { "buf": "slot_mapping" @@ -29886,12 +29861,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 851968 + "state": "kv.13" }, { - "state": "kv", - "offset": 852224 + "state": "kv.13", + "offset": 256 }, { "buf": "block_table" @@ -30139,12 +30113,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 917504 + "state": "kv.14" }, { - "state": "kv", - "offset": 917760 + "state": "kv.14", + "offset": 256 }, { "buf": "slot_mapping" @@ -30170,12 +30143,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 917504 + "state": "kv.14" }, { - "state": "kv", - "offset": 917760 + "state": "kv.14", + "offset": 256 }, { "buf": "block_table" @@ -30423,12 +30395,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 983040 + "state": "kv.15" }, { - "state": "kv", - "offset": 983296 + "state": "kv.15", + "offset": 256 }, { "buf": "slot_mapping" @@ -30454,12 +30425,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 983040 + "state": "kv.15" }, { - "state": "kv", - "offset": 983296 + "state": "kv.15", + "offset": 256 }, { "buf": "block_table" @@ -30707,12 +30677,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1048576 + "state": "kv.16" }, { - "state": "kv", - "offset": 1048832 + "state": "kv.16", + "offset": 256 }, { "buf": "slot_mapping" @@ -30738,12 +30707,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1048576 + "state": "kv.16" }, { - "state": "kv", - "offset": 1048832 + "state": "kv.16", + "offset": 256 }, { "buf": "block_table" @@ -30991,12 +30959,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1114112 + "state": "kv.17" }, { - "state": "kv", - "offset": 1114368 + "state": "kv.17", + "offset": 256 }, { "buf": "slot_mapping" @@ -31022,12 +30989,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1114112 + "state": "kv.17" }, { - "state": "kv", - "offset": 1114368 + "state": "kv.17", + "offset": 256 }, { "buf": "block_table" @@ -31275,12 +31241,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1179648 + "state": "kv.18" }, { - "state": "kv", - "offset": 1179904 + "state": "kv.18", + "offset": 256 }, { "buf": "slot_mapping" @@ -31306,12 +31271,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1179648 + "state": "kv.18" }, { - "state": "kv", - "offset": 1179904 + "state": "kv.18", + "offset": 256 }, { "buf": "block_table" @@ -31559,12 +31523,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1245184 + "state": "kv.19" }, { - "state": "kv", - "offset": 1245440 + "state": "kv.19", + "offset": 256 }, { "buf": "slot_mapping" @@ -31590,12 +31553,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1245184 + "state": "kv.19" }, { - "state": "kv", - "offset": 1245440 + "state": "kv.19", + "offset": 256 }, { "buf": "block_table" @@ -31843,12 +31805,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1310720 + "state": "kv.20" }, { - "state": "kv", - "offset": 1310976 + "state": "kv.20", + "offset": 256 }, { "buf": "slot_mapping" @@ -31874,12 +31835,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1310720 + "state": "kv.20" }, { - "state": "kv", - "offset": 1310976 + "state": "kv.20", + "offset": 256 }, { "buf": "block_table" @@ -32127,12 +32087,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1376256 + "state": "kv.21" }, { - "state": "kv", - "offset": 1376512 + "state": "kv.21", + "offset": 256 }, { "buf": "slot_mapping" @@ -32158,12 +32117,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1376256 + "state": "kv.21" }, { - "state": "kv", - "offset": 1376512 + "state": "kv.21", + "offset": 256 }, { "buf": "block_table" @@ -32411,12 +32369,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1441792 + "state": "kv.22" }, { - "state": "kv", - "offset": 1442048 + "state": "kv.22", + "offset": 256 }, { "buf": "slot_mapping" @@ -32442,12 +32399,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1441792 + "state": "kv.22" }, { - "state": "kv", - "offset": 1442048 + "state": "kv.22", + "offset": 256 }, { "buf": "block_table" @@ -32695,12 +32651,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1507328 + "state": "kv.23" }, { - "state": "kv", - "offset": 1507584 + "state": "kv.23", + "offset": 256 }, { "buf": "slot_mapping" @@ -32726,12 +32681,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1507328 + "state": "kv.23" }, { - "state": "kv", - "offset": 1507584 + "state": "kv.23", + "offset": 256 }, { "buf": "block_table" @@ -32979,12 +32933,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1572864 + "state": "kv.24" }, { - "state": "kv", - "offset": 1573120 + "state": "kv.24", + "offset": 256 }, { "buf": "slot_mapping" @@ -33010,12 +32963,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1572864 + "state": "kv.24" }, { - "state": "kv", - "offset": 1573120 + "state": "kv.24", + "offset": 256 }, { "buf": "block_table" @@ -33263,12 +33215,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1638400 + "state": "kv.25" }, { - "state": "kv", - "offset": 1638656 + "state": "kv.25", + "offset": 256 }, { "buf": "slot_mapping" @@ -33294,12 +33245,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1638400 + "state": "kv.25" }, { - "state": "kv", - "offset": 1638656 + "state": "kv.25", + "offset": 256 }, { "buf": "block_table" @@ -33547,12 +33497,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1703936 + "state": "kv.26" }, { - "state": "kv", - "offset": 1704192 + "state": "kv.26", + "offset": 256 }, { "buf": "slot_mapping" @@ -33578,12 +33527,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1703936 + "state": "kv.26" }, { - "state": "kv", - "offset": 1704192 + "state": "kv.26", + "offset": 256 }, { "buf": "block_table" @@ -33831,12 +33779,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1769472 + "state": "kv.27" }, { - "state": "kv", - "offset": 1769728 + "state": "kv.27", + "offset": 256 }, { "buf": "slot_mapping" @@ -33862,12 +33809,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1769472 + "state": "kv.27" }, { - "state": "kv", - "offset": 1769728 + "state": "kv.27", + "offset": 256 }, { "buf": "block_table" @@ -34115,12 +34061,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1835008 + "state": "kv.28" }, { - "state": "kv", - "offset": 1835264 + "state": "kv.28", + "offset": 256 }, { "buf": "slot_mapping" @@ -34146,12 +34091,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1835008 + "state": "kv.28" }, { - "state": "kv", - "offset": 1835264 + "state": "kv.28", + "offset": 256 }, { "buf": "block_table" @@ -34399,12 +34343,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1900544 + "state": "kv.29" }, { - "state": "kv", - "offset": 1900800 + "state": "kv.29", + "offset": 256 }, { "buf": "slot_mapping" @@ -34430,12 +34373,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1900544 + "state": "kv.29" }, { - "state": "kv", - "offset": 1900800 + "state": "kv.29", + "offset": 256 }, { "buf": "block_table" @@ -34683,12 +34625,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1966080 + "state": "kv.30" }, { - "state": "kv", - "offset": 1966336 + "state": "kv.30", + "offset": 256 }, { "buf": "slot_mapping" @@ -34714,12 +34655,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1966080 + "state": "kv.30" }, { - "state": "kv", - "offset": 1966336 + "state": "kv.30", + "offset": 256 }, { "buf": "block_table" @@ -34967,12 +34907,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 2031616 + "state": "kv.31" }, { - "state": "kv", - "offset": 2031872 + "state": "kv.31", + "offset": 256 }, { "buf": "slot_mapping" @@ -34998,12 +34937,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 2031616 + "state": "kv.31" }, { - "state": "kv", - "offset": 2031872 + "state": "kv.31", + "offset": 256 }, { "buf": "block_table" @@ -35251,12 +35189,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 2097152 + "state": "kv.32" }, { - "state": "kv", - "offset": 2097408 + "state": "kv.32", + "offset": 256 }, { "buf": "slot_mapping" @@ -35282,12 +35219,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 2097152 + "state": "kv.32" }, { - "state": "kv", - "offset": 2097408 + "state": "kv.32", + "offset": 256 }, { "buf": "block_table" @@ -35535,12 +35471,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 2162688 + "state": "kv.33" }, { - "state": "kv", - "offset": 2162944 + "state": "kv.33", + "offset": 256 }, { "buf": "slot_mapping" @@ -35566,12 +35501,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 2162688 + "state": "kv.33" }, { - "state": "kv", - "offset": 2162944 + "state": "kv.33", + "offset": 256 }, { "buf": "block_table" @@ -35819,12 +35753,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 2228224 + "state": "kv.34" }, { - "state": "kv", - "offset": 2228480 + "state": "kv.34", + "offset": 256 }, { "buf": "slot_mapping" @@ -35850,12 +35783,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 2228224 + "state": "kv.34" }, { - "state": "kv", - "offset": 2228480 + "state": "kv.34", + "offset": 256 }, { "buf": "block_table" @@ -36103,12 +36035,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 2293760 + "state": "kv.35" }, { - "state": "kv", - "offset": 2294016 + "state": "kv.35", + "offset": 256 }, { "buf": "slot_mapping" @@ -36134,12 +36065,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 2293760 + "state": "kv.35" }, { - "state": "kv", - "offset": 2294016 + "state": "kv.35", + "offset": 256 }, { "buf": "block_table" diff --git a/examples/qwen3-4b.json b/examples/qwen3-4b.json index 21931136..a58ab33c 100644 --- a/examples/qwen3-4b.json +++ b/examples/qwen3-4b.json @@ -17,7 +17,42 @@ "KV_SCALE_COUNT": 72, "MAX_TOKENS": 2048, "MAX_SEQS": 256, - "KV_BYTES_PER_TOKEN": 147456, + "KV.0_BYTES_PER_TOKEN": 4096, + "KV.1_BYTES_PER_TOKEN": 4096, + "KV.2_BYTES_PER_TOKEN": 4096, + "KV.3_BYTES_PER_TOKEN": 4096, + "KV.4_BYTES_PER_TOKEN": 4096, + "KV.5_BYTES_PER_TOKEN": 4096, + "KV.6_BYTES_PER_TOKEN": 4096, + "KV.7_BYTES_PER_TOKEN": 4096, + "KV.8_BYTES_PER_TOKEN": 4096, + "KV.9_BYTES_PER_TOKEN": 4096, + "KV.10_BYTES_PER_TOKEN": 4096, + "KV.11_BYTES_PER_TOKEN": 4096, + "KV.12_BYTES_PER_TOKEN": 4096, + "KV.13_BYTES_PER_TOKEN": 4096, + "KV.14_BYTES_PER_TOKEN": 4096, + "KV.15_BYTES_PER_TOKEN": 4096, + "KV.16_BYTES_PER_TOKEN": 4096, + "KV.17_BYTES_PER_TOKEN": 4096, + "KV.18_BYTES_PER_TOKEN": 4096, + "KV.19_BYTES_PER_TOKEN": 4096, + "KV.20_BYTES_PER_TOKEN": 4096, + "KV.21_BYTES_PER_TOKEN": 4096, + "KV.22_BYTES_PER_TOKEN": 4096, + "KV.23_BYTES_PER_TOKEN": 4096, + "KV.24_BYTES_PER_TOKEN": 4096, + "KV.25_BYTES_PER_TOKEN": 4096, + "KV.26_BYTES_PER_TOKEN": 4096, + "KV.27_BYTES_PER_TOKEN": 4096, + "KV.28_BYTES_PER_TOKEN": 4096, + "KV.29_BYTES_PER_TOKEN": 4096, + "KV.30_BYTES_PER_TOKEN": 4096, + "KV.31_BYTES_PER_TOKEN": 4096, + "KV.32_BYTES_PER_TOKEN": 4096, + "KV.33_BYTES_PER_TOKEN": 4096, + "KV.34_BYTES_PER_TOKEN": 4096, + "KV.35_BYTES_PER_TOKEN": 4096, "RMS_NORM_EPS": 9.999999974752427e-07, "NUM_HEADS": 32, "NUM_KV_HEADS": 8, @@ -34,8 +69,113 @@ } }, "states": { - "kv": { - "bytes_per_token": "KV_BYTES_PER_TOKEN" + "kv.0": { + "bytes_per_token": "KV.0_BYTES_PER_TOKEN" + }, + "kv.1": { + "bytes_per_token": "KV.1_BYTES_PER_TOKEN" + }, + "kv.2": { + "bytes_per_token": "KV.2_BYTES_PER_TOKEN" + }, + "kv.3": { + "bytes_per_token": "KV.3_BYTES_PER_TOKEN" + }, + "kv.4": { + "bytes_per_token": "KV.4_BYTES_PER_TOKEN" + }, + "kv.5": { + "bytes_per_token": "KV.5_BYTES_PER_TOKEN" + }, + "kv.6": { + "bytes_per_token": "KV.6_BYTES_PER_TOKEN" + }, + "kv.7": { + "bytes_per_token": "KV.7_BYTES_PER_TOKEN" + }, + "kv.8": { + "bytes_per_token": "KV.8_BYTES_PER_TOKEN" + }, + "kv.9": { + "bytes_per_token": "KV.9_BYTES_PER_TOKEN" + }, + "kv.10": { + "bytes_per_token": "KV.10_BYTES_PER_TOKEN" + }, + "kv.11": { + "bytes_per_token": "KV.11_BYTES_PER_TOKEN" + }, + "kv.12": { + "bytes_per_token": "KV.12_BYTES_PER_TOKEN" + }, + "kv.13": { + "bytes_per_token": "KV.13_BYTES_PER_TOKEN" + }, + "kv.14": { + "bytes_per_token": "KV.14_BYTES_PER_TOKEN" + }, + "kv.15": { + "bytes_per_token": "KV.15_BYTES_PER_TOKEN" + }, + "kv.16": { + "bytes_per_token": "KV.16_BYTES_PER_TOKEN" + }, + "kv.17": { + "bytes_per_token": "KV.17_BYTES_PER_TOKEN" + }, + "kv.18": { + "bytes_per_token": "KV.18_BYTES_PER_TOKEN" + }, + "kv.19": { + "bytes_per_token": "KV.19_BYTES_PER_TOKEN" + }, + "kv.20": { + "bytes_per_token": "KV.20_BYTES_PER_TOKEN" + }, + "kv.21": { + "bytes_per_token": "KV.21_BYTES_PER_TOKEN" + }, + "kv.22": { + "bytes_per_token": "KV.22_BYTES_PER_TOKEN" + }, + "kv.23": { + "bytes_per_token": "KV.23_BYTES_PER_TOKEN" + }, + "kv.24": { + "bytes_per_token": "KV.24_BYTES_PER_TOKEN" + }, + "kv.25": { + "bytes_per_token": "KV.25_BYTES_PER_TOKEN" + }, + "kv.26": { + "bytes_per_token": "KV.26_BYTES_PER_TOKEN" + }, + "kv.27": { + "bytes_per_token": "KV.27_BYTES_PER_TOKEN" + }, + "kv.28": { + "bytes_per_token": "KV.28_BYTES_PER_TOKEN" + }, + "kv.29": { + "bytes_per_token": "KV.29_BYTES_PER_TOKEN" + }, + "kv.30": { + "bytes_per_token": "KV.30_BYTES_PER_TOKEN" + }, + "kv.31": { + "bytes_per_token": "KV.31_BYTES_PER_TOKEN" + }, + "kv.32": { + "bytes_per_token": "KV.32_BYTES_PER_TOKEN" + }, + "kv.33": { + "bytes_per_token": "KV.33_BYTES_PER_TOKEN" + }, + "kv.34": { + "bytes_per_token": "KV.34_BYTES_PER_TOKEN" + }, + "kv.35": { + "bytes_per_token": "KV.35_BYTES_PER_TOKEN" } }, "buffers": { @@ -69,7 +209,7 @@ "kind": "input", "fill": "slot", "domain": { - "index_into": "kv" + "index_into": "kv.0" } }, "block_table": { @@ -80,7 +220,7 @@ ], "kind": "input", "domain": { - "index_into": "kv", + "index_into": "kv.0", "stride": "KV_BLOCK_SIZE" } }, @@ -4186,12 +4326,12 @@ "sha256": "73748b54059552f5983322f7dedc36ed349b38ad6fb9318301bb4965b1fe49aa" }, "argmax": { - "source": "argmax.cubin", - "sha256": "25374575bd35dbd9fe07f90982d0da0c2d4a72cc6ac88df5ec92cdb62a3beb16" + "source": "hf:Pegainfer/kern-kernels/blobs/ebe4ba79544ddfac43ee2dcf6a7f5299aba1124e176c201133d4ef9400e3dc25", + "sha256": "ebe4ba79544ddfac43ee2dcf6a7f5299aba1124e176c201133d4ef9400e3dc25" }, "embedding": { - "source": "embedding.cubin", - "sha256": "1c3dd3e591d872aed259d7ef59d44649061f77a4247a5df864a7fbf0ccbde92f" + "source": "hf:Pegainfer/kern-kernels/blobs/868e93ae6e46e36a11c56af1fae49aeff9fcdfdc79f64ca99475a309e721e9e7", + "sha256": "868e93ae6e46e36a11c56af1fae49aeff9fcdfdc79f64ca99475a309e721e9e7" }, "reduce_segments": { "source": "reduce_segments.cubin", @@ -4218,7 +4358,7 @@ "sha256": "116de812ac5a3a186f76fa239d4c16abbed6b8b0af06d16e1baec0aefe8fe8fa" }, "weight_prep": { - "source": "weight_prep.cubin", + "source": "hf:Pegainfer/kern-kernels/blobs/040f51b01c421b9c69a2090034ff1bc4741e40c02f5ea52bd04a3e62e5c1af4d", "sha256": "040f51b01c421b9c69a2090034ff1bc4741e40c02f5ea52bd04a3e62e5c1af4d" } }, @@ -4621,7 +4761,7 @@ "i64": "QKV_DIM" }, { - "i64": 1179648 + "i64": 32768 }, { "i64": 256 @@ -4791,7 +4931,7 @@ "param": 8 }, { - "i64": 1179648 + "i64": 32768 }, { "i64": 2048 @@ -4800,7 +4940,7 @@ "i64": 256 }, { - "i64": 1179648 + "i64": 32768 }, { "i64": 2048 @@ -5022,7 +5162,7 @@ "param": 8 }, { - "i64": 1179648 + "i64": 32768 }, { "i64": 2048 @@ -5031,7 +5171,7 @@ "i64": 256 }, { - "i64": 1179648 + "i64": 32768 }, { "i64": 2048 @@ -5182,7 +5322,7 @@ "param": 8 }, { - "i64": 1179648 + "i64": 32768 }, { "i64": 2048 @@ -5191,7 +5331,7 @@ "i64": 256 }, { - "i64": 1179648 + "i64": 32768 }, { "i64": 2048 @@ -5715,10 +5855,10 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv" + "state": "kv.0" }, { - "state": "kv", + "state": "kv.0", "offset": 256 }, { @@ -5744,10 +5884,10 @@ "buf": "q_n" }, { - "state": "kv" + "state": "kv.0" }, { - "state": "kv", + "state": "kv.0", "offset": 256 }, { @@ -5995,12 +6135,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 65536 + "state": "kv.1" }, { - "state": "kv", - "offset": 65792 + "state": "kv.1", + "offset": 256 }, { "buf": "slot_mapping" @@ -6026,12 +6165,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 65536 + "state": "kv.1" }, { - "state": "kv", - "offset": 65792 + "state": "kv.1", + "offset": 256 }, { "buf": "block_table" @@ -6279,12 +6417,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 131072 + "state": "kv.2" }, { - "state": "kv", - "offset": 131328 + "state": "kv.2", + "offset": 256 }, { "buf": "slot_mapping" @@ -6310,12 +6447,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 131072 + "state": "kv.2" }, { - "state": "kv", - "offset": 131328 + "state": "kv.2", + "offset": 256 }, { "buf": "block_table" @@ -6563,12 +6699,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 196608 + "state": "kv.3" }, { - "state": "kv", - "offset": 196864 + "state": "kv.3", + "offset": 256 }, { "buf": "slot_mapping" @@ -6594,12 +6729,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 196608 + "state": "kv.3" }, { - "state": "kv", - "offset": 196864 + "state": "kv.3", + "offset": 256 }, { "buf": "block_table" @@ -6847,12 +6981,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 262144 + "state": "kv.4" }, { - "state": "kv", - "offset": 262400 + "state": "kv.4", + "offset": 256 }, { "buf": "slot_mapping" @@ -6878,12 +7011,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 262144 + "state": "kv.4" }, { - "state": "kv", - "offset": 262400 + "state": "kv.4", + "offset": 256 }, { "buf": "block_table" @@ -7131,12 +7263,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 327680 + "state": "kv.5" }, { - "state": "kv", - "offset": 327936 + "state": "kv.5", + "offset": 256 }, { "buf": "slot_mapping" @@ -7162,12 +7293,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 327680 + "state": "kv.5" }, { - "state": "kv", - "offset": 327936 + "state": "kv.5", + "offset": 256 }, { "buf": "block_table" @@ -7415,12 +7545,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 393216 + "state": "kv.6" }, { - "state": "kv", - "offset": 393472 + "state": "kv.6", + "offset": 256 }, { "buf": "slot_mapping" @@ -7446,12 +7575,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 393216 + "state": "kv.6" }, { - "state": "kv", - "offset": 393472 + "state": "kv.6", + "offset": 256 }, { "buf": "block_table" @@ -7699,12 +7827,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 458752 + "state": "kv.7" }, { - "state": "kv", - "offset": 459008 + "state": "kv.7", + "offset": 256 }, { "buf": "slot_mapping" @@ -7730,12 +7857,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 458752 + "state": "kv.7" }, { - "state": "kv", - "offset": 459008 + "state": "kv.7", + "offset": 256 }, { "buf": "block_table" @@ -7983,12 +8109,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 524288 + "state": "kv.8" }, { - "state": "kv", - "offset": 524544 + "state": "kv.8", + "offset": 256 }, { "buf": "slot_mapping" @@ -8014,12 +8139,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 524288 + "state": "kv.8" }, { - "state": "kv", - "offset": 524544 + "state": "kv.8", + "offset": 256 }, { "buf": "block_table" @@ -8267,12 +8391,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 589824 + "state": "kv.9" }, { - "state": "kv", - "offset": 590080 + "state": "kv.9", + "offset": 256 }, { "buf": "slot_mapping" @@ -8298,12 +8421,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 589824 + "state": "kv.9" }, { - "state": "kv", - "offset": 590080 + "state": "kv.9", + "offset": 256 }, { "buf": "block_table" @@ -8551,12 +8673,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 655360 + "state": "kv.10" }, { - "state": "kv", - "offset": 655616 + "state": "kv.10", + "offset": 256 }, { "buf": "slot_mapping" @@ -8582,12 +8703,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 655360 + "state": "kv.10" }, { - "state": "kv", - "offset": 655616 + "state": "kv.10", + "offset": 256 }, { "buf": "block_table" @@ -8835,12 +8955,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 720896 + "state": "kv.11" }, { - "state": "kv", - "offset": 721152 + "state": "kv.11", + "offset": 256 }, { "buf": "slot_mapping" @@ -8866,12 +8985,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 720896 + "state": "kv.11" }, { - "state": "kv", - "offset": 721152 + "state": "kv.11", + "offset": 256 }, { "buf": "block_table" @@ -9119,12 +9237,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 786432 + "state": "kv.12" }, { - "state": "kv", - "offset": 786688 + "state": "kv.12", + "offset": 256 }, { "buf": "slot_mapping" @@ -9150,12 +9267,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 786432 + "state": "kv.12" }, { - "state": "kv", - "offset": 786688 + "state": "kv.12", + "offset": 256 }, { "buf": "block_table" @@ -9403,12 +9519,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 851968 + "state": "kv.13" }, { - "state": "kv", - "offset": 852224 + "state": "kv.13", + "offset": 256 }, { "buf": "slot_mapping" @@ -9434,12 +9549,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 851968 + "state": "kv.13" }, { - "state": "kv", - "offset": 852224 + "state": "kv.13", + "offset": 256 }, { "buf": "block_table" @@ -9687,12 +9801,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 917504 + "state": "kv.14" }, { - "state": "kv", - "offset": 917760 + "state": "kv.14", + "offset": 256 }, { "buf": "slot_mapping" @@ -9718,12 +9831,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 917504 + "state": "kv.14" }, { - "state": "kv", - "offset": 917760 + "state": "kv.14", + "offset": 256 }, { "buf": "block_table" @@ -9971,12 +10083,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 983040 + "state": "kv.15" }, { - "state": "kv", - "offset": 983296 + "state": "kv.15", + "offset": 256 }, { "buf": "slot_mapping" @@ -10002,12 +10113,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 983040 + "state": "kv.15" }, { - "state": "kv", - "offset": 983296 + "state": "kv.15", + "offset": 256 }, { "buf": "block_table" @@ -10255,12 +10365,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1048576 + "state": "kv.16" }, { - "state": "kv", - "offset": 1048832 + "state": "kv.16", + "offset": 256 }, { "buf": "slot_mapping" @@ -10286,12 +10395,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1048576 + "state": "kv.16" }, { - "state": "kv", - "offset": 1048832 + "state": "kv.16", + "offset": 256 }, { "buf": "block_table" @@ -10539,12 +10647,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1114112 + "state": "kv.17" }, { - "state": "kv", - "offset": 1114368 + "state": "kv.17", + "offset": 256 }, { "buf": "slot_mapping" @@ -10570,12 +10677,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1114112 + "state": "kv.17" }, { - "state": "kv", - "offset": 1114368 + "state": "kv.17", + "offset": 256 }, { "buf": "block_table" @@ -10823,12 +10929,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1179648 + "state": "kv.18" }, { - "state": "kv", - "offset": 1179904 + "state": "kv.18", + "offset": 256 }, { "buf": "slot_mapping" @@ -10854,12 +10959,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1179648 + "state": "kv.18" }, { - "state": "kv", - "offset": 1179904 + "state": "kv.18", + "offset": 256 }, { "buf": "block_table" @@ -11107,12 +11211,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1245184 + "state": "kv.19" }, { - "state": "kv", - "offset": 1245440 + "state": "kv.19", + "offset": 256 }, { "buf": "slot_mapping" @@ -11138,12 +11241,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1245184 + "state": "kv.19" }, { - "state": "kv", - "offset": 1245440 + "state": "kv.19", + "offset": 256 }, { "buf": "block_table" @@ -11391,12 +11493,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1310720 + "state": "kv.20" }, { - "state": "kv", - "offset": 1310976 + "state": "kv.20", + "offset": 256 }, { "buf": "slot_mapping" @@ -11422,12 +11523,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1310720 + "state": "kv.20" }, { - "state": "kv", - "offset": 1310976 + "state": "kv.20", + "offset": 256 }, { "buf": "block_table" @@ -11675,12 +11775,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1376256 + "state": "kv.21" }, { - "state": "kv", - "offset": 1376512 + "state": "kv.21", + "offset": 256 }, { "buf": "slot_mapping" @@ -11706,12 +11805,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1376256 + "state": "kv.21" }, { - "state": "kv", - "offset": 1376512 + "state": "kv.21", + "offset": 256 }, { "buf": "block_table" @@ -11959,12 +12057,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1441792 + "state": "kv.22" }, { - "state": "kv", - "offset": 1442048 + "state": "kv.22", + "offset": 256 }, { "buf": "slot_mapping" @@ -11990,12 +12087,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1441792 + "state": "kv.22" }, { - "state": "kv", - "offset": 1442048 + "state": "kv.22", + "offset": 256 }, { "buf": "block_table" @@ -12243,12 +12339,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1507328 + "state": "kv.23" }, { - "state": "kv", - "offset": 1507584 + "state": "kv.23", + "offset": 256 }, { "buf": "slot_mapping" @@ -12274,12 +12369,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1507328 + "state": "kv.23" }, { - "state": "kv", - "offset": 1507584 + "state": "kv.23", + "offset": 256 }, { "buf": "block_table" @@ -12527,12 +12621,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1572864 + "state": "kv.24" }, { - "state": "kv", - "offset": 1573120 + "state": "kv.24", + "offset": 256 }, { "buf": "slot_mapping" @@ -12558,12 +12651,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1572864 + "state": "kv.24" }, { - "state": "kv", - "offset": 1573120 + "state": "kv.24", + "offset": 256 }, { "buf": "block_table" @@ -12811,12 +12903,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1638400 + "state": "kv.25" }, { - "state": "kv", - "offset": 1638656 + "state": "kv.25", + "offset": 256 }, { "buf": "slot_mapping" @@ -12842,12 +12933,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1638400 + "state": "kv.25" }, { - "state": "kv", - "offset": 1638656 + "state": "kv.25", + "offset": 256 }, { "buf": "block_table" @@ -13095,12 +13185,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1703936 + "state": "kv.26" }, { - "state": "kv", - "offset": 1704192 + "state": "kv.26", + "offset": 256 }, { "buf": "slot_mapping" @@ -13126,12 +13215,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1703936 + "state": "kv.26" }, { - "state": "kv", - "offset": 1704192 + "state": "kv.26", + "offset": 256 }, { "buf": "block_table" @@ -13379,12 +13467,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1769472 + "state": "kv.27" }, { - "state": "kv", - "offset": 1769728 + "state": "kv.27", + "offset": 256 }, { "buf": "slot_mapping" @@ -13410,12 +13497,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1769472 + "state": "kv.27" }, { - "state": "kv", - "offset": 1769728 + "state": "kv.27", + "offset": 256 }, { "buf": "block_table" @@ -13663,12 +13749,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1835008 + "state": "kv.28" }, { - "state": "kv", - "offset": 1835264 + "state": "kv.28", + "offset": 256 }, { "buf": "slot_mapping" @@ -13694,12 +13779,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1835008 + "state": "kv.28" }, { - "state": "kv", - "offset": 1835264 + "state": "kv.28", + "offset": 256 }, { "buf": "block_table" @@ -13947,12 +14031,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1900544 + "state": "kv.29" }, { - "state": "kv", - "offset": 1900800 + "state": "kv.29", + "offset": 256 }, { "buf": "slot_mapping" @@ -13978,12 +14061,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1900544 + "state": "kv.29" }, { - "state": "kv", - "offset": 1900800 + "state": "kv.29", + "offset": 256 }, { "buf": "block_table" @@ -14231,12 +14313,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1966080 + "state": "kv.30" }, { - "state": "kv", - "offset": 1966336 + "state": "kv.30", + "offset": 256 }, { "buf": "slot_mapping" @@ -14262,12 +14343,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1966080 + "state": "kv.30" }, { - "state": "kv", - "offset": 1966336 + "state": "kv.30", + "offset": 256 }, { "buf": "block_table" @@ -14515,12 +14595,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 2031616 + "state": "kv.31" }, { - "state": "kv", - "offset": 2031872 + "state": "kv.31", + "offset": 256 }, { "buf": "slot_mapping" @@ -14546,12 +14625,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 2031616 + "state": "kv.31" }, { - "state": "kv", - "offset": 2031872 + "state": "kv.31", + "offset": 256 }, { "buf": "block_table" @@ -14799,12 +14877,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 2097152 + "state": "kv.32" }, { - "state": "kv", - "offset": 2097408 + "state": "kv.32", + "offset": 256 }, { "buf": "slot_mapping" @@ -14830,12 +14907,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 2097152 + "state": "kv.32" }, { - "state": "kv", - "offset": 2097408 + "state": "kv.32", + "offset": 256 }, { "buf": "block_table" @@ -15083,12 +15159,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 2162688 + "state": "kv.33" }, { - "state": "kv", - "offset": 2162944 + "state": "kv.33", + "offset": 256 }, { "buf": "slot_mapping" @@ -15114,12 +15189,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 2162688 + "state": "kv.33" }, { - "state": "kv", - "offset": 2162944 + "state": "kv.33", + "offset": 256 }, { "buf": "block_table" @@ -15367,12 +15441,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 2228224 + "state": "kv.34" }, { - "state": "kv", - "offset": 2228480 + "state": "kv.34", + "offset": 256 }, { "buf": "slot_mapping" @@ -15398,12 +15471,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 2228224 + "state": "kv.34" }, { - "state": "kv", - "offset": 2228480 + "state": "kv.34", + "offset": 256 }, { "buf": "block_table" @@ -15651,12 +15723,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 2293760 + "state": "kv.35" }, { - "state": "kv", - "offset": 2294016 + "state": "kv.35", + "offset": 256 }, { "buf": "slot_mapping" @@ -15682,12 +15753,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 2293760 + "state": "kv.35" }, { - "state": "kv", - "offset": 2294016 + "state": "kv.35", + "offset": 256 }, { "buf": "block_table" @@ -15962,10 +16032,10 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv" + "state": "kv.0" }, { - "state": "kv", + "state": "kv.0", "offset": 256 }, { @@ -15991,10 +16061,10 @@ "buf": "q_n" }, { - "state": "kv" + "state": "kv.0" }, { - "state": "kv", + "state": "kv.0", "offset": 256 }, { @@ -16239,12 +16309,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 65536 + "state": "kv.1" }, { - "state": "kv", - "offset": 65792 + "state": "kv.1", + "offset": 256 }, { "buf": "slot_mapping" @@ -16270,12 +16339,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 65536 + "state": "kv.1" }, { - "state": "kv", - "offset": 65792 + "state": "kv.1", + "offset": 256 }, { "buf": "block_table" @@ -16520,12 +16588,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 131072 + "state": "kv.2" }, { - "state": "kv", - "offset": 131328 + "state": "kv.2", + "offset": 256 }, { "buf": "slot_mapping" @@ -16551,12 +16618,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 131072 + "state": "kv.2" }, { - "state": "kv", - "offset": 131328 + "state": "kv.2", + "offset": 256 }, { "buf": "block_table" @@ -16801,12 +16867,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 196608 + "state": "kv.3" }, { - "state": "kv", - "offset": 196864 + "state": "kv.3", + "offset": 256 }, { "buf": "slot_mapping" @@ -16832,12 +16897,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 196608 + "state": "kv.3" }, { - "state": "kv", - "offset": 196864 + "state": "kv.3", + "offset": 256 }, { "buf": "block_table" @@ -17082,12 +17146,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 262144 + "state": "kv.4" }, { - "state": "kv", - "offset": 262400 + "state": "kv.4", + "offset": 256 }, { "buf": "slot_mapping" @@ -17113,12 +17176,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 262144 + "state": "kv.4" }, { - "state": "kv", - "offset": 262400 + "state": "kv.4", + "offset": 256 }, { "buf": "block_table" @@ -17363,12 +17425,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 327680 + "state": "kv.5" }, { - "state": "kv", - "offset": 327936 + "state": "kv.5", + "offset": 256 }, { "buf": "slot_mapping" @@ -17394,12 +17455,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 327680 + "state": "kv.5" }, { - "state": "kv", - "offset": 327936 + "state": "kv.5", + "offset": 256 }, { "buf": "block_table" @@ -17644,12 +17704,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 393216 + "state": "kv.6" }, { - "state": "kv", - "offset": 393472 + "state": "kv.6", + "offset": 256 }, { "buf": "slot_mapping" @@ -17675,12 +17734,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 393216 + "state": "kv.6" }, { - "state": "kv", - "offset": 393472 + "state": "kv.6", + "offset": 256 }, { "buf": "block_table" @@ -17925,12 +17983,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 458752 + "state": "kv.7" }, { - "state": "kv", - "offset": 459008 + "state": "kv.7", + "offset": 256 }, { "buf": "slot_mapping" @@ -17956,12 +18013,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 458752 + "state": "kv.7" }, { - "state": "kv", - "offset": 459008 + "state": "kv.7", + "offset": 256 }, { "buf": "block_table" @@ -18206,12 +18262,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 524288 + "state": "kv.8" }, { - "state": "kv", - "offset": 524544 + "state": "kv.8", + "offset": 256 }, { "buf": "slot_mapping" @@ -18237,12 +18292,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 524288 + "state": "kv.8" }, { - "state": "kv", - "offset": 524544 + "state": "kv.8", + "offset": 256 }, { "buf": "block_table" @@ -18487,12 +18541,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 589824 + "state": "kv.9" }, { - "state": "kv", - "offset": 590080 + "state": "kv.9", + "offset": 256 }, { "buf": "slot_mapping" @@ -18518,12 +18571,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 589824 + "state": "kv.9" }, { - "state": "kv", - "offset": 590080 + "state": "kv.9", + "offset": 256 }, { "buf": "block_table" @@ -18768,12 +18820,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 655360 + "state": "kv.10" }, { - "state": "kv", - "offset": 655616 + "state": "kv.10", + "offset": 256 }, { "buf": "slot_mapping" @@ -18799,12 +18850,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 655360 + "state": "kv.10" }, { - "state": "kv", - "offset": 655616 + "state": "kv.10", + "offset": 256 }, { "buf": "block_table" @@ -19049,12 +19099,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 720896 + "state": "kv.11" }, { - "state": "kv", - "offset": 721152 + "state": "kv.11", + "offset": 256 }, { "buf": "slot_mapping" @@ -19080,12 +19129,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 720896 + "state": "kv.11" }, { - "state": "kv", - "offset": 721152 + "state": "kv.11", + "offset": 256 }, { "buf": "block_table" @@ -19330,12 +19378,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 786432 + "state": "kv.12" }, { - "state": "kv", - "offset": 786688 + "state": "kv.12", + "offset": 256 }, { "buf": "slot_mapping" @@ -19361,12 +19408,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 786432 + "state": "kv.12" }, { - "state": "kv", - "offset": 786688 + "state": "kv.12", + "offset": 256 }, { "buf": "block_table" @@ -19611,12 +19657,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 851968 + "state": "kv.13" }, { - "state": "kv", - "offset": 852224 + "state": "kv.13", + "offset": 256 }, { "buf": "slot_mapping" @@ -19642,12 +19687,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 851968 + "state": "kv.13" }, { - "state": "kv", - "offset": 852224 + "state": "kv.13", + "offset": 256 }, { "buf": "block_table" @@ -19892,12 +19936,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 917504 + "state": "kv.14" }, { - "state": "kv", - "offset": 917760 + "state": "kv.14", + "offset": 256 }, { "buf": "slot_mapping" @@ -19923,12 +19966,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 917504 + "state": "kv.14" }, { - "state": "kv", - "offset": 917760 + "state": "kv.14", + "offset": 256 }, { "buf": "block_table" @@ -20173,12 +20215,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 983040 + "state": "kv.15" }, { - "state": "kv", - "offset": 983296 + "state": "kv.15", + "offset": 256 }, { "buf": "slot_mapping" @@ -20204,12 +20245,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 983040 + "state": "kv.15" }, { - "state": "kv", - "offset": 983296 + "state": "kv.15", + "offset": 256 }, { "buf": "block_table" @@ -20454,12 +20494,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1048576 + "state": "kv.16" }, { - "state": "kv", - "offset": 1048832 + "state": "kv.16", + "offset": 256 }, { "buf": "slot_mapping" @@ -20485,12 +20524,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1048576 + "state": "kv.16" }, { - "state": "kv", - "offset": 1048832 + "state": "kv.16", + "offset": 256 }, { "buf": "block_table" @@ -20735,12 +20773,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1114112 + "state": "kv.17" }, { - "state": "kv", - "offset": 1114368 + "state": "kv.17", + "offset": 256 }, { "buf": "slot_mapping" @@ -20766,12 +20803,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1114112 + "state": "kv.17" }, { - "state": "kv", - "offset": 1114368 + "state": "kv.17", + "offset": 256 }, { "buf": "block_table" @@ -21016,12 +21052,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1179648 + "state": "kv.18" }, { - "state": "kv", - "offset": 1179904 + "state": "kv.18", + "offset": 256 }, { "buf": "slot_mapping" @@ -21047,12 +21082,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1179648 + "state": "kv.18" }, { - "state": "kv", - "offset": 1179904 + "state": "kv.18", + "offset": 256 }, { "buf": "block_table" @@ -21297,12 +21331,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1245184 + "state": "kv.19" }, { - "state": "kv", - "offset": 1245440 + "state": "kv.19", + "offset": 256 }, { "buf": "slot_mapping" @@ -21328,12 +21361,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1245184 + "state": "kv.19" }, { - "state": "kv", - "offset": 1245440 + "state": "kv.19", + "offset": 256 }, { "buf": "block_table" @@ -21578,12 +21610,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1310720 + "state": "kv.20" }, { - "state": "kv", - "offset": 1310976 + "state": "kv.20", + "offset": 256 }, { "buf": "slot_mapping" @@ -21609,12 +21640,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1310720 + "state": "kv.20" }, { - "state": "kv", - "offset": 1310976 + "state": "kv.20", + "offset": 256 }, { "buf": "block_table" @@ -21859,12 +21889,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1376256 + "state": "kv.21" }, { - "state": "kv", - "offset": 1376512 + "state": "kv.21", + "offset": 256 }, { "buf": "slot_mapping" @@ -21890,12 +21919,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1376256 + "state": "kv.21" }, { - "state": "kv", - "offset": 1376512 + "state": "kv.21", + "offset": 256 }, { "buf": "block_table" @@ -22140,12 +22168,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1441792 + "state": "kv.22" }, { - "state": "kv", - "offset": 1442048 + "state": "kv.22", + "offset": 256 }, { "buf": "slot_mapping" @@ -22171,12 +22198,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1441792 + "state": "kv.22" }, { - "state": "kv", - "offset": 1442048 + "state": "kv.22", + "offset": 256 }, { "buf": "block_table" @@ -22421,12 +22447,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1507328 + "state": "kv.23" }, { - "state": "kv", - "offset": 1507584 + "state": "kv.23", + "offset": 256 }, { "buf": "slot_mapping" @@ -22452,12 +22477,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1507328 + "state": "kv.23" }, { - "state": "kv", - "offset": 1507584 + "state": "kv.23", + "offset": 256 }, { "buf": "block_table" @@ -22702,12 +22726,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1572864 + "state": "kv.24" }, { - "state": "kv", - "offset": 1573120 + "state": "kv.24", + "offset": 256 }, { "buf": "slot_mapping" @@ -22733,12 +22756,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1572864 + "state": "kv.24" }, { - "state": "kv", - "offset": 1573120 + "state": "kv.24", + "offset": 256 }, { "buf": "block_table" @@ -22983,12 +23005,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1638400 + "state": "kv.25" }, { - "state": "kv", - "offset": 1638656 + "state": "kv.25", + "offset": 256 }, { "buf": "slot_mapping" @@ -23014,12 +23035,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1638400 + "state": "kv.25" }, { - "state": "kv", - "offset": 1638656 + "state": "kv.25", + "offset": 256 }, { "buf": "block_table" @@ -23264,12 +23284,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1703936 + "state": "kv.26" }, { - "state": "kv", - "offset": 1704192 + "state": "kv.26", + "offset": 256 }, { "buf": "slot_mapping" @@ -23295,12 +23314,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1703936 + "state": "kv.26" }, { - "state": "kv", - "offset": 1704192 + "state": "kv.26", + "offset": 256 }, { "buf": "block_table" @@ -23545,12 +23563,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1769472 + "state": "kv.27" }, { - "state": "kv", - "offset": 1769728 + "state": "kv.27", + "offset": 256 }, { "buf": "slot_mapping" @@ -23576,12 +23593,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1769472 + "state": "kv.27" }, { - "state": "kv", - "offset": 1769728 + "state": "kv.27", + "offset": 256 }, { "buf": "block_table" @@ -23826,12 +23842,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1835008 + "state": "kv.28" }, { - "state": "kv", - "offset": 1835264 + "state": "kv.28", + "offset": 256 }, { "buf": "slot_mapping" @@ -23857,12 +23872,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1835008 + "state": "kv.28" }, { - "state": "kv", - "offset": 1835264 + "state": "kv.28", + "offset": 256 }, { "buf": "block_table" @@ -24107,12 +24121,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1900544 + "state": "kv.29" }, { - "state": "kv", - "offset": 1900800 + "state": "kv.29", + "offset": 256 }, { "buf": "slot_mapping" @@ -24138,12 +24151,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1900544 + "state": "kv.29" }, { - "state": "kv", - "offset": 1900800 + "state": "kv.29", + "offset": 256 }, { "buf": "block_table" @@ -24388,12 +24400,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1966080 + "state": "kv.30" }, { - "state": "kv", - "offset": 1966336 + "state": "kv.30", + "offset": 256 }, { "buf": "slot_mapping" @@ -24419,12 +24430,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1966080 + "state": "kv.30" }, { - "state": "kv", - "offset": 1966336 + "state": "kv.30", + "offset": 256 }, { "buf": "block_table" @@ -24669,12 +24679,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 2031616 + "state": "kv.31" }, { - "state": "kv", - "offset": 2031872 + "state": "kv.31", + "offset": 256 }, { "buf": "slot_mapping" @@ -24700,12 +24709,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 2031616 + "state": "kv.31" }, { - "state": "kv", - "offset": 2031872 + "state": "kv.31", + "offset": 256 }, { "buf": "block_table" @@ -24950,12 +24958,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 2097152 + "state": "kv.32" }, { - "state": "kv", - "offset": 2097408 + "state": "kv.32", + "offset": 256 }, { "buf": "slot_mapping" @@ -24981,12 +24988,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 2097152 + "state": "kv.32" }, { - "state": "kv", - "offset": 2097408 + "state": "kv.32", + "offset": 256 }, { "buf": "block_table" @@ -25231,12 +25237,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 2162688 + "state": "kv.33" }, { - "state": "kv", - "offset": 2162944 + "state": "kv.33", + "offset": 256 }, { "buf": "slot_mapping" @@ -25262,12 +25267,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 2162688 + "state": "kv.33" }, { - "state": "kv", - "offset": 2162944 + "state": "kv.33", + "offset": 256 }, { "buf": "block_table" @@ -25512,12 +25516,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 2228224 + "state": "kv.34" }, { - "state": "kv", - "offset": 2228480 + "state": "kv.34", + "offset": 256 }, { "buf": "slot_mapping" @@ -25543,12 +25546,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 2228224 + "state": "kv.34" }, { - "state": "kv", - "offset": 2228480 + "state": "kv.34", + "offset": 256 }, { "buf": "block_table" @@ -25793,12 +25795,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 2293760 + "state": "kv.35" }, { - "state": "kv", - "offset": 2294016 + "state": "kv.35", + "offset": 256 }, { "buf": "slot_mapping" @@ -25824,12 +25825,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 2293760 + "state": "kv.35" }, { - "state": "kv", - "offset": 2294016 + "state": "kv.35", + "offset": 256 }, { "buf": "block_table" @@ -26155,10 +26155,10 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv" + "state": "kv.0" }, { - "state": "kv", + "state": "kv.0", "offset": 256 }, { @@ -26184,10 +26184,10 @@ "buf": "q_n" }, { - "state": "kv" + "state": "kv.0" }, { - "state": "kv", + "state": "kv.0", "offset": 256 }, { @@ -26435,12 +26435,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 65536 + "state": "kv.1" }, { - "state": "kv", - "offset": 65792 + "state": "kv.1", + "offset": 256 }, { "buf": "slot_mapping" @@ -26466,12 +26465,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 65536 + "state": "kv.1" }, { - "state": "kv", - "offset": 65792 + "state": "kv.1", + "offset": 256 }, { "buf": "block_table" @@ -26719,12 +26717,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 131072 + "state": "kv.2" }, { - "state": "kv", - "offset": 131328 + "state": "kv.2", + "offset": 256 }, { "buf": "slot_mapping" @@ -26750,12 +26747,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 131072 + "state": "kv.2" }, { - "state": "kv", - "offset": 131328 + "state": "kv.2", + "offset": 256 }, { "buf": "block_table" @@ -27003,12 +26999,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 196608 + "state": "kv.3" }, { - "state": "kv", - "offset": 196864 + "state": "kv.3", + "offset": 256 }, { "buf": "slot_mapping" @@ -27034,12 +27029,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 196608 + "state": "kv.3" }, { - "state": "kv", - "offset": 196864 + "state": "kv.3", + "offset": 256 }, { "buf": "block_table" @@ -27287,12 +27281,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 262144 + "state": "kv.4" }, { - "state": "kv", - "offset": 262400 + "state": "kv.4", + "offset": 256 }, { "buf": "slot_mapping" @@ -27318,12 +27311,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 262144 + "state": "kv.4" }, { - "state": "kv", - "offset": 262400 + "state": "kv.4", + "offset": 256 }, { "buf": "block_table" @@ -27571,12 +27563,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 327680 + "state": "kv.5" }, { - "state": "kv", - "offset": 327936 + "state": "kv.5", + "offset": 256 }, { "buf": "slot_mapping" @@ -27602,12 +27593,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 327680 + "state": "kv.5" }, { - "state": "kv", - "offset": 327936 + "state": "kv.5", + "offset": 256 }, { "buf": "block_table" @@ -27855,12 +27845,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 393216 + "state": "kv.6" }, { - "state": "kv", - "offset": 393472 + "state": "kv.6", + "offset": 256 }, { "buf": "slot_mapping" @@ -27886,12 +27875,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 393216 + "state": "kv.6" }, { - "state": "kv", - "offset": 393472 + "state": "kv.6", + "offset": 256 }, { "buf": "block_table" @@ -28139,12 +28127,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 458752 + "state": "kv.7" }, { - "state": "kv", - "offset": 459008 + "state": "kv.7", + "offset": 256 }, { "buf": "slot_mapping" @@ -28170,12 +28157,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 458752 + "state": "kv.7" }, { - "state": "kv", - "offset": 459008 + "state": "kv.7", + "offset": 256 }, { "buf": "block_table" @@ -28423,12 +28409,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 524288 + "state": "kv.8" }, { - "state": "kv", - "offset": 524544 + "state": "kv.8", + "offset": 256 }, { "buf": "slot_mapping" @@ -28454,12 +28439,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 524288 + "state": "kv.8" }, { - "state": "kv", - "offset": 524544 + "state": "kv.8", + "offset": 256 }, { "buf": "block_table" @@ -28707,12 +28691,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 589824 + "state": "kv.9" }, { - "state": "kv", - "offset": 590080 + "state": "kv.9", + "offset": 256 }, { "buf": "slot_mapping" @@ -28738,12 +28721,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 589824 + "state": "kv.9" }, { - "state": "kv", - "offset": 590080 + "state": "kv.9", + "offset": 256 }, { "buf": "block_table" @@ -28991,12 +28973,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 655360 + "state": "kv.10" }, { - "state": "kv", - "offset": 655616 + "state": "kv.10", + "offset": 256 }, { "buf": "slot_mapping" @@ -29022,12 +29003,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 655360 + "state": "kv.10" }, { - "state": "kv", - "offset": 655616 + "state": "kv.10", + "offset": 256 }, { "buf": "block_table" @@ -29275,12 +29255,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 720896 + "state": "kv.11" }, { - "state": "kv", - "offset": 721152 + "state": "kv.11", + "offset": 256 }, { "buf": "slot_mapping" @@ -29306,12 +29285,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 720896 + "state": "kv.11" }, { - "state": "kv", - "offset": 721152 + "state": "kv.11", + "offset": 256 }, { "buf": "block_table" @@ -29559,12 +29537,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 786432 + "state": "kv.12" }, { - "state": "kv", - "offset": 786688 + "state": "kv.12", + "offset": 256 }, { "buf": "slot_mapping" @@ -29590,12 +29567,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 786432 + "state": "kv.12" }, { - "state": "kv", - "offset": 786688 + "state": "kv.12", + "offset": 256 }, { "buf": "block_table" @@ -29843,12 +29819,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 851968 + "state": "kv.13" }, { - "state": "kv", - "offset": 852224 + "state": "kv.13", + "offset": 256 }, { "buf": "slot_mapping" @@ -29874,12 +29849,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 851968 + "state": "kv.13" }, { - "state": "kv", - "offset": 852224 + "state": "kv.13", + "offset": 256 }, { "buf": "block_table" @@ -30127,12 +30101,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 917504 + "state": "kv.14" }, { - "state": "kv", - "offset": 917760 + "state": "kv.14", + "offset": 256 }, { "buf": "slot_mapping" @@ -30158,12 +30131,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 917504 + "state": "kv.14" }, { - "state": "kv", - "offset": 917760 + "state": "kv.14", + "offset": 256 }, { "buf": "block_table" @@ -30411,12 +30383,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 983040 + "state": "kv.15" }, { - "state": "kv", - "offset": 983296 + "state": "kv.15", + "offset": 256 }, { "buf": "slot_mapping" @@ -30442,12 +30413,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 983040 + "state": "kv.15" }, { - "state": "kv", - "offset": 983296 + "state": "kv.15", + "offset": 256 }, { "buf": "block_table" @@ -30695,12 +30665,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1048576 + "state": "kv.16" }, { - "state": "kv", - "offset": 1048832 + "state": "kv.16", + "offset": 256 }, { "buf": "slot_mapping" @@ -30726,12 +30695,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1048576 + "state": "kv.16" }, { - "state": "kv", - "offset": 1048832 + "state": "kv.16", + "offset": 256 }, { "buf": "block_table" @@ -30979,12 +30947,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1114112 + "state": "kv.17" }, { - "state": "kv", - "offset": 1114368 + "state": "kv.17", + "offset": 256 }, { "buf": "slot_mapping" @@ -31010,12 +30977,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1114112 + "state": "kv.17" }, { - "state": "kv", - "offset": 1114368 + "state": "kv.17", + "offset": 256 }, { "buf": "block_table" @@ -31263,12 +31229,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1179648 + "state": "kv.18" }, { - "state": "kv", - "offset": 1179904 + "state": "kv.18", + "offset": 256 }, { "buf": "slot_mapping" @@ -31294,12 +31259,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1179648 + "state": "kv.18" }, { - "state": "kv", - "offset": 1179904 + "state": "kv.18", + "offset": 256 }, { "buf": "block_table" @@ -31547,12 +31511,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1245184 + "state": "kv.19" }, { - "state": "kv", - "offset": 1245440 + "state": "kv.19", + "offset": 256 }, { "buf": "slot_mapping" @@ -31578,12 +31541,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1245184 + "state": "kv.19" }, { - "state": "kv", - "offset": 1245440 + "state": "kv.19", + "offset": 256 }, { "buf": "block_table" @@ -31831,12 +31793,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1310720 + "state": "kv.20" }, { - "state": "kv", - "offset": 1310976 + "state": "kv.20", + "offset": 256 }, { "buf": "slot_mapping" @@ -31862,12 +31823,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1310720 + "state": "kv.20" }, { - "state": "kv", - "offset": 1310976 + "state": "kv.20", + "offset": 256 }, { "buf": "block_table" @@ -32115,12 +32075,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1376256 + "state": "kv.21" }, { - "state": "kv", - "offset": 1376512 + "state": "kv.21", + "offset": 256 }, { "buf": "slot_mapping" @@ -32146,12 +32105,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1376256 + "state": "kv.21" }, { - "state": "kv", - "offset": 1376512 + "state": "kv.21", + "offset": 256 }, { "buf": "block_table" @@ -32399,12 +32357,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1441792 + "state": "kv.22" }, { - "state": "kv", - "offset": 1442048 + "state": "kv.22", + "offset": 256 }, { "buf": "slot_mapping" @@ -32430,12 +32387,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1441792 + "state": "kv.22" }, { - "state": "kv", - "offset": 1442048 + "state": "kv.22", + "offset": 256 }, { "buf": "block_table" @@ -32683,12 +32639,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1507328 + "state": "kv.23" }, { - "state": "kv", - "offset": 1507584 + "state": "kv.23", + "offset": 256 }, { "buf": "slot_mapping" @@ -32714,12 +32669,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1507328 + "state": "kv.23" }, { - "state": "kv", - "offset": 1507584 + "state": "kv.23", + "offset": 256 }, { "buf": "block_table" @@ -32967,12 +32921,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1572864 + "state": "kv.24" }, { - "state": "kv", - "offset": 1573120 + "state": "kv.24", + "offset": 256 }, { "buf": "slot_mapping" @@ -32998,12 +32951,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1572864 + "state": "kv.24" }, { - "state": "kv", - "offset": 1573120 + "state": "kv.24", + "offset": 256 }, { "buf": "block_table" @@ -33251,12 +33203,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1638400 + "state": "kv.25" }, { - "state": "kv", - "offset": 1638656 + "state": "kv.25", + "offset": 256 }, { "buf": "slot_mapping" @@ -33282,12 +33233,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1638400 + "state": "kv.25" }, { - "state": "kv", - "offset": 1638656 + "state": "kv.25", + "offset": 256 }, { "buf": "block_table" @@ -33535,12 +33485,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1703936 + "state": "kv.26" }, { - "state": "kv", - "offset": 1704192 + "state": "kv.26", + "offset": 256 }, { "buf": "slot_mapping" @@ -33566,12 +33515,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1703936 + "state": "kv.26" }, { - "state": "kv", - "offset": 1704192 + "state": "kv.26", + "offset": 256 }, { "buf": "block_table" @@ -33819,12 +33767,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1769472 + "state": "kv.27" }, { - "state": "kv", - "offset": 1769728 + "state": "kv.27", + "offset": 256 }, { "buf": "slot_mapping" @@ -33850,12 +33797,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1769472 + "state": "kv.27" }, { - "state": "kv", - "offset": 1769728 + "state": "kv.27", + "offset": 256 }, { "buf": "block_table" @@ -34103,12 +34049,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1835008 + "state": "kv.28" }, { - "state": "kv", - "offset": 1835264 + "state": "kv.28", + "offset": 256 }, { "buf": "slot_mapping" @@ -34134,12 +34079,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1835008 + "state": "kv.28" }, { - "state": "kv", - "offset": 1835264 + "state": "kv.28", + "offset": 256 }, { "buf": "block_table" @@ -34387,12 +34331,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1900544 + "state": "kv.29" }, { - "state": "kv", - "offset": 1900800 + "state": "kv.29", + "offset": 256 }, { "buf": "slot_mapping" @@ -34418,12 +34361,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1900544 + "state": "kv.29" }, { - "state": "kv", - "offset": 1900800 + "state": "kv.29", + "offset": 256 }, { "buf": "block_table" @@ -34671,12 +34613,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 1966080 + "state": "kv.30" }, { - "state": "kv", - "offset": 1966336 + "state": "kv.30", + "offset": 256 }, { "buf": "slot_mapping" @@ -34702,12 +34643,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 1966080 + "state": "kv.30" }, { - "state": "kv", - "offset": 1966336 + "state": "kv.30", + "offset": 256 }, { "buf": "block_table" @@ -34955,12 +34895,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 2031616 + "state": "kv.31" }, { - "state": "kv", - "offset": 2031872 + "state": "kv.31", + "offset": 256 }, { "buf": "slot_mapping" @@ -34986,12 +34925,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 2031616 + "state": "kv.31" }, { - "state": "kv", - "offset": 2031872 + "state": "kv.31", + "offset": 256 }, { "buf": "block_table" @@ -35239,12 +35177,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 2097152 + "state": "kv.32" }, { - "state": "kv", - "offset": 2097408 + "state": "kv.32", + "offset": 256 }, { "buf": "slot_mapping" @@ -35270,12 +35207,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 2097152 + "state": "kv.32" }, { - "state": "kv", - "offset": 2097408 + "state": "kv.32", + "offset": 256 }, { "buf": "block_table" @@ -35523,12 +35459,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 2162688 + "state": "kv.33" }, { - "state": "kv", - "offset": 2162944 + "state": "kv.33", + "offset": 256 }, { "buf": "slot_mapping" @@ -35554,12 +35489,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 2162688 + "state": "kv.33" }, { - "state": "kv", - "offset": 2162944 + "state": "kv.33", + "offset": 256 }, { "buf": "block_table" @@ -35807,12 +35741,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 2228224 + "state": "kv.34" }, { - "state": "kv", - "offset": 2228480 + "state": "kv.34", + "offset": 256 }, { "buf": "slot_mapping" @@ -35838,12 +35771,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 2228224 + "state": "kv.34" }, { - "state": "kv", - "offset": 2228480 + "state": "kv.34", + "offset": 256 }, { "buf": "block_table" @@ -36091,12 +36023,11 @@ "offset": "QKV_V_OFFSET_BYTES" }, { - "state": "kv", - "offset": 2293760 + "state": "kv.35" }, { - "state": "kv", - "offset": 2294016 + "state": "kv.35", + "offset": 256 }, { "buf": "slot_mapping" @@ -36122,12 +36053,11 @@ "buf": "q_n" }, { - "state": "kv", - "offset": 2293760 + "state": "kv.35" }, { - "state": "kv", - "offset": 2294016 + "state": "kv.35", + "offset": 256 }, { "buf": "block_table" diff --git a/tools/gen_qwen3_decode.py b/tools/gen_qwen3_decode.py index 03d673ab..f9b12f3f 100644 --- a/tools/gen_qwen3_decode.py +++ b/tools/gen_qwen3_decode.py @@ -16,9 +16,9 @@ 被 Triton 特化进 binary 不再传参）——这里 pin 的是 decode 实例。 - GEMM 是 runtime 特判（symbol 前缀 `extern:`，cublasLt）；embedding 是 待写的 Triton 占位。 -- KV state 布局从 vLLM 的逐层池改为层交织 `[page][layer][16][8][2][128]` - （同一批 kernel，靠 stride 参数 ×LAYERS 和 state offset 字面量适配）， - bytes_per_token = 36*2*8*128*2 = 147456。 +- KV 是每层一个 paged state `kv.`（vLLM 的逐层池布局，bytes_per_token + = 2*8*128*2 = 4096），页号由 pool 统一发；一份切开的 manifest 每段只带 + 自己那些层的 state。 - 发射前对挖矿数据做结构断言：q/k/v 在 qkv 中的视图偏移、residual 全程 同址、逐层权重互异、KV 池 k/v 相距 256B、cache/attention 共享同一 KV 池与 scale 指针、unified 与 reduce 共享 segm 缓冲——连线是手写的， @@ -65,7 +65,7 @@ sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) import mine_capture as mc -from handwritten import hw # tools/handwritten.py: build + pin handwritten cubins +from kernels.index import variant # noqa: E402 HIDDEN = 2560 LAYERS = 36 @@ -96,12 +96,13 @@ TAPS = {0: 0, 8: 1, 16: 2, 24: 3, 32: 4} # target_layer_ids [1,9,17,25,33]-1 DRAFT_KV_DIM = DRAFT_LAYERS * 2 * KV_DIM # 融合 KV GEMM 输出行宽 10240 -# 层交织 KV 布局（vLLM 逐层池 -> 我们的单 state）： -# 一个 block(16 token) 在一层里占 16*8*2*128 = 32768 elems；层间连续。 +# target KV：每层一个 paged state `kv.`，即 vLLM 的逐层池布局 +# `[page][16][8][2][128]`；pool 给所有 paged state 同一套页号。一个 +# block(16 token) 在一层里占 16*8*2*128 = 32768 elems。draft 仍是一个 +# 层交织的 state（`draft_kv`，stride × 层数）。 BLOCK_ELEMS_PER_LAYER = 16 * KV_HEADS * 2 * HEAD_DIM -BLOCK_STRIDE = LAYERS * BLOCK_ELEMS_PER_LAYER # 传给 kernel 的 elems -LAYER_KV_BYTES = BLOCK_ELEMS_PER_LAYER * BF16 # state offset 步长 -KV_BYTES_PER_TOKEN = LAYERS * 2 * KV_DIM * BF16 # 147456 +LAYER_KV_BYTES = BLOCK_ELEMS_PER_LAYER * BF16 # draft_kv 的层 offset 步长 +LAYER_KV_BYTES_PER_TOKEN = 2 * KV_DIM * BF16 # 4096 DRAFT_BLOCK_STRIDE = DRAFT_LAYERS * BLOCK_ELEMS_PER_LAYER DRAFT_KV_BYTES_PER_TOKEN = DRAFT_LAYERS * 2 * KV_DIM * BF16 # 20480 V_BYTE_OFF = 2 * HEAD_DIM # 挖矿实测 k/v 相距 256B @@ -388,7 +389,7 @@ def scr(name): def step(symbol, params, block, grid, args, shared_mem=None, cubin=None, - sha256=None): + sha256=None, label=None): s = {"entry": symbol, "params": params, "block": block, "grid": [_e(g) for g in grid], "args": args} if shared_mem is not None: @@ -397,16 +398,18 @@ def step(symbol, params, block, grid, args, shared_mem=None, cubin=None, s["cubin"] = cubin if sha256 is not None: s["sha256"] = sha256 + if label is not None: + s["label"] = label return s def single(symbol, params, block, grid, shared_mem=None, cubin=None, - sha256=None): + sha256=None, label=None): """单步实现，恒等布线：接口即该核的 launch ABI。""" return {"params": params, "impl": {"launches": [step(symbol, params, block, grid, [a(i) for i in range(len(params))], - shared_mem, cubin, sha256)]}} + shared_mem, cubin, sha256, label)]}} # 结构输入的先验（domain）：接模型的人才知道 buffer 是页表不是激活。 @@ -415,8 +418,8 @@ def single(symbol, params, block, grid, shared_mem=None, cubin=None, DOMAINS = { "token_ids": TOKEN_DOMAIN, "positions": {"index_into": "rope.cos_sin_cache"}, - "slot_mapping": {"index_into": "kv"}, - "block_table": {"index_into": "kv", "stride": 16}, # vLLM block_size + "slot_mapping": {"index_into": "kv.0"}, + "block_table": {"index_into": "kv.0", "stride": 16}, # vLLM block_size "seq_lens": {"min": 1}, "cu_seqlens_q": {"min": 0, "max": "tokens", "monotone": True}, "next_token": TOKEN_DOMAIN, @@ -822,8 +825,8 @@ def fused(label, x, w): [buf(x), i64(HIDDEN), buf("residual"), buf(w), f32(eps), T, i32(HIDDEN), i64(HIDDEN)]) - def forward(attn_kernel, tail, mp="model.", layers=LAYERS, kv_state="kv", - block_stride=BLOCK_STRIDE, scales="kv_scales", taps=False, + def forward(attn_kernel, tail, mp="model.", layers=LAYERS, kv_state=None, + block_stride=BLOCK_ELEMS_PER_LAYER, scales="kv_scales", taps=False, lm_head_w="lm_head.weight", final_norm_w="model.norm.weight", ids="token_ids"): """embed + 逐层的直线 dispatch 表，target/draft 共用（几何同构， @@ -848,7 +851,7 @@ def forward(attn_kernel, tail, mp="model.", layers=LAYERS, kv_state="kv", for i in range(layers): p = f"{mp}layers.{i}." l = f"l{i}." - koff = i * LAYER_KV_BYTES + st, koff = (f"kv.{i}", 0) if kv_state is None else (kv_state, i * LAYER_KV_BYTES) ks, vs = buf(scales, i * 8), buf(scales, i * 8 + 4) last = i + 1 == layers ds += [ @@ -867,14 +870,14 @@ def forward(attn_kernel, tail, mp="model.", layers=LAYERS, kv_state="kv", # [8]=value 行距=QKV_DIM（v 视图在融合 qkv 里；decode # capture 的 1024 同样是 tokens=1 下的假常量） [buf("k_n"), buf("qkv", (Q_DIM + KV_DIM) * BF16), - state(kv_state, koff), state(kv_state, koff + V_BYTE_OFF), + state(st, koff), state(st, koff + V_BYTE_OFF), buf("slot_mapping"), ks, vs, i64(KV_DIM), i64(QKV_DIM), i64(block_stride), i64(2 * HEAD_DIM), i64(0), i64(0), i64(KV_HEADS * 2 * HEAD_DIM), i64(0), i64(0)]), d(l + "attn", attn_kernel, [buf("attn_out"), buf("q_n"), - state(kv_state, koff), state(kv_state, koff + V_BYTE_OFF), + state(st, koff), state(st, koff + V_BYTE_OFF), buf("block_table"), buf("seq_lens"), f32(scale), ks, vs, f32(1.0), f32(0.0), i64(MAX_BLOCKS), i64(Q_DIM), i64(HEAD_DIM), i64(Q_DIM), @@ -1000,7 +1003,7 @@ def draft_precompute(): ] return ds - states = {"kv": {"bytes_per_token": KV_BYTES_PER_TOKEN}} + states = {f"kv.{i}": {"bytes_per_token": LAYER_KV_BYTES_PER_TOKEN} for i in range(LAYERS)} programs = { # prefill: one sequence, as many rows as the chunk holds (tokens); # decode: the bs=1 contract, 3D split-KV kernels; decode_batch: up From 59590c7821c7509846445cae0ce2705f50876a98 Mon Sep 17 00:00:00 2001 From: JinYan Su <751080330@qq.com> Date: Wed, 7 Oct 2026 15:44:10 +0000 Subject: [PATCH 4/6] manifest: a cut stage carries only the states it touches and says how much of each it uses A stage keeps a state whole or drops it, so a whole-model state is carried by every stage that touches a piece of it. `kern cut` now prints, per stage, the regions (distinct call offsets) of each kept state its calls touch against the whole manifest's; a share below one is memory held for other stages, reported, not refused. A page or slot table whose state a stage dropped indexes a kept state of the same kind instead of losing its domain (and with it its place in the protocol). Gate (tray07, qwen3-4b per-layer kv cut before l18): each stage keeps 18 states, all used whole (the single-state manifest: `kv` 36 of 72 regions on both stages); pages per stage without --capacity 122993 -> 245760; the chain gate (7 prompts, chunk 512 and 64) and the 2-process serve gate are bitwise identical as before; sweep unchanged within noise. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/kern-manifest/src/cut.rs | 38 ++++++++++++++++++++++++++++--- crates/kern-manifest/tests/cut.rs | 10 +++++++- crates/kern-run/src/bin/kern.rs | 9 ++++++++ docs/manifest.md | 7 ++++++ docs/roadmap.md | 3 ++- docs/serve.md | 8 ++++--- 6 files changed, 67 insertions(+), 8 deletions(-) diff --git a/crates/kern-manifest/src/cut.rs b/crates/kern-manifest/src/cut.rs index 1023c6fe..c5b3e4ea 100644 --- a/crates/kern-manifest/src/cut.rs +++ b/crates/kern-manifest/src/cut.rs @@ -30,7 +30,13 @@ //! The prompt reaches the step program's first token the way it does //! unsplit; a chunk program that hands a token back itself is refused, so //! a pipeline's first token always comes from a step. Everything a stage -//! does not use is dropped, and every stage verifies. +//! does not use is dropped, and every stage verifies. A state is kept whole +//! or not at all, so a state laid out over the whole model (one KV state +//! for every layer) is carried whole by every stage that touches a piece of +//! it; [`state_use`] says how much of each one a stage uses, and a +//! generator that gives each piece its own state (one per layer) lets a +//! stage carry only its own. A table whose state a stage dropped indexes a +//! kept state of the same kind: a pool hands every paged state the same ids. use std::collections::{BTreeMap, BTreeSet}; @@ -356,9 +362,17 @@ fn stage( let states: BTreeMap = calls.iter().flat_map(|c| state_args(c)).map(|(st, _)| (st.to_string(), m.states[st].clone())).collect(); let names: BTreeSet = buffers.keys().chain(states.keys()).cloned().collect(); + // A table hands the same ids to every state of its kind: one naming a + // state this stage dropped names a kept state of that kind instead. + let kind = |s: &State| (s.bytes_per_token > 0, s.bytes_per_seq > 0, s.is_owned()); for b in buffers.values_mut() { - if b.domain.as_ref().and_then(|d| d.index_into.as_ref()).is_some_and(|t| !names.contains(t)) { - b.domain = None; + let Some(t) = b.domain.as_ref().and_then(|d| d.index_into.clone()).filter(|t| !names.contains(t)) else { + continue; + }; + let to = m.states.get(&t).and_then(|old| states.iter().find(|(_, s)| kind(s) == kind(old))).map(|(n, _)| n); + match (to, b.domain.as_mut()) { + (Some(n), Some(d)) => d.index_into = Some(n.clone()), + _ => b.domain = None, } } let ops: BTreeMap = @@ -468,6 +482,24 @@ fn live_in<'a>(m: &'a Manifest, calls: &'a [Call]) -> BTreeSet<&'a str> { live } +/// How much of each state a stage keeps it uses: per state, the regions +/// (distinct call offsets) the stage's calls touch and those the whole +/// manifest's calls touch. A stage allocates a state whole, so a share +/// below one is memory it holds for another stage's regions: a state laid +/// out over the whole model rather than one per piece. +pub fn state_use(whole: &Manifest, stage: &Manifest) -> Vec<(String, usize, usize)> { + let regions = |m: &Manifest, st: &str| -> BTreeSet { + m.programs + .values() + .flat_map(|p| &p.calls) + .flat_map(state_args) + .filter(|(s, _)| *s == st) + .map(|(_, o)| o) + .collect() + }; + stage.states.keys().map(|st| (st.clone(), regions(stage, st).len(), regions(whole, st).len())).collect() +} + /// Every state region `c` touches. fn state_args(c: &Call) -> impl Iterator { c.args.iter().filter_map(|a| match a { diff --git a/crates/kern-manifest/tests/cut.rs b/crates/kern-manifest/tests/cut.rs index 748f1135..51acd8e4 100644 --- a/crates/kern-manifest/tests/cut.rs +++ b/crates/kern-manifest/tests/cut.rs @@ -3,7 +3,7 @@ use std::collections::BTreeMap; -use kern_manifest::cut::cut; +use kern_manifest::cut::{cut, state_use}; use kern_manifest::types::{Arg, Dir, Manifest, Module, ParamType}; use kern_manifest::{verify, Protocol, Verified}; use serde_json::{json, Value}; @@ -268,6 +268,14 @@ fn qwen3_cuts_at_a_layer_into_stages_that_each_serve() { ); assert!(tail.buffers.contains_key("lm_head.weight") && !head.buffers.contains_key("lm_head.weight")); assert!(!head.buffers.contains_key("model.layers.18.self_attn.qkv_proj.weight")); + // A KV state per layer: each stage keeps its 18 and uses them whole, + // and its tables index one it kept. + for (st, first) in [(head, "kv.0"), (tail, "kv.18")] { + let used = state_use(&qwen3(), st); + assert_eq!((used.len(), used.iter().all(|(_, a, b)| a == b)), (18, true)); + let table = |b: &str| st.buffers[b].domain.as_ref().and_then(|d| d.index_into.clone()); + assert_eq!((table("block_table"), table("slot_mapping")), (Some(first.into()), Some(first.into()))); + } // The head stages tokens and hands nothing back; the tail is fed // activations and samples. let ph = Protocol::check_unsampled(head).unwrap_or_else(|e| panic!("{e}")); diff --git a/crates/kern-run/src/bin/kern.rs b/crates/kern-run/src/bin/kern.rs index d98e1ee0..f9214c1d 100644 --- a/crates/kern-run/src/bin/kern.rs +++ b/crates/kern-run/src/bin/kern.rs @@ -162,6 +162,15 @@ fn cut(manifest: &Path, at: &[usize], module: &Path, out: &Path) -> Result<()> { st.buffers.values().filter(|b| b.kind == kern_manifest::types::BufferKind::Weight).count(), calls.join(", ") ); + let (whole, part): (Vec<_>, Vec<_>) = + kern_manifest::cut::state_use(&m, st).into_iter().partition(|(_, a, b)| a == b); + tracing::info!("stage {s}: {} of {} states used whole", whole.len(), whole.len() + part.len()); + for (name, a, b) in part { + tracing::warn!( + "stage {s}: state `{name}` is allocated whole and its calls touch {a} of its {b} regions (~{:.0}% of its bytes)", + 100.0 * a as f64 / b as f64 + ); + } } tracing::info!("cut `{}` into {} stages", stages[0].cut.as_ref().map_or("", |c| c.id.as_str()), stages.len()); Ok(()) diff --git a/docs/manifest.md b/docs/manifest.md index fb438dab..220eae69 100644 --- a/docs/manifest.md +++ b/docs/manifest.md @@ -402,6 +402,13 @@ kern-serve 只拿 `Protocol` 驱动 runtime，不读 JSON（CI 用 grep 保证 `var × 常数`）；output / carry 只许在末段写；一个 (state, offset) 区域 只许一段碰（按层切 KV 时各段各持自己那些层）。做不到的切点报错并说明 哪条不满足，不会静默切出错的东西。 +- state 整个留或整个丢：一段只要碰到一个 state 的一块就分配整个。`kern cut` + 逐段打印每个 state 被本段碰到的区域数 / 整份碰到的区域数（区域 = 不同的 + call offset，按等大估字节），不满的就是给别段占着的内存（报告，不拒）。 + 所以生成器把 KV 写成每层一个 state（qwen3-4b：`kv.0`…`kv.35`），各段只带 + 自己那些层；生成器不知道流水线。页表 / slot 的 `index_into` 指向一个本段 + 丢掉的 state 时改指本段留下的同类 state（pool 给所有 paged state 同一套 + 页号）。 - 边界处插一对 send / recv：上游段末尾 `pp.wait_empty → pp.put.… → pp.post_full`，下游段开头 `pp.wait_full → pp.take.… → pp.post_empty`；邮箱 `pp..box`（`u8[256 + 最大载荷]`）在下游 GPU 上， diff --git a/docs/roadmap.md b/docs/roadmap.md index 872c5e50..c44e78ff 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -63,7 +63,8 @@ kern 在内部 RSI 里是 kernel agent 的反馈环境（`kern test` 裁判、`k | 级 | 内容 | 门禁 | |---|---|---| | PP1 ✅ | `kern cut`（按 call 下标切、活跃性找边界、send/recv 邮箱核、`clock` fill）+ kern-serve 一段一进程的 prefill-only 流水（head 持唯一的池，TCP 发 item，GPU 邮箱传激活） | **2026-10-07 tray07 过**：qwen3-4b 2 段，各段依次跑 logits/token 与整份逐位同（chunk 512/64）；两进程首 token 与单卡同；8193 token conc1 TTFT 313 → 174 ms（serve.md「流水线」） | -| PP2 | 每段只分自己那些层的 KV（现在按整份 `bytes_per_token`）；decode 走流水；per-seq state（GDN）与吐 token 的 chunk program | | +| PP1.1 ✅ | qwen3-4b 的 KV 每层一个 state，`kern cut` 每段只带自己碰的 state、逐段报告 state 用了几成 | **2026-10-07 tray07 过**：`kern test qwen3-4b` 位一致、`kern run` 文本同；chain / serve gate 位一致；每段页数 122,993 → 245,760 | +| PP2 | decode 走流水；K3 的 state 拆分；per-seq state（GDN）与吐 token 的 chunk program | | ## 协议线 diff --git a/docs/serve.md b/docs/serve.md index 02c8e101..a3cbc822 100644 --- a/docs/serve.md +++ b/docs/serve.md @@ -605,9 +605,11 @@ stop、fault 消息：任何错误进程退出，连接断了对端退出；设 | | | | | | | 1 | 9.52 | 2.13 | 0.73 | - | 单请求内 chunk 之间就流水起来（stage 0 算第 k+1 块时 stage 1 算第 k 块），所以长 prompt conc1 也 -接近 2×；1 块的 prompt 没有可重叠的东西，略慢（邮箱 + 两段 launch）。v1 的限制：每段按整份的 -`bytes_per_token` 分 KV（只用自己那些层，浪费一半）；只认纯 paged 的 manifest；chunk program -自己吐 token 的模型拒切；上游那份 box 是不用的占位。 +接近 2×；1 块的 prompt 没有可重叠的东西，略慢（邮箱 + 两段 launch）。KV 每层一个 state 之后（同日， +`examples/qwen3-4b.json` 重生成，`kern test qwen3-4b` 位一致、`kern run` 文本同），每段只带自己的 +18 层：不给 `--capacity` 时每段 122,993 → 245,760 页（2.0×）；切点、gate、扫描数字与上表一致 +（`~/bench_results/2026-10-07-kern-pp-v1/perlayer/`）。v1 的限制：只认纯 paged 的 manifest； +chunk program 自己吐 token 的模型拒切；上游那份 box 是不用的占位。 ## 没做（按需要加） From 0bbf81a88f5ca85e069e9ab2dfce5972a494ac4c Mon Sep 17 00:00:00 2001 From: JinYan Su <751080330@qq.com> Date: Wed, 7 Oct 2026 15:50:11 +0000 Subject: [PATCH 5/6] qwen3: the example hash baselines follow the per-layer kv The regenerated qwen3-4b examples carry one kv state per layer and the registry builds of argmax, embedding and weight_prep; kern test and the pipeline gates on PR #42 cover the change. Co-Authored-By: Claude Opus 5.5 (1M context) --- tools/test_qwen_constants.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tools/test_qwen_constants.py b/tools/test_qwen_constants.py index 2b67087d..9ffe542e 100644 --- a/tools/test_qwen_constants.py +++ b/tools/test_qwen_constants.py @@ -9,12 +9,13 @@ ROOT = pathlib.Path(__file__).resolve().parent.parent # Canonical JSON hashes of the checked-in examples (last: unified attention -# reads the kv state, `inout` -> `in`, 2026-09-16). Changes to kernel wiring, +# reads the kv state, `inout` -> `in`, 2026-09-16; qwen3-4b: one kv state per +# layer and registry builds of argmax / embedding / weight_prep, 2026-10-07). Changes to kernel wiring, # capacities, float literals or even artifact pins require review. BASELINES = { - "qwen3-4b-dspark.json": "444001bf6fa6991128128a878c0cc139b8ecf71800c6ccfea0d344c1ad169133", - "qwen3-4b-silu-mined.json": "e14198af226849f46826bc624c4a6e8a208a4e8d7b160ffc611b817d2915d7b8", - "qwen3-4b.json": "792666b476938520a21598bec6bdf5461cb62dd8fdf52cd5998b33a7388c2e69", + "qwen3-4b-dspark.json": "c72a03b5c0f817e742befcffa1e8241bc13f92c4f3458743a8ee27ccc6e4d30c", + "qwen3-4b-silu-mined.json": "3cd79ecd0d0d54c830f842644138b5dafb92c88efcf283709535b096665fe23e", + "qwen3-4b.json": "b54ac6fa2766667782626ec56b8d02ceb46b250eee7f3cf1a6d79561e5d58e84", "qwen3.8-27b-dflash2.json": "134839dd1b25fc8fb5a5cb97391a02576d2624f2f0ccc5431af6e10c9dd679dd", "qwen3.8-27b.json": "a9cc996a7548dabc756b30f9ef848459c2a5c2043075b79e11091054d1a30443" } From f83b8ace6eaaa07a8066e7f947ad09b6c1b28a93 Mon Sep 17 00:00:00 2001 From: JinYan Su <751080330@qq.com> Date: Wed, 7 Oct 2026 15:54:05 +0000 Subject: [PATCH 6/6] dsv41: ar_trace skips the clock fill like the other outputs Co-Authored-By: Claude Opus 5.5 (1M context) --- tools/dsv41/attention/ar_trace/src/main.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/dsv41/attention/ar_trace/src/main.rs b/tools/dsv41/attention/ar_trace/src/main.rs index 3c56175a..7a9015b0 100644 --- a/tools/dsv41/attention/ar_trace/src/main.rs +++ b/tools/dsv41/attention/ar_trace/src/main.rs @@ -104,7 +104,7 @@ fn main() -> anyhow::Result<()> { Fill::CuSeqlens => vec![0, rows as i64], Fill::SpanAt => vec![0], Fill::Blocks => anyhow::bail!("unexpected collective row blocks in DP manifest"), - Fill::Tokens | Fill::Count | Fill::Error => continue, + Fill::Tokens | Fill::Count | Fill::Error | Fill::Clock => continue, }; rt.write_input_at(&f.name, &f.encode(&values), &vars)?; } @@ -159,7 +159,7 @@ fn main() -> anyhow::Result<()> { Fill::CuSeqlens => vec![0, 6], Fill::SpanAt => vec![0], Fill::Blocks => anyhow::bail!("unexpected blocks"), - Fill::Tokens | Fill::Count | Fill::Error => continue, + Fill::Tokens | Fill::Count | Fill::Error | Fill::Clock => continue, }; rt.write_input_at(&f.name, &f.encode(&values), &dv)?; }