diff --git a/crates/kern-pool/src/pages.rs b/crates/kern-pool/src/pages.rs index c066144..f0167d6 100644 --- a/crates/kern-pool/src/pages.rs +++ b/crates/kern-pool/src/pages.rs @@ -552,6 +552,13 @@ impl Pool { lock(&self.inner).remapped } + /// Whether a remap is planned or in flight: what [`Denied::Remapping`] + /// told the last caller to wait for. + pub fn remapping(&self) -> bool { + let g = lock(&self.inner); + g.pending.is_some() || g.in_flight + } + /// The remap waiting to be executed, if any; from here until /// [`Pool::complete`] one is in flight and no other is planned. pub fn take_pending(&self) -> Option { diff --git a/crates/kern-runtime/src/lease.rs b/crates/kern-runtime/src/lease.rs index 038fb14..4d8b839 100644 --- a/crates/kern-runtime/src/lease.rs +++ b/crates/kern-runtime/src/lease.rs @@ -254,6 +254,26 @@ impl Runtime { Ok(()) } + /// Land every remap, blocking until the thread has run the one in + /// flight and whatever the pool plans after it: the ask a + /// [`Denied::Remapping`] refused is answered on the next call. + pub fn settle(&mut self) -> Result<()> { + self.ctx.bind_to_thread()?; + self.poll()?; + while self.pool.remapping() { + match self.remaps.done.recv() { + Ok(Ok(plan)) => { + self.zero_fresh(&plan)?; + self.pool.complete(plan); + } + Ok(Err(e)) => return Err(e), + Err(_) => bail!(Cuda, "the remap thread is gone"), + } + self.poll()?; + } + Ok(()) + } + /// Zero every chunk `plan` mapped. pub(crate) fn zero_fresh(&mut self, plan: &Remap) -> Result<()> { let chunk = self.pool.chunk() as usize; diff --git a/crates/kern-serve/src/scheduler.rs b/crates/kern-serve/src/scheduler.rs index 9a3c094..5641f10 100644 --- a/crates/kern-serve/src/scheduler.rs +++ b/crates/kern-serve/src/scheduler.rs @@ -76,8 +76,11 @@ //! accepted past `max_tokens` or the stop leaves a state no next turn //! continues, which is not kept. The prompt's snapshot is an //! optimization, never a reason to fail its request: `Busy` makes room -//! as a lease does, and when nothing is left to make room with it is -//! skipped. A `Busy` lease makes room and retries until it fits or +//! as a lease does, `Remapping` waits for the slot being carved (this +//! is the one place that waits: the state cannot be snapshotted later, +//! and the wait is the remap's own milliseconds), and when nothing is +//! left to make room with it is skipped. A `Busy` lease makes room and +//! retries until it fits or //! nothing is left: the least recently hit snapshot is parked into the //! host tier (`--host-gib`: pinned DRAM per rank, `Tray::park`, all of //! its members' pieces or none; the coldest parked ones are dropped when @@ -720,9 +723,11 @@ impl KernScheduler { } /// The `key.len()` tokens `row` holds, which `key` names, as a snapshot - /// of its recurrent state. `Busy` makes room and retries; with nothing - /// left to make room with, a remap in flight (it lands between steps, - /// not here) or any other denial, the request runs on without it. + /// of its recurrent state. `Busy` makes room and retries, `Remapping` + /// waits for the slot on its way (the prefill just read its output + /// back, so the stream the remap waits on is drained; what is left is + /// the remap's own milliseconds); with nothing left to make room with + /// or any other denial, the request runs on without it. fn keep_prompt(&mut self, row: &mut Row, key: &[i64], id: RequestId) -> Result<()> { loop { match self.tray.checkpoint(row, key.len()) { @@ -732,6 +737,7 @@ impl KernScheduler { return Ok(()); } Err(Error::Denied(Denied::Busy)) if self.make_room()? => {} + Err(Error::Denied(Denied::Remapping)) => self.tray.settle()?, Err(Error::Denied(d)) => { debug!(request = %id, tokens = key.len(), denied = %d, "history not kept"); self.stats.prompt_skips += 1; diff --git a/crates/kern-serve/src/tray.rs b/crates/kern-serve/src/tray.rs index 97015d0..6918c7f 100644 --- a/crates/kern-serve/src/tray.rs +++ b/crates/kern-serve/src/tray.rs @@ -685,6 +685,12 @@ impl Tray { self.each(row.by_mut(), |rt, _, l| rt.checkpoint(l, len)) } + /// Every rank's remaps landed, blocking until they are: a lease or a + /// checkpoint denied `Remapping` is asked again after this. + pub fn settle(&mut self) -> Result<(), Error> { + self.ranks.iter_mut().try_for_each(Runtime::settle) + } + /// The first `len` tokens of a finished row as a snapshot; nothing is copied. pub fn retire(&mut self, row: Row, len: usize) -> Snapshot { let base = self.groups.members(row.owner.0).start; diff --git a/docs/roadmap.md b/docs/roadmap.md index 4ef4508..209e327 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -25,7 +25,7 @@ manifest,与 qwen3 同一条流水)。两条线并行,每级带门禁。 | K1 ✅ | 前缀缓存 = checkpoint 表:`Checkpoint` = (精确长度, 页链上的一个节点, 可选 state slot),页链按引用计数还页(`Arc`,一页一节点,一条序列的所有 checkpoint 共一条链);`Prefix` 表按页哈希链索引,序列自带滚动 `Chain`。纯 KV 模型每整页留一个、零拷贝;带循环状态的模型只在请求结束 `retire`(slot 易主,不拷),中途分叉不命中 | AgentX trace 回放(`crates/kern-run/examples/agentx_replay.rs`,393 session / 98 827 请求,2026-09-02):纯 KV 页 64 命中 98.6%(缓存 7M token 时 98.1%),extend p50 640 / p90 5056 / p99 40704;带状态页 64 + 1024 slot 96.6%,页 784 + 130 slot(qwen3.8-27b 实机形状)91.0%。GPU 门禁在 `serve.md`:qwen3-4b / qwen3.8-27b 命中后 warm 与 cold 的 64 个 greedy token 一致 | | K1b ✅ | 物理内存一个池:分页 state 与 per-seq state 各占一段虚拟地址,物理块(`cuMemCreate`,2 MiB 粒度的整数倍、≤ 最小对象的一半、封顶 64 MiB;最小对象不足两个粒度时直接取封顶,2026-09-14 起,见 runtime.md)谁用谁 map,块留在上次用它的地方,一类用光才从另一类的空闲对象上拆(`Remap`,后台线程等 stream 事件后 unmap → map → access,主线程收下时清零),`seqs.max` 只管每步 batch 行数、slot 从 `seqs.max + 2` 起按需长;拒绝分 `Remapping` / `Busy` / `ExceedsPool`。不做 vLLM 式"每个块边界都存一份 state"(它靠 TP 把 state 切小才付得起,且块边界与公域前缀的边界对不上;K3 EP-only 一份 605 MB)。实测(tray03 GB300,2026-09-02,`tools/vmm_bench.py`):`cuMemMap` 1.3 µs/块、`cuMemSetAccess` ~30 µs/块、`cuMemUnmap` ~20 µs/块,都按块数计不按字节,所以块要尽量大:2 MiB 块时拼一个 qwen3.8 slot 2.3 ms / 一份 K3 state 8.9 ms,32 MiB 块时 0.2 / 0.7 ms;首次访问多 ~20 µs;连续 map/unmap 时同卡的带宽循环慢 3% | 回放(页 784 + 147 MiB slot,qwen3.8-27b 实机形状,2026-09-02):同样的活跃负载(并发 32、页不设限 → 1024 GiB)命中率 91.0% → 96.9%,slot 峰值 2641、73 578 次 remap;单卡真实预算 250 GiB 并发 32 为 93.1%(页成了瓶颈,17 个请求等页),并发 16 为 95.8%;纯 KV 的 qwen3-4b 形状(147 KB/token、页 16)在 250 GiB = 1.9M token 下并发 32 只有 84.2%(2180 个请求等页)——单卡装不下 32 个 20 万 token 的会话,这是容量不是缓存的事。GPU:qwen3-4b `kern test` 位一致;qwen3-4b / qwen3.8-27b 的 serve 门禁输出与 K1 时逐字相同;qwen3.8 上 200 个不同请求结束后 slot 从 130 长出去、之后的输出与 cold 一致(`serve.md`)。64 MiB 块(2026-09-14 tray18):DSV4.1 Flash EP4 `--max-seqs 8` 下 416 个 checkpoint 把 slot 长到 424,380 次 remap 中 4 次真搬块(各 unmap 41 / map 43),之后 16 条 greedy 输出与冷启动逐字节相同;qwen3-4b `kern test` 位一致;四 rank state 池从 13–16 s 到 0.5 s,页数少 2.2%(`~/bench_results/2026-09-14-kern-wcps-weights`) | | K1c | 显式断点:请求可标断点(system prompt 末尾),prefill 把 chunk 末尾裁到断点后 `checkpoint`;带状态模型 `Busy` 只淘汰带 state 的 entry;"每 N 页自动存一份"留作旋钮,等有需要它的流量再加 | 共享 system prompt 的两个不同 session,第二个命中到断点 | -| K1d ✅ | history 末尾快照(2026-10-08):带状态模型在 `Request::history_tokens` 处停一次 prefill 并 `checkpoint`(半页 + slot 拷贝);分界由 pegainfer chat 层量(最后一条消息带 / 不带 generation prompt 渲染两遍,尾巴走 `vllm_xargs.generation_prompt_tokens`,pegainfer #1140),scheduler 不碰 template。**未做**:`Pool::checkpoint` 遇 `Remapping` 直接跳过(AgentX 3 min:69 留 / 42 跳,全是 remap 在途),应等它落地或 admit 时预留 slot;影响有限——跳过一轮后下一轮退回命中上上轮的快照,实测只少命中 1.3 个点的 prompt(10.1 万 token),0 次 miss 由此造成 | AgentX 3 min A/B(tray08,`~/bench_results/2026-10-08-kern-serve-agentx-3min`):27 → 83 请求,38.7 → 243 tok/s,TTFT p50 11.8 s → 0.2 s,0 → 7.79M hit token(aiperf 侧 cached 92.8%,上限 97.9%) | +| K1d ✅ | history 末尾快照(2026-10-08):带状态模型在 `Request::history_tokens` 处停一次 prefill 并 `checkpoint`(半页 + slot 拷贝);分界由 pegainfer chat 层量(最后一条消息带 / 不带 generation prompt 渲染两遍,尾巴走 `vllm_xargs.generation_prompt_tokens`,pegainfer #1140),scheduler 不碰 template。**2026-10-09**:`Remapping` 时 `Tray::settle` 阻塞等 remap 落地再重试(3 min A/B 里 69 留 / 42 跳全是 remap 在途;跳过一轮只少命中上一轮的增量,实测 1.3 个点、10.1 万 token,0 次 miss 由此造成)。同负载重跑:112 留 / 0 跳,remap 44 → 85,命中 7.79M → 8.00M token,aiperf cached 92.8% → 94.1%,84 请求 245 tok/s、TTFT p50 157 ms(`bin/kern-serve-c`) | AgentX 3 min A/B(tray08,`~/bench_results/2026-10-08-kern-serve-agentx-3min`):27 → 83 请求,38.7 → 243 tok/s,TTFT p50 11.8 s → 0.2 s,0 → 7.79M hit token(aiperf 侧 cached 92.8%,上限 97.9%) | | K2 ✅ | fork = 引用计数 +1 + 状态快照拷贝:`Runtime::fork(&mut parent, len, tokens)` 从活着的序列分叉,整页共享、半页拷给孩子、per-seq state 拷进新 slot(带状态模型只能在父的当前位置分叉);与 `lease_from` 同一套 `Pool` 决策,父不用先 checkpoint;`lease_from` 也接受纯 KV checkpoint 的更浅整页 | **2026-09-03 tray07 通过**(`k3_golden --fork 12 --seqs 4 --mixed --distinct`,93 层 EP4,4 卡):第 12 步从行 0 分出两个孩子——喂同样 token 的"孪生"28 步逐 token 与行 0 一致(4 个 rank 都是),喂随机 token 的"走失"28 步与它从零跑的参照一致,行 0 对 oracle 仍 38/40 且错的两步(15、16)与 E2b/E3 相同,即孩子写自己的页、父不受影响;4 层 EP1 同样通过。参照 batch 与被测 batch 在同一步同样分叉,行与行的对照始终在同一 bucket(bucket 一变,随机行的近平局就翻,最初的"分叉后各行分歧"全是这个) | | K3 ✅ | session 睡/醒到本 tray DRAM:`reserve_host` 一次 pin 一块(绑到卡所在 NUMA 节点),`park(checkpoint)` 把页和 slot 拷到 host 链上(同 session 再睡只拷新增的页),`wake` → `Waking` → `awake` 落地了才交出 `Lease`;拷贝走 transfer stream,compute stream 从不等它;`Prefix` 分 resident / parked 两层,纯 KV 链条目按页增长、部分命中只醒需要的页;kern-serve `--host-gib` | **2026-09-03 tray08 通过**:qwen3.8-27b 形状 98k token = 6.12 GiB,park 34 ms / wake 31 ms(180 / 197 GiB/s,逐字节验证;pinned 块落在另一颗 Grace 上时 53 / 52 ms,所以 runtime 自己绑 NUMA);qwen3-4b 2500 页 5.49 GiB 31 / 28 ms。kern-serve:8 路 decode 下每秒 ~10 个 3.5 GiB 的 wake,token 间隔 p50 3.03 → 3.08 ms(+1.7%),p90/p99 不变;命中后 warm / cold 64 token 逐字一致。AgentX 回放(qwen3.8 形状,单卡 250 GiB + host 512 GiB,并发 32)命中 93.1% → 95.6%,p99 extend 430k → 208k。命中请求本身的 p90 85 ms 是 prefill 尾块在长上下文上的 40 ms(K5 的事) | | K3b ✅ | kern-pool 重写(设计 `pool.md`):`Node` 的页自创建起冻结(半页 checkpoint 拷给 checkpoint,写者只有 lease),`Storage` trait 下 `Store` = `Checkpoint`、`Store` = `Parked` 一套链和引用计数,device 节点带 host 副本的 `Weak` twin(同 session 再睡、醒来再睡都不再拷),`Prefix` 换成 token 键的 radix tree(条目持有 store、`Hit` 持有 clone,没有 id、没有 `Chain`),wake 回到 resident 再 `restore`;kern-serve stats 与回放按 tier 报命中(`resident_hits` / `host_hits` 与 token 数) | 2026-09-14:`cargo test -p kern-pool` 三个 property test(字节级 lease/checkpoint/restore/fork/retire、park/wake 字节与 host 分区、索引对线性扫描)全绿;AgentX 回放 qwen3.8 形状(`~/bench_results/2026-09-14-pool-radix-replay/`)不带 host 93.1%(同 K1b),host 512 GiB **95.6% → 96.4%**,p99 extend 208k → 130k,1 276 个请求从 host 醒来(4.4 亿 token,输入的 2.1%),0 次 device 淘汰 | diff --git a/docs/runtime.md b/docs/runtime.md index 422676d..d702efa 100644 --- a/docs/runtime.md +++ b/docs/runtime.md @@ -43,7 +43,8 @@ per-seq state(`bytes_per_seq`)**共用一份物理块预算**(`kern-pool` 才 unmap)。计划由 runtime 的后台线程执行:先等 stream 上记的事件(此前入队的 kernel 都过了才 unmap),完成后主线程在下一次 `lease` / `checkpoint` / `lease_from` 里收下、把新 map 的块在 stream 上清零,新页 / 新 slot 才可租。同一 -时刻最多一个计划在飞。租不到时三种拒绝分工明确:`Remapping`(计划已定,落地后再 +时刻最多一个计划在飞;`Runtime::settle` 阻塞到在飞的和随后计划的都落地,给过不了 +这一刻的调用方(history 末尾的 checkpoint)用。租不到时三种拒绝分工明确:`Remapping`(计划已定,落地后再 问,别淘汰)、`Busy`(有被持有的对象挡路,淘汰点什么)、`ExceedsPool`(怎么摆都 放不下)。`seqs.max` 只限一步 batch 的行数;slot 从 `seqs.max + 2` 个起按需长 (session 睡着时它的 checkpoint 拿着 slot,活跃请求再要就从空闲页拆), diff --git a/docs/serve.md b/docs/serve.md index cb29083..c91d439 100644 --- a/docs/serve.md +++ b/docs/serve.md @@ -156,8 +156,9 @@ tok/s,普通模式 353 / 2048 / 6800——交叉点在 bs 16–32,v4 没有 请求结束时 `Runtime::retire`——结束序列的 state slot 原样成为 checkpoint 的,不拷。"续着上一轮 整段上下文"的 prompt 命中后者;回放录下来的回复而不是我们生成的回复的下一轮(AgentX、改过历史 的对话)命中前者,只付上一轮 prompt 之后的部分;同一 prompt 重发也命中前者,只付尾巴。history - 末尾的快照拿不到 slot 时按租约的规则腾地方,腾不出(或 remap 在途)就不留,请求照跑(stats 行 - 的 `prompt_checkpoints` / `prompt_skips`)。 + 末尾的快照拿不到 slot 时按租约的规则腾地方,remap 在途就等它落地(`Tray::settle`,唯一一处 + 阻塞等 remap:这个 state 过了这一刻就留不下了,而 prefill 刚读回输出、stream 已空,等的只是 + remap 本身的几毫秒),腾不出就不留,请求照跑(stats 行的 `prompt_checkpoints` / `prompt_skips`)。 slot 从 manifest 的 `seqs.max + 2` 个起,与 KV 页共用一份显存预算按需互换(K1b): 睡着的 session 的 checkpoint 拿着 slot,活跃请求再要 slot 就从空闲页拆,页不够 再从空闲 slot 拆回来;租约 `Busy` 时按最久未命中淘汰,`Remapping` 时等它落地