From 4c2e063ff047174b4c2d82ed46d34ff291aad794 Mon Sep 17 00:00:00 2001 From: gilby Date: Thu, 29 Oct 2026 00:29:33 +0100 Subject: [PATCH] glm cuda: prompt chunks of 4,096 rows, and the DFlash2 context buffers sized to match A 2,048-row chunk leaves the prompt step's per-chunk costs a larger share of a cold prompt. 4,096 rows still fits four 500K windows at TP=3, and at TP=2 without a drafter, and the output does not change. The drafter's prompt taps and context buffers took MAX_CTX = 2048 rows, which a wider chunk overran at startup with a drafter loaded; MAX_CTX now follows prefill_rows. Fixes #1570 --- zig/src/families/glm/cuda_state.zig | 2 +- zig/src/families/glm/dflash.zig | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/zig/src/families/glm/cuda_state.zig b/zig/src/families/glm/cuda_state.zig index 68cdd0c29b..70f3d9ec90 100644 --- a/zig/src/families/glm/cuda_state.zig +++ b/zig/src/families/glm/cuda_state.zig @@ -16,7 +16,7 @@ const mlp = @import("cuda_mlp.zig"); const kda_block = @import("cuda_kda_block.zig"); /// A prompt chunk's rows (decode.PREFILL_ROWS). -pub const prefill_rows = 2048; +pub const prefill_rows = 4096; /// A decode window's rows (Engine's max_rows). pub const window_rows = 8; /// Prompts one prompt pass takes together (cuda_prompts): a prompt buffer's logits rows, one a prompt. diff --git a/zig/src/families/glm/dflash.zig b/zig/src/families/glm/dflash.zig index 6b67d76489..d69289181f 100644 --- a/zig/src/families/glm/dflash.zig +++ b/zig/src/families/glm/dflash.zig @@ -27,7 +27,7 @@ pub const CH: usize = 16; // key chunks of the chunked attention (df_attn_part) pub const CW: usize = 2 * TOPK; // a row's gathered ints: 16 ids, then 16 values' bits pub const MAX_S: usize = 16; // streams one block pass drafts for pub const MAX_M: usize = MAX_S * BLOCK; // block rows at most -pub const MAX_CTX: usize = 2048; // context rows one ingest takes (a prompt chunk's) +pub const MAX_CTX: usize = @import("cuda_state.zig").prefill_rows; // context rows one ingest takes (a prompt chunk's) pub const STEP_ROWS: usize = 32; // a verify's tap rows (the shared windows' rows at most) pub const TAP_ROW: usize = taps_mod.layers.len * D * 2; // bytes of one row's taps