fix(#4621): apr chat sizes its device KV to the context (D3-SPLIT C, root 3)

crates/apr-cli/src/commands/chat_load_tokenizers.rs

@@ -266,7 +266,8 @@ fn try_init_gguf_cuda(
             return Ok((None, true));
         }
     };
-    match OwnedQuantizedModelCuda::new(owned, 0) {
+    // #3715: the turn length is unknown at load; size the device KV to the context.
+    match OwnedQuantizedModelCuda::for_session(owned, 0) {
         Ok(cuda_model) => {
             println!(
                 "{}",
@@ -348,7 +349,7 @@ fn try_init_apr_cuda(
     };
     // #3955: read what loaded BEFORE the model moves into the CUDA wrapper.
     let qtypes = loaded_weight_qtypes(&model);
-    let cuda_model = match OwnedQuantizedModelCuda::new(model, 0) {
+    let cuda_model = match OwnedQuantizedModelCuda::for_session(model, 0) {
         Ok(m) => m,
         Err(e) => {
             eprintln!("[APR CUDA init failed: {e}, will use CPU]");

crates/apr-cli/src/commands/serve/handler_gpu_completion.rs

@@ -563,6 +563,11 @@ fn no_vocabulary(what: &str) -> CliError {
 /// explicit override must still beat the flag. The 2048 default now applies
 /// only when neither is set.
 ///
+/// K1 (#4603): with neither set this now returns `None`, and the caller sizes the
+/// KV to the model's context within free VRAM (`OwnedQuantizedModelCuda::for_serving`).
+/// `serve::run` writes `REALIZR_CONTEXT_LENGTH` only for an operator-set flag, so the
+/// 4096 default no longer caps a 32K model's KV at 4096.
+///
 /// NOTE ON SCOPE. This is NOT the same root as #2774 even though both land in
 /// the same allocation. #2774 is a VRAM budget computed before the weights are
 /// resident; this is a flag written to one name and read from another. They are
@@ -571,13 +576,39 @@ fn no_vocabulary(what: &str) -> CliError {
 /// while `--context-length` was ignored the batched KV was half-sized, which is
 /// the only reason the 7B appeared to survive c=4 on a 24 GB card at all.
 #[cfg(any(all(feature = "inference", feature = "cuda"), test))]
-fn resolve_serve_max_seq_len(explicit_override: Option<&str>, context_length: Option<&str>) -> usize {
-    const DEFAULT_MAX_SEQ_LEN: usize = 2048;
+fn resolve_serve_max_seq_len(
+    explicit_override: Option<&str>,
+    context_length: Option<&str>,
+) -> Option<usize> {
     explicit_override
         .and_then(|v| v.parse::<usize>().ok())
         .or_else(|| context_length.and_then(|v| v.parse::<usize>().ok()))
         .filter(|&n| n > 0)
-        .unwrap_or(DEFAULT_MAX_SEQ_LEN)
+}
+
+/// GH-129 + #2762 + K1 (#4603): build the CUDA model a server runs, with its KV sized
+/// from `REALIZR_MAX_SEQ_LEN` / `--context-length` when one is set, else from the
+/// model's context within free VRAM. Every CUDA serve path (GGUF, APR, SafeTensors)
+/// takes this; the APR and SafeTensors paths used `OwnedQuantizedModelCuda::new`, a
+/// fixed 2048, which no flag could move.
+#[cfg(all(feature = "inference", feature = "cuda"))]
+fn serving_cuda_model(
+    model: realizar::gguf::OwnedQuantizedModel,
+) -> std::result::Result<realizar::gguf::OwnedQuantizedModelCuda, realizar::gguf::CudaInitError> {
+    use realizar::gguf::OwnedQuantizedModelCuda;
+    match resolve_serve_max_seq_len(
+        std::env::var("REALIZR_MAX_SEQ_LEN").ok().as_deref(),
+        std::env::var("REALIZR_CONTEXT_LENGTH").ok().as_deref(),
+    ) {
+        Some(n) => {
+            println!("  Max sequence length: {n}");
+            OwnedQuantizedModelCuda::with_max_seq_len(model, 0, n)
+        },
+        None => {
+            println!("  Max sequence length: the model's context, within free VRAM (K1)");
+            OwnedQuantizedModelCuda::for_serving(model, 0)
+        },
+    }
 }
 
 /// Start GGUF server with CUDA acceleration (PAR-111).
@@ -625,18 +656,11 @@ fn start_gguf_server_cuda(
         "Enabling optimized CUDA acceleration (PAR-111)...".cyan()
     );
 
-    // GH-129 + #2762: resolve the KV-cache context length.
-    let max_seq_len = resolve_serve_max_seq_len(
-        std::env::var("REALIZR_MAX_SEQ_LEN").ok().as_deref(),
-        std::env::var("REALIZR_CONTEXT_LENGTH").ok().as_deref(),
-    );
-    println!("  Max sequence length: {max_seq_len}");
-
     // #4254: measured before the model moves into CUDA. Without it `/v1/effective-config`
     // on a CUDA server reported quantization, path and parameter count all null.
     let model_source = measured_model_source(&quantized_model, Some(&mapped_model), config);
 
-    match OwnedQuantizedModelCuda::with_max_seq_len(quantized_model, 0, max_seq_len) {
+    match serving_cuda_model(quantized_model) {
         Ok(mut cuda_model) => {
             preload_gpu_weights(&mut cuda_model);
 
@@ -785,7 +809,7 @@ mod ctx_length_2762_tests {
     fn context_length_flag_reaches_the_kv_cache() {
         assert_eq!(
             resolve_serve_max_seq_len(None, Some("4096")),
-            4096,
+            Some(4096),
             "--context-length is written to REALIZR_CONTEXT_LENGTH and must be \
              the KV cache's max_len; ignoring it sizes the batched KV stride \
              from a constant (#2762)"
@@ -797,24 +821,28 @@ mod ctx_length_2762_tests {
     /// 7.4 GB of unified memory silently gets 4096 back.
     #[test]
     fn explicit_override_still_beats_the_flag() {
-        assert_eq!(resolve_serve_max_seq_len(Some("1024"), Some("4096")), 1024);
+        assert_eq!(
+            resolve_serve_max_seq_len(Some("1024"), Some("4096")),
+            Some(1024)
+        );
     }
 
-    /// DISCRIMINATION: stays GREEN both before and after. With neither set the
-    /// historical default is unchanged, so this fix moves no default.
+    /// K1 (#4603): with neither set there is no fixed length. RED before K1, which
+    /// returned 2048 here -- and `serve::run` always wrote the 4096 default, so a
+    /// Qwen3-8B (40960 context) was served from a 4096-position KV.
     #[test]
-    fn default_is_unchanged_when_nothing_is_set() {
-        assert_eq!(resolve_serve_max_seq_len(None, None), 2048);
+    fn nothing_set_means_size_from_the_model() {
+        assert_eq!(resolve_serve_max_seq_len(None, None), None);
     }
 
     /// A garbled value must not resolve to 0 -- `num_kv_heads * 0 * head_dim`
     /// is a zero-length KV cache, which the allocator accepts and the attention
-    /// kernel then reads out of.
+    /// kernel then reads out of. It falls back to sizing from the model.
     #[test]
-    fn junk_and_zero_fall_back_to_the_default() {
-        assert_eq!(resolve_serve_max_seq_len(None, Some("banana")), 2048);
-        assert_eq!(resolve_serve_max_seq_len(None, Some("0")), 2048);
-        assert_eq!(resolve_serve_max_seq_len(Some("0"), None), 2048);
+    fn junk_and_zero_fall_back_to_the_model() {
+        assert_eq!(resolve_serve_max_seq_len(None, Some("banana")), None);
+        assert_eq!(resolve_serve_max_seq_len(None, Some("0")), None);
+        assert_eq!(resolve_serve_max_seq_len(Some("0"), None), None);
     }
 }
 

crates/apr-cli/src/commands/serve/handlers_include_01.rs

@@ -67,7 +67,7 @@ fn start_apr_server_gpu(
 
     println!("{}", "Enabling fused CUDA acceleration (GH-87)...".cyan());
 
-    let mut cuda_model = OwnedQuantizedModelCuda::new(quantized, 0)
+    let mut cuda_model = serving_cuda_model(quantized)
         .map_err(|e| CliError::InferenceFailed(format!("CUDA init failed: {e}")))?;
 
     preload_gpu_weights(&mut cuda_model);
@@ -262,7 +262,7 @@ fn start_safetensors_server_gpu(
 
     println!("{}", "Enabling fused CUDA acceleration (GH-88)...".cyan());
 
-    let mut cuda_model = OwnedQuantizedModelCuda::new(quantized, 0)
+    let mut cuda_model = serving_cuda_model(quantized)
         .map_err(|e| CliError::InferenceFailed(format!("CUDA init failed: {e}")))?;
 
     preload_gpu_weights(&mut cuda_model);

crates/apr-cli/src/commands/serve/mod.rs

@@ -301,8 +301,13 @@ pub(crate) fn run(model_path: &Path, config: &ServerConfig) -> Result<()> {
         eprintln!("[PMAT-297] Thread pool config: {e} (may already be initialized)");
     }
 
-    // GH-286: Set env vars for realizr's KV cache and FP8 control
-    std::env::set_var("REALIZR_CONTEXT_LENGTH", config.context_length.to_string());
+    // GH-286: Set env vars for realizr's KV cache and FP8 control.
+    // K1 (#4603): only an operator-set `--context-length` is written. The 4096 default
+    // written unconditionally sized every GPU KV to 4096 whatever the model's context;
+    // unset, the GGUF CUDA path sizes the KV to the model's context within free VRAM.
+    if config.context_length != types::DEFAULT_CONTEXT_LENGTH {
+        std::env::set_var("REALIZR_CONTEXT_LENGTH", config.context_length.to_string());
+    }
     if config.no_fp8_cache {
         std::env::set_var("REALIZR_NO_FP8_CACHE", "1");
     }

crates/aprender-serve/src/api/tests/mod.rs

@@ -56,6 +56,8 @@ mod sse_stream_stop_markers_4340; // #4340: <|im_end|> must never reach a live S
 mod native_im_end_4344; // #4344: /generate, /batch/generate, /stream/generate, /realize/* cut spelled-out <|im_end|>
 mod serve_dead_routes_3991; // aprender#3991: every generation route GET / lists must generate; text is the completion
 mod native_routes_2376; // aprender#2376: native routes on a quantized server, KV-cache budget, sampling fields
+#[cfg(feature = "cuda")]
+mod serve_kv_limit_k1; // K1 (#4603): serve KV = model context; at-limit 200 whole, past it a 400
 mod router_flags; // --no-cors / --no-metrics must change HTTP behaviour, not just the banner
 mod ollama_compat_http; // Dogfood 0.63.0 (#2396/#2402): /api/tags|show|version routed, stream:true is NDJSON, /realize/* stops fabricating
 mod embed_and_envelope_2376; // aprender#2376(1 seventh route, 7, 8) + #2396(2): embeddings on a quantized server, one error envelope, / and /ready

crates/aprender-serve/src/api/tests/serve_kv_limit_k1.rs

@@ -0,0 +1,203 @@
+//! K1: `apr serve` sizes the device KV from the model's context, so a prompt at
+//! the limit is answered whole and one past it is refused with a 400.
+//!
+//! Before K1 serve fixed the KV at 4096 whatever the model declared, so on a
+//! model whose context is larger a prompt past 4096 tokens overflowed the cache
+//! and came back as a 500. This builds an 8192-context model (above the old
+//! 4096, so a return to the fixed size fails the KV assertion), serves it
+//! through the real router and batch scheduler on the device, and checks three
+//! things. At the limit it answers 200 with every requested token. At the limit
+//! and one past it, it answers 400 and says the prompt was refused whole.
+//!
+//! The model is synthetic (no file), so the test runs on any CUDA runner.
+//! Without a device it panics: a run that could not reach the device measured
+//! nothing, so it is a failure, never a pass (L25). It is not `#[ignore]`d,
+//! because `cuda-unit` runs the cuda-only lib tests without `--ignored`.
+
+use crate::api::{create_router, AppState};
+use crate::gguf::test_helpers::create_test_model_with_config;
+use crate::gguf::{
+    ArchConstraints, GGUFConfig, OwnedQKVWeights, OwnedQuantizedModel, OwnedQuantizedModelCuda,
+    OwnedQuantizedTensor,
+};
+use axum::body::Body;
+use axum::http::{Request, StatusCode};
+use tower::ServiceExt;
+
+/// Above serve's pre-K1 fixed 4096, so a fixed size cannot pass.
+const CONTEXT: usize = 8192;
+const VOCAB: usize = 256;
+/// Generation per request. At the limit, prompt + this == `CONTEXT`.
+const MAX_TOKENS: usize = 4;
+
+fn config() -> GGUFConfig {
+    GGUFConfig {
+        architecture: "llama".to_string(),
+        constraints: ArchConstraints::from_architecture("llama"),
+        hidden_dim: 256,
+        intermediate_dim: 512,
+        num_heads: 4,
+        num_kv_heads: 4,
+        num_layers: 1,
+        vocab_size: VOCAB,
+        rope_theta: 10000.0,
+        context_length: CONTEXT,
+        eps: 1e-5,
+        rope_type: 0,
+        explicit_head_dim: None,
+        query_pre_attn_scalar: None,
+        bos_token_id: None,
+        eos_token_id: None,
+    }
+}
+
+/// One token per `a`: the only single-character entries are the letters, and the
+/// greedy matcher has no longer match for a run of `a`.
+fn vocab() -> Vec<String> {
+    let mut v: Vec<String> = vec!["<unk>".to_string()];
+    v.extend(('a'..='z').map(String::from));
+    v.extend((v.len()..VOCAB).map(|i| format!("<t{i}>")));
+    v
+}
+
+/// Q4_K weights small and centred on zero (d = 2^-10, dmin = 2^-7, pseudo-random
+/// scales and quants). The shared `create_q4k_test_data` pattern is all-positive
+/// at d = 1.0, which saturates a forward pass until the device parity gate
+/// (rightly) refuses it; this one keeps activations near unit scale.
+fn q4k(in_dim: usize, out_dim: usize, seed: u64) -> OwnedQuantizedTensor {
+    let blocks = in_dim.div_ceil(256);
+    let mut data = vec![0u8; out_dim * blocks * 144];
+    let mut x = seed.wrapping_mul(6_364_136_223_846_793_005).wrapping_add(1);
+    for block in data.chunks_exact_mut(144) {
+        block[0..2].copy_from_slice(&0x1400_u16.to_le_bytes());
+        block[2..4].copy_from_slice(&0x2000_u16.to_le_bytes());
+        for b in &mut block[4..] {
+            x = x
+                .wrapping_mul(6_364_136_223_846_793_005)
+                .wrapping_add(1_442_695_040_888_963_407);
+            *b = (x >> 56) as u8;
+        }
+    }
+    OwnedQuantizedTensor {
+        data,
+        in_dim,
+        out_dim,
+        qtype: 12,
+    }
+}
+
+/// A llama-shaped model with separate Q/K/V, as a llama GGUF loads: the
+/// GPU-resident path serve generates on takes nothing else
+/// (`supports_gpu_resident`).
+fn model() -> OwnedQuantizedModel {
+    let c = config();
+    let (h, kv, ffn) = (
+        c.hidden_dim,
+        c.num_kv_heads * (c.hidden_dim / c.num_heads),
+        c.intermediate_dim,
+    );
+    let mut model = create_test_model_with_config(&c);
+    for (i, layer) in model.layers.iter_mut().enumerate() {
+        let s = 10 * i as u64;
+        layer.qkv_weight = OwnedQKVWeights::Separate {
+            q: q4k(h, h, s + 1),
+            k: q4k(h, kv, s + 2),
+            v: q4k(h, kv, s + 3),
+        };
+        layer.attn_output_weight = q4k(h, h, s + 4);
+        layer.ffn_up_weight = q4k(h, ffn, s + 5);
+        layer.ffn_down_weight = q4k(ffn, h, s + 6);
+        layer.ffn_gate_weight = Some(q4k(h, ffn, s + 7));
+        // llama normalises before the FFN; the shared fixture leaves it out.
+        layer.ffn_norm_weight = Some(vec![1.0; h]);
+    }
+    model.lm_head_weight = q4k(h, VOCAB, 99);
+    model
+}
+
+/// The router `apr serve` builds for a CUDA GGUF: the model sized by
+/// `for_serving`, behind the continuous-batching scheduler. Panics without a
+/// CUDA device: an unreachable device is not measured, never a pass (L25).
+fn served() -> (axum::Router, usize) {
+    let cuda = match OwnedQuantizedModelCuda::for_serving(model(), 0) {
+        Ok(m) => m,
+        Err(e) => panic!("K1 not measured: CUDA model init failed: {}", e.error),
+    };
+    let kv = cuda.executor().max_kv_len();
+    let state = AppState::with_cuda_model_and_vocab(cuda, vocab()).expect("state");
+    let model = state.cuda_model().expect("the CUDA model").clone();
+    let tx = crate::api::cuda_batch_scheduler::spawn_cuda_batch_scheduler(
+        model,
+        crate::api::cuda_batch_scheduler::CudaBatchConfig::default(),
+        crate::api::InFlightCounter::new(),
+    );
+    (create_router(state.with_cuda_batch_tx(tx)), kv)
+}
+
+async fn complete(app: &axum::Router, prompt_tokens: usize) -> (StatusCode, serde_json::Value) {
+    let body = serde_json::json!({
+        "model": "k1",
+        "prompt": "a".repeat(prompt_tokens),
+        "max_tokens": MAX_TOKENS,
+        "temperature": 0.0,
+    });
+    let response = app
+        .clone()
+        .oneshot(
+            Request::builder()
+                .method("POST")
+                .uri("/v1/completions")
+                .header("content-type", "application/json")
+                .body(Body::from(body.to_string()))
+                .expect("request"),
+        )
+        .await
+        .expect("the router answers");
+    let status = response.status();
+    let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)
+        .await
+        .expect("body");
+    let json = serde_json::from_slice(&bytes)
+        .unwrap_or_else(|_| serde_json::json!({ "raw": String::from_utf8_lossy(&bytes) }));
+    (status, json)
+}
+
+#[tokio::test(flavor = "multi_thread")]
+async fn k1_serve_kv_is_the_model_context_and_the_limit_is_a_clean_400() {
+    let (app, kv) = served();
+    assert_eq!(
+        kv, CONTEXT,
+        "K1: the serving KV is the model's context, not a fixed size"
+    );
+
+    // At the limit: prompt + generation fill the context exactly. Answered
+    // whole: 200, every requested token, the prompt counted as sent.
+    let at_limit = CONTEXT - MAX_TOKENS;
+    let (status, json) = complete(&app, at_limit).await;
+    assert_eq!(
+        status,
+        StatusCode::OK,
+        "at the limit ({at_limit} + {MAX_TOKENS}): {json}"
+    );
+    assert_eq!(json["usage"]["prompt_tokens"], at_limit, "{json}");
+    assert_eq!(
+        json["usage"]["completion_tokens"], MAX_TOKENS,
+        "full output: {json}"
+    );
+    assert!(json["choices"][0]["text"].is_string(), "{json}");
+    eprintln!(
+        "K1 at-limit: 200, prompt {at_limit}, completion {MAX_TOKENS}, finish {}",
+        json["choices"][0]["finish_reason"]
+    );
+
+    // At and past the limit: refused before it reaches the device, as a 400
+    // that says so. Not a 500, and not a truncated 200.
+    for prompt in [CONTEXT, CONTEXT + 1] {
+        let (status, json) = complete(&app, prompt).await;
+        assert_eq!(status, StatusCode::BAD_REQUEST, "prompt {prompt}: {json}");
+        let text = json.to_string();
+        assert!(text.contains("context exceeds"), "prompt {prompt}: {text}");
+        assert!(text.contains("refused whole"), "prompt {prompt}: {text}");
+        eprintln!("K1 over-limit: prompt {prompt} → 400 {text}");
+    }
+}