DO NOT MERGE: C25 revert probe for #4630 (serve KV back to fixed 4096)

crates/apr-cli/src/commands/serve/handler_gpu_completion.rs

@@ -563,6 +563,11 @@ fn no_vocabulary(what: &str) -> CliError {
 /// explicit override must still beat the flag. The 2048 default now applies
 /// only when neither is set.
 ///
+/// K1 (#4603): with neither set this now returns `None`, and the caller sizes the
+/// KV to the model's context within free VRAM (`OwnedQuantizedModelCuda::for_serving`).
+/// `serve::run` writes `REALIZR_CONTEXT_LENGTH` only for an operator-set flag, so the
+/// 4096 default no longer caps a 32K model's KV at 4096.
+///
 /// NOTE ON SCOPE. This is NOT the same root as #2774 even though both land in
 /// the same allocation. #2774 is a VRAM budget computed before the weights are
 /// resident; this is a flag written to one name and read from another. They are
@@ -571,13 +576,39 @@ fn no_vocabulary(what: &str) -> CliError {
 /// while `--context-length` was ignored the batched KV was half-sized, which is
 /// the only reason the 7B appeared to survive c=4 on a 24 GB card at all.
 #[cfg(any(all(feature = "inference", feature = "cuda"), test))]
-fn resolve_serve_max_seq_len(explicit_override: Option<&str>, context_length: Option<&str>) -> usize {
-    const DEFAULT_MAX_SEQ_LEN: usize = 2048;
+fn resolve_serve_max_seq_len(
+    explicit_override: Option<&str>,
+    context_length: Option<&str>,
+) -> Option<usize> {
     explicit_override
         .and_then(|v| v.parse::<usize>().ok())
         .or_else(|| context_length.and_then(|v| v.parse::<usize>().ok()))
         .filter(|&n| n > 0)
-        .unwrap_or(DEFAULT_MAX_SEQ_LEN)
+}
+
+/// GH-129 + #2762 + K1 (#4603): build the CUDA model a server runs, with its KV sized
+/// from `REALIZR_MAX_SEQ_LEN` / `--context-length` when one is set, else from the
+/// model's context within free VRAM. Every CUDA serve path (GGUF, APR, SafeTensors)
+/// takes this; the APR and SafeTensors paths used `OwnedQuantizedModelCuda::new`, a
+/// fixed 2048, which no flag could move.
+#[cfg(all(feature = "inference", feature = "cuda"))]
+fn serving_cuda_model(
+    model: realizar::gguf::OwnedQuantizedModel,
+) -> std::result::Result<realizar::gguf::OwnedQuantizedModelCuda, realizar::gguf::CudaInitError> {
+    use realizar::gguf::OwnedQuantizedModelCuda;
+    match resolve_serve_max_seq_len(
+        std::env::var("REALIZR_MAX_SEQ_LEN").ok().as_deref(),
+        std::env::var("REALIZR_CONTEXT_LENGTH").ok().as_deref(),
+    ) {
+        Some(n) => {
+            println!("  Max sequence length: {n}");
+            OwnedQuantizedModelCuda::with_max_seq_len(model, 0, n)
+        },
+        None => {
+            println!("  Max sequence length: the model's context, within free VRAM (K1)");
+            OwnedQuantizedModelCuda::for_serving(model, 0)
+        },
+    }
 }
 
 /// Start GGUF server with CUDA acceleration (PAR-111).
@@ -625,14 +656,7 @@ fn start_gguf_server_cuda(
         "Enabling optimized CUDA acceleration (PAR-111)...".cyan()
     );
 
-    // GH-129 + #2762: resolve the KV-cache context length.
-    let max_seq_len = resolve_serve_max_seq_len(
-        std::env::var("REALIZR_MAX_SEQ_LEN").ok().as_deref(),
-        std::env::var("REALIZR_CONTEXT_LENGTH").ok().as_deref(),
-    );
-    println!("  Max sequence length: {max_seq_len}");
-
-    match OwnedQuantizedModelCuda::with_max_seq_len(quantized_model, 0, max_seq_len) {
+    match serving_cuda_model(quantized_model) {
         Ok(mut cuda_model) => {
             preload_gpu_weights(&mut cuda_model);
 
@@ -780,7 +804,7 @@ mod ctx_length_2762_tests {
     fn context_length_flag_reaches_the_kv_cache() {
         assert_eq!(
             resolve_serve_max_seq_len(None, Some("4096")),
-            4096,
+            Some(4096),
             "--context-length is written to REALIZR_CONTEXT_LENGTH and must be \
              the KV cache's max_len; ignoring it sizes the batched KV stride \
              from a constant (#2762)"
@@ -792,24 +816,28 @@ mod ctx_length_2762_tests {
     /// 7.4 GB of unified memory silently gets 4096 back.
     #[test]
     fn explicit_override_still_beats_the_flag() {
-        assert_eq!(resolve_serve_max_seq_len(Some("1024"), Some("4096")), 1024);
+        assert_eq!(
+            resolve_serve_max_seq_len(Some("1024"), Some("4096")),
+            Some(1024)
+        );
     }
 
-    /// DISCRIMINATION: stays GREEN both before and after. With neither set the
-    /// historical default is unchanged, so this fix moves no default.
+    /// K1 (#4603): with neither set there is no fixed length. RED before K1, which
+    /// returned 2048 here -- and `serve::run` always wrote the 4096 default, so a
+    /// Qwen3-8B (40960 context) was served from a 4096-position KV.
     #[test]
-    fn default_is_unchanged_when_nothing_is_set() {
-        assert_eq!(resolve_serve_max_seq_len(None, None), 2048);
+    fn nothing_set_means_size_from_the_model() {
+        assert_eq!(resolve_serve_max_seq_len(None, None), None);
     }
 
     /// A garbled value must not resolve to 0 -- `num_kv_heads * 0 * head_dim`
     /// is a zero-length KV cache, which the allocator accepts and the attention
-    /// kernel then reads out of.
+    /// kernel then reads out of. It falls back to sizing from the model.
     #[test]
-    fn junk_and_zero_fall_back_to_the_default() {
-        assert_eq!(resolve_serve_max_seq_len(None, Some("banana")), 2048);
-        assert_eq!(resolve_serve_max_seq_len(None, Some("0")), 2048);
-        assert_eq!(resolve_serve_max_seq_len(Some("0"), None), 2048);
+    fn junk_and_zero_fall_back_to_the_model() {
+        assert_eq!(resolve_serve_max_seq_len(None, Some("banana")), None);
+        assert_eq!(resolve_serve_max_seq_len(None, Some("0")), None);
+        assert_eq!(resolve_serve_max_seq_len(Some("0"), None), None);
     }
 }
 

crates/apr-cli/src/commands/serve/handlers_include_01.rs

@@ -67,7 +67,7 @@ fn start_apr_server_gpu(
 
     println!("{}", "Enabling fused CUDA acceleration (GH-87)...".cyan());
 
-    let mut cuda_model = OwnedQuantizedModelCuda::new(quantized, 0)
+    let mut cuda_model = serving_cuda_model(quantized)
         .map_err(|e| CliError::InferenceFailed(format!("CUDA init failed: {e}")))?;
 
     preload_gpu_weights(&mut cuda_model);
@@ -262,7 +262,7 @@ fn start_safetensors_server_gpu(
 
     println!("{}", "Enabling fused CUDA acceleration (GH-88)...".cyan());
 
-    let mut cuda_model = OwnedQuantizedModelCuda::new(quantized, 0)
+    let mut cuda_model = serving_cuda_model(quantized)
         .map_err(|e| CliError::InferenceFailed(format!("CUDA init failed: {e}")))?;
 
     preload_gpu_weights(&mut cuda_model);

crates/apr-cli/src/commands/serve/mod.rs

@@ -296,8 +296,13 @@ pub(crate) fn run(model_path: &Path, config: &ServerConfig) -> Result<()> {
         eprintln!("[PMAT-297] Thread pool config: {e} (may already be initialized)");
     }
 
-    // GH-286: Set env vars for realizr's KV cache and FP8 control
-    std::env::set_var("REALIZR_CONTEXT_LENGTH", config.context_length.to_string());
+    // GH-286: Set env vars for realizr's KV cache and FP8 control.
+    // K1 (#4603): only an operator-set `--context-length` is written. The 4096 default
+    // written unconditionally sized every GPU KV to 4096 whatever the model's context;
+    // unset, the GGUF CUDA path sizes the KV to the model's context within free VRAM.
+    if config.context_length != types::DEFAULT_CONTEXT_LENGTH {
+        std::env::set_var("REALIZR_CONTEXT_LENGTH", config.context_length.to_string());
+    }
     if config.no_fp8_cache {
         std::env::set_var("REALIZR_NO_FP8_CACHE", "1");
     }

crates/aprender-serve/src/api/batch.rs

@@ -271,7 +271,7 @@ fn try_apr_generate(
         .generate_with_cache(&prompt_ids, &gen_config)
         .map_err(|e| {
             api_err(
-                StatusCode::INTERNAL_SERVER_ERROR,
+                crate::api::generation_error_status(&e),
                 format!("APR generation failed: {e}"),
             )
         })?;
@@ -321,7 +321,7 @@ fn registry_generate(
 
     let generated = model
         .generate(&prompt, &config)
-        .map_err(|e| api_err(StatusCode::INTERNAL_SERVER_ERROR, e))?;
+        .map_err(|e| api_err(crate::api::generation_error_status(&e), e))?;
 
     let token_ids: Vec<u32> = generated
         .iter()
@@ -515,7 +515,7 @@ fn try_cuda_batch_generate(
             .generate_gpu_resident(&prompt_ids, &q_config)
             .map_err(|e| {
                 api_err(
-                    StatusCode::INTERNAL_SERVER_ERROR,
+                    crate::api::generation_error_status(&e),
                     format!("CUDA generation failed: {e}"),
                 )
             })?;
@@ -622,7 +622,7 @@ fn try_apr_batch_generate(
             .generate_with_cache(&prompt_ids, &gen_config)
             .map_err(|e| {
                 api_err(
-                    StatusCode::INTERNAL_SERVER_ERROR,
+                    crate::api::generation_error_status(&e),
                     format!("APR generation failed: {e}"),
                 )
             })?;
@@ -682,7 +682,7 @@ fn registry_batch_generate(
         let prompt: Vec<usize> = prompt_ids.iter().map(|&id| id as usize).collect();
         let generated = model
             .generate(&prompt, &config)
-            .map_err(|e| api_err(StatusCode::INTERNAL_SERVER_ERROR, e))?;
+            .map_err(|e| api_err(crate::api::generation_error_status(&e), e))?;
         let token_ids: Vec<u32> = generated
             .iter()
             .map(|&id| {

crates/aprender-serve/src/api/batch_processing.rs

@@ -355,7 +355,7 @@ pub async fn gpu_batch_completions_handler(
                 Ok(tokens) => results.push(tokens),
                 Err(e) => {
                     return Err((
-                        StatusCode::INTERNAL_SERVER_ERROR,
+                        crate::api::generation_error_status(&e),
                         Json(ErrorResponse {
                             error: format!("Generation failed: {e}"),
                         }),
@@ -492,7 +492,7 @@ fn try_cuda_generate(
         .generate_gpu_resident(&prompt_ids, &q_config)
         .map_err(|e| {
             api_err(
-                StatusCode::INTERNAL_SERVER_ERROR,
+                crate::api::generation_error_status(&e),
                 format!("CUDA generation failed: {e}"),
             )
         })?;

crates/aprender-serve/src/api/cuda_chat_backend.rs

@@ -61,7 +61,7 @@ fn try_safetensors_cuda_backend(
             let msg = format!("SafeTensors CUDA generation failed: {e}");
             return Some(
                 (
-                    axum::http::StatusCode::INTERNAL_SERVER_ERROR,
+                    crate::api::generation_error_status(&e),
                     axum::Json(serde_json::json!({"error": msg})),
                 )
                     .into_response(),
@@ -118,8 +118,12 @@ async fn try_cuda_backend(
     };
     // GH-319: Use actual model architecture for chat template detection
     let arch_hint = state.model_architecture();
+    // D5: refused before any path (streaming, batch scheduler, direct) takes the
+    // model, so the client gets a 400 rather than a 500 or a mid-stream error.
+    let tokenized =
+        tokenize_chat_prompt(&tokenizer, &request.messages, arch_hint.as_deref(), request.thinking(), state);
     let prompt_ids =
-        match tokenize_chat_prompt(&tokenizer, &request.messages, arch_hint.as_deref(), request.thinking(), state) {
+        match fit_serving_context(state, tokenized) {
             Ok(ids) => ids,
             Err(r) => return Some(r),
         };
@@ -246,7 +250,7 @@ async fn try_cuda_backend(
         let generate_start = std::time::Instant::now();
         let generated = match dense_cuda_turn(&mut cuda_model, &prompt_ids, &q_config, |_| true) {
             Ok(g) => g,
-            Err(e) => return Some(fail_response(state, StatusCode::INTERNAL_SERVER_ERROR, e)),
+            Err(e) => return Some(fail_response(state, crate::api::generation_error_status(&e), e)),
         };
         let _ = timing_tx.send(phase_split(&mut cuda_model, generate_start));
         let tokens: Vec<u32> = generated.iter().skip(prompt_tokens).copied().collect();

crates/aprender-serve/src/api/gpu_completions_handler.rs

@@ -62,7 +62,7 @@ fn try_gpu_completions(
     })?;
     let generated = gpu_model
         .generate(&prompt, &gpu_config)
-        .map_err(|e| rerr(state, StatusCode::INTERNAL_SERVER_ERROR, e))?;
+        .map_err(|e| rerr(state, crate::api::generation_error_status(&e), e))?;
 
     let token_ids: Vec<u32> = generated
         .iter()
@@ -246,7 +246,7 @@ fn registry_completions(
 
     let generated = model
         .generate(&prompt, &config)
-        .map_err(|e| rerr(state, StatusCode::INTERNAL_SERVER_ERROR, e))?;
+        .map_err(|e| rerr(state, crate::api::generation_error_status(&e), e))?;
     let token_ids: Vec<u32> = generated
         .iter()
         .skip(prompt_tokens)
@@ -383,6 +383,13 @@ async fn try_cuda_gguf_completions(
         return Err(rerr(state, StatusCode::BAD_REQUEST, "Prompt cannot be empty"));
     }
     let prompt_tokens = prompt_ids.len();
+    // D5: a 400 before the batch scheduler sees it (its errors come back as strings).
+    if let Some(msg) = state
+        .serving_context()
+        .and_then(|ctx| super::serve_context_refusal(prompt_tokens, ctx))
+    {
+        return Err(rerr(state, StatusCode::BAD_REQUEST, msg));
+    }
 
     let eos = state.cached_eos_token_id.unwrap_or(151643);
     let q_config = QuantizedGenerateConfig {
@@ -610,7 +617,7 @@ async fn completions_inner(
             model.generate_gpu_resident_logprobs(
                 &prompt_ids.iter().map(|&id| id as u32).collect::<Vec<_>>(),
                 &config,
-            ).map_err(|e| rerr(&state, StatusCode::INTERNAL_SERVER_ERROR, e))?
+            ).map_err(|e| rerr(&state, super::generation_error_status(&e), e))?
         };
         let prompt_len = prompt_ids.len();
         let gen_tokens: Vec<u32> = result.tokens[prompt_len..].to_vec();
@@ -699,7 +706,7 @@ pub async fn logprobs_handler(
         model.generate_gpu_resident_logprobs(
             &prompt_ids.iter().map(|&x| x as u32).collect::<Vec<_>>(),
             &config,
-        ).map_err(|e| rerr(&state, StatusCode::INTERNAL_SERVER_ERROR, e))?
+        ).map_err(|e| rerr(&state, crate::api::generation_error_status(&e), e))?
     };
 
     let prompt_len = prompt_ids.len();
@@ -766,7 +773,7 @@ pub async fn perplexity_handler(
     // realizr#203: Run BOTH paths for comparison during development
     let ppl_sequential = model
         .perplexity_gpu_resident(&token_ids)
-        .map_err(|e| rerr(&state, StatusCode::INTERNAL_SERVER_ERROR, e))?;
+        .map_err(|e| rerr(&state, crate::api::generation_error_status(&e), e))?;
     let ppl_batched = model.perplexity_gpu_batched(&token_ids).ok();
 
     drop(model);

crates/aprender-serve/src/api/mod.rs

@@ -235,6 +235,10 @@ pub struct AppState {
     qwen35_session: Option<Arc<Qwen35Served>>,
     /// GH-330: Cached EOS token ID (avoids RwLock in hot path)
     cached_eos_token_id: Option<u32>,
+    /// D5 (ruling-3715-2010): positions a dense CUDA serve turn can reach, the
+    /// model's context capped by the device KV cache. Cached at construction so
+    /// the pre-flight length check never waits on the scheduler's write lock.
+    cached_serving_context: Option<usize>,
     /// GH-152: Enable verbose request/response logging
     verbose: bool,
     /// GH-103: Enable inference tracing (propagates into QuantizedGenerateConfig.trace)
@@ -377,6 +381,27 @@ pub(crate) fn generation_error_status(err: &RealizarError) -> StatusCode {
     }
 }
 
+/// D5 (ruling-3715-2010): the refusal for a prompt that cannot fit a serve turn,
+/// or `None` when it fits with room to answer.
+///
+/// `serving_context` is the model's context capped by the device KV cache
+/// ([`AppState::serving_context`]). A prompt at or past it used to reach the GPU
+/// forward and come back as a 500 ("the device KV cache holds 4096"), which
+/// tells the client the server broke. It is decided by the request alone, so it
+/// is a 400 and says which limit it hit.
+pub(crate) fn serve_context_refusal(
+    prompt_tokens: usize,
+    serving_context: usize,
+) -> Option<String> {
+    (prompt_tokens >= serving_context).then(|| {
+        format!(
+            "context exceeds {serving_context}: the prompt is {prompt_tokens} tokens and this \
+             server holds {serving_context} positions per turn (the model's context, capped by \
+             the GPU KV cache). It was refused whole, not truncated."
+        )
+    })
+}
+
 include!("mod_app_state_gpu.rs");
 include!("mod_app_state_qwen35.rs");
 include!("mod_create_demo.rs");