diff --git a/Cargo.toml b/Cargo.toml index 9d2db4927f..7db1b1c0c1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -624,6 +624,14 @@ gates = [ # silence. RED when evidence/perf041//marker.json is absent, is not # PASS, or is older than perf-matrix.yaml `witness.max_age_days`. "scripts/check_perf041_marker.sh", + # #3076: apr's F16/BF16 CPU matvec against the pinned llama.cpp, greedy, on the + # template-free models its committed receipts name (evidence/pmat-3076-f16-bf16-matvec/). + # A model absent from the release host is a FAIL. This list is + # [package.metadata.dogfood].gates: only `scripts/dogfood.sh --phase pre-publish` + # executes it, at release time, on the host that has the models. Nothing on the PR path + # runs this entry. The PR-time half, the comparison helpers' case table, is + # scripts/check_float16_greedy_parity.sh, which guard_tree.sh dispatches. + "scripts/float16_greedy_parity.sh", ] # The interfaces this release SHIPS, each paired with the e2e target that proves diff --git a/crates/apr-cli/src/commands/hex.rs b/crates/apr-cli/src/commands/hex.rs index 0df93181fb..015dd0775d 100644 --- a/crates/apr-cli/src/commands/hex.rs +++ b/crates/apr-cli/src/commands/hex.rs @@ -116,6 +116,19 @@ fn format_display_name(fmt: FileFormat) -> &'static str { } } +/// The format a model file's magic bytes identify, for an error message: +/// "APR", "GGUF", "SafeTensors", or "model" when they identify none or the +/// file cannot be read (#3661). Reads at most 9 bytes. +pub(crate) fn detected_model_format(path: &Path) -> &'static str { + use std::io::Read; + let mut head = Vec::with_capacity(9); + let read = std::fs::File::open(path).and_then(|f| f.take(9).read_to_end(&mut head)); + match read { + Ok(_) => detect_format(&head).map_or("model", format_display_name), + Err(_) => "model", + } +} + // ============================================================================ // Main entry point // ============================================================================ diff --git a/crates/apr-cli/src/commands/inference_output.rs b/crates/apr-cli/src/commands/inference_output.rs index ff185c912d..fd84665dd6 100644 --- a/crates/apr-cli/src/commands/inference_output.rs +++ b/crates/apr-cli/src/commands/inference_output.rs @@ -327,7 +327,8 @@ fn execute_with_realizar( .with_top_p(options.top_p) .with_seed(options.seed) .with_repeat_penalty(options.repeat_penalty) - .with_repeat_last_n(options.repeat_last_n); + .with_repeat_last_n(options.repeat_last_n) + .with_force_chat_template(options.chat_template); if options.no_gpu { config = config.without_gpu(); diff --git a/crates/apr-cli/src/commands/inspect.rs b/crates/apr-cli/src/commands/inspect.rs index f3dc223705..6fc74333ac 100644 --- a/crates/apr-cli/src/commands/inspect.rs +++ b/crates/apr-cli/src/commands/inspect.rs @@ -378,7 +378,7 @@ fn run_rosetta_inspect( let rosetta = RosettaStone::new(); let report = rosetta .inspect(path) - .map_err(|e| CliError::InvalidFormat(format!("Inspection failed: {e}")))?; + .map_err(|e| CliError::invalid_model_file(path, "Inspection failed", &e))?; if json_output { output_rosetta_json(path, &report); diff --git a/crates/apr-cli/src/commands/mod.rs b/crates/apr-cli/src/commands/mod.rs index 9476956249..0c2be4309d 100644 --- a/crates/apr-cli/src/commands/mod.rs +++ b/crates/apr-cli/src/commands/mod.rs @@ -106,6 +106,9 @@ pub(crate) mod mcp; pub(crate) mod merge; #[cfg(feature = "training")] pub(crate) mod model_config; +// #3661: a model parse failure names the format its magic bytes identify. +#[cfg(test)] +mod model_file_error_tests_3661; #[cfg(feature = "training")] pub(crate) mod monitor; #[cfg(feature = "dev")] diff --git a/crates/apr-cli/src/commands/model_file_error_tests_3661.rs b/crates/apr-cli/src/commands/model_file_error_tests_3661.rs new file mode 100644 index 0000000000..d19581fc11 --- /dev/null +++ b/crates/apr-cli/src/commands/model_file_error_tests_3661.rs @@ -0,0 +1,143 @@ +//! #3661: a model file that fails to parse is named by what its magic bytes say it is. +//! +//! `CliError::InvalidFormat` Displays "Invalid APR format", and `apr inspect` and +//! `apr tensors` wrapped every parse failure in it, so a truncated GGUF was reported +//! as an invalid APR file. Each row drives the real command function and renders the +//! error exactly as `apr` prints it (`error: {e}`, main.rs), then checks three things: +//! the format the message names, exit code 4, and that the core error's +//! "Invalid model format:" prefix never appears under the CLI's own. + +use crate::commands::{inspect, tensors}; +use crate::error::CliError; +use std::path::{Path, PathBuf}; + +/// A GGUF written by aprender's own exporter, cut off inside its metadata. +/// That is the issue's input: a real GGUF truncated mid-header. +fn truncated_gguf(dir: &Path) -> PathBuf { + use aprender::format::gguf::{export_tensors_to_gguf, GgmlType, GgufTensor, GgufValue}; + let tensors = vec![GgufTensor { + name: "token_embd.weight".to_string(), + shape: vec![4, 8], + dtype: GgmlType::F32, + data: vec![0u8; 4 * 8 * 4], + }]; + let metadata = vec![ + ( + "general.architecture".to_string(), + GgufValue::String("llama".to_string()), + ), + ( + "general.name".to_string(), + GgufValue::String("truncated-3661".to_string()), + ), + ]; + let mut bytes = Vec::new(); + export_tensors_to_gguf(&mut bytes, &tensors, &metadata).expect("write GGUF"); + // 24 bytes of fixed header (magic, version, two counts), then the first key's u64 + // length: cut 4 bytes into it, so the reader hits the issue's own failure, + // "Unexpected EOF reading u64". + let path = dir.join("t.gguf"); + std::fs::write(&path, &bytes[..28]).expect("write truncated GGUF"); + path +} + +/// 4 KiB of noise named `.safetensors`, opening with the issue's own magic bytes +/// (`8e44f12f`). Its first u64 is far past any real header length, so no format matches. +fn random_safetensors(dir: &Path) -> PathBuf { + let mut state: u32 = 0x3661; + let mut bytes: Vec = vec![0x8e, 0x44, 0xf1, 0x2f, 0x9d, 0x27, 0x6b, 0xb3]; + while bytes.len() < 4096 { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + bytes.push((state >> 24) as u8); + } + let path = dir.join("t.safetensors"); + std::fs::write(&path, &bytes).expect("write random bytes"); + path +} + +/// A real APR v2 file cut to 32 bytes, less than its 64-byte header. +fn truncated_apr(dir: &Path) -> PathBuf { + use aprender::format::v2::{AprV2Metadata, AprV2Writer}; + let mut writer = AprV2Writer::new(AprV2Metadata::new("truncated-3661")); + writer.add_f32_tensor("w", vec![2, 2], &[0.0, 1.0, 2.0, 3.0]); + let bytes = writer.write().expect("write APR v2"); + let path = dir.join("t.apr"); + std::fs::write(&path, &bytes[..32]).expect("write truncated APR"); + path +} + +fn inspect_err(path: &Path) -> CliError { + inspect::run(path, false, false, false, false, false).expect_err("inspect must refuse") +} + +fn tensors_err(path: &Path) -> CliError { + tensors::run(path, false, None, false, 0).expect_err("tensors must refuse") +} + +#[test] +fn a_parse_failure_names_the_format_the_magic_bytes_identify() { + let dir = tempfile::tempdir().expect("tempdir"); + let gguf = truncated_gguf(dir.path()); + let st = random_safetensors(dir.path()); + let apr = truncated_apr(dir.path()); + + // (row, the error, the prefix the rendered line must start with) + let rows: Vec<(&str, CliError, &str)> = vec![ + ( + "inspect truncated GGUF", + inspect_err(&gguf), + "error: Invalid GGUF file: ", + ), + ( + "tensors truncated GGUF", + tensors_err(&gguf), + "error: Invalid GGUF file: ", + ), + ( + "inspect random .safetensors", + inspect_err(&st), + "error: Invalid model file: ", + ), + ( + "tensors random .safetensors", + tensors_err(&st), + "error: Invalid model file: ", + ), + // A truncated .apr still says APR, so the fix is not "delete the word". + ( + "inspect truncated APR", + inspect_err(&apr), + "error: Invalid APR ", + ), + ( + "tensors truncated APR", + tensors_err(&apr), + "error: Invalid APR file: ", + ), + ]; + + let mut failures = Vec::new(); + for (row, err, want_prefix) in &rows { + let rendered = format!("error: {err}"); + let is_apr = want_prefix.starts_with("error: Invalid APR"); + if !rendered.starts_with(want_prefix) { + failures.push(format!("{row}: wanted `{want_prefix}…`, got `{rendered}`")); + } + if !is_apr && rendered.contains("Invalid APR") { + failures.push(format!("{row}: calls a non-APR file APR: `{rendered}`")); + } + if rendered.contains("Invalid model format") { + failures.push(format!( + "{row}: the core prefix leaked under the CLI's: `{rendered}`" + )); + } + if err.exit_code_value() != 4 { + failures.push(format!("{row}: exit {} (wanted 4)", err.exit_code_value())); + } + } + assert!( + failures.is_empty(), + "#3661 rows failed:\n{}", + failures.join("\n") + ); +} diff --git a/crates/apr-cli/src/commands/profile_safetensors.rs b/crates/apr-cli/src/commands/profile_safetensors.rs index 8ef4572d56..dab880ef4b 100644 --- a/crates/apr-cli/src/commands/profile_safetensors.rs +++ b/crates/apr-cli/src/commands/profile_safetensors.rs @@ -29,7 +29,7 @@ fn profile_safetensors_real( let rosetta = aprender::format::rosetta::RosettaStone::new(); let report = rosetta .inspect(path) - .map_err(|e| CliError::InvalidFormat(format!("Inspection failed: {e}")))?; + .map_err(|e| CliError::invalid_model_file(path, "Inspection failed", &e))?; let num_layers = report .tensors diff --git a/crates/apr-cli/src/commands/run.rs b/crates/apr-cli/src/commands/run.rs index d56b9a967c..f6d82433b0 100644 --- a/crates/apr-cli/src/commands/run.rs +++ b/crates/apr-cli/src/commands/run.rs @@ -169,6 +169,9 @@ pub(crate) struct RunOptions { pub repeat_last_n: usize, /// Process prompt tokens one-by-one (debug prefill) pub split_prompt: bool, + /// #3672: apply the model's chat template (`--chat`, or an instruct/chat source name) + /// even when its metadata and file name say base model. The prompt itself stays raw. + pub chat_template: bool, /// `--stream`: emit one NDJSON event per generated token. /// /// Known here (not only at the print site) because streaming is the one @@ -202,6 +205,7 @@ impl Default for RunOptions { repeat_penalty: 1.0, repeat_last_n: 64, split_prompt: false, + chat_template: false, stream: false, } } diff --git a/crates/apr-cli/src/commands/run_entry.rs b/crates/apr-cli/src/commands/run_entry.rs index e62666df49..2ed50c2cb7 100644 --- a/crates/apr-cli/src/commands/run_entry.rs +++ b/crates/apr-cli/src/commands/run_entry.rs @@ -42,6 +42,8 @@ pub(crate) fn run( repeat_penalty: f32, repeat_last_n: usize, split_prompt: bool, + // #3672: apply the model's chat template once, in realizar; the prompt is raw text. + chat_template: bool, ) -> Result<()> { // GH-516: Warn on --language/--task since whisper integration is not yet wired up if language.is_some() { @@ -105,6 +107,7 @@ pub(crate) fn run( repeat_penalty, repeat_last_n, split_prompt, + chat_template, stream, }; diff --git a/crates/apr-cli/src/commands/serve/chat.rs b/crates/apr-cli/src/commands/serve/chat.rs index f7596598be..f8190817df 100644 --- a/crates/apr-cli/src/commands/serve/chat.rs +++ b/crates/apr-cli/src/commands/serve/chat.rs @@ -74,8 +74,14 @@ pub(crate) fn load_safetensors_tokenizer(path: &Path) -> Option").ok()?; + // Create BPE tokenizer with vocab and merge rules. #3609: the unknown token is the one + // tokenizer.json DECLARES (`model.unk_token`, null for byte-level models like Qwen), + // not a literal "" every model is assumed to have. + let unk_token = json + .get("model") + .and_then(|m| m.get("unk_token")) + .and_then(|v| v.as_str()); + let tokenizer = realizar::tokenizer::BPETokenizer::new(vocab.clone(), merges, unk_token).ok()?; Some(SafeTensorsTokenizerInfo { tokenizer: std::sync::Arc::new(tokenizer), diff --git a/crates/apr-cli/src/commands/serve/handler_gpu_completion.rs b/crates/apr-cli/src/commands/serve/handler_gpu_completion.rs index 910ceafcb6..07e390a622 100644 --- a/crates/apr-cli/src/commands/serve/handler_gpu_completion.rs +++ b/crates/apr-cli/src/commands/serve/handler_gpu_completion.rs @@ -418,7 +418,7 @@ fn start_gguf_server(model_path: &Path, config: &ServerConfig) -> Result<()> { .green() ); - let vocab = extract_gguf_vocab(&mapped_model, quantized_model.config().vocab_size); + let vocab = extract_gguf_vocab(&mapped_model)?; // PERF-021 / N4 / I-2: a request has a RESOLUTION, and it is reported. // @@ -463,26 +463,27 @@ fn start_gguf_server(model_path: &Path, config: &ServerConfig) -> Result<()> { run_cpu_server(quantized_model, vocab, Some(mapped_model), config, Some(offload)) } -/// Extract vocabulary from GGUF model, falling back to placeholder tokens. +/// Extract the vocabulary from a GGUF model, or refuse to serve it (#3609). /// -/// GH-226: When the GGUF lacks `tokenizer.ggml.tokens` metadata, the placeholder -/// vocabulary must include `` so that `BPETokenizer::new` can find it. -/// Without this, the serve path fails with "Unknown token '' not in vocabulary" -/// and the server exits before binding — causing "Server failed to become ready". -fn extract_gguf_vocab( - mapped_model: &realizar::gguf::MappedGGUFModel, - vocab_size: usize, -) -> Vec { - mapped_model.model.vocabulary().unwrap_or_else(|| { - eprintln!("Warning: No vocabulary in GGUF, using placeholder tokens"); - let mut vocab: Vec = (0..vocab_size).map(|i| format!("token{i}")).collect(); - // GH-226: Ensure is present for BPETokenizer compatibility. - // Use slot 0 (standard convention for unknown token). - if !vocab.is_empty() { - vocab[0] = "".to_string(); - } - vocab - }) +/// GH-226 once filled a missing `tokenizer.ggml.tokens` with `token0..tokenN` and put +/// `` in slot 0, because `BPETokenizer::new` required one. So a model with NO +/// vocabulary loaded, while a model with a real vocabulary that lacked `` (Qwen3.5) +/// was refused: the less-specified input was the one accepted. The unknown token is +/// now optional, and a model with no vocabulary refuses by name. Placeholder tokens +/// are not a tokenizer. +fn extract_gguf_vocab(mapped_model: &realizar::gguf::MappedGGUFModel) -> Result> { + mapped_model + .model + .vocabulary() + .ok_or_else(|| no_vocabulary("the GGUF (no tokenizer.ggml.tokens)")) +} + +/// #3609: the refusal every serve path gives for a model with no vocabulary. +fn no_vocabulary(what: &str) -> CliError { + CliError::ModelLoadFailed(format!( + "{what} has no vocabulary to tokenize with, so there is nothing to serve; \ + refusing to substitute placeholder tokens (#3609)" + )) } /// #2762: resolve the KV-cache context length for the GGUF + CUDA serve path. @@ -651,7 +652,7 @@ fn start_gguf_server_cuda( let quantized_model = OwnedQuantizedModel::from_mapped(&mapped_model).map_err(|e| { CliError::ModelLoadFailed(format!("Failed to rebuild quantized model: {e}")) })?; - let vocab = extract_gguf_vocab(&mapped_model, quantized_model.config().vocab_size); + let vocab = extract_gguf_vocab(&mapped_model)?; // CUDA init failed and this process fell back to CPU. The offload // report travels with it UNCHANGED, so `/v1/effective-config` shows // `gpu_layers_resolved` beside `backend_loaded: ["cpu"]` — which is diff --git a/crates/apr-cli/src/commands/serve/handlers.rs b/crates/apr-cli/src/commands/serve/handlers.rs index c6b7d51fe2..ded665a5b4 100644 --- a/crates/apr-cli/src/commands/serve/handlers.rs +++ b/crates/apr-cli/src/commands/serve/handlers.rs @@ -594,16 +594,13 @@ fn wgpu_smoke_test( } /// PMAT-340: Extract the real vocab from the GGUF for tokenize/detokenize. +/// #3609: a GGUF with no vocabulary refuses by name (see `extract_gguf_vocab`). #[cfg(feature = "wgpu")] -fn wgpu_vocabulary(mapped: &realizar::gguf::MappedGGUFModel, vocab_size: usize) -> Vec { - mapped.model.vocabulary().unwrap_or_else(|| { - eprintln!("Warning: No vocabulary in GGUF, using placeholder"); - let mut v: Vec = (0..vocab_size).map(|i| format!("token{i}")).collect(); - if !v.is_empty() { - v[0] = "".to_string(); - } - v - }) +fn wgpu_vocabulary(mapped: &realizar::gguf::MappedGGUFModel) -> Result> { + mapped + .model + .vocabulary() + .ok_or_else(|| no_vocabulary("the GGUF (no tokenizer.ggml.tokens)")) } /// PMAT-341: Build a BPE tokenizer from the GGUF merge rules. @@ -822,7 +819,7 @@ fn serve_wgpu_backend( println!("{}", "Starting WGPU inference server...".cyan()); // PMAT-340: Extract real vocab from GGUF for tokenization/detokenization - let vocab = wgpu_vocabulary(mapped, vocab_size); + let vocab = wgpu_vocabulary(mapped)?; // PMAT-341: Extract BPE merge rules for proper tokenization let merges = mapped.model.merge_rules().unwrap_or_default(); println!( @@ -1443,15 +1440,7 @@ fn try_apr_quantized_cpu(model_path: &Path, config: &ServerConfig) -> Result<()> let vocab = mapped .metadata .get_embedded_vocabulary() - .unwrap_or_else(|| { - let vocab_size = mapped.metadata.vocab_size.unwrap_or(32000); - eprintln!("Warning: No embedded vocabulary in APR, using placeholder tokens"); - let mut v: Vec = (0..vocab_size).map(|i| format!("token{i}")).collect(); - if !v.is_empty() { - v[0] = "".to_string(); - } - v - }); + .ok_or_else(|| no_vocabulary("the APR file (no embedded vocabulary)"))?; println!("{}", "Q4K CPU inference ready".green()); diff --git a/crates/apr-cli/src/commands/serve/handlers_include_01.rs b/crates/apr-cli/src/commands/serve/handlers_include_01.rs index 537efcdb9b..c03979450b 100644 --- a/crates/apr-cli/src/commands/serve/handlers_include_01.rs +++ b/crates/apr-cli/src/commands/serve/handlers_include_01.rs @@ -50,15 +50,10 @@ fn start_apr_server_gpu( ); // Extract vocabulary from embedded APR metadata - let vocab = mapped.metadata.get_embedded_vocabulary().unwrap_or_else(|| { - let vocab_size = mapped.metadata.vocab_size.unwrap_or(32000); - eprintln!("Warning: No embedded vocabulary in APR, using placeholder tokens"); - let mut v: Vec = (0..vocab_size).map(|i| format!("token{i}")).collect(); - if !v.is_empty() { - v[0] = "".to_string(); - } - v - }); + let vocab = mapped + .metadata + .get_embedded_vocabulary() + .ok_or_else(|| no_vocabulary("the APR file (no embedded vocabulary)"))?; // GH-88: Extract merge rules for proper BPE tokenization (HuggingFace models) let merges = mapped.metadata.get_embedded_merges(); @@ -126,13 +121,9 @@ fn start_apr_q4k_server_gpu( .and_then(|m| m.load_embedded_tokenizer()) .map(|t| (t.id_to_token.clone(), None)) }) - .unwrap_or_else(|| { - println!( - "{}", - "Warning: No vocabulary found, using placeholder tokens".yellow() - ); - ((0..151936).map(|i| format!("token{i}")).collect(), None) - }); + .ok_or_else(|| { + no_vocabulary("the APR model (no sibling tokenizer.json, no embedded tokenizer)") + })?; println!(" Vocab: {} tokens", vocab.len()); if let Some(eos) = eos_id { @@ -200,15 +191,10 @@ fn start_safetensors_server_gpu( ); // Extract vocabulary from embedded APR metadata - let vocab = mapped.metadata.get_embedded_vocabulary().unwrap_or_else(|| { - let vocab_size = mapped.metadata.vocab_size.unwrap_or(32000); - eprintln!("Warning: No embedded vocabulary in APR, using placeholder tokens"); - let mut v: Vec = (0..vocab_size).map(|i| format!("token{i}")).collect(); - if !v.is_empty() { - v[0] = "".to_string(); - } - v - }); + let vocab = mapped + .metadata + .get_embedded_vocabulary() + .ok_or_else(|| no_vocabulary("the APR file (no embedded vocabulary)"))?; // GH-88: Extract merge rules for proper BPE tokenization (HuggingFace models) let merges = mapped.metadata.get_embedded_merges(); @@ -281,15 +267,10 @@ fn start_safetensors_server_cpu_quantized( ); // Extract vocabulary from embedded APR metadata - let vocab = mapped.metadata.get_embedded_vocabulary().unwrap_or_else(|| { - let vocab_size = mapped.metadata.vocab_size.unwrap_or(32000); - eprintln!("Warning: No embedded vocabulary in APR, using placeholder tokens"); - let mut v: Vec = (0..vocab_size).map(|i| format!("token{i}")).collect(); - if !v.is_empty() { - v[0] = "".to_string(); - } - v - }); + let vocab = mapped + .metadata + .get_embedded_vocabulary() + .ok_or_else(|| no_vocabulary("the APR file (no embedded vocabulary)"))?; let _ = std::fs::remove_file(&tmp_apr); diff --git a/crates/apr-cli/src/commands/tensors.rs b/crates/apr-cli/src/commands/tensors.rs index a8ee833d39..b80e19cb97 100644 --- a/crates/apr-cli/src/commands/tensors.rs +++ b/crates/apr-cli/src/commands/tensors.rs @@ -119,7 +119,7 @@ pub(crate) fn run( // Call library function let result = list_tensors(path, options) - .map_err(|e| CliError::InvalidFormat(format!("Failed to list tensors: {e}")))?; + .map_err(|e| CliError::invalid_model_file(path, "Failed to list tensors", &e))?; // Output results if json_output { diff --git a/crates/apr-cli/src/dispatch_run.rs b/crates/apr-cli/src/dispatch_run.rs index 350f23d021..809ee1d37b 100644 --- a/crates/apr-cli/src/dispatch_run.rs +++ b/crates/apr-cli/src/dispatch_run.rs @@ -41,28 +41,12 @@ fn dispatch_run( } else { trace_level }; - let merged_prompt = prompt.or(positional_prompt).cloned(); - // GH-638: Auto-detect chat template from model name when --chat not explicit. - // Instruct/Chat models (Qwen-Instruct, LLaMA-Instruct, Mistral-Instruct, etc.) - // need ChatML wrapping for correct output. Without it, the model ignores the - // prompt structure and produces garbled responses. - let use_chat = chat || { - let src_lower = source.to_lowercase(); - merged_prompt.is_some() - && (src_lower.contains("instruct") || src_lower.contains("chat")) - }; - let effective_prompt = if use_chat { - merged_prompt - .as_ref() - .map(|p| format!("<|im_start|>user\n{p}<|im_end|>\n<|im_start|>assistant\n")) - } else { - merged_prompt - }; + let (run_prompt, chat_template) = run_prompt_and_chat(prompt, positional_prompt, source, chat); run::run( source, input, - effective_prompt.as_deref(), + run_prompt.as_deref(), max_tokens, stream, language, @@ -86,9 +70,34 @@ fn dispatch_run( repeat_penalty, repeat_last_n, split_prompt, + chat_template, ) } +/// The prompt `apr run` hands to realizar, and whether realizar must apply the chat +/// template to it. +/// +/// #3672: this used to return the prompt already wrapped in hard-coded ChatML whenever +/// `--chat` was given or the source name said instruct/chat (GH-638). realizar's +/// `prepare_tokens` then applied the model's own template on top, escaping the inner +/// special tokens (`<\u{200B}|`), so every instruct `apr run` fed the model a user turn +/// nested inside a user turn: 52 prompt tokens where llama.cpp has 21, and ChatML even for +/// a model whose template is not ChatML. The prompt now stays raw text. The flag, `--chat` +/// or the GH-638 name heuristic, reaches realizar as `force_chat_template`, and the model's +/// own template is applied exactly once. +fn run_prompt_and_chat( + prompt: Option<&String>, + positional_prompt: Option<&String>, + source: &str, + chat: bool, +) -> (Option, bool) { + let merged_prompt = prompt.or(positional_prompt).cloned(); + let src_lower = source.to_lowercase(); + let chat_template = chat + || (merged_prompt.is_some() && (src_lower.contains("instruct") || src_lower.contains("chat"))); + (merged_prompt, chat_template) +} + /// Build server config and launch serve. #[allow(clippy::too_many_arguments)] fn dispatch_serve( diff --git a/crates/apr-cli/src/error.rs b/crates/apr-cli/src/error.rs index 18b2ad7ce7..586eb6bbad 100644 --- a/crates/apr-cli/src/error.rs +++ b/crates/apr-cli/src/error.rs @@ -49,6 +49,21 @@ pub enum CliError { #[error("Invalid input: {0}")] InvalidInput(String), + /// A model file that failed to parse, named by what its magic bytes say + /// it is: "GGUF", "SafeTensors", "APR", or "model" when they match none. + /// + /// #3661: `apr inspect` and `apr tensors` wrapped every parse failure in + /// [`CliError::InvalidFormat`], so a truncated GGUF was reported as + /// "Invalid APR format". Build it with [`CliError::invalid_model_file`]. + /// Shares exit code 4, the same failure class. + #[error("Invalid {format} file: {message}")] + InvalidModelFile { + /// The format the file's magic bytes identify, or "model". + format: &'static str, + /// What failed, without the core error's own "Invalid model format:" prefix. + message: String, + }, + /// IO error #[error("IO error: {0}")] Io(#[from] std::io::Error), @@ -131,7 +146,7 @@ impl CliError { pub fn exit_code_value(&self) -> u8 { match self { Self::FileNotFound(_) | Self::NotAFile(_) => 3, - Self::InvalidFormat(_) | Self::InvalidInput(_) => 4, + Self::InvalidFormat(_) | Self::InvalidInput(_) | Self::InvalidModelFile { .. } => 4, Self::Io(_) => 7, Self::ValidationFailed(_) => 5, Self::Aprender(_) => 1, @@ -155,6 +170,28 @@ impl CliError { } } +impl CliError { + /// A parse failure of the model file at `path` (#3661). + /// + /// The format is read from the file's magic bytes, never assumed. A + /// core `FormatError` contributes its message, not its Display, so the + /// result never reads "Invalid … file: …: Invalid model format: …". + pub(crate) fn invalid_model_file( + path: &std::path::Path, + context: &str, + e: &aprender::error::AprenderError, + ) -> Self { + let inner = match e { + aprender::error::AprenderError::FormatError { message } => message.clone(), + other => other.to_string(), + }; + Self::InvalidModelFile { + format: crate::commands::hex::detected_model_format(path), + message: format!("{context}: {inner}"), + } + } +} + impl From for CliError { fn from(e: aprender::error::AprenderError) -> Self { Self::Aprender(e.to_string()) diff --git a/crates/apr-cli/src/lib_execute_export_convert.rs b/crates/apr-cli/src/lib_execute_export_convert.rs index c8d9e5c4cc..6ecab9f4d2 100644 --- a/crates/apr-cli/src/lib_execute_export_convert.rs +++ b/crates/apr-cli/src/lib_execute_export_convert.rs @@ -290,72 +290,54 @@ } // ========================================================================= - // --chat flag logic: effective_prompt with ChatML wrapping + // --chat: the prompt reaches realizar RAW; the flag carries the intent (#3672) // ========================================================================= + // + // These replace three tests that re-implemented the old ChatML pre-wrap inline and + // asserted on their own copy: they could not fail, and they pinned the double + // template as the contract. - /// Test that --chat flag wraps prompt in ChatML format (verified via parse) + /// An instruct source auto-enables the template, and the prompt is NOT pre-wrapped: + /// realizar applies the model's own template exactly once. #[test] - fn test_chat_flag_chatml_wrapping_logic() { - // We cannot call execute_command with --chat on a non-existent model - // without error, but we can verify the ChatML wrapping logic directly. - let prompt = "What is the meaning of life?"; - let chat = true; - - let effective_prompt = if chat { - Some(format!( - "<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n", - prompt - )) - } else { - Some(prompt.to_string()) - }; - - assert!(effective_prompt - .as_ref() - .expect("prompt should exist") - .starts_with("<|im_start|>user\n")); - assert!(effective_prompt - .as_ref() - .expect("prompt should exist") - .ends_with("<|im_start|>assistant\n")); - assert!(effective_prompt - .as_ref() - .expect("prompt should exist") - .contains("What is the meaning of life?")); + fn test_3672_instruct_source_sets_the_flag_and_leaves_the_prompt_raw() { + let prompt = "What is the meaning of life?".to_string(); + let (run_prompt, chat) = + run_prompt_and_chat(Some(&prompt), None, "Qwen2.5-0.5B-Instruct-f16.gguf", false); + assert_eq!(run_prompt.as_deref(), Some("What is the meaning of life?")); + assert!(chat, "an instruct source must ask realizar for the chat template"); + assert!( + !run_prompt.unwrap_or_default().contains("<|im_start|>"), + "a pre-wrapped prompt is templated a second time by realizar (#3672)" + ); } - /// Test that without --chat, prompt is passed through unchanged + /// `--chat` on a source whose name says nothing still asks for the template. #[test] - fn test_no_chat_flag_passthrough() { - let prompt = Some("Hello world".to_string()); - let chat = false; - - let effective_prompt = if chat { - prompt - .as_ref() - .map(|p| format!("<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n", p)) - } else { - prompt.clone() - }; - - assert_eq!(effective_prompt, Some("Hello world".to_string())); + fn test_3672_explicit_chat_flag_on_a_plain_source() { + let prompt = "Hello".to_string(); + let (run_prompt, chat) = run_prompt_and_chat(Some(&prompt), None, "model.gguf", true); + assert_eq!(run_prompt.as_deref(), Some("Hello")); + assert!(chat); } - /// Test that --chat with no prompt produces None + /// A base model with no `--chat` gets neither a wrap nor the flag. #[test] - fn test_chat_flag_no_prompt() { - let prompt: Option = None; - let chat = true; - - let effective_prompt = if chat { - prompt - .as_ref() - .map(|p| format!("<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n", p)) - } else { - prompt.clone() - }; + fn test_3672_base_source_without_chat_is_untouched() { + let prompt = "Hello world".to_string(); + let (run_prompt, chat) = run_prompt_and_chat(Some(&prompt), None, "smollm-base.gguf", false); + assert_eq!(run_prompt.as_deref(), Some("Hello world")); + assert!(!chat); + } - assert!(effective_prompt.is_none()); + /// No prompt: nothing to template; `-p` wins over the positional prompt. + #[test] + fn test_3672_prompt_precedence_and_absence() { + let (none, chat) = run_prompt_and_chat(None, None, "x-instruct.gguf", false); + assert!(none.is_none() && !chat, "the name heuristic needs a prompt to template"); + let (p, pos) = ("from -p".to_string(), "positional".to_string()); + let (got, _) = run_prompt_and_chat(Some(&p), Some(&pos), "m.gguf", false); + assert_eq!(got.as_deref(), Some("from -p")); } // ========================================================================= diff --git a/crates/aprender-core/src/format/safetensors.rs b/crates/aprender-core/src/format/safetensors.rs index ff6337fe5d..c042174e70 100644 --- a/crates/aprender-core/src/format/safetensors.rs +++ b/crates/aprender-core/src/format/safetensors.rs @@ -18,8 +18,13 @@ fn ggml_dtype_element_size(dtype: u32) -> f64 { /// List tensors from GGUF file bytes fn list_tensors_gguf(data: &[u8], options: TensorListOptions) -> Result { + // #3661: a FormatError contributes its message, not its Display, or the + // result reads "Invalid model format: Failed to parse GGUF: Invalid model format: …". let reader = GgufReader::from_bytes(data.to_vec()).map_err(|e| AprenderError::FormatError { - message: format!("Failed to parse GGUF: {e}"), + message: match e { + AprenderError::FormatError { message } => format!("Failed to parse GGUF: {message}"), + other => format!("Failed to parse GGUF: {other}"), + }, })?; // #2569: every row below asserts that `size_bytes` of tensor data exist at a diff --git a/crates/aprender-orchestrate/src/agent/driver/realizar.rs b/crates/aprender-orchestrate/src/agent/driver/realizar.rs index e18fb8eb33..f5b92b25be 100644 --- a/crates/aprender-orchestrate/src/agent/driver/realizar.rs +++ b/crates/aprender-orchestrate/src/agent/driver/realizar.rs @@ -86,6 +86,7 @@ impl LlmDriver for RealizarDriver { trace_steps: None, verbose: false, use_mock_backend: false, + force_chat_template: false, stop_tokens: vec![], }; diff --git a/crates/aprender-serve/book/src/examples/api-server.md b/crates/aprender-serve/book/src/examples/api-server.md index ac4bc3f8bc..d62940cca5 100644 --- a/crates/aprender-serve/book/src/examples/api-server.md +++ b/crates/aprender-serve/book/src/examples/api-server.md @@ -31,7 +31,10 @@ let config = ModelConfig { eps: 1e-5, }; let model = Model::new(config)?; -let tokenizer = BPETokenizer::new(vocab, merges, "")?; +// The unknown token is the model's own, if it has one: LLaMA-style vocabularies declare +// ``, byte-level ones (GPT-2, Qwen) have none. BPETokenizer requires neither (#3609). +let unk = realizar::tokenizer::vocabulary_unk_token(&vocab); // Some("") or None +let tokenizer = BPETokenizer::new(vocab, merges, unk)?; // Create server let state = AppState::new(model, tokenizer); diff --git a/crates/aprender-serve/examples/model_cache.rs b/crates/aprender-serve/examples/model_cache.rs index d97ff30163..e5c29d0125 100644 --- a/crates/aprender-serve/examples/model_cache.rs +++ b/crates/aprender-serve/examples/model_cache.rs @@ -35,7 +35,10 @@ fn create_demo_model(vocab_size: usize) -> (Model, BPETokenizer) { } }) .collect(); - let tokenizer = BPETokenizer::new(vocab, vec![], "").expect("Failed to create tokenizer"); + // This demo vocabulary declares `` at id 0, so it passes it. The unknown token is + // the model's own: a byte-level vocabulary (GPT-2, Qwen) has none and passes `None` (#3609). + let tokenizer = + BPETokenizer::new(vocab, vec![], Some("")).expect("Failed to create tokenizer"); (model, tokenizer) } diff --git a/crates/aprender-serve/src/api/mod_app_state_gpu.rs b/crates/aprender-serve/src/api/mod_app_state_gpu.rs index acee3111ff..253765eddc 100644 --- a/crates/aprender-serve/src/api/mod_app_state_gpu.rs +++ b/crates/aprender-serve/src/api/mod_app_state_gpu.rs @@ -28,7 +28,8 @@ impl AppState { } }) .collect(); - let tokenizer = BPETokenizer::new(vocab, vec![], "")?; + let unk = crate::tokenizer::vocabulary_unk_token(&vocab); // #3609: never a literal + let tokenizer = BPETokenizer::new(vocab, vec![], unk)?; let (audit_logger, audit_sink) = create_audit_state(); Ok(Self { @@ -85,7 +86,8 @@ impl AppState { cached_model: crate::gguf::OwnedQuantizedModelCachedSync, vocab: Vec, ) -> Result { - let tokenizer = BPETokenizer::new(vocab, vec![], "")?; + let unk = crate::tokenizer::vocabulary_unk_token(&vocab); // #3609: never a literal + let tokenizer = BPETokenizer::new(vocab, vec![], unk)?; let (audit_logger, audit_sink) = create_audit_state(); Ok(Self { @@ -140,7 +142,8 @@ impl AppState { quantized_model: crate::gguf::OwnedQuantizedModel, vocab: Vec, ) -> Result { - let tokenizer = BPETokenizer::new(vocab, vec![], "")?; + let unk = crate::tokenizer::vocabulary_unk_token(&vocab); // #3609: never a literal + let tokenizer = BPETokenizer::new(vocab, vec![], unk)?; // PMAT-181: Cache architecture for chat template auto-detection. // Qwen3 models get Qwen3NoThinkTemplate (disables thinking mode). @@ -190,7 +193,8 @@ impl AppState { /// Create application state with CUDA-optimized model for high-performance GPU inference (PAR-111) /// - /// This uses the `OwnedQuantizedModelCuda` wrapper which achieves 755+ tok/s (2.6x Ollama) by: + /// This uses the `OwnedQuantizedModelCuda` wrapper, the fast GPU path (measured + /// throughput lives in `docs/BEATS.md`, not here), by: /// - Pre-uploading all weights to GPU via `preload_weights_gpu()` /// - Using batched workspaces for efficient inference /// - GPU-resident KV cache to avoid CPU→GPU transfers @@ -208,7 +212,8 @@ impl AppState { cuda_model: crate::gguf::OwnedQuantizedModelCuda, vocab: Vec, ) -> Result { - let tokenizer = BPETokenizer::new(vocab, vec![], "")?; + let unk = crate::tokenizer::vocabulary_unk_token(&vocab); // #3609: never a literal + let tokenizer = BPETokenizer::new(vocab, vec![], unk)?; // PMAT-073: Cache architecture at construction to avoid RwLock in hot path. // model_architecture() was blocking HTTP handlers for ~2s due to read lock // contention with the batch scheduler's write lock. @@ -265,7 +270,8 @@ impl AppState { vocab: Vec, merges: Vec<(String, String)>, ) -> Result { - let tokenizer = BPETokenizer::with_merges(vocab, merges, "")?; + let unk = crate::tokenizer::vocabulary_unk_token(&vocab); // #3609: never a literal + let tokenizer = BPETokenizer::with_merges(vocab, merges, unk)?; let arch = Some(cuda_model.model().config.architecture.clone()); let eos = cuda_model.model().config.eos_token_id; @@ -314,7 +320,7 @@ impl AppState { /// Create application state with APR Transformer for SafeTensors/APR inference (PMAT-SERVE-FIX-001) /// /// This enables the `/generate` and `/batch/generate` endpoints for SafeTensors and APR models. - /// Uses F32 weights for inference, achieving ~1-10 tok/s on CPU. + /// Uses F32 weights for inference on CPU (the slow path; see `docs/BEATS.md` for measurements). /// /// # Arguments /// @@ -328,7 +334,8 @@ impl AppState { transformer: crate::apr_transformer::AprTransformer, vocab: Vec, ) -> Result { - let tokenizer = BPETokenizer::new(vocab, vec![], "")?; + let unk = crate::tokenizer::vocabulary_unk_token(&vocab); // #3609: never a literal + let tokenizer = BPETokenizer::new(vocab, vec![], unk)?; let (audit_logger, audit_sink) = create_audit_state(); Ok(Self { @@ -453,7 +460,7 @@ impl AppState { /// Get the CUDA-optimized model for high-performance GPU inference (PAR-111) /// - /// Returns the model wrapper that achieves 755+ tok/s (2.6x Ollama) by using: + /// Returns the fast GPU model wrapper (measured throughput lives in `docs/BEATS.md`), which uses: /// - Pre-uploaded GPU weights /// - Batched workspaces /// - GPU-resident KV cache @@ -555,7 +562,8 @@ impl AppState { vocab: Vec, eos_id: Option, ) -> Result { - let tokenizer = BPETokenizer::new(vocab, vec![], "")?; + let unk = crate::tokenizer::vocabulary_unk_token(&vocab); // #3609: never a literal + let tokenizer = BPETokenizer::new(vocab, vec![], unk)?; let (audit_logger, audit_sink) = create_audit_state(); Ok(Self { model: None, @@ -601,7 +609,8 @@ impl AppState { model: crate::safetensors_cuda::SafeTensorsCudaModel, vocab: Vec, ) -> Result { - let tokenizer = BPETokenizer::new(vocab, vec![], "")?; + let unk = crate::tokenizer::vocabulary_unk_token(&vocab); // #3609: never a literal + let tokenizer = BPETokenizer::new(vocab, vec![], unk)?; let metrics = Arc::new(MetricsCollector::new()); let (audit_logger, audit_sink) = create_audit_state(); diff --git a/crates/aprender-serve/src/api/mod_app_state_new.rs b/crates/aprender-serve/src/api/mod_app_state_new.rs index d5984e9c8a..18b11b0b8c 100644 --- a/crates/aprender-serve/src/api/mod_app_state_new.rs +++ b/crates/aprender-serve/src/api/mod_app_state_new.rs @@ -227,8 +227,9 @@ impl AppState { } }) .collect(); + let unk = crate::tokenizer::vocabulary_unk_token(&vocab); // #3609: never a literal let tokenizer = - BPETokenizer::new(vocab, vec![], "").expect("Failed to create tokenizer"); + BPETokenizer::new(vocab, vec![], unk).expect("Failed to create tokenizer"); let (audit_logger, audit_sink) = create_audit_state(); Self { @@ -298,7 +299,8 @@ impl AppState { } }) .collect(); - let tokenizer = BPETokenizer::new(vocab, vec![], "")?; + let unk = crate::tokenizer::vocabulary_unk_token(&vocab); // #3609: never a literal + let tokenizer = BPETokenizer::new(vocab, vec![], unk)?; // Create demo APR model (real inference, not mock) // Simple model: sum of inputs with bias @@ -424,7 +426,8 @@ impl AppState { } }) .collect(); - let tokenizer = BPETokenizer::new(vocab, vec![], "")?; + let unk = crate::tokenizer::vocabulary_unk_token(&vocab); // #3609: never a literal + let tokenizer = BPETokenizer::new(vocab, vec![], unk)?; let (audit_logger, audit_sink) = create_audit_state(); Ok(Self { @@ -480,7 +483,8 @@ impl AppState { gpu_model: crate::gpu::GpuModel, vocab: Vec, ) -> Result { - let tokenizer = BPETokenizer::new(vocab, vec![], "")?; + let unk = crate::tokenizer::vocabulary_unk_token(&vocab); // #3609: never a literal + let tokenizer = BPETokenizer::new(vocab, vec![], unk)?; let (audit_logger, audit_sink) = create_audit_state(); Ok(Self { @@ -545,7 +549,8 @@ impl AppState { } }) .collect(); - let tokenizer = BPETokenizer::new(vocab, vec![], "")?; + let unk = crate::tokenizer::vocabulary_unk_token(&vocab); // #3609: never a literal + let tokenizer = BPETokenizer::new(vocab, vec![], unk)?; let (audit_logger, audit_sink) = create_audit_state(); Ok(Self { diff --git a/crates/aprender-serve/src/cli/mod_server_commands.rs b/crates/aprender-serve/src/cli/mod_server_commands.rs index 81d5d99c7e..a4e0fcf4d8 100644 --- a/crates/aprender-serve/src/cli/mod_server_commands.rs +++ b/crates/aprender-serve/src/cli/mod_server_commands.rs @@ -6,6 +6,20 @@ mod server_commands { use super::Result; + /// #3609: the refusal every serve path gives for a model with no vocabulary. + /// + /// These paths used to substitute `token0..tokenN` and serve that. Placeholder tokens are + /// not a tokenizer: a model with no vocabulary refuses by name, it does not load. + fn no_vocabulary(what: &str) -> crate::error::RealizarError { + crate::error::RealizarError::UnsupportedOperation { + operation: "load_vocabulary".to_string(), + reason: format!( + "{what} has no vocabulary to tokenize with, so there is nothing to serve; \ + refusing to substitute placeholder tokens (#3609)" + ), + } + } + /// Result of preparing server state (returned by `prepare_serve_state`) pub struct PreparedServer { /// The prepared AppState for the server @@ -95,12 +109,10 @@ mod server_commands { println!(" Layers: {}", quantized_model.layers.len()); // Extract vocabulary from GGUF for proper token decoding - let vocab = mapped.model.vocabulary().unwrap_or_else(|| { - eprintln!(" Warning: No vocabulary in GGUF, using placeholder tokens"); - (0..quantized_model.config.vocab_size) - .map(|i| format!("token{i}")) - .collect() - }); + let vocab = mapped + .model + .vocabulary() + .ok_or_else(|| no_vocabulary("the GGUF (no tokenizer.ggml.tokens)"))?; println!(" Vocab loaded: {} tokens", vocab.len()); println!(); @@ -336,14 +348,9 @@ mod server_commands { } })?; - let vocab_size = cuda_model.config().vocab_size; - #[allow(clippy::map_unwrap_or)] let vocab = crate::apr::AprV2Model::load_tokenizer_from_sibling(model_path_obj) .map(|(v, _, _)| v) - .unwrap_or_else(|| { - println!(" Warning: No tokenizer.json found, using simple vocabulary"); - (0..vocab_size).map(|i| format!("token{i}")).collect() - }); + .ok_or_else(|| no_vocabulary("the SafeTensors model (no sibling tokenizer.json)"))?; println!(" Vocab size: {}", vocab.len()); @@ -377,15 +384,9 @@ mod server_commands { println!(" Layers: {}", transformer.config.num_layers); println!(" Hidden: {}", transformer.config.hidden_dim); - #[allow(clippy::map_unwrap_or)] let vocab = crate::apr::AprV2Model::load_tokenizer_from_sibling(model_path_obj) .map(|(v, _, _)| v) - .unwrap_or_else(|| { - println!(" Warning: No tokenizer.json found, using simple vocabulary"); - (0..transformer.config.vocab_size) - .map(|i| format!("token{i}")) - .collect() - }); + .ok_or_else(|| no_vocabulary("the SafeTensors model (no sibling tokenizer.json)"))?; println!(" Vocab size: {}", vocab.len()); println!(" Mode: CPU (F32 inference)"); @@ -449,12 +450,7 @@ mod server_commands { .and_then(|m| m.load_embedded_tokenizer()) .map(|t| t.id_to_token.clone()) }) - .unwrap_or_else(|| { - println!(" Warning: No vocabulary found, using simple vocabulary"); - (0..model.config.vocab_size) - .map(|i| format!("token{i}")) - .collect() - }); + .ok_or_else(|| no_vocabulary("the APR model (no sibling tokenizer.json, no embedded tokenizer)"))?; println!(" Vocab size: {}", vocab.len()); println!(" Mode: CPU (per-tensor scratch dequant)"); @@ -494,10 +490,7 @@ mod server_commands { let vocab = apr_v2.as_ref() .and_then(|m| m.metadata().get_embedded_vocabulary()) .or_else(|| crate::apr::AprV2Model::load_tokenizer_from_sibling(path).map(|(v, _, _)| v)) - .unwrap_or_else(|| { - println!(" Warning: No tokenizer found"); - (0..model.config.vocab_size).map(|i| format!("token{i}")).collect() - }); + .ok_or_else(|| no_vocabulary("the APR model (no embedded vocabulary, no sibling tokenizer.json)"))?; println!(" Vocab size: {}", vocab.len()); // PMAT-785: fail-closed quant gate at GPU-resident construction. An APR @@ -556,10 +549,7 @@ mod server_commands { .and_then(|m| m.load_embedded_tokenizer()) .map(|t| (t.id_to_token.clone(), None)) }) - .unwrap_or_else(|| { - println!(" Warning: No vocabulary found, using simple vocabulary"); - ((0..151936).map(|i| format!("token{i}")).collect(), None) - }); + .ok_or_else(|| no_vocabulary("the APR model (no sibling tokenizer.json, no embedded tokenizer)"))?; println!(" Vocab: {} tokens", vocab.len()); if let Some(eos) = eos_id { diff --git a/crates/aprender-serve/src/gguf/inference/float16_dot.rs b/crates/aprender-serve/src/gguf/inference/float16_dot.rs new file mode 100644 index 0000000000..a8c6d33b04 --- /dev/null +++ b/crates/aprender-serve/src/gguf/inference/float16_dot.rs @@ -0,0 +1,180 @@ +// #3076: F16 / BF16 weight-row · f32 activation dot products for `float16_matmul`. +// +// The row loop used to decode each weight through a `fn(u16) -> f32` pointer: one indirect +// call per element, which nothing could inline or vectorise. Measured on a Threadripper 7960X +// (Zen 4, 24 cores), Qwen2.5-0.5B-Instruct-f16 decoded at ~10 tok/s, and the 272 MB LM head +// read at ~27 GB/s. These kernels read the row's bytes directly: AVX2 + F16C + FMA on x86_64 +// when the CPU has them (runtime-detected), and a chunked decode-then-dot everywhere else. +// +// After, same host and load: 21.2 tok/s at the default 48 threads, 68.6 at RAYON_NUM_THREADS=8, and +// the LM head at 2.7-3.0 ms; greedy output matches the pinned llama.cpp (receipts, method and +// raw numbers: evidence/pmat-3076-f16-bf16-matvec/). + +/// The 2-byte float format a weight row is stored in. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum Float16Kind { + /// IEEE 754 binary16 (GGUF type 1). + F16, + /// bfloat16, the top half of an f32 (GGUF type 30). + Bf16, +} + +/// Dot product of a row of little-endian 2-byte floats with `x`. +/// +/// Covers `min(row.len() / 2, x.len())` elements: a trailing odd byte is not an element, which +/// is exactly what the per-element `offset + 1 < data.len()` check it replaces admitted. +pub(super) fn float16_row_dot(kind: Float16Kind, row: &[u8], x: &[f32]) -> f32 { + let n = (row.len() / 2).min(x.len()); + let (row, x) = (&row[..n * 2], &x[..n]); + + #[cfg(target_arch = "x86_64")] + { + if is_x86_feature_detected!("avx2") && is_x86_feature_detected!("fma") { + match kind { + Float16Kind::F16 if is_x86_feature_detected!("f16c") => { + // SAFETY: avx2, fma and f16c verified at runtime; `row.len() == 2 * x.len()`. + return unsafe { f16_row_dot_avx2(row, x) }; + }, + Float16Kind::Bf16 => { + // SAFETY: avx2 and fma verified at runtime; `row.len() == 2 * x.len()`. + return unsafe { bf16_row_dot_avx2(row, x) }; + }, + Float16Kind::F16 => {}, + } + } + } + float16_row_dot_portable(kind, row, x) +} + +/// A bfloat16 is the top 16 bits of an f32. +#[inline] +fn bf16_bits_to_f32(bits: u16) -> f32 { + f32::from_bits(u32::from(bits) << 16) +} + +/// Decode one element (`bits` is the little-endian u16 of the weight). +#[inline] +fn decode_float16(kind: Float16Kind, bits: u16) -> f32 { + match kind { + Float16Kind::F16 => crate::quantize::f16_to_f32_lut(bits), + Float16Kind::Bf16 => bf16_bits_to_f32(bits), + } +} + +/// The non-x86 path, and the x86 path on a CPU without the features above: decode a chunk into +/// a stack buffer, then accumulate it in eight independent lanes the compiler can vectorise. +pub(super) fn float16_row_dot_portable(kind: Float16Kind, row: &[u8], x: &[f32]) -> f32 { + const CHUNK: usize = 64; + let mut buf = [0.0f32; CHUNK]; + let mut lanes = [0.0f32; 8]; + let mut tail = 0.0f32; + for (bytes, xs) in row.chunks(CHUNK * 2).zip(x.chunks(CHUNK)) { + let m = xs.len(); + for (w, b) in buf[..m].iter_mut().zip(bytes.chunks_exact(2)) { + *w = decode_float16(kind, u16::from_le_bytes([b[0], b[1]])); + } + let (ws, xs8) = (buf[..m].chunks_exact(8), xs.chunks_exact(8)); + let (w_rem, x_rem) = (ws.remainder(), xs8.remainder()); + for (w8, x8) in ws.zip(xs8) { + for l in 0..8 { + lanes[l] += w8[l] * x8[l]; + } + } + tail += w_rem.iter().zip(x_rem).map(|(w, x)| w * x).sum::(); + } + lanes.iter().sum::() + tail +} + +/// The shared AVX2 loop: four accumulators of fused multiply-adds over 32 elements, then +/// 8 at a time, then a scalar tail. A macro rather than a generic function, so that each +/// expansion sits inside a kernel whose target features include its `$load8`, and the 8-lane +/// load inlines. A closure would not inherit those features, and would cost a call per 8 +/// weights. +#[cfg(target_arch = "x86_64")] +macro_rules! avx2_float16_row_dot { + ($row:ident, $x:ident, $load8:ident, $decode1:expr) => {{ + use std::arch::x86_64::{ + _mm256_add_ps, _mm256_castps256_ps128, _mm256_extractf128_ps, _mm256_fmadd_ps, + _mm256_loadu_ps, _mm256_setzero_ps, _mm_add_ps, _mm_cvtss_f32, _mm_hadd_ps, + }; + let n = $x.len(); + let (rp, xp) = ($row.as_ptr(), $x.as_ptr()); + let (mut a0, mut a1, mut a2, mut a3) = + (_mm256_setzero_ps(), _mm256_setzero_ps(), _mm256_setzero_ps(), _mm256_setzero_ps()); + let mut i = 0; + // `i + 32 <= n` (then `i + 8 <= n`) bounds every 16-byte `row` load at byte `2 * i` + // and every 8-float `x` load at `i`, given `row.len() == 2 * x.len()`. + while i + 32 <= n { + a0 = _mm256_fmadd_ps($load8(rp.add(2 * i)), _mm256_loadu_ps(xp.add(i)), a0); + a1 = _mm256_fmadd_ps($load8(rp.add(2 * i + 16)), _mm256_loadu_ps(xp.add(i + 8)), a1); + a2 = _mm256_fmadd_ps($load8(rp.add(2 * i + 32)), _mm256_loadu_ps(xp.add(i + 16)), a2); + a3 = _mm256_fmadd_ps($load8(rp.add(2 * i + 48)), _mm256_loadu_ps(xp.add(i + 24)), a3); + i += 32; + } + while i + 8 <= n { + a0 = _mm256_fmadd_ps($load8(rp.add(2 * i)), _mm256_loadu_ps(xp.add(i)), a0); + i += 8; + } + let acc = _mm256_add_ps(_mm256_add_ps(a0, a1), _mm256_add_ps(a2, a3)); + let s = _mm_add_ps(_mm256_castps256_ps128(acc), _mm256_extractf128_ps::<1>(acc)); + let s = _mm_hadd_ps(s, s); + let mut sum = _mm_cvtss_f32(_mm_hadd_ps(s, s)); + for j in i..n { + sum += $decode1(u16::from_le_bytes([$row[2 * j], $row[2 * j + 1]])) * $x[j]; + } + sum + }}; +} + +/// 8 F16 weights at `p` as f32 lanes (`vcvtph2ps`). +/// +/// # Safety +/// The CPU must support avx2 and f16c, and `p..p + 16` must be readable. +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "avx2,f16c")] +#[inline] +unsafe fn load8_f16(p: *const u8) -> std::arch::x86_64::__m256 { + use std::arch::x86_64::{__m128i, _mm256_cvtph_ps, _mm_loadu_si128}; + // SAFETY: the caller guarantees 16 readable bytes at `p` and the target features. + unsafe { _mm256_cvtph_ps(_mm_loadu_si128(p.cast::<__m128i>())) } +} + +/// 8 BF16 weights at `p` as f32 lanes: widen u16 to u32, shift into the top half. +/// +/// # Safety +/// The CPU must support avx2, and `p..p + 16` must be readable. +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "avx2")] +#[inline] +unsafe fn load8_bf16(p: *const u8) -> std::arch::x86_64::__m256 { + use std::arch::x86_64::{ + __m128i, _mm256_castsi256_ps, _mm256_cvtepu16_epi32, _mm256_slli_epi32, _mm_loadu_si128, + }; + // SAFETY: the caller guarantees 16 readable bytes at `p` and the target feature. + unsafe { + let wide = _mm256_cvtepu16_epi32(_mm_loadu_si128(p.cast::<__m128i>())); + _mm256_castsi256_ps(_mm256_slli_epi32::<16>(wide)) + } +} + +/// F16 row dot on AVX2 + F16C + FMA. +/// +/// # Safety +/// The CPU must support avx2, fma and f16c, and `row.len()` must be `2 * x.len()`. +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "avx2,fma,f16c")] +unsafe fn f16_row_dot_avx2(row: &[u8], x: &[f32]) -> f32 { + // SAFETY: loads are bounded as the macro documents; features are this fn's contract. + unsafe { avx2_float16_row_dot!(row, x, load8_f16, crate::quantize::f16_to_f32_lut) } +} + +/// BF16 row dot on AVX2 + FMA. +/// +/// # Safety +/// The CPU must support avx2 and fma, and `row.len()` must be `2 * x.len()`. +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "avx2,fma")] +unsafe fn bf16_row_dot_avx2(row: &[u8], x: &[f32]) -> f32 { + // SAFETY: loads are bounded as the macro documents; features are this fn's contract. + unsafe { avx2_float16_row_dot!(row, x, load8_bf16, bf16_bits_to_f32) } +} diff --git a/crates/aprender-serve/src/gguf/inference/float16_dot_tests.rs b/crates/aprender-serve/src/gguf/inference/float16_dot_tests.rs new file mode 100644 index 0000000000..924a87ae56 --- /dev/null +++ b/crates/aprender-serve/src/gguf/inference/float16_dot_tests.rs @@ -0,0 +1,177 @@ +//! #3076: the F16/BF16 row dot that replaced `float16_matmul`'s per-element `fn`-pointer loop. +//! +//! Every length below is checked against an f64 reference, which reaches every branch of the +//! kernels: the 32-wide unrolled loop, the 8-wide loop, the scalar tail, and the portable +//! path's 64-element chunks and remainders. `float16_matmul` itself is checked against a +//! transcription of the loop it replaced, including a truncated weight buffer. + +use super::matmul::{float16_matmul, float16_row_dot, float16_row_dot_portable, Float16Kind}; + +/// Deterministic values in roughly [-2, 2]: a 64-bit LCG, so a failure reproduces exactly. +fn values(n: usize, seed: u64) -> Vec { + let mut s = seed.wrapping_mul(6_364_136_223_846_793_005).wrapping_add(1); + (0..n) + .map(|_| { + s = s + .wrapping_mul(6_364_136_223_846_793_005) + .wrapping_add(1_442_695_040_888_963_407); + ((s >> 40) as f32 / (1u64 << 24) as f32) * 4.0 - 2.0 + }) + .collect() +} + +fn encode(kind: Float16Kind, w: &[f32]) -> Vec { + w.iter() + .flat_map(|&v| match kind { + Float16Kind::F16 => half::f16::from_f32(v).to_bits().to_le_bytes(), + Float16Kind::Bf16 => half::bf16::from_f32(v).to_bits().to_le_bytes(), + }) + .collect() +} + +fn decode(kind: Float16Kind, b: [u8; 2]) -> f32 { + let bits = u16::from_le_bytes(b); + match kind { + Float16Kind::F16 => half::f16::from_bits(bits).to_f32(), + Float16Kind::Bf16 => half::bf16::from_bits(bits).to_f32(), + } +} + +/// (exact dot in f64 of the decoded weights, sum of |w * x| for the error bound). +fn reference(kind: Float16Kind, row: &[u8], x: &[f32]) -> (f64, f64) { + row.chunks_exact(2) + .zip(x) + .fold((0.0, 0.0), |(dot, mag), (b, &xv)| { + let p = f64::from(decode(kind, [b[0], b[1]])) * f64::from(xv); + (dot + p, mag + p.abs()) + }) +} + +const LENGTHS: [usize; 16] = [ + 0, 1, 7, 8, 9, 31, 32, 33, 63, 64, 65, 100, 129, 896, 4864, 4867, +]; + +fn assert_matches_reference(kind: Float16Kind, dot: fn(Float16Kind, &[u8], &[f32]) -> f32) { + for (k, &n) in LENGTHS.iter().enumerate() { + let row = encode(kind, &values(n, 2 * k as u64 + 1)); + let x = values(n, 2 * k as u64 + 2); + let (exact, mag) = reference(kind, &row, &x); + let got = f64::from(dot(kind, &row, &x)); + // f32 accumulation over n terms: error well inside 1e-5 of the magnitude sum here. + assert!( + (got - exact).abs() <= 1e-5 * mag + 1e-6, + "{kind:?} n={n}: got {got}, exact {exact}, |w*x| sum {mag}" + ); + } +} + +#[test] +fn test_3076_f16_row_dot_matches_the_f64_reference_at_every_length() { + assert_matches_reference(Float16Kind::F16, float16_row_dot); +} + +#[test] +fn test_3076_bf16_row_dot_matches_the_f64_reference_at_every_length() { + assert_matches_reference(Float16Kind::Bf16, float16_row_dot); +} + +/// The fallback must be right on its own: on an AVX2 host `float16_row_dot` never reaches it. +#[test] +fn test_3076_portable_path_matches_the_f64_reference_at_every_length() { + assert_matches_reference(Float16Kind::F16, float16_row_dot_portable); + assert_matches_reference(Float16Kind::Bf16, float16_row_dot_portable); +} + +/// Non-finite and subnormal weights reach the sum the way the scalar loop let them. +#[test] +fn test_3076_specials_propagate_through_every_lane_position() { + for kind in [Float16Kind::F16, Float16Kind::Bf16] { + let (nan, inf) = match kind { + Float16Kind::F16 => (0x7E00u16, 0x7C00u16), + Float16Kind::Bf16 => (0x7FC0u16, 0x7F80u16), + }; + // Position 3 sits in the 32-wide loop, 37 in the 8-wide loop, 42 in the tail. + for pos in [3usize, 37, 42] { + let mut row = encode(kind, &vec![0.5; 43]); + row[2 * pos..2 * pos + 2].copy_from_slice(&nan.to_le_bytes()); + assert!( + float16_row_dot(kind, &row, &vec![1.0; 43]).is_nan(), + "{kind:?} NaN at {pos}" + ); + row[2 * pos..2 * pos + 2].copy_from_slice(&inf.to_le_bytes()); + let got = float16_row_dot(kind, &row, &vec![1.0; 43]); + assert!( + got.is_infinite() && got > 0.0, + "{kind:?} +inf at {pos}: {got}" + ); + } + } + // The smallest F16 subnormal is 2^-24, not zero: 40 of them times 2^20 is 40 / 16 = 2.5. + let row: Vec = std::iter::repeat_n(0x0001u16.to_le_bytes(), 40) + .flatten() + .collect(); + let got = float16_row_dot(Float16Kind::F16, &row, &vec![(1u32 << 20) as f32; 40]); + assert!((got - 2.5).abs() < 1e-6, "F16 subnormals: {got}"); +} + +/// The element count is `min(row.len() / 2, x.len())`: an odd trailing byte is not an element. +#[test] +fn test_3076_element_count_is_whole_elements_of_the_shorter_operand() { + let kind = Float16Kind::F16; + let row = encode(kind, &[1.0; 20]); + let mut odd = row.clone(); + odd.push(0x3C); // half an element: must be ignored, not read as a weight + assert_eq!(float16_row_dot(kind, &odd, &[2.0; 20]), 40.0); + assert_eq!(float16_row_dot(kind, &row, &[2.0; 9]), 18.0); + assert_eq!(float16_row_dot(kind, &row[..10], &[2.0; 20]), 10.0); +} + +/// `float16_matmul` against the loop it replaced, transcribed: one element at a time, an +/// element counted only when both of its bytes are inside `data`. +fn pre_3076_float16_matmul( + kind: Float16Kind, + input: &[f32], + data: &[u8], + in_dim: usize, + out_dim: usize, + seq_len: usize, +) -> Vec { + let mut out = Vec::with_capacity(seq_len * out_dim); + for s in 0..seq_len { + let x = &input[s * in_dim..(s + 1) * in_dim]; + for row in 0..out_dim { + let mut sum = 0.0f64; + for (col, &xv) in x.iter().enumerate() { + let offset = row * in_dim * 2 + col * 2; + if offset + 1 < data.len() { + sum += + f64::from(decode(kind, [data[offset], data[offset + 1]])) * f64::from(xv); + } + } + out.push(sum); + } + } + out +} + +#[test] +fn test_3076_float16_matmul_matches_the_pre_3076_loop_including_a_truncated_buffer() { + let (in_dim, out_dim, seq_len) = (37, 5, 3); + let input = values(in_dim * seq_len, 7); + for kind in [Float16Kind::F16, Float16Kind::Bf16] { + let full = encode(kind, &values(in_dim * out_dim, 8)); + // Full buffer, then one cut mid-row that also leaves an odd trailing byte. + for data in [&full[..], &full[..full.len() - 2 * in_dim - 21]] { + let got = float16_matmul(&input, data, in_dim, out_dim, seq_len, kind); + let want = pre_3076_float16_matmul(kind, &input, data, in_dim, out_dim, seq_len); + assert_eq!(got.len(), want.len()); + for (i, (&g, &w)) in got.iter().zip(&want).enumerate() { + assert!( + (f64::from(g) - w).abs() <= 1e-5 * w.abs().max(1.0), + "{kind:?} len={} out[{i}]: got {g}, want {w}", + data.len() + ); + } + } + } +} diff --git a/crates/aprender-serve/src/gguf/inference/matmul_fused.rs b/crates/aprender-serve/src/gguf/inference/matmul_fused.rs index c810b80719..056d623107 100644 --- a/crates/aprender-serve/src/gguf/inference/matmul_fused.rs +++ b/crates/aprender-serve/src/gguf/inference/matmul_fused.rs @@ -1,15 +1,22 @@ +include!("float16_dot.rs"); + /// CPU matmul for 2-byte-per-element float formats (BF16, F16) /// Shared by BF16 and F16 paths — same structure, different decode. -fn float16_matmul( +/// +/// #3076: each row is one `float16_row_dot` over the row's bytes. The per-element loop it +/// replaces called the decode through a `fn` pointer. A row that runs past the end of `data` +/// is dotted over the whole elements that exist, as that loop's per-element bounds check did. +pub(super) fn float16_matmul( input: &[f32], data: &[u8], in_dim: usize, out_dim: usize, seq_len: usize, - decode: fn(u16) -> f32, + kind: Float16Kind, ) -> Vec { use rayon::prelude::*; + let row_bytes = in_dim * 2; let mut all_output = Vec::with_capacity(seq_len * out_dim); for s in 0..seq_len { let x = &input[s * in_dim..(s + 1) * in_dim]; @@ -17,16 +24,9 @@ fn float16_matmul( let row_output: Vec = (0..out_dim) .into_par_iter() .map(|row| { - let row_byte_start = row * in_dim * 2; - let mut sum = 0.0f32; - for col in 0..in_dim { - let offset = row_byte_start + col * 2; - if offset + 1 < data.len() { - let bits = u16::from_le_bytes([data[offset], data[offset + 1]]); - sum += decode(bits) * x[col]; - } - } - sum + let start = (row * row_bytes).min(data.len()); + let end = (start + row_bytes).min(data.len()); + float16_row_dot(kind, &data[start..end], x) }) .collect(); @@ -169,12 +169,22 @@ impl OwnedQuantizedModel { let data = &weight.data; match weight.qtype { GGUF_TYPE_F32 => Ok(self.fused_matmul_f32(input, data, in_dim, out_dim, seq_len)), - GGUF_TYPE_BF16 => Ok(float16_matmul(input, data, in_dim, out_dim, seq_len, |b| { - f32::from_bits((b as u32) << 16) - })), - GGUF_TYPE_F16 => Ok(float16_matmul(input, data, in_dim, out_dim, seq_len, |b| { - half::f16::from_bits(b).to_f32() - })), + GGUF_TYPE_BF16 => Ok(float16_matmul( + input, + data, + in_dim, + out_dim, + seq_len, + Float16Kind::Bf16, + )), + GGUF_TYPE_F16 => Ok(float16_matmul( + input, + data, + in_dim, + out_dim, + seq_len, + Float16Kind::F16, + )), GGUF_TYPE_Q4_0 | GGUF_TYPE_Q8_0 => { self.fused_matmul_q4_q8(input, weight, in_dim, out_dim, seq_len) }, diff --git a/crates/aprender-serve/src/gguf/inference/mod.rs b/crates/aprender-serve/src/gguf/inference/mod.rs index e9541c9bf6..a57f376a60 100644 --- a/crates/aprender-serve/src/gguf/inference/mod.rs +++ b/crates/aprender-serve/src/gguf/inference/mod.rs @@ -23,6 +23,8 @@ mod attention_flash_tests; #[cfg(test)] mod attention_gqa_tests; #[cfg(test)] +mod float16_dot_tests; +#[cfg(test)] mod generate_quantized_topk_tests; #[cfg(test)] mod generate_quantized_topp_tests; diff --git a/crates/aprender-serve/src/infer/mod.rs b/crates/aprender-serve/src/infer/mod.rs index 40c75ed64f..699948c133 100644 --- a/crates/aprender-serve/src/infer/mod.rs +++ b/crates/aprender-serve/src/infer/mod.rs @@ -83,6 +83,10 @@ pub struct InferenceConfig { /// INTERNAL: Use mock backend for testing (PMAT-COV-95) #[doc(hidden)] pub use_mock_backend: bool, + /// #3672: apply the model's chat template even when neither its metadata nor its file + /// name marks it as an instruct model (`apr run --chat`). The prompt stays raw text: + /// the template is applied once, by `prepare_tokens`, never pre-wrapped by a caller. + pub force_chat_template: bool, } impl InferenceConfig { @@ -111,6 +115,7 @@ impl InferenceConfig { verbose: false, stop_tokens: Vec::new(), use_mock_backend: false, + force_chat_template: false, } } @@ -191,6 +196,13 @@ impl InferenceConfig { self } + /// #3672: apply the chat template even when metadata and file name say base model. + #[must_use] + pub fn with_force_chat_template(mut self, force: bool) -> Self { + self.force_chat_template = force; + self + } + /// Enable inference tracing #[must_use] pub fn with_trace(mut self, trace: bool) -> Self { @@ -353,7 +365,7 @@ fn prepare_tokens_gguf(config: &InferenceConfig, prompt: &str) -> Result Result< // Detect instruct model from architecture or filename let arch_lower = architecture.to_lowercase(); - let is_instruct = arch_lower.contains("instruct") + let is_instruct = config.force_chat_template + || arch_lower.contains("instruct") || model_name.to_lowercase().contains("instruct") || matches!( arch_lower.as_str(), @@ -535,7 +548,7 @@ fn prepare_tokens_apr(config: &InferenceConfig, prompt: &str) -> Result, /// ID to token mapping id_to_token: HashMap, - /// Unknown token ID - unk_token_id: u32, + /// The model's unknown token, or `None` when it has none (#3609). + /// + /// An unknown token belongs to a model family, not to BPE. LLaMA-style vocabularies + /// declare ``; byte-level ones (GPT-2, Qwen) need none, because every byte has a + /// token, and Qwen3.5 declares none. When this is `None`, construction has proved + /// that every byte has a token, so `encode` never needs a stand-in. + unk_token_id: Option, /// Cached vocabulary size (PMAT-805: vLLM pattern) vocab_size: usize, /// Cached maximum token ID (PMAT-805: vLLM pattern) @@ -151,15 +156,21 @@ impl BPETokenizer { /// /// * `vocab` - List of tokens (index = token ID) /// * `merges` - List of merge pairs in priority order - /// * `unk_token` - Unknown token string + /// * `unk_token` - The model's unknown token if it has one (`""` or + /// `Some("")`), or `None` if it has none. Having one is a property of the + /// model, not a requirement of this type (#3609). /// /// # Errors /// - /// Returns error if vocabulary is empty or unknown token not found - pub fn new( + /// Returns an error if the vocabulary is empty, or if `unk_token` names a token the + /// vocabulary does not contain. With no unknown token, also returns an error naming + /// the first byte that has neither a `<0xNN>` token nor a byte-level glyph token, + /// because `encode` would have nothing to emit for it. Absent means absent: no + /// stand-in is synthesised and no input is dropped. + pub fn new<'a>( vocab: Vec, _merges: Vec<(String, String)>, - unk_token: &str, + unk_token: impl Into>, ) -> Result { if vocab.is_empty() { return Err(RealizarError::UnsupportedOperation { @@ -181,12 +192,31 @@ impl BPETokenizer { } let unk_token_id = - *token_to_id - .get(unk_token) - .ok_or_else(|| RealizarError::UnsupportedOperation { - operation: "create_bpe_tokenizer".to_string(), - reason: format!("Unknown token '{unk_token}' not in vocabulary"), - })?; + match unk_token.into() { + Some(unk) => Some(*token_to_id.get(unk).ok_or_else(|| { + RealizarError::UnsupportedOperation { + operation: "create_bpe_tokenizer".to_string(), + reason: format!( + "unknown token '{unk}' was named, but the vocabulary does not contain it" + ), + } + })?), + None => { + if let Some(byte) = + (0u8..=255).find(|&b| byte_token_id(&token_to_id, b).is_none()) + { + return Err(RealizarError::UnsupportedOperation { + operation: "create_bpe_tokenizer".to_string(), + reason: format!( + "the model has no unknown token, and byte 0x{byte:02X} has neither a \ + '<0x{byte:02X}>' token nor a byte-level glyph token, so encode would \ + have nothing to emit for it" + ), + }); + } + None + }, + }; // PMAT-805: Cache vocabulary properties at construction (vLLM pattern) // This avoids repeated HashMap operations during inference @@ -209,12 +239,28 @@ impl BPETokenizer { /// HuggingFace vocabularies require merge-based BPE (not greedy longest-match). /// When merge rules are provided, `encode()` delegates to `bpe_encode()` from /// `apr::tokenizer` which handles special tokens atomically and applies merges. - pub fn with_merges( + pub fn with_merges<'a>( vocab: Vec, merges: Vec<(String, String)>, - unk_token: &str, + unk_token: impl Into>, ) -> Result { let mut tokenizer = Self::new(vocab, vec![], unk_token)?; + // #3609: the merge path (`bpe_encode`) maps each byte to its byte-level glyph and + // nothing else, so with no unknown token every glyph must be in the vocabulary, + // or that byte would be dropped from the input. + if tokenizer.unk_token_id.is_none() { + if let Some(byte) = + (0u8..=255).find(|&b| byte_glyph_id(&tokenizer.token_to_id, b).is_none()) + { + return Err(RealizarError::UnsupportedOperation { + operation: "create_bpe_tokenizer".to_string(), + reason: format!( + "the model has no unknown token, and byte 0x{byte:02X} has no byte-level \ + glyph token, which merge-based encoding needs" + ), + }); + } + } // Extract special tokens from vocabulary for atomic tokenization let special_tokens: HashMap = tokenizer .token_to_id @@ -317,12 +363,7 @@ impl BPETokenizer { let ch_len = ch.len_utf8(); for byte in remaining[..ch_len].bytes() { - let byte_token = format!("<0x{byte:02X}>"); - if let Some(&id) = self.token_to_id.get(&byte_token) { - tokens.push(id); - } else { - tokens.push(self.unk_token_id); - } + tokens.push(self.byte_fallback_id(byte)); } remaining = &remaining[ch_len..]; } @@ -433,6 +474,27 @@ impl BPETokenizer { self.token_to_id.get(token).copied() } + /// The unknown token's ID, or `None` when the model has none (#3609). + #[must_use] + pub fn unk_token_id(&self) -> Option { + self.unk_token_id + } + + /// The token for one byte that no vocabulary entry covers: its `<0xNN>` token, else the + /// model's unknown token, else its byte-level glyph (#3609). With no unknown token, + /// `new` proved that every byte has a `<0xNN>` or a glyph token, so this cannot miss. + fn byte_fallback_id(&self, byte: u8) -> u32 { + if let Some(&id) = self.token_to_id.get(&format!("<0x{byte:02X}>")) { + return id; + } + if let Some(unk) = self.unk_token_id { + return unk; + } + byte_glyph_id(&self.token_to_id, byte).expect( + "BPETokenizer::new proved every byte encodable without an unknown token (#3609)", + ) + } + /// Get token for a token ID #[must_use] pub fn get_token(&self, id: u32) -> Option<&str> { @@ -450,6 +512,38 @@ impl BPETokenizer { } } +/// The token for one raw byte that no vocabulary token covers: SentencePiece's `<0xNN>` +/// byte-fallback token, else the GPT-2 byte-level glyph (#3609). +fn byte_token_id(token_to_id: &HashMap, byte: u8) -> Option { + token_to_id + .get(&format!("<0x{byte:02X}>")) + .copied() + .or_else(|| byte_glyph_id(token_to_id, byte)) +} + +/// The GPT-2 byte-level glyph token for one byte (`é`'s 0xC3 → `Ã`), if the vocabulary has it. +fn byte_glyph_id(token_to_id: &HashMap, byte: u8) -> Option { + let glyph = crate::gguf::utils::gpt2_byte_to_unicode(byte); + token_to_id + .get(glyph.encode_utf8(&mut [0u8; 4]) as &str) + .copied() +} + +/// The unknown token to pass to [`BPETokenizer::new`] when the caller has only a token list. +/// +/// That is the vocabulary's own `` entry if it has one, and `None` if it does not. +/// Nothing is synthesised: a vocabulary without `` (Qwen3.5, GPT-2) gets no unknown +/// token, and `BPETokenizer::new` then proves that every byte has a token (#3609). +/// +/// This finds the token by NAME. It is the interim for callers that receive a `Vec` +/// and nothing else. The model's DECLARATION (`tokenizer.ggml.unknown_token_id`, or +/// `tokenizer.json`'s `unk_token`) is the design, and threading it through every loader +/// is #3675. +#[must_use] +pub fn vocabulary_unk_token(vocab: &[String]) -> Option<&'static str> { + vocab.iter().any(|t| t == "").then_some("") +} + /// Viterbi algorithm state: `(best_score, best_token)` at each position type ViterbiState = (Vec, Vec>); diff --git a/crates/aprender-serve/src/tokenizer_04.rs b/crates/aprender-serve/src/tokenizer_04.rs index 1771c11c29..edbac1fdbd 100644 --- a/crates/aprender-serve/src/tokenizer_04.rs +++ b/crates/aprender-serve/src/tokenizer_04.rs @@ -6,3 +6,7 @@ mod tokenizer_tests_bpe_encode; #[cfg(test)] #[path = "tokenizer_tests_sentencepiece_viterbi.rs"] mod tokenizer_tests_sentencepiece_viterbi; + +#[cfg(test)] +#[path = "tokenizer_tests_unk_3609.rs"] +mod tokenizer_tests_unk_3609; diff --git a/crates/aprender-serve/src/tokenizer_tests_unk_3609.rs b/crates/aprender-serve/src/tokenizer_tests_unk_3609.rs new file mode 100644 index 0000000000..8bed72618b --- /dev/null +++ b/crates/aprender-serve/src/tokenizer_tests_unk_3609.rs @@ -0,0 +1,161 @@ +//! #3609: an unknown token is a property of a model family, not of BPE. +//! +//! LLaMA-style vocabularies declare ``. Byte-level ones (GPT-2, Qwen) have no need +//! for one, because every byte already has a token, and Qwen3.5 declares none at all. +//! These rows pin both directions of the fix: a model WITH an unknown token still +//! resolves it (the fix is not "ignore unk"), and a model WITHOUT one tokenizes instead +//! of being refused. With no unknown token, nothing is synthesised and nothing is +//! dropped: a byte with no token is refused by name at construction. +//! +//! The two `*.gguf-header` fixtures are REAL headers, cut down by +//! `tests/fixtures/gguf-header-slices/generate.py`: every key and scalar value verbatim, +//! only the per-token arrays sliced (see that directory's MANIFEST.json). + +use crate::gguf::{GGUFModel, GGUFValue}; +use crate::tokenizer::{vocabulary_unk_token, BPETokenizer}; + +const QWEN35_HEADER: &[u8] = + include_bytes!("../tests/fixtures/gguf-header-slices/qwen3.5-0.8b.gguf-header"); +const TINYLLAMA_HEADER: &[u8] = + include_bytes!("../tests/fixtures/gguf-header-slices/tinyllama-1.1b-chat.gguf-header"); + +const UNK_ID_KEY: &str = "tokenizer.ggml.unknown_token_id"; +const EOS_ID_KEY: &str = "tokenizer.ggml.eos_token_id"; + +fn header(bytes: &[u8]) -> GGUFModel { + GGUFModel::from_bytes(bytes).expect("a real GGUF header parses") +} + +fn u32_key(model: &GGUFModel, key: &str) -> Option { + match model.metadata.get(key) { + Some(GGUFValue::UInt32(v)) => Some(*v), + _ => None, + } +} + +/// All 256 GPT-2 byte-level glyphs: the first 256 tokens of a GPT-2/Qwen vocabulary. +fn byte_level_vocab() -> Vec { + (0u8..=255) + .map(|b| crate::gguf::utils::gpt2_byte_to_unicode(b).to_string()) + .collect() +} + +fn id_of(tok: &BPETokenizer, token: &str) -> u32 { + tok.get_token_id(token).expect("token in vocabulary") +} + +// --------------------------------------------------------------------------- +// The real Qwen3.5 header: declares EOS, declares no unknown token, has no ``. +// --------------------------------------------------------------------------- + +#[test] +fn qwen35_real_header_declares_eos_and_no_unknown_token() { + let model = header(QWEN35_HEADER); + assert_eq!(u32_key(&model, EOS_ID_KEY), Some(248_046), "the real EOS id, verbatim"); + assert!( + !model.metadata.contains_key(UNK_ID_KEY), + "Qwen3.5 declares no tokenizer.ggml.unknown_token_id at all" + ); + let vocab = model.vocabulary().expect("the header carries a vocabulary"); + assert!(!vocab.iter().any(|t| t == ""), "and has no token"); +} + +/// THE row #3609 is about. Mutation: restore the unconditional +/// `token_to_id.get("").ok_or_else(..)` in `BPETokenizer::new` and this goes RED. +#[test] +fn qwen35_real_vocabulary_builds_a_tokenizer_with_no_unknown_token() { + let vocab = header(QWEN35_HEADER).vocabulary().expect("vocabulary"); + let unk = vocabulary_unk_token(&vocab); + assert_eq!(unk, None, "nothing named , so nothing is passed"); + let tok = BPETokenizer::new(vocab, vec![], unk) + .expect("a real vocabulary without is not refused (#3609)"); + assert_eq!(tok.unk_token_id(), None, "absent means absent: no synthesised "); + // With no unknown token, a character the vocabulary lacks (CJK here) goes through + // the per-byte glyph fallback and round-trips. `é` is NOT a row: the greedy path + // matches it as the byte-0xE9 glyph token before any fallback runs. That is a + // pre-existing greedy-path defect, independent of the unknown token, filed as #3677. + for text in ["Hi there", "世界", "a\nb"] { + let ids = tok.encode(text); + assert!(!ids.is_empty(), "{text:?} encodes to something"); + assert_eq!(tok.decode(&ids).expect("decode"), text, "{text:?} round-trips"); + } +} + +// --------------------------------------------------------------------------- +// The real TinyLlama header: declares unknown_token_id = 0 = ``. It must still +// resolve (the other direction: the fix is not "ignore unk"). +// --------------------------------------------------------------------------- + +#[test] +fn tinyllama_real_header_declares_unk_and_it_still_resolves() { + let model = header(TINYLLAMA_HEADER); + assert_eq!(u32_key(&model, UNK_ID_KEY), Some(0), "the real declaration, verbatim"); + let vocab = model.vocabulary().expect("vocabulary"); + assert_eq!(vocab[0], "", "id 0 is the declared "); + let unk = vocabulary_unk_token(&vocab); + assert_eq!(unk, Some("")); + let tok = BPETokenizer::new(vocab, vec![], unk).expect("tokenizer"); + assert_eq!(tok.unk_token_id(), Some(0), "the declared unknown token still resolves"); +} + +// --------------------------------------------------------------------------- +// The case table on small vocabularies. +// --------------------------------------------------------------------------- + +#[test] +fn declared_unknown_token_is_still_emitted_for_an_unencodable_byte() { + let tok = BPETokenizer::new(vec!["".into(), "a".into()], vec![], "") + .expect("tokenizer"); + assert_eq!(tok.encode("é"), vec![0, 0], "unchanged: two bytes, two "); +} + +#[test] +fn a_named_unknown_token_missing_from_the_vocabulary_is_refused() { + let err = BPETokenizer::new(vec!["a".into(), "b".into()], vec![], "") + .expect_err("naming an unknown token the vocabulary lacks is an error"); + assert!(err.to_string().contains("''"), "names the token: {err}"); +} + +#[test] +fn no_unknown_token_and_byte_level_glyphs_encode_every_byte() { + let tok = BPETokenizer::new(byte_level_vocab(), vec![], None).expect("tokenizer"); + // '世' is E4 B8 96: not a token, so each byte takes its glyph. + let glyphs: Vec = [0xE4u8, 0xB8, 0x96] + .iter() + .map(|&b| id_of(&tok, &crate::gguf::utils::gpt2_byte_to_unicode(b).to_string())) + .collect(); + assert_eq!(tok.encode("世"), glyphs); + assert_eq!(tok.decode(&tok.encode("世")).expect("decode"), "世"); +} + +#[test] +fn no_unknown_token_and_byte_fallback_tokens_encode_every_byte() { + let vocab: Vec = (0u8..=255).map(|b| format!("<0x{b:02X}>")).collect(); + let tok = BPETokenizer::new(vocab, vec![], None).expect("tokenizer"); + assert_eq!( + tok.encode("é"), + vec![id_of(&tok, "<0xC3>"), id_of(&tok, "<0xA9>")] + ); +} + +/// The cop's row: absent means absent at encode time too. No unknown token, and one byte +/// with neither a `<0xNN>` token nor a glyph token, is refused by name. It is never +/// dropped and never emitted as nothing. +#[test] +fn no_unknown_token_and_a_byte_with_no_token_is_refused_by_name() { + let glyph_a9 = crate::gguf::utils::gpt2_byte_to_unicode(0xA9).to_string(); + let vocab: Vec = byte_level_vocab() + .into_iter() + .filter(|t| *t != glyph_a9) + .collect(); + let err = BPETokenizer::new(vocab, vec![], None) + .expect_err("a byte nothing can encode must refuse the tokenizer"); + assert!(err.to_string().contains("0xA9"), "names the byte: {err}"); +} + +#[test] +fn vocabulary_unk_token_names_only_an_unk_that_is_present() { + assert_eq!(vocabulary_unk_token(&["".into(), "a".into()]), Some("")); + assert_eq!(vocabulary_unk_token(&["a".into(), "b".into()]), None); + assert_eq!(vocabulary_unk_token(&[]), None); +} diff --git a/crates/aprender-serve/tests/fixtures/gguf-header-slices/MANIFEST.json b/crates/aprender-serve/tests/fixtures/gguf-header-slices/MANIFEST.json new file mode 100644 index 0000000000..889f7ab19f --- /dev/null +++ b/crates/aprender-serve/tests/fixtures/gguf-header-slices/MANIFEST.json @@ -0,0 +1,56 @@ +{ + "qwen3.5-0.8b.gguf-header": { + "keep": 1024, + "keys": 46, + "output_bytes": 39063, + "output_sha256": "0855f96190fcfced331768da28b0bd59117c1a825175220b9de1dddac38cb082", + "sliced_arrays": { + "tokenizer.ggml.merges": { + "kept": 1024, + "source_len": 247587 + }, + "tokenizer.ggml.token_type": { + "kept": 1024, + "source_len": 248320 + }, + "tokenizer.ggml.tokens": { + "kept": 1024, + "source_len": 248320 + } + }, + "source_file": "Qwen3.5-0.8B-IQ4_XS.gguf", + "source_gguf_version": 3, + "source_header_bytes": 10943082, + "source_header_sha256": "2eb5e86ecd53b7ce44da7d43c5b20cdd8962ef87444b9e4f963d0f0a0fdf5d36", + "source_tensor_count": 320 + }, + "tinyllama-1.1b-chat.gguf-header": { + "keep": 512, + "keys": 23, + "output_bytes": 18724, + "output_sha256": "e0e4c5887f9b5de6837620b3caa9681e0db92724aaa7a3fa849ee81a1fde71e9", + "sliced_arrays": { + "tokenizer.ggml.merges": { + "kept": 512, + "source_len": 61249 + }, + "tokenizer.ggml.scores": { + "kept": 512, + "source_len": 32000 + }, + "tokenizer.ggml.token_type": { + "kept": 512, + "source_len": 32000 + }, + "tokenizer.ggml.tokens": { + "kept": 512, + "source_len": 32000 + } + }, + "source_file": "tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf", + "source_gguf_version": 3, + "source_header_bytes": 1697530, + "source_header_sha256": "2d44149d9173954e4dfb8b1eca58823d0951e88ac35de4e56abe71f6454b39a9", + "source_tensor_count": 201 + } +} diff --git a/crates/aprender-serve/tests/fixtures/gguf-header-slices/generate.py b/crates/aprender-serve/tests/fixtures/gguf-header-slices/generate.py new file mode 100644 index 0000000000..dace075a29 --- /dev/null +++ b/crates/aprender-serve/tests/fixtures/gguf-header-slices/generate.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +"""Cut a REAL GGUF header down to a committable fixture (#3609). + +A GGUF's metadata section carries its tokenizer: the vocabulary, the merges, and +the keys that say which special tokens the model declares. The real Qwen3.5-0.8B +header is 10.9 MB (248,320 tokens, 247,587 merges), too large to commit. What +the #3609 tests need from it is the real KEY SET and real values: Qwen3.5 +declares `tokenizer.ggml.eos_token_id` and has NO `tokenizer.ggml.unknown_token_id` +key at all, while TinyLlama declares `unknown_token_id = 0` (``). + +So the output is the source header with: + * every key, and every scalar and string value, copied byte-for-byte; + * the tokenizer's per-token arrays (tokens, token_type, scores, merges) cut + to their first --keep entries, which is the ONLY change; + * tensor_count = 0 and no tensor data (a header, not a model). + +The truncation is named here and in MANIFEST.json, never hidden. MANIFEST.json +records the source file's name, the sha256 of its header bytes, and the output's +sha256, so a reader can check that the fixture came from what it claims to. + +usage: generate.py SOURCE.gguf OUT.gguf-header --keep N +""" +import argparse +import hashlib +import json +import os +import struct +import sys + +# The per-token arrays: the only values this script changes. +SLICED = { + "tokenizer.ggml.tokens", + "tokenizer.ggml.token_type", + "tokenizer.ggml.scores", + "tokenizer.ggml.merges", +} +SCALAR_SIZE = {0: 1, 1: 1, 2: 2, 3: 2, 4: 4, 5: 4, 6: 4, 7: 1, 10: 8, 11: 8, 12: 8} +STRING, ARRAY = 8, 9 + + +def read_exact(f, n): + b = f.read(n) + if len(b) != n: + sys.exit(f"truncated source: wanted {n} bytes, got {len(b)}") + return b + + +def read_string_raw(f): + """The string's length prefix and bytes, verbatim.""" + head = read_exact(f, 8) + (n,) = struct.unpack("` vocabulary. It instead fails securely with `no_vocabulary` when a model has no tokenizer. The required prose changes to the book and model_cache examples have been made. The Qwen3.5 falsification test correctly uses a truncated real GGUF header to verify that a model with no unknown token builds without synthesis and without dropping input.", + "findings": [ + { + "claim": "unk becomes Option; absent means absent. No synthesis in either direction. `BPETokenizer::new` accepts `unk_token: impl Into>`.", + "file": "crates/aprender-serve/src/tokenizer.rs", + "grounding": "cited" + }, + { + "claim": "The `vocab[0] = \"\"` placeholder for an absent vocabulary goes too. A GGUF with no vocabulary refuses by name.", + "file": "crates/apr-cli/src/commands/serve/handler_gpu_completion.rs", + "grounding": "cited" + }, + { + "claim": "When the glyph for a byte is ALSO absent, encode must refuse with an error naming the byte. It must never drop the input or emit nothing. Tested during construction.", + "file": "crates/aprender-serve/src/tokenizer.rs", + "grounding": "cited" + }, + { + "claim": "Prose changes in the same PR removing the belief that the vocabulary must include ``.", + "file": "crates/aprender-serve/book/src/examples/api-server.md", + "grounding": "cited" + }, + { + "claim": "Mutation: reintroduce the `ok_or_else` -> the Qwen3.5 fixture goes RED. Fixture is a real Qwen3.5 GGUF header without `unknown_token_id`.", + "file": "crates/aprender-serve/src/tokenizer_tests_unk_3609.rs", + "grounding": "cited" + } + ], + "raw_bytes": 5514, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-high", + "model_measured": "gemini-3.1-pro-high", + "model_source": "measured", + "family": "gemini" + }, + { + "lane": 2, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff satisfies all acceptance criteria for PMAT-3609. The unknown token is now optional (absent means absent), placeholder synthesis is removed, models with no vocabulary correctly refuse to load, the required prose changes are made, and fixture tests verify the behavior in both directions (Qwen3.5 and TinyLlama). The missing glyph guard is also implemented and tested.", + "findings": [ + { + "claim": "`unk` becomes `Option`; absent means absent.", + "file": "crates/aprender-serve/src/tokenizer.rs", + "grounding": "cited" + }, + { + "claim": "No synthesis (fallback to placeholder tokens removed).", + "file": "crates/apr-cli/src/commands/serve/handler_gpu_completion.rs", + "grounding": "cited" + }, + { + "claim": "Model with no vocabulary refuses.", + "file": "crates/apr-cli/src/commands/serve/handler_gpu_completion.rs", + "grounding": "cited" + }, + { + "claim": "Prose changes are present in book and examples.", + "file": "crates/aprender-serve/book/src/examples/api-server.md", + "grounding": "cited" + }, + { + "claim": "Fixture tests for Qwen3.5 (no unknown token) and TinyLlama (with ``) are included.", + "file": "crates/aprender-serve/src/tokenizer_tests_unk_3609.rs", + "grounding": "cited" + }, + { + "claim": "Guard: missing glyph refused by name.", + "file": "crates/aprender-serve/src/tokenizer.rs", + "grounding": "cited" + } + ], + "raw_bytes": 4806, + "err_bytes": 100, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-low", + "model_measured": "gemini-3.1-pro-low", + "model_source": "measured", + "family": "gemini" + }, + { + "lane": 3, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff fully satisfies all acceptance criteria of PMAT-3609. The BPE tokenizer unknown token parameter (`unk_token`) was changed from a hardcoded compulsory string requirement (`\"\"`) to an `Option`, matching model family realities where byte-level models like Qwen3.5 do not declare an unknown token. Additionally, all fallback mechanisms that generated dummy placeholder `token0..tokenN` vocabularies with `` at slot 0 were replaced with explicit `no_vocabulary` refusal errors. Documentation and example references were updated, real Qwen3.5 and TinyLlama GGUF header fixtures were added, and comprehensive unit tests were added to verify behavior in both directions (models with `` and models without). No gates were weakened, no extra unrequested changes were introduced, and tests verify the implementation.", + "findings": [ + { + "claim": "BPETokenizer::new parameter unk_token changed to impl Into> and unk_token_id changed to Option to allow model families without an unknown token (such as Qwen3.5) to construct without requiring a synthesised '' token.", + "file": "crates/aprender-serve/src/tokenizer.rs", + "fix": "pub fn new<'a>(vocab: Vec, _merges: Vec<(String, String)>, unk_token: impl Into>) -> Result", + "grounding": "cited", + "line": 168 + }, + { + "claim": "Placeholder token substitution (`vocab[0] = \"\"`) on missing vocabulary in GGUF models was removed and replaced with explicit `no_vocabulary` refusal by name.", + "file": "crates/apr-cli/src/commands/serve/handler_gpu_completion.rs", + "fix": "fn extract_gguf_vocab(mapped_model: &realizar::gguf::MappedGGUFModel) -> Result> { mapped_model.model.vocabulary().ok_or_else(|| no_vocabulary(\"the GGUF (no tokenizer.ggml.tokens)\")) }", + "grounding": "cited", + "line": 473 + }, + { + "claim": "qwen35_real_vocabulary_builds_a_tokenizer_with_no_unknown_token verifies that real Qwen3.5 vocabulary without '' token constructs a tokenizer with unk_token_id = None without failing or synthesising a fake '' token.", + "file": "crates/aprender-serve/src/tokenizer_tests_unk_3609.rs", + "fix": "let tok = BPETokenizer::new(vocab, vec![], unk).expect(\"a real vocabulary without is not refused (#3609)\");", + "grounding": "cited", + "line": 65 + }, + { + "claim": "Existing tokenizer unit tests pass with the optional unk_token changes.", + "command": "cargo test -p aprender-serve --lib tokenizer_tests", + "file": "crates/aprender-serve/src/tokenizer.rs", + "fix": "Verified via cargo test execution", + "grounding": "measured", + "line": 156, + "output": "running 45 tests\ntest tokenizer_04::tokenizer_tests_bpe_encode::test_bpe_encode_basic ... ok\ntest tokenizer_04::tokenizer_tests_bpe_encode::test_bpe_encode_empty ... ok\ntest tokenizer_04::tokenizer_tests_bpe_encode::test_bpe_encode_hierarchy ... ok\ntest tokenizer_04::tokenizer_tests_bpe_encode::test_bpe_encode_no_matching_merges ... ok\ntest tokenizer_04::tokenizer_tests_bpe_encode::test_bpe_encode_special_tokens ... ok\ntest tokenizer_04::tokenizer_tests_sentencepiece_viterbi::test_viterbi_all_unk ... ok\ntest tokenizer_04::tokenizer_tests_sentencepiece_viterbi::test_viterbi_bpe_encode ... ok\ntest tokenizer_04::tokenizer_tests_sentencepiece_viterbi::test_viterbi_empty_input ... ok\ntest tokenizer_04::tokenizer_tests_sentencepiece_viterbi::test_viterbi_greedy_match ... ok\ntest tokenizer_04::tokenizer_tests_sentencepiece_viterbi::test_viterbi_impossible_string ... ok\ntest tokenizer_04::tokenizer_tests_sentencepiece_viterbi::test_viterbi_multiple_candidate_matches ... ok\ntest tokenizer_04::tokenizer_tests_sentencepiece_viterbi::test_viterbi_prefers_higher_score ... ok\ntest tokenizer_04::tokenizer_tests_sentencepiece_viterbi::test_viterbi_real_tokens ... ok\ntest tokenizer_04::tokenizer_tests_sentencepiece_viterbi::test_viterbi_single_char ... ok\ntest tokenizer_04::tokenizer_tests_sentencepiece_viterbi::test_viterbi_space_token ... ok\ntest tokenizer_04::tokenizer_tests_sentencepiece_viterbi::test_viterbi_unseen_char ... ok\ntest tokenizer::tests::test_bpe_decode ... ok\ntest tokenizer::tests::test_bpe_decode_empty ... ok\ntest tokenizer::tests::test_bpe_decode_unknown_token ... ok\ntest tokenizer::tests::test_bpe_encode ... ok\ntest tokenizer::tests::test_bpe_encode_empty ... ok\ntest tokenizer::tests::test_bpe_encode_unknown_chars ... ok\ntest tokenizer::tests::test_bpe_tokenizer_creation ... ok\ntest tokenizer::tests::test_bpe_tokenizer_empty_vocab ... ok\ntest tokenizer::tests::test_bpe_tokenizer_invalid_unk ... ok\ntest tokenizer::tests::test_bpe_tokenizer_properties ... ok\ntest tokenizer::tests::test_bpe_tokenizer_vocab_helpers ... ok\ntest tokenizer::tests::test_bpe_with_merges ... ok\ntest tokenizer::tests::test_bpe_with_merges_empty ... ok\ntest tokenizer::tests::test_bpe_with_merges_no_matches ... ok\ntest tokenizer::tests::test_bpe_with_merges_non_ascii ... ok\ntest tokenizer::tests::test_byte_fallback ... ok\ntest tokenizer::tests::test_get_token ... ok\ntest tokenizer::tests::test_get_token_id ... ok\ntest tokenizer::tests::test_latin1_byte_level_roundtrip ... ok\ntest tokenizer::tests::test_special_tokens ... ok\ntest tokenizer::tests::test_tokenizer_trait_implementation ... ok\ntest tokenizer::tests::test_unknown_token ... ok\ntest tokenizer::tests::test_vocab_size ... ok\ntest tokenizer::tests::test_sentencepiece_score ... ok\ntest tokenizer::tests::test_sentencepiece_tokenization ... ok\ntest tokenizer::tests::test_viterbi_best_sequence ... ok\ntest tokenizer::tests::test_viterbi_decoding ... ok\ntest tokenizer::tests::test_viterbi_empty_vocab ... ok\ntest tokenizer::tests::test_viterbi_tokenizer_trait ... ok\n\ntest result: ok. 45 passed; 0 failed; 0 ignored; 0 measured; 7238 filtered out; finished in 0.05s" + } + ], + "raw_bytes": 18741, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.6-flash-high", + "model_measured": "gemini-3.6-flash-high", + "model_source": "measured", + "family": "gemini" + } + ], + "dissent": [], + "dedup": [ + { + "file": "crates/apr-cli/src/commands/serve/handler_gpu_completion.rs", + "line": null, + "lanes_agreeing": [ + 1, + 2 + ], + "claims": [ + "Model with no vocabulary refuses.", + "No synthesis (fallback to placeholder tokens removed).", + "The `vocab[0] = \"\"` placeholder for an absent vocabulary goes too. A GGUF with no vocabulary refuses by name." + ] + }, + { + "file": "crates/apr-cli/src/commands/serve/handler_gpu_completion.rs", + "line": 473, + "lanes_agreeing": [ + 3 + ], + "claims": [ + "Placeholder token substitution (`vocab[0] = \"\"`) on missing vocabulary in GGUF models was removed and replaced with explicit `no_vocabulary` refusal by name." + ] + }, + { + "file": "crates/aprender-serve/book/src/examples/api-server.md", + "line": null, + "lanes_agreeing": [ + 1, + 2 + ], + "claims": [ + "Prose changes are present in book and examples.", + "Prose changes in the same PR removing the belief that the vocabulary must include ``." + ] + }, + { + "file": "crates/aprender-serve/src/tokenizer.rs", + "line": null, + "lanes_agreeing": [ + 1, + 2 + ], + "claims": [ + "Guard: missing glyph refused by name.", + "When the glyph for a byte is ALSO absent, encode must refuse with an error naming the byte. It must never drop the input or emit nothing. Tested during construction.", + "`unk` becomes `Option`; absent means absent.", + "unk becomes Option; absent means absent. No synthesis in either direction. `BPETokenizer::new` accepts `unk_token: impl Into>`." + ] + }, + { + "file": "crates/aprender-serve/src/tokenizer.rs", + "line": 156, + "lanes_agreeing": [ + 3 + ], + "claims": [ + "Existing tokenizer unit tests pass with the optional unk_token changes." + ] + }, + { + "file": "crates/aprender-serve/src/tokenizer.rs", + "line": 168, + "lanes_agreeing": [ + 3 + ], + "claims": [ + "BPETokenizer::new parameter unk_token changed to impl Into> and unk_token_id changed to Option to allow model families without an unknown token (such as Qwen3.5) to construct without requiring a synthesised '' token." + ] + }, + { + "file": "crates/aprender-serve/src/tokenizer_tests_unk_3609.rs", + "line": null, + "lanes_agreeing": [ + 1, + 2 + ], + "claims": [ + "Fixture tests for Qwen3.5 (no unknown token) and TinyLlama (with ``) are included.", + "Mutation: reintroduce the `ok_or_else` -> the Qwen3.5 fixture goes RED. Fixture is a real Qwen3.5 GGUF header without `unknown_token_id`." + ] + }, + { + "file": "crates/aprender-serve/src/tokenizer_tests_unk_3609.rs", + "line": 65, + "lanes_agreeing": [ + 3 + ], + "claims": [ + "qwen35_real_vocabulary_builds_a_tokenizer_with_no_unknown_token verifies that real Qwen3.5 vocabulary without '' token constructs a tokenizer with unk_token_id = None without failing or synthesising a fake '' token." + ] + } + ], + "uncovered": [], + "coverage_source": "lanes", + "partial": false, + "partial_reasons": [], + "auto_merge": { + "checked": false, + "was_armed": false, + "disarmed": false, + "note": "no --pr given: nothing to disarm" + }, + "lint": { + "ok": true, + "output": "receipt complete: kind=artifact lanes=3 author=claude-opus-5/claude" + } +} diff --git a/docs/audits/quorum-PMAT-3673.json b/docs/audits/quorum-PMAT-3673.json new file mode 100644 index 0000000000..8201a77d7e --- /dev/null +++ b/docs/audits/quorum-PMAT-3673.json @@ -0,0 +1,207 @@ +{ + "ticket": "PMAT-3673", + "base": "origin/main", + "base_resolved": "origin/main", + "base_note": "no origin/origin/main exists; judged against the local ref", + "head": "eebd642c85a3cbe974d8177a58c72b6c0d0e1801", + "diff_sha256": "33accb56a764c6bc6884689eff02ff0eb49f61261d941278d7aad0d1bbae58f8", + "width": 3, + "executor": "agy", + "prompt_mode": "inline", + "prompt_bytes": 37168, + "author": { + "model": "claude-opus-5", + "family": "claude", + "source": "flag" + }, + "agreed": true, + "lanes": [ + { + "lane": 1, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The PR successfully implements the integration batching procedure. `scripts/batch_fold.sh` accurately implements the fold rules, strictly distinguishing between generated/non-generated conflicts. The `--regen` flag executes correctly and checks fixed points. The comprehensive `scripts/check_batch_fold.sh` correctly prevents tampering through its self-test mutation capability. `APR-RELEASE-001` documentation covers all procedures accurately.", + "findings": [ + { + "claim": "The script resolves ONLY the specified generated set on conflicts, takes '--theirs' for them, and skips on any other real conflicts.", + "command": "bash check_batch_fold.sh", + "file": "scripts/batch_fold.sh", + "grounding": "measured", + "line": 139, + "output": "=== batch_fold.sh folds only what it may, and says so (check_batch_fold.sh) ===\nok row 1 a branch touching no generated path folds, generated=[]\nok row 2 a conflict ONLY in roadmap.yaml folds, marked for regeneration\nok row 3 an independent code change folds\nok row 4 a conflict in real code is SKIPPED, never resolved by picking a side" + }, + { + "claim": "README conflicts are strictly partitioned using readme_counts_only to fold only CONTRACT_COUNT block differences and SKIP prose conflicts.", + "command": "bash check_batch_fold.sh", + "file": "scripts/batch_fold.sh", + "grounding": "measured", + "line": 61, + "output": "ok row 8 a README conflict ONLY in a CONTRACT_COUNT block folds, marked for regeneration\nok row 9 ...and the other side's clean README prose edit is kept\nok row 10 ...and the count block stays numeric (well-formed for readme_sync)\nok row 11 a folded, unregenerated generated set is never silent: REGEN REQUIRED, exit 3 (rc=3)\nok row 12 a README conflict OUTSIDE a count block is a real conflict: SKIP" + }, + { + "claim": "§13 explicitly defines the batching procedure as default, covers generated-set regeneration, assembly, what needs a new quorum, and red batches.", + "file": "docs/specifications/APR-RELEASE-001-train-and-build-kaizen.md", + "grounding": "cited", + "line": 1001 + }, + { + "claim": "The self-test case table exists, tests exact mutant cases that would violate the ticket (like taking all conflicts or ignoring README prose conflicts), and successfully fails on them.", + "command": "bash check_batch_fold.sh --self-test", + "file": "scripts/check_batch_fold.sh", + "grounding": "measured", + "line": 150, + "output": "=== check_batch_fold.sh --self-test: each wrong folder must turn the table RED ===\nok mutant RED: README treated as wholly generated (the draft's rule) (row 8 a README conflict ONLY in a CONTRACT_COUNT block folds, marked for regenerati)\nok mutant RED: every conflict taken, as if generated (row 4 a conflict in real code is SKIPPED, never resolved by picking a side)\nok mutant RED: a clean fold never marks the set stale (row 14 a CLEAN fold that touches the census still marks the set stale (exit 3, rc=0))\nok mutant RED: a failed fixed-point check ignored (row 21 a failed fixed-point check is exit 2 and names the check (rc=0))\nSELF-TEST PASSED: 4 mutants, all RED" + } + ], + "raw_bytes": 8554, + "err_bytes": 100, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-high", + "model_measured": "gemini-3.1-pro-high", + "model_source": "measured", + "family": "gemini" + }, + { + "lane": 2, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff perfectly implements PMAT-3673. `scripts/batch_fold.sh` correctly resolves only the generated set and the README `CONTRACT_COUNT` blocks, skipping branches with real conflicts and leaving no half-merged state. The regeneration mechanism (`--regen`) correctly asserts the three fixed points and commits. The required exit codes are implemented, and an unregenerated set gracefully fails with exit code 3. The `check_batch_fold.sh` table properly covers these cases and is wired into CI as a `check_*.sh` script. The updates to the release specification fully define all requested rules (assembly, left-out, quorum, push/disarm, red batch, and the VOID rule).", + "findings": [], + "raw_bytes": 3477, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-low", + "model_measured": "gemini-3.1-pro-low", + "model_source": "measured", + "family": "gemini" + }, + { + "lane": 3, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "Independent Review Lane 3: Verdict PASS for PMAT-3673.\n\nThe diff completely satisfies all ticket requirements:\n1. `scripts/batch_fold.sh` correctly implements folding of receipted PR branches into `release/-batch`, resolving only conflicts in the generated set ({roadmap.yaml, census.json, contracts.nt, shapes.ttl, README.md CONTRACT_COUNT blocks}) and aborting/reporting SKIP (exit 1) for any real conflict elsewhere. Fixed points are asserted upon `--regen`, returning exit 0 on clean completion, exit 2 on usage/git/regen error, and exit 3 when regeneration is required.\n2. `scripts/check_batch_fold.sh` provides a comprehensive 21-row test table verifying all folding and regeneration edge cases, along with a self-test verifying 4 mutants are caught (RED).\n3. `APR-RELEASE-001-train-and-build-kaizen.md` is updated with Revision 2026-09-21, amended Rule 9, amended §8 stop, and full §13 integration batch specifications (including VOID review lane rules and falsifiers FX-B1..FX-B3).\n4. All shell scripts pass `bash -n` and `shellcheck -S error`.", + "findings": [ + { + "claim": "check_batch_fold.sh test table passes all 21 test rows and --self-test catches all 4 mutants", + "command": "bash /tmp/test_batch_fold/check_batch_fold.sh && bash /tmp/test_batch_fold/check_batch_fold.sh --self-test", + "file": "scripts/check_batch_fold.sh", + "grounding": "measured", + "output": "=== batch_fold.sh folds only what it may, and says so (check_batch_fold.sh) ===\nok row 1 a branch touching no generated path folds, generated=[]\nok row 2 a conflict ONLY in roadmap.yaml folds, marked for regeneration\nok row 3 an independent code change folds\nok row 4 a conflict in real code is SKIPPED, never resolved by picking a side\nok row 5 any SKIP makes the exit 1 (rc=1)\nok row 6 the aborted merge leaves no half-merged state (dirty=0)\nok row 7 the skipped branch's code did not leak in (lib=y)\nok row 8 a README conflict ONLY in a CONTRACT_COUNT block folds, marked for regeneration\nok row 9 ...and the other side's clean README prose edit is kept\nok row 10 ...and the count block stays numeric (well-formed for readme_sync)\nok row 11 a folded, unregenerated generated set is never silent: REGEN REQUIRED, exit 3 (rc=3)\nok row 12 a README conflict OUTSIDE a count block is a real conflict: SKIP\nok row 13 ...and the README keeps the side already in the batch\nok row 14 a CLEAN fold that touches the census still marks the set stale (exit 3, rc=3)\nok row 15 a dirty batch worktree is refused (exit 2, rc=2)\nok row 16 a branch that does not exist is ERROR, exit 2 (rc=2)\nok row 17 --regen regenerates once, asserts the fixed points, commits (rc=0)\nok row 18 ...the last commit is the regeneration (batch: regenerate the generated set once (batch_fold.sh --regen))\nok row 19 ...roadmap aggregated and README counts rewritten\nok row 20 ...and the tree is clean after it (dirty=0)\nok row 21 a failed fixed-point check is exit 2 and names the check (rc=2)\n21 rows, PASS\nPASS\n=== check_batch_fold.sh --self-test: each wrong folder must turn the table RED ===\nok mutant RED: README treated as wholly generated (the draft's rule) (row 8 a README conflict ONLY in a CONTRACT_COUNT block folds, marked for regenerati)\nok mutant RED: every conflict taken, as if generated (row 4 a conflict in real code is SKIPPED, never resolved by picking a side)\nok mutant RED: a clean fold never marks the set stale (row 14 a CLEAN fold that touches the census still marks the set stale (exit 3, rc=0))\nok mutant RED: a failed fixed-point check ignored (row 21 a failed fixed-point check is exit 2 and names the check (rc=0))\nSELF-TEST PASSED: 4 mutants, all RED" + }, + { + "claim": "scripts/batch_fold.sh and scripts/check_batch_fold.sh pass bash syntax checking and shellcheck with zero errors", + "command": "bash -n /tmp/test_batch_fold/batch_fold.sh /tmp/test_batch_fold/check_batch_fold.sh && shellcheck -S error /tmp/test_batch_fold/batch_fold.sh /tmp/test_batch_fold/check_batch_fold.sh", + "file": "scripts/batch_fold.sh", + "grounding": "cited", + "output": "", + "grounding_downgraded": "measured-without-command-or-output" + }, + { + "claim": "APR-RELEASE-001 is updated with §13 defining batching procedure, disarm/close rules, VOID review-lane rule, and falsifiers FX-B1..FX-B3", + "file": "docs/specifications/APR-RELEASE-001-train-and-build-kaizen.md", + "grounding": "cited" + } + ], + "raw_bytes": 11296, + "err_bytes": 100, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.6-flash-high", + "model_measured": "gemini-3.6-flash-high", + "model_source": "measured", + "family": "gemini" + } + ], + "dissent": [], + "dedup": [ + { + "file": "docs/specifications/APR-RELEASE-001-train-and-build-kaizen.md", + "line": null, + "lanes_agreeing": [ + 3 + ], + "claims": [ + "APR-RELEASE-001 is updated with §13 defining batching procedure, disarm/close rules, VOID review-lane rule, and falsifiers FX-B1..FX-B3" + ] + }, + { + "file": "docs/specifications/APR-RELEASE-001-train-and-build-kaizen.md", + "line": 1001, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "§13 explicitly defines the batching procedure as default, covers generated-set regeneration, assembly, what needs a new quorum, and red batches." + ] + }, + { + "file": "scripts/batch_fold.sh", + "line": null, + "lanes_agreeing": [ + 3 + ], + "claims": [ + "scripts/batch_fold.sh and scripts/check_batch_fold.sh pass bash syntax checking and shellcheck with zero errors" + ] + }, + { + "file": "scripts/batch_fold.sh", + "line": 61, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "README conflicts are strictly partitioned using readme_counts_only to fold only CONTRACT_COUNT block differences and SKIP prose conflicts." + ] + }, + { + "file": "scripts/batch_fold.sh", + "line": 139, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "The script resolves ONLY the specified generated set on conflicts, takes '--theirs' for them, and skips on any other real conflicts." + ] + }, + { + "file": "scripts/check_batch_fold.sh", + "line": null, + "lanes_agreeing": [ + 3 + ], + "claims": [ + "check_batch_fold.sh test table passes all 21 test rows and --self-test catches all 4 mutants" + ] + }, + { + "file": "scripts/check_batch_fold.sh", + "line": 150, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "The self-test case table exists, tests exact mutant cases that would violate the ticket (like taking all conflicts or ignoring README prose conflicts), and successfully fails on them." + ] + } + ], + "uncovered": [], + "coverage_source": "lanes", + "partial": false, + "partial_reasons": [], + "auto_merge": { + "checked": true, + "was_armed": false, + "disarmed": false, + "note": "auto-merge not armed" + }, + "lint": { + "ok": true, + "output": "receipt complete: kind=artifact lanes=3 author=claude-opus-5/claude" + } +} diff --git a/docs/roadmaps/entries/PMAT-3076.yaml b/docs/roadmaps/entries/PMAT-3076.yaml new file mode 100644 index 0000000000..59fe9734e9 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3076.yaml @@ -0,0 +1,17 @@ +- id: PMAT-3076 + github_issue: 3076 + item_type: task + title: 'F16/BF16 GGUF CPU matmul has no SIMD path — unquantized models appear to hang' + status: planned + priority: high + assigned_to: null + created: 2026-09-21T07:52:58Z + updated: 2026-09-21T07:52:58Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'ACCEPTANCE (hand-entered; #3076 has no done_when section, so this quotes its last plan comment verbatim, minted on the PR branch so the AD-04 quorum can run). "(1) the GGUF CPU path gets a SIMD f16/bf16 matvec (dequant-to-f32 tiles through the existing fused dispatcher) with a greedy-parity gate against llama.cpp on an unquantized ≤1B GGUF, so an F16/BF16 model decodes at a measured tok/s instead of appearing to hang; (2) until that lands the loader prints a one-line notice that the F16/BF16 path is scalar and slow rather than silently crawling." Since (1) lands here, (2) is moot. DEVIATION from the issue body: it suggests an integration test "with a wall-clock assertion"; the project bans wall-clock assertions in required checks, so the regression guard is behavioural (kernel parity against an f64 reference at lengths hitting every loop and tail, and float16_matmul against a transcription of the pre-fix loop including a truncated buffer), and tok/s is measured and reported rather than asserted. Not in scope: prefill running one position at a time at decode rate (#2787 / #2801). EVIDENCE (added after the round-1 quorum found the llama.cpp gate missing): scripts/float16_greedy_parity.sh is the greedy-parity gate against the pinned llama.cpp (llama_pin.toml build_commit), on template-free copies of an unquantized ≤1B F16 and BF16 GGUF, because apr double-templates instruct prompts (#3672); --baseline separates divergences this change introduces from ones that predate it. Receipts, the negative control (a deliberately broken binary FAILs 12/12) and the measured before/after decode tok/s with model sha256, host and binaries are in evidence/pmat-3076-f16-bf16-matvec/. WIRING (cop ruling, no workflow edit): the model-backed run is declared in Cargo.toml [package.metadata.dogfood] and runs bare (release mode: the committed receipts are the baseline and pin each model sha256; a model absent from the host is a FAIL) in scripts/dogfood.sh --phase pre-publish; the comparison helpers live in scripts/float16_parity_lib.sh and their case table runs on every PR as scripts/check_float16_greedy_parity.sh through guard_tree.sh.' diff --git a/docs/roadmaps/entries/PMAT-3609.yaml b/docs/roadmaps/entries/PMAT-3609.yaml new file mode 100644 index 0000000000..83cf5d0352 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3609.yaml @@ -0,0 +1,24 @@ +- id: PMAT-3609 + github_issue: 3609 + item_type: task + title: 'The BPE tokenizer requires a '''' token that Qwen3.5 does not have — a universal unknown-token is an assumption, not a fact' + status: in_progress + priority: high + assigned_to: aprender-62 + created: 2026-09-21T09:00:00Z + updated: 2026-09-21T09:00:00Z + spec: null + acceptance_criteria: + - 'Operator ruling on #3609 (2026-09-20), constraint 1, verbatim: "`unk` becomes `Option`; absent means absent. No synthesis in either direction. That is stronger than the original framing and deliberately so: the `vocab[0] = ""` placeholder for an absent vocabulary goes too. A GGUF with no vocabulary refuses by name — it does not load. Today it loads, which is the inversion this ticket is about; removing only the refusal on the present-but-lacking side would leave the permissive fallback intact and simply move the asymmetry. The two-direction falsifier already proposed covers both: a model with an unknown token still resolves it (so the fix is not \"ignore unk\"), and a model with no vocabulary now refuses (so the fix is not \"synthesise one\")."' + - 'Constraint 2, verbatim: "Prose changes in the same PR, or the belief is reproduced. `book/src/examples/api-server.md` and `examples/model_cache.rs` both repeat \"vocabulary must include ``\" — a property of our constructor stated as a property of the data. Run `pmat query` for `` and `unknown_token`; every hit is either fixed or listed in the PR body as intentionally kept. A hit that is neither is the defect''s next copy."' + - 'Constraint 3, verbatim: "Mutation: reintroduce the `ok_or_else` → the Qwen3.5 fixture goes RED. A fix with no red-turning mutation is a claim. The fixture must be a real Qwen3.5 GGUF header — the family declares `tokenizer.ggml.eos_token_id` and no `unknown_token_id` key at all, which is the actual input, not a synthetic one."' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'Taken by aprender-62, sequenced ahead by the cop (aprender-04). Two interpretation decisions, sent to the cop before implementation: (1) THE FIXTURE. The real Qwen3.5-0.8B header is 10.9 MB (248,320 tokens, 247,587 merges), too large to commit. The fixture is a header-only GGUF generated from the real file: every one of its 46 keys and every scalar value verbatim (eos 248046, padding 248055, model gpt2, pre qwen35, the chat template; no unknown_token_id and no bos key, exactly as in the real file), with only the three arrays (tokens, token_type, merges) cut to a leading slice. Qwen''s first 256 tokens are its byte-level glyphs, so the slice still covers every byte. The generator is committed next to it and records the source header''s sha256. (2) WHERE THE UNKNOWN TOKEN COMES FROM at the vocabulary-only constructors, which receive a Vec and nothing else: the vocabulary''s own `` entry when it has one, otherwise none. Nothing is synthesised, and every model that loads today keeps its unknown token. Threading `tokenizer.ggml.unknown_token_id` through every loader is a filed follow-up, not a silent gap. + + The cop ruled both decisions YES on 2026-09-21. It also added a guard, verbatim: "When the glyph for a byte is ALSO absent, encode must refuse with an error naming the byte. It must never drop the input or emit nothing. Put a row for it in the case table (a vocab without that glyph and without unk → error). Absent means absent at encode time too." How it is met: `encode` returns an infallible Vec and has dozens of callers, so the refusal happens at CONSTRUCTION. With no unknown token, `BPETokenizer::new` refuses, naming the first byte that has neither a `<0xNN>` token nor a byte-level glyph token. `with_merges` also requires every glyph, because the merge path maps bytes to glyphs only. `encode` therefore cannot meet an unencodable byte, and the no-drop guarantee holds on every path. Case row: `no_unknown_token_and_a_byte_with_no_token_is_refused_by_name` (a 256-glyph vocabulary minus byte 0xA9''s glyph → error naming 0xA9). With the unknown token present, encode is unchanged: `<0xNN>`, then the unknown token. The glyph fallback applies only when there is no unknown token. Follow-up filed before the PR: #3675 (thread the DECLARED unknown token through every loader; by-name `vocabulary_unk_token` is the interim, and `apr serve`''s chat path already uses tokenizer.json''s declared `model.unk_token`). Found while testing, filed separately because it is independent of the unknown token: #3677 (the greedy path matches Latin-1 characters such as `é` as byte-level glyph tokens, so `é` decodes to U+FFFD). The Qwen round-trip rows therefore use ASCII, CJK and newline, not `é`. Same class but outside the query''s hits, so not in this PR: `apr showcase`''s pipeline substitutes token ids [1,2,3,4,5] when a model has no vocabulary (commands/showcase/pipeline.rs:165). + + Repo-layout facts a reviewer needs, each checked at the PR head (round 2 lane 1 misread all three, and every one is refuted by `cargo check -p apr-cli` rc=0 and `cargo test -p apr-cli --lib` 7283 passed at e5b2e7541). (1) `realizar` IS aprender-serve: crates/aprender-serve/Cargo.toml:42-43 is `[lib] name = "realizar"`, so `realizar::tokenizer::BPETokenizer` in apr-cli and `crate::tokenizer::BPETokenizer` in aprender-serve are the SAME type, and `realizar::tokenizer::vocabulary_unk_token` (the book, api-server.md) is the function this PR adds. (2) `no_vocabulary` in apr-cli is defined ONCE, at crates/apr-cli/src/commands/serve/handler_gpu_completion.rs:482, and is visible to every serve site because these files are one module: handlers.rs:1618 is `include!("handler_apr_cpu_completion.rs")`, handlers.rs:1619 is `include!("handler_gpu_completion.rs")`, and handler_apr_cpu_completion.rs:869 is `include!("handlers_include_01.rs")`. (3) aprender-serve''s cli has its own `no_vocabulary` (cli/mod_server_commands.rs:13, returning RealizarError), inside the same `mod server_commands` as its six call sites.' diff --git a/docs/roadmaps/entries/PMAT-3646.yaml b/docs/roadmaps/entries/PMAT-3646.yaml new file mode 100644 index 0000000000..a06966c712 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3646.yaml @@ -0,0 +1,23 @@ +- id: PMAT-3646 + github_issue: 3646 + item_type: task + title: check_pr_review_counts.sh is RED on main (6 rows disagree) and check_pr_review_receipt.sh has no caller — both dark since PRQ-013 moved the receipt job, classified out of --no-cargo by a comment + status: planned + priority: high + assigned_to: null + created: 2026-09-21T06:00:51Z + updated: 2026-09-21T06:00:51Z + spec: null + acceptance_criteria: + - 'Issue #3646 "What lands" item 1, verbatim: "Fix the 6 disagreeing rows in PR-REVIEW-SKILL-002''s files (or the derivation), then reword the one comment so the guard leaves the cargo population and `guard-tree` runs it; delete its ledger line in the same commit (shrink-only)."' + - 'Issue #3646 "What lands" item 2, verbatim: "Decide `check_pr_review_receipt.sh`''s caller: either `pr-review-quorum.yml` invokes it on the PR''s receipt (a workflow edit — operator''s call), or it is folded into `check_pr_review_arm4.sh`, which is already wired with args. Delete its ledger line with whichever lands."' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'SCOPE OF THIS PR: item 1 only. The PR says `Refs #3646`, not a closing keyword, and carries a `keep-open:` line for item 2. check_pr_review_receipt.sh keeps its line in scripts/unwired_guards_baseline.txt on purpose: neither of item 2''s two options lands here. The workflow edit is the operator''s call, and the fold is moot because check_pr_review_arm4.sh already invokes it with a receipt (see Item 2 below). The ledger entry''s own comment says so ("NOT dark: check_pr_review_arm4.sh runs it … Ledgered only because this meta-guard counts a DIRECT workflow invocation"). A reviewer should judge item 2 as OPEN BY DESIGN, not as missed. Handed over by aprender-b3 (2026-09-21); claimed on the issue. Measured on origin/main @ 237fbc32f per the issue: `bash scripts/check_pr_review_counts.sh` rc=1 "--- 6 row(s) disagree with the tree ---"; `bash scripts/check_pr_review_receipt.sh` rc=1 "A run over zero receipts is not a pass (S6.3: a missing receipt is RED)." The ledger line for each guard is added by #3647 (scripts/unwired_guards_baseline.txt, shrink-only); it is deleted in the commit that wires that guard, after #3647 merges. Item 1 lands first; item 2 lands here only if it folds into check_pr_review_arm4.sh with no workflow edit, otherwise the ticket stays open for the operator''s call and the PR says Refs, not Closes. + + What the item-1 PR does. (a) The 6 RED rows were ci.yml sites whose sentences PRQ-013 deleted along with the receipt job (roadmap: "STATED LOSS"), so the rows go. The bats_tests derivation goes too, because no file states that count any more; ci.yml''s fixture_rows site goes from 2 to 1; and scripts/check_pr_review_wiring.sh''s "43-row" joins the table with a self-test row of its own. (b) The :196 comment is reworded so CARGO_RE stops matching: `guard_tree.sh --dry-run --no-cargo` prints `run: scripts/check_pr_review_counts.sh`, and restoring the old wording moves it back to --cargo-only (mutation-verified). (c) The --list mode of mutate-guard.sh and mutate_quorum_arm.sh requires only awk. Before this it demanded bats/jq/minisign/check-jsonschema, so wiring the counts guard into guard-tree, which runs on clean-room HOSTS including the gx10/yoga docker runners, would have turned a dark guard into an ENV-red job (reproduced: rc=2 under a PATH without those four). Sweeps still refuse without them (rc=1). + + Item 2, measured: check_pr_review_receipt.sh HAS a caller with a receipt. check_pr_review_arm4.sh runs `bash "$guard" "$scratch"` (A3) and `bash "$guard" "$best_dir"` (A4) with guard defaulting to it, and pr-review-quorum.yml:141 runs arm4. So there is nothing to fold. What remains is whether guard_tree.sh and the unwired ledger should treat a guard reachable only through another wired guard as wired. That is recorded on the issue and not changed here. LEDGER LINE (added after the 0.69 batch landed): check_pr_review_counts.sh entered scripts/unwired_guards_baseline.txt through #3644 in batch #3669 (squash a877fa056), after this branch was cut, so the deletion criterion 1 names could only be made once main carried the line. Main is merged into this branch (the only conflict was the generated roadmap.yaml, regenerated from entries/), and the entry and its four comment lines are deleted. On the squash-merge this is one commit with the fix. Measured on the merged tree: `bash scripts/check_guards_are_wired.sh` rc 0, "did not grow (1 removed) vs a877fa056", 149 scanned, 5 named by no workflow; `bash scripts/check_pr_review_counts.sh` rc 0, "PASS every stated count equals the derived one"; `--self-test` 16/16; `bash scripts/guard_tree.sh --no-cargo` 78 checks, 0 failed (76 before this branch), including "PASS scripts/check_pr_review_counts.sh [self-test]" and "[run]".' diff --git a/docs/roadmaps/entries/PMAT-3657.yaml b/docs/roadmaps/entries/PMAT-3657.yaml new file mode 100644 index 0000000000..5cd8e755e7 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3657.yaml @@ -0,0 +1,19 @@ +- id: PMAT-3657 + github_issue: 3657 + item_type: task + title: 'publish_strict.sh hardcodes the publish universe size (74) — derive it from the cargo metadata that builds publish-order.txt' + status: in_progress + priority: medium + assigned_to: aprender-62 + created: 2026-09-21T10:40:00Z + updated: 2026-09-21T10:40:00Z + spec: null + acceptance_criteria: + - 'Issue #3657 "What lands" 1, verbatim: "Derive the expected size from the same source that builds `publish-order.txt` (`cargo metadata` at the tag + the facade list), and assert `ORDER == universe` as **sets** (no missing, no extra, no duplicate). The script already checks membership one way."' + - '"What lands" 2, verbatim: "Extend `scripts/check_release_scripts_derive_identity.sh` (or its case table) so a bare crate-count literal in an assertion is refused, with a planted-literal row."' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'Assigned to aprender-62 by the cop (aprender-04). Not a T-4 blocker for 0.69, per the issue. THE SOURCE. publish-order.txt is a committed file with no generator in the tree (ported in #3582); the universe it must equal is scripts/lib/cascade_universe.py, which runs `cargo metadata --no-deps` over EVERY workspace this repo publishes from (`.` and `crates/facades`) at the tag checkout ($WT) and drops `publish = false`. publish_strict.sh already read that universe into EXPECT; this change makes N=${#EXPECT[@]} the size, so the size comes from the same cargo metadata, and the facade names in ORDER are checked against the facade workspace''s own packages. HOW IT IS MET, publish_strict.sh: the three 74 assertions (lines 39, 41, 90 at d0771c5c5) are replaced by `N=${#EXPECT[@]}`, a non-empty check, and three SET checks, each naming the crates: duplicates in ORDER (`sort | uniq -d`), ORDER entries not in the universe, and universe entries not in ORDER ("so never uploaded"). The progress and final strings print $N ("$i/$N", "$n/$N live", "$N/$N"). The final verification is `[ "$n" -eq "$N" ]`. The dependency-order proof and the publish loop are unchanged. check_release_scripts_derive_identity.sh gets rule R3: on a non-comment line, no numeric test (-eq -ne -lt -le -gt -ge == !=) against an integer literal of two or more digits, on either side. Case rows 12-20 are new: the PLANTED row is publish_strict.sh''s own 0.68.2 line 39 verbatim; five more RED shapes (a wc -l count, an unquoted counter, the literal on the left, (( n == 74 )), a quoted "74"); three clean shapes (a count against the derived $N plus `-lt 3` and `-eq 0`; sleep/seq/cut numbers; a comment). The one-digit exemption is deliberate: a retry bound or an exit code is one digit, a crate universe is not. `--help` printed a hardcoded `sed -n 2,24p`; it now prints the leading comment block. MEASURED at this commit: `bash scripts/check_release_scripts_derive_identity.sh --self-test` 23/23 ok (was 14). The guard over the PRE-FIX scripts/release (d0771c5c5) exits 1 with R3 naming lines 39, 41 and 90; over this tree it exits 0 ("8 file(s) … no literal count"). MUTANT with R3''s hits forced empty: rows 12-17 go RED ("rc=0 (wanted 1)"), and no other row changes. END TO END in a private clone (its own refs, tag v9.9.9 in the clone only, RELEASE_AP in scratch, `--plan` reads the sparse index and uploads nothing): the fixed script plans 74 rows, rc 0. NEGATIVE CONTROLS, each committed and re-tagged in the clone, each rc 1: aprender-tsp removed from the order gives "in the universe but not in the publish order, so never uploaded: aprender-tsp"; apr-format appended gives "crate(s) twice in the publish order: apr-format"; not-a-crate appended gives "in the publish order but not in the universe: not-a-crate". THE DEFECT ITSELF: aprender-tsp set `publish = false` AND dropped from the order, a legitimate universe of 73. The pre-fix script (d0771c5c5) stops with "STOP publish: order=73 universe=73, expected 74/74", rc 1. The fixed script plans 73 rows, rc 0. bashrs: 0 errors on the guard (baseline 0). publish_strict.sh has 4 errors before and after, all pre-existing (the change adds none).' diff --git a/docs/roadmaps/entries/PMAT-3661.yaml b/docs/roadmaps/entries/PMAT-3661.yaml new file mode 100644 index 0000000000..dc8fa3f907 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3661.yaml @@ -0,0 +1,21 @@ +- id: PMAT-3661 + github_issue: 3661 + item_type: task + title: 'apr inspect/tensors print "Invalid APR format" for GGUF and SafeTensors failures — CliError::InvalidFormat hardcodes the format name' + status: in_progress + priority: medium + assigned_to: aprender-62 + created: 2026-09-21T10:45:00Z + updated: 2026-09-21T10:45:00Z + spec: null + acceptance_criteria: + - 'Issue #3661 done_when 1, verbatim: "`apr inspect` and `apr tensors` on a truncated GGUF and on a random-bytes `.safetensors` do not call the input an APR file unless its magic bytes are APR''s. Exit code stays 4 (same failure class; `lint_exit_convention_tests` unchanged)." Amended 2026-09-21 by the filer, before implementation, verbatim: "this previously read \"print no \\\"APR\\\" in the error\". Core''s unknown-magic message ends with \"Supported formats: APR (.apr), GGUF (.gguf), SafeTensors (.safetensors)\", which names APR without calling the file one. The criterion is about the claim, not the substring."' + - 'done_when 2, verbatim: "A case-table test drives each of the four inputs above through the real CLI error path and asserts the message names the detected format, or \"model\" when detection failed. It must fail on today''s Display (mutation: restore `\"Invalid APR format: {0}\"`, and the test goes RED)."' + - 'done_when 3, verbatim: "A truncated `.apr` still says APR, so the fix doesn''t just delete the word."' + - 'done_when 4, verbatim: "No double prefix (\"Invalid model format: … Invalid model format: …\") in any of the four messages."' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'Claimed on the issue by aprender-62 (2026-09-21 08:17Z). HOW IT IS MET. New variant `CliError::InvalidModelFile { format, message }`, Display "Invalid {format} file: {message}", exit code 4 (the same arm as InvalidFormat and InvalidInput). It is built only by `CliError::invalid_model_file(path, context, &e)`, which (a) reads the format from the file''s magic bytes through `commands::hex::detected_model_format`, a new 9-byte wrapper over hex.rs''s existing `detect_format`/`format_display_name` that returns "APR", "GGUF", "SafeTensors", or "model" when the bytes match none or the file cannot be read; and (b) takes a core `FormatError`''s message, not its Display, so the core "Invalid model format:" prefix does not appear under the CLI''s own. Three call sites move to it: inspect.rs `run_rosetta_inspect`, tensors.rs `run`, and profile_safetensors.rs (the same `Inspection failed` wrap). `CliError::InvalidFormat` is unchanged, so every other site and the tests that pin "Invalid APR format" (lint_exit_convention_tests, lint_family_guard, beat_run, grad_norm) are untouched. The core double prefix is fixed at its source: aprender-core format/safetensors.rs (include!d into format/tensors.rs) wrapped `GgufReader::from_bytes`''s error as `Failed to parse GGUF: {e}`, and `e` is itself a FormatError whose Display is "Invalid model format: …"; it now takes the inner message. WHAT THE FOUR MESSAGES SAY NOW (the case table renders `error: {e}` exactly as main.rs prints it): inspect truncated GGUF "error: Invalid GGUF file: Inspection failed: Unexpected EOF reading u64 at offset 24"; tensors truncated GGUF "error: Invalid GGUF file: Failed to list tensors: Failed to parse GGUF: Unexpected EOF reading u64 at offset 24"; inspect random .safetensors "error: Invalid model file: Inspection failed: SafeTensors open failed: Invalid SafeTensors file: metadata length 12928470711364830350 exceeds file size"; tensors random .safetensors "error: Invalid model file: Failed to list tensors: Unknown model format: magic bytes 8e44f12f. Supported formats: APR (.apr), GGUF (.gguf), SafeTensors (.safetensors)". A truncated .apr: inspect "error: Invalid APR format: File too small to contain valid APR header (need 64 bytes)" (inspect''s native APR path, unchanged); tensors "error: Invalid APR file: Failed to list tensors: Failed to parse APR v2: Invalid header: file too small". FIXTURES, all built in the test: a GGUF written by aprender''s own `export_tensors_to_gguf`, cut to 28 bytes (4 bytes into the first key''s u64 length, so the reader hits the issue''s own "Unexpected EOF reading u64"); 4 KiB of deterministic noise named .safetensors, opening with the issue''s magic `8e44f12f`; and an `AprV2Writer` file cut to 32 bytes, under its 64-byte header. The case table is `commands/model_file_error_tests_3661.rs`, with 6 rows. It calls the real `inspect::run` / `tensors::run`. For each row it asserts: the rendered prefix; no "Invalid APR" unless the magic is APR''s; no "Invalid model format" anywhere, which is stronger than "not twice"; and `exit_code_value() == 4`. MUTANTS, each RED: M1, the three call sites restored to `CliError::InvalidFormat(format!(…: {e}))` (the pre-fix code at d0771c5c5), fails 14 checks and reproduces the issue''s strings, e.g. "error: Invalid APR format: Failed to list tensors: Invalid model format: Failed to parse GGUF: Unexpected EOF reading u64 at offset 24". M2, the new variant''s Display restored to "Invalid APR format: {message}", fails 9 checks: every non-APR row. M3, the core wrap restored to `Failed to parse GGUF: {e}`, fails 1 check: the tensors-GGUF row''s "Failed to parse GGUF: Invalid model format: …". With all restored: 1 passed. MEASURED at this commit: `cargo test -p apr-cli --lib` 7294 passed, 0 failed, 12 ignored; `cargo test -p aprender-core --lib format::tensors` 131 passed; `cargo clippy -p apr-cli --lib -- -D warnings` rc 0 and `cargo clippy -p aprender-core --lib -- -D warnings` rc 0 (`--tests` on apr-cli fails only in pre-existing tests/falsification_crux_*.rs and nf4_classifier.rs, none in a touched file); `cargo fmt --all -- --check` rc 0; `scripts/check_complexity_ratchet.sh` "none new, none grown"; `bash scripts/guard_tree.sh --no-cargo` 76 checks, 0 failed. NOT DONE: a run of a freshly built `apr` binary. The host was at load 57 on 48 cores and a release build of the facade would have added to it; the table renders the same `error: {e}` line main.rs prints and asserts the exit-code mapping directly. FOLLOW-UP, filed before the PR as #3691: six more sites name GGUF or SafeTensors in their own message and still wrap it in `InvalidFormat` (hex.rs:323/350, safe_tensors_header.rs:180/182, trace_likely_has_repeated.rs:267/332). They are outside this issue''s done_when, which names inspect and tensors.' diff --git a/docs/roadmaps/entries/PMAT-3671.yaml b/docs/roadmaps/entries/PMAT-3671.yaml new file mode 100644 index 0000000000..19c801e102 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3671.yaml @@ -0,0 +1,21 @@ +- id: PMAT-3671 + github_issue: 3671 + item_type: task + title: 'check_fleet_pv_shapes_gate.sh: reason=pv-env (exit 0) covers a pinned pv that PANICS — only pv rc 2 is fleet state; any other no-JSON rc must stay RED' + status: in_progress + priority: high + assigned_to: aprender-62 + created: 2026-09-21T10:15:00Z + updated: 2026-09-21T10:15:00Z + spec: null + acceptance_criteria: + - 'Issue #3671 done_when 1, verbatim: "pv''s own exit code is captured before the parser runs. `reason=pv-env`, exit 0, applies **only** when pv''s rc is 2 and it produced no JSON."' + - 'done_when 2, verbatim: "A pinned pv that produces no JSON with any other rc (101 panic, 134/139 signal, 1 with no output) goes RED, not `UNMEASURED`. It exits non-zero, and the row names pv''s rc and its first stderr line."' + - 'done_when 3, verbatim: "Case-table rows: a panic stub (101, no JSON) → RED. The existing rc-2-no-JSON stub stays `UNMEASURED reason=pv-env`. A mutant that removes the pv-rc check sends the panic row back to `UNMEASURED` (the row discriminates)."' + - 'done_when 4, verbatim: "The other fleet-state branches (`no-pin`, `no-binary`, `pin-mismatch`, `incapable`, `judge-env`) are unchanged."' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'Found by aprender-62 cross-inspecting the andon commit 984fc76a9 (PMAT-3670) in the 0.69 batch #3669; the cop ruled it a separate ticket for after the batch. HOW IT IS MET. `judge` captures pv''s own exit as `pvrc` (the parser still receives it, for its message) and the verdict `case` is keyed on the PAIR `:`: `2:2` is `UNMEASURED reason=pv-env` exit 0, unchanged; `2:*` is `FAIL runner=… reason=pv-no-verdict … pv exited … pv stderr: ` to stderr, exit 1. The arms for parser rc 0 (PASS), 1 (FAIL) and other (judge-env) are unchanged, and the no-pin / no-binary / pin-mismatch / incapable branches return before pv runs, so they are untouched. RED is FAIL (exit 1), not ENV (exit 2): a panic caused by a contract in the tree is tree state. SIGNAL ROW. The case table uses SIGKILL (rc 137, `kill -KILL $$` in the stub), a real signal death that writes no core file on a runner; 134/139 are the same `2:*` arm, since the arm matches any pv rc but 2. MEASURED at this commit: `bash scripts/check_fleet_pv_shapes_gate.sh --self-test` 19/19 ok (was 15). The new rows are 13-15 (panic 101, SIGKILL 137, rc 1 with no output: each RED naming pv''s rc and stderr, never UNMEASURED) and 16 (mutant `2:2)` -> `2:*)`, i.e. pv-env keyed on the parser alone: the panic stub goes back to `UNMEASURED reason=pv-env`, exit 0, so the panic row discriminates). BEFORE THE FIX, the same self-test with the new rows: rows 13-15 FAIL with "UNMEASURED runner=probe-runner reason=pv-env … ENV no JSON verdict (rc=101) … panicked", rc 0, reproducing the issue''s table. ALSO FIXED, same rows: the existing pv-env mutant (row 12) was VACUOUS. Its sed replaced the arm''s opening single quote with a double quote, so the mutant script was a bash syntax error (`bash -n` rc 2 at "line 146: syntax error near unexpected token `(`") and "exited non-zero" without reaching the arm it claimed to test. Both mutants now keep the arm''s text through a sed backreference, are refused unless `bash -n` passes, and assert the specific outcome (row 12: exit 2 and `ENV (rc=2)`; row 16: exit 0 and `UNMEASURED … reason=pv-env`). `--help` printed a hardcoded `sed -n 2,44p`, which cut the header off once it grew; it now prints the leading comment block, found by awk. `bashrs lint`: 0 errors (baseline 0). `bash scripts/guard_tree.sh --no-cargo`: 76 checks, 0 failed.' diff --git a/docs/roadmaps/entries/PMAT-3672.yaml b/docs/roadmaps/entries/PMAT-3672.yaml new file mode 100644 index 0000000000..4de86bc6da --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3672.yaml @@ -0,0 +1,17 @@ +- id: PMAT-3672 + github_issue: 3672 + item_type: task + title: 'apr run applies the chat template TWICE: the prompt reaches the model as an escaped user turn inside a user turn' + status: planned + priority: high + assigned_to: null + created: 2026-09-21T11:00:21Z + updated: 2026-09-21T11:00:21Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'ACCEPTANCE (hand-entered from issue #3672 done_when, verbatim; minted on the PR branch so the AD-04 quorum can run). (1) apr run -v on Qwen2.5-0.5B-Instruct-f16.gguf with a plain prompt shows exactly one template application and no \u{200B} in formatted_prompt. (2) The prompt token count equals llama.cpp''s for the same rendered conversation (21 for the prompt above, with no system turn). (3) A test fails if the prompt is templated twice (mutation: restore the upstream wrap → RED). ROOT CAUSE (issue comment): crates/apr-cli/src/dispatch_run.rs (GH-638) pre-wrapped the prompt in hard-coded ChatML, and realizar prepare_tokens (GH-278) then applied the model''s own template again. Fix: the prompt stays raw; --chat (or the GH-638 instruct/chat source-name heuristic) reaches realizar as InferenceConfig::force_chat_template, which prepare_tokens_{gguf,safetensors,apr} OR into its single template decision.' diff --git a/docs/roadmaps/entries/PMAT-3673.yaml b/docs/roadmaps/entries/PMAT-3673.yaml new file mode 100644 index 0000000000..4e0e102b9e --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3673.yaml @@ -0,0 +1,17 @@ +- id: PMAT-3673 + github_issue: 3673 + item_type: task + title: 'Batching is the default merge path: scripts/batch_fold.sh + APR-RELEASE-001 §13 (operator 2026-09-21)' + status: planned + priority: critical + assigned_to: null + created: 2026-09-21T08:23:21Z + updated: 2026-09-21T08:23:21Z + spec: docs/specifications/APR-RELEASE-001-train-and-build-kaizen.md + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'ACCEPTANCE (quoted from issue #3673 "What lands"): 1. scripts/batch_fold.sh — fold receipted PR branches into one release/-batch branch, git merge --no-ff in the given order: a conflict ONLY in the generated set {docs/roadmaps/roadmap.yaml, contracts/census.json, contracts/contracts.nt, contracts/shapes.ttl, the CONTRACT_COUNT blocks of README.md} is taken and marked for regeneration; a conflict ANYWHERE else — including README prose outside a count block — aborts that branch''s merge and reports SKIP with the paths, never resolved by picking a side; --regen regenerates the generated set ONCE with the tree-built pv (scripts/pv_bin.sh), asserts the fixed points (aggregate --check, pv extract --check, readme_sync.sh --check), and commits the regeneration; exit 0 all folded; 1 any SKIP; 2 usage/git/regeneration failure; a folded-but-unregenerated generated set is never silent; --self-test case table, wired into CI. 2. APR-RELEASE-001 §13 — the batching procedure as the default path (cop prompt §7): assembly, generated-set regeneration, left-out rules, what needs a new quorum (only content new to the batch), constituent disarm/close, a red batch, and the review-lane VOID rule while paiml-implement#317 is open. AUTHOR NOTES (not criteria): the case table lives in scripts/check_batch_fold.sh (guard-tree runs it) because a --self-test case inside batch_fold.sh made check_guards_are_wired.sh RED; §13 required amending hard rule 9 ("No hand-squash of batches") and the matching §8 stop, which the new default contradicts.' diff --git a/docs/roadmaps/entries/PMAT-3696.yaml b/docs/roadmaps/entries/PMAT-3696.yaml new file mode 100644 index 0000000000..b472149d3a --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3696.yaml @@ -0,0 +1,19 @@ +- id: PMAT-3696 + github_issue: 3696 + item_type: task + title: 'publish_strict.sh: a blank line in publish-order.txt stops the cascade naming an EMPTY crate — refuse it by line number' + status: in_progress + priority: low + assigned_to: aprender-62 + created: 2026-09-21T11:39:00Z + updated: 2026-09-21T11:39:00Z + spec: null + acceptance_criteria: + - 'Issue #3696 done_when 1, verbatim: "An empty crate name is refused **by name**, before any set check: `STOP publish: blank line N in publish-order.txt` (every blank or whitespace-only line, each by its number), rc 1, and no `bad array subscript`."' + - 'done_when 2, verbatim: "Measured in a private clone: a blank line at line 36 gives exactly that message; the clean file still plans 74 rows, rc 0."' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'Found by aprender-7a seat-filling #3657; the cop ruled it a small 0.70 row stacked on #3657''s 6fd241645, which is not touched (this commit is its child). SCOPE: scripts/release/publish_strict.sh (4 lines before the mapfile) and this fragment. HOW IT IS MET: awk lists every line of publish-order.txt that is empty or whitespace-only, by number; any hit stops with `die "blank line in publish-order.txt: an empty crate name is not a crate"` before ORDER is built, so no empty string reaches the set checks or bash''s associative-array subscript. publish_strict.sh has no --self-test on purpose (one would make it a guard that must be workflow-invoked), so the proof is end to end. MEASURED in a private clone (its own refs; tag v9.9.9 in the clone only; RELEASE_AP in scratch; --plan reads the sparse index and uploads nothing). BEFORE, at 6fd241645, a blank line at line 36: rc 1, "scripts/release/publish_strict.sh: line 47: EXPECT: bad array subscript" and "STOP publish: in the publish order but not in the universe: " (an empty name). AFTER, at this commit: the clean file gives rc 0 and 74 rows; a blank line at 36 gives rc 1, "STOP publish: blank line 36 in publish-order.txt: an empty crate name is not a crate", with 0 subscript errors; a whitespace-only line at 3 plus a blank line (now at 37) gives rc 1, "STOP publish: blank line 3, 37 in publish-order.txt: an empty crate name is not a crate". `bash scripts/check_release_scripts_derive_identity.sh`: PASS (no literal count, no identity literal).' diff --git a/docs/roadmaps/roadmap.yaml b/docs/roadmaps/roadmap.yaml index 3103603d5b..997bd80d0e 100644 --- a/docs/roadmaps/roadmap.yaml +++ b/docs/roadmaps/roadmap.yaml @@ -17150,6 +17150,23 @@ roadmap: - orch:fable - ci notes: null +- id: PMAT-3076 + github_issue: 3076 + item_type: task + title: 'F16/BF16 GGUF CPU matmul has no SIMD path — unquantized models appear to hang' + status: planned + priority: high + assigned_to: null + created: 2026-09-21T07:52:58Z + updated: 2026-09-21T07:52:58Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'ACCEPTANCE (hand-entered; #3076 has no done_when section, so this quotes its last plan comment verbatim, minted on the PR branch so the AD-04 quorum can run). "(1) the GGUF CPU path gets a SIMD f16/bf16 matvec (dequant-to-f32 tiles through the existing fused dispatcher) with a greedy-parity gate against llama.cpp on an unquantized ≤1B GGUF, so an F16/BF16 model decodes at a measured tok/s instead of appearing to hang; (2) until that lands the loader prints a one-line notice that the F16/BF16 path is scalar and slow rather than silently crawling." Since (1) lands here, (2) is moot. DEVIATION from the issue body: it suggests an integration test "with a wall-clock assertion"; the project bans wall-clock assertions in required checks, so the regression guard is behavioural (kernel parity against an f64 reference at lengths hitting every loop and tail, and float16_matmul against a transcription of the pre-fix loop including a truncated buffer), and tok/s is measured and reported rather than asserted. Not in scope: prefill running one position at a time at decode rate (#2787 / #2801). EVIDENCE (added after the round-1 quorum found the llama.cpp gate missing): scripts/float16_greedy_parity.sh is the greedy-parity gate against the pinned llama.cpp (llama_pin.toml build_commit), on template-free copies of an unquantized ≤1B F16 and BF16 GGUF, because apr double-templates instruct prompts (#3672); --baseline separates divergences this change introduces from ones that predate it. Receipts, the negative control (a deliberately broken binary FAILs 12/12) and the measured before/after decode tok/s with model sha256, host and binaries are in evidence/pmat-3076-f16-bf16-matvec/. WIRING (cop ruling, no workflow edit): the model-backed run is declared in Cargo.toml [package.metadata.dogfood] and runs bare (release mode: the committed receipts are the baseline and pin each model sha256; a model absent from the host is a FAIL) in scripts/dogfood.sh --phase pre-publish; the comparison helpers live in scripts/float16_parity_lib.sh and their case table runs on every PR as scripts/check_float16_greedy_parity.sh through guard_tree.sh.' - id: PMAT-3118 github_issue: 3118 item_type: task @@ -19672,6 +19689,30 @@ roadmap: labels: - kind:code notes: 'ACCEPTANCE (hand-entered from gh#3605 done_when; see paiml-mcp-agent-toolkit#1414). Done when: contracts/refusal-receipt-v1.yaml exists with a shape over removed_by, and the value space is a CLOSED SET rather than free text: pattern ^(v[0-9]+\\.[0-9]+|never|unscheduled)$. The point of the closed set is that legitimately-nothing stays SAYABLE and CHECKABLE — never and unscheduled are honest answers with a shape behind them, instead of a required field manufacturing a fake version. Fixtures cover each arm of the alternation plus a rejecting case. STOP: do not admit free text; do not make the field optional, which would make absence indistinguishable from a deliberate never.' +- id: PMAT-3609 + github_issue: 3609 + item_type: task + title: 'The BPE tokenizer requires a '''' token that Qwen3.5 does not have — a universal unknown-token is an assumption, not a fact' + status: in_progress + priority: high + assigned_to: aprender-62 + created: 2026-09-21T09:00:00Z + updated: 2026-09-21T09:00:00Z + spec: null + acceptance_criteria: + - 'Operator ruling on #3609 (2026-09-20), constraint 1, verbatim: "`unk` becomes `Option`; absent means absent. No synthesis in either direction. That is stronger than the original framing and deliberately so: the `vocab[0] = ""` placeholder for an absent vocabulary goes too. A GGUF with no vocabulary refuses by name — it does not load. Today it loads, which is the inversion this ticket is about; removing only the refusal on the present-but-lacking side would leave the permissive fallback intact and simply move the asymmetry. The two-direction falsifier already proposed covers both: a model with an unknown token still resolves it (so the fix is not \"ignore unk\"), and a model with no vocabulary now refuses (so the fix is not \"synthesise one\")."' + - 'Constraint 2, verbatim: "Prose changes in the same PR, or the belief is reproduced. `book/src/examples/api-server.md` and `examples/model_cache.rs` both repeat \"vocabulary must include ``\" — a property of our constructor stated as a property of the data. Run `pmat query` for `` and `unknown_token`; every hit is either fixed or listed in the PR body as intentionally kept. A hit that is neither is the defect''s next copy."' + - 'Constraint 3, verbatim: "Mutation: reintroduce the `ok_or_else` → the Qwen3.5 fixture goes RED. A fix with no red-turning mutation is a claim. The fixture must be a real Qwen3.5 GGUF header — the family declares `tokenizer.ggml.eos_token_id` and no `unknown_token_id` key at all, which is the actual input, not a synthetic one."' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'Taken by aprender-62, sequenced ahead by the cop (aprender-04). Two interpretation decisions, sent to the cop before implementation: (1) THE FIXTURE. The real Qwen3.5-0.8B header is 10.9 MB (248,320 tokens, 247,587 merges), too large to commit. The fixture is a header-only GGUF generated from the real file: every one of its 46 keys and every scalar value verbatim (eos 248046, padding 248055, model gpt2, pre qwen35, the chat template; no unknown_token_id and no bos key, exactly as in the real file), with only the three arrays (tokens, token_type, merges) cut to a leading slice. Qwen''s first 256 tokens are its byte-level glyphs, so the slice still covers every byte. The generator is committed next to it and records the source header''s sha256. (2) WHERE THE UNKNOWN TOKEN COMES FROM at the vocabulary-only constructors, which receive a Vec and nothing else: the vocabulary''s own `` entry when it has one, otherwise none. Nothing is synthesised, and every model that loads today keeps its unknown token. Threading `tokenizer.ggml.unknown_token_id` through every loader is a filed follow-up, not a silent gap. + + The cop ruled both decisions YES on 2026-09-21. It also added a guard, verbatim: "When the glyph for a byte is ALSO absent, encode must refuse with an error naming the byte. It must never drop the input or emit nothing. Put a row for it in the case table (a vocab without that glyph and without unk → error). Absent means absent at encode time too." How it is met: `encode` returns an infallible Vec and has dozens of callers, so the refusal happens at CONSTRUCTION. With no unknown token, `BPETokenizer::new` refuses, naming the first byte that has neither a `<0xNN>` token nor a byte-level glyph token. `with_merges` also requires every glyph, because the merge path maps bytes to glyphs only. `encode` therefore cannot meet an unencodable byte, and the no-drop guarantee holds on every path. Case row: `no_unknown_token_and_a_byte_with_no_token_is_refused_by_name` (a 256-glyph vocabulary minus byte 0xA9''s glyph → error naming 0xA9). With the unknown token present, encode is unchanged: `<0xNN>`, then the unknown token. The glyph fallback applies only when there is no unknown token. Follow-up filed before the PR: #3675 (thread the DECLARED unknown token through every loader; by-name `vocabulary_unk_token` is the interim, and `apr serve`''s chat path already uses tokenizer.json''s declared `model.unk_token`). Found while testing, filed separately because it is independent of the unknown token: #3677 (the greedy path matches Latin-1 characters such as `é` as byte-level glyph tokens, so `é` decodes to U+FFFD). The Qwen round-trip rows therefore use ASCII, CJK and newline, not `é`. Same class but outside the query''s hits, so not in this PR: `apr showcase`''s pipeline substitutes token ids [1,2,3,4,5] when a model has no vocabulary (commands/showcase/pipeline.rs:165). + + Repo-layout facts a reviewer needs, each checked at the PR head (round 2 lane 1 misread all three, and every one is refuted by `cargo check -p apr-cli` rc=0 and `cargo test -p apr-cli --lib` 7283 passed at e5b2e7541). (1) `realizar` IS aprender-serve: crates/aprender-serve/Cargo.toml:42-43 is `[lib] name = "realizar"`, so `realizar::tokenizer::BPETokenizer` in apr-cli and `crate::tokenizer::BPETokenizer` in aprender-serve are the SAME type, and `realizar::tokenizer::vocabulary_unk_token` (the book, api-server.md) is the function this PR adds. (2) `no_vocabulary` in apr-cli is defined ONCE, at crates/apr-cli/src/commands/serve/handler_gpu_completion.rs:482, and is visible to every serve site because these files are one module: handlers.rs:1618 is `include!("handler_apr_cpu_completion.rs")`, handlers.rs:1619 is `include!("handler_gpu_completion.rs")`, and handler_apr_cpu_completion.rs:869 is `include!("handlers_include_01.rs")`. (3) aprender-serve''s cli has its own `no_vocabulary` (cli/mod_server_commands.rs:13, returning RealizarError), inside the same `mod server_commands` as its six call sites.' - id: PMAT-3618 github_issue: 3618 item_type: task @@ -19757,6 +19798,29 @@ roadmap: labels: - kind:code notes: 'ACCEPTANCE (hand-entered from issue #3644 "What lands"; minted on the PR branch so the AD-04 quorum can run). 1. check_guards_are_wired.sh drops name: lines before the invocation match, in dispatcher_wired() and the per-guard scan; rows 8-10 (the exact ci.yml step-name fixture must NOT count as a dispatch; a quoted run: line still does; a step named after a guard wires nothing); the mutation (remove the drop) turns rows 8 and 10 RED. 2. check_pathonly_devdeps_unused_in_src.sh: tomllib -> tomli -> ENV rc=2 naming interpreter and runner; the scanner ends with a SCAN-DONE trailer the shell REQUIRES; five death rows incl. the whole guard under the intel shape; comments reworded so CARGO_RE no longer classifies it out of --no-cargo, so guard-tree runs it (bare run on the tree: rc 0, 3 s). 3. unwired_guards_baseline.txt records the other three dark guards as APERTURE REVEALS (set-aperture gains a NAME-entry admission: a file the comparand carries, in a diff that changes the owning guard; six case rows, each admission beside its refusal) with the reason each may only leave; #3646 owns the pr_review pair. No workflow file is edited.' +- id: PMAT-3646 + github_issue: 3646 + item_type: task + title: check_pr_review_counts.sh is RED on main (6 rows disagree) and check_pr_review_receipt.sh has no caller — both dark since PRQ-013 moved the receipt job, classified out of --no-cargo by a comment + status: planned + priority: high + assigned_to: null + created: 2026-09-21T06:00:51Z + updated: 2026-09-21T06:00:51Z + spec: null + acceptance_criteria: + - 'Issue #3646 "What lands" item 1, verbatim: "Fix the 6 disagreeing rows in PR-REVIEW-SKILL-002''s files (or the derivation), then reword the one comment so the guard leaves the cargo population and `guard-tree` runs it; delete its ledger line in the same commit (shrink-only)."' + - 'Issue #3646 "What lands" item 2, verbatim: "Decide `check_pr_review_receipt.sh`''s caller: either `pr-review-quorum.yml` invokes it on the PR''s receipt (a workflow edit — operator''s call), or it is folded into `check_pr_review_arm4.sh`, which is already wired with args. Delete its ledger line with whichever lands."' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'SCOPE OF THIS PR: item 1 only. The PR says `Refs #3646`, not a closing keyword, and carries a `keep-open:` line for item 2. check_pr_review_receipt.sh keeps its line in scripts/unwired_guards_baseline.txt on purpose: neither of item 2''s two options lands here. The workflow edit is the operator''s call, and the fold is moot because check_pr_review_arm4.sh already invokes it with a receipt (see Item 2 below). The ledger entry''s own comment says so ("NOT dark: check_pr_review_arm4.sh runs it … Ledgered only because this meta-guard counts a DIRECT workflow invocation"). A reviewer should judge item 2 as OPEN BY DESIGN, not as missed. Handed over by aprender-b3 (2026-09-21); claimed on the issue. Measured on origin/main @ 237fbc32f per the issue: `bash scripts/check_pr_review_counts.sh` rc=1 "--- 6 row(s) disagree with the tree ---"; `bash scripts/check_pr_review_receipt.sh` rc=1 "A run over zero receipts is not a pass (S6.3: a missing receipt is RED)." The ledger line for each guard is added by #3647 (scripts/unwired_guards_baseline.txt, shrink-only); it is deleted in the commit that wires that guard, after #3647 merges. Item 1 lands first; item 2 lands here only if it folds into check_pr_review_arm4.sh with no workflow edit, otherwise the ticket stays open for the operator''s call and the PR says Refs, not Closes. + + What the item-1 PR does. (a) The 6 RED rows were ci.yml sites whose sentences PRQ-013 deleted along with the receipt job (roadmap: "STATED LOSS"), so the rows go. The bats_tests derivation goes too, because no file states that count any more; ci.yml''s fixture_rows site goes from 2 to 1; and scripts/check_pr_review_wiring.sh''s "43-row" joins the table with a self-test row of its own. (b) The :196 comment is reworded so CARGO_RE stops matching: `guard_tree.sh --dry-run --no-cargo` prints `run: scripts/check_pr_review_counts.sh`, and restoring the old wording moves it back to --cargo-only (mutation-verified). (c) The --list mode of mutate-guard.sh and mutate_quorum_arm.sh requires only awk. Before this it demanded bats/jq/minisign/check-jsonschema, so wiring the counts guard into guard-tree, which runs on clean-room HOSTS including the gx10/yoga docker runners, would have turned a dark guard into an ENV-red job (reproduced: rc=2 under a PATH without those four). Sweeps still refuse without them (rc=1). + + Item 2, measured: check_pr_review_receipt.sh HAS a caller with a receipt. check_pr_review_arm4.sh runs `bash "$guard" "$scratch"` (A3) and `bash "$guard" "$best_dir"` (A4) with guard defaulting to it, and pr-review-quorum.yml:141 runs arm4. So there is nothing to fold. What remains is whether guard_tree.sh and the unwired ledger should treat a guard reachable only through another wired guard as wired. That is recorded on the issue and not changed here. LEDGER LINE (added after the 0.69 batch landed): check_pr_review_counts.sh entered scripts/unwired_guards_baseline.txt through #3644 in batch #3669 (squash a877fa056), after this branch was cut, so the deletion criterion 1 names could only be made once main carried the line. Main is merged into this branch (the only conflict was the generated roadmap.yaml, regenerated from entries/), and the entry and its four comment lines are deleted. On the squash-merge this is one commit with the fix. Measured on the merged tree: `bash scripts/check_guards_are_wired.sh` rc 0, "did not grow (1 removed) vs a877fa056", 149 scanned, 5 named by no workflow; `bash scripts/check_pr_review_counts.sh` rc 0, "PASS every stated count equals the derived one"; `--self-test` 16/16; `bash scripts/guard_tree.sh --no-cargo` 78 checks, 0 failed (76 before this branch), including "PASS scripts/check_pr_review_counts.sh [self-test]" and "[run]".' - id: PMAT-3651 github_issue: 3651 item_type: task @@ -19791,6 +19855,25 @@ roadmap: labels: - kind:code notes: 'ACCEPTANCE (hand-entered from issue #3656 done_when, verbatim; minted on the PR branch so the AD-04 quorum can run). Done when: (1) get_tensor_f32 on any IQ tensor returns Err naming the tensor and its type; no path in aprender-core returns approximated IQ values; dequantize_iq_approximate is deleted. (2) apr convert … --quantize q4k and apr import on Qwen3.5-0.8B-IQ4_XS.gguf exit non-zero and write no output file. (3) A unit test fails if an approximation arm is restored (mutation: restore the 16..=23 arm → RED). Not in scope: real IQ dequantization in aprender-core (aprender-serve has dequantize_iq*, core cannot depend on serve; the route is PP-QUANT-001 #3421). Why it rides with PMAT-3601: once IQ tensors are sized, raw apr import reaches map_gguf_dtype, whose Q8_0/Q5_K refusal says "cannot represent exactly" — a GH-375 fallback trigger — so without this fix an IQ file could import invented weights where it used to be refused.' +- id: PMAT-3657 + github_issue: 3657 + item_type: task + title: 'publish_strict.sh hardcodes the publish universe size (74) — derive it from the cargo metadata that builds publish-order.txt' + status: in_progress + priority: medium + assigned_to: aprender-62 + created: 2026-09-21T10:40:00Z + updated: 2026-09-21T10:40:00Z + spec: null + acceptance_criteria: + - 'Issue #3657 "What lands" 1, verbatim: "Derive the expected size from the same source that builds `publish-order.txt` (`cargo metadata` at the tag + the facade list), and assert `ORDER == universe` as **sets** (no missing, no extra, no duplicate). The script already checks membership one way."' + - '"What lands" 2, verbatim: "Extend `scripts/check_release_scripts_derive_identity.sh` (or its case table) so a bare crate-count literal in an assertion is refused, with a planted-literal row."' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'Assigned to aprender-62 by the cop (aprender-04). Not a T-4 blocker for 0.69, per the issue. THE SOURCE. publish-order.txt is a committed file with no generator in the tree (ported in #3582); the universe it must equal is scripts/lib/cascade_universe.py, which runs `cargo metadata --no-deps` over EVERY workspace this repo publishes from (`.` and `crates/facades`) at the tag checkout ($WT) and drops `publish = false`. publish_strict.sh already read that universe into EXPECT; this change makes N=${#EXPECT[@]} the size, so the size comes from the same cargo metadata, and the facade names in ORDER are checked against the facade workspace''s own packages. HOW IT IS MET, publish_strict.sh: the three 74 assertions (lines 39, 41, 90 at d0771c5c5) are replaced by `N=${#EXPECT[@]}`, a non-empty check, and three SET checks, each naming the crates: duplicates in ORDER (`sort | uniq -d`), ORDER entries not in the universe, and universe entries not in ORDER ("so never uploaded"). The progress and final strings print $N ("$i/$N", "$n/$N live", "$N/$N"). The final verification is `[ "$n" -eq "$N" ]`. The dependency-order proof and the publish loop are unchanged. check_release_scripts_derive_identity.sh gets rule R3: on a non-comment line, no numeric test (-eq -ne -lt -le -gt -ge == !=) against an integer literal of two or more digits, on either side. Case rows 12-20 are new: the PLANTED row is publish_strict.sh''s own 0.68.2 line 39 verbatim; five more RED shapes (a wc -l count, an unquoted counter, the literal on the left, (( n == 74 )), a quoted "74"); three clean shapes (a count against the derived $N plus `-lt 3` and `-eq 0`; sleep/seq/cut numbers; a comment). The one-digit exemption is deliberate: a retry bound or an exit code is one digit, a crate universe is not. `--help` printed a hardcoded `sed -n 2,24p`; it now prints the leading comment block. MEASURED at this commit: `bash scripts/check_release_scripts_derive_identity.sh --self-test` 23/23 ok (was 14). The guard over the PRE-FIX scripts/release (d0771c5c5) exits 1 with R3 naming lines 39, 41 and 90; over this tree it exits 0 ("8 file(s) … no literal count"). MUTANT with R3''s hits forced empty: rows 12-17 go RED ("rc=0 (wanted 1)"), and no other row changes. END TO END in a private clone (its own refs, tag v9.9.9 in the clone only, RELEASE_AP in scratch, `--plan` reads the sparse index and uploads nothing): the fixed script plans 74 rows, rc 0. NEGATIVE CONTROLS, each committed and re-tagged in the clone, each rc 1: aprender-tsp removed from the order gives "in the universe but not in the publish order, so never uploaded: aprender-tsp"; apr-format appended gives "crate(s) twice in the publish order: apr-format"; not-a-crate appended gives "in the publish order but not in the universe: not-a-crate". THE DEFECT ITSELF: aprender-tsp set `publish = false` AND dropped from the order, a legitimate universe of 73. The pre-fix script (d0771c5c5) stops with "STOP publish: order=73 universe=73, expected 74/74", rc 1. The fixed script plans 73 rows, rc 0. bashrs: 0 errors on the guard (baseline 0). publish_strict.sh has 4 errors before and after, all pre-existing (the change adds none).' - id: PMAT-3658 github_issue: 3658 item_type: task @@ -19808,6 +19891,27 @@ roadmap: labels: - kind:code notes: 'ACCEPTANCE — issue #3658 done_when as AMENDED 2026-09-21 06:10Z (the amendment supersedes the original list; operator instruction quoted there: "you will fix ASAP this is P0, and NO roadmap shouldn''t even have 10 minutes testing"). (1) A squash-shaped merge-group fixture (one parent, main moved, a leaf crate touched) gives tier=quick with that crate, not full. The same fixture with a roadmap entry only gives tier=none. Both are case-table rows. (2) Mutation: restoring the HEAD^2 gate sends the squash group back to full (a RED row). (3) A failing git diff HEAD^1..HEAD still gives tier=full (the fail-closed row is kept). (4) On a pull_request: roadmap + audit paths only -> tier=none; the same with a deleted docs path -> quick; roadmap plus any non-docs path -> quick; docs/specifications/ only -> quick. (5) A push of a squash landing (one parent, no reflog) -> quick over HEAD^1..HEAD, not full. (6) [AMENDMENT 2, 2026-09-21 06:55Z, supersedes amendment 1 item 6] ci.yml changes in exactly two places, and nothing else in the workflow changes: (a) one step, if: tier == none, that logs the reason through env:; (b) the pull_request touched-list line (git diff ... > "$RUNNER_TEMP/touched.txt" in the tier step) gains --no-renames, so both sides of every rename are touched paths. diff_into in ci_test_tier.sh gains the same flag. Rows: default --name-only on a rename fixture lists only the new path (the trap is real); the rename into docs/audits/ gives quick with the crate it left; a mutant without --no-renames gives none (RED); and a file-shaped row asserts the ci.yml line carries the flag. The other touched-list builder with the same shape (ci.yml:2783, the ci_gpu_touched feed) is pre-existing, has a different consumer, is out of scope, and is #3667. (7) After landing: the next 5 merge-group runs'' shard-1 logs show a tier= other than full for the docs-only and small-crate groups. This is observed after the merge, and the issue stays open until it is. SCOPE ADDITIONS in the amendment: A. docs-only diffs (every path under docs/roadmaps/ or docs/audits/ and present at HEAD; never a delete) and an empty diff get tier=none; B. a push landing a squash uses HEAD^1..HEAD. NOT IN SCOPE: guard-cargo and the reusable ci/* jobs on docs-only changes (the next row). Refs #3650, #3643.' +- id: PMAT-3661 + github_issue: 3661 + item_type: task + title: 'apr inspect/tensors print "Invalid APR format" for GGUF and SafeTensors failures — CliError::InvalidFormat hardcodes the format name' + status: in_progress + priority: medium + assigned_to: aprender-62 + created: 2026-09-21T10:45:00Z + updated: 2026-09-21T10:45:00Z + spec: null + acceptance_criteria: + - 'Issue #3661 done_when 1, verbatim: "`apr inspect` and `apr tensors` on a truncated GGUF and on a random-bytes `.safetensors` do not call the input an APR file unless its magic bytes are APR''s. Exit code stays 4 (same failure class; `lint_exit_convention_tests` unchanged)." Amended 2026-09-21 by the filer, before implementation, verbatim: "this previously read \"print no \\\"APR\\\" in the error\". Core''s unknown-magic message ends with \"Supported formats: APR (.apr), GGUF (.gguf), SafeTensors (.safetensors)\", which names APR without calling the file one. The criterion is about the claim, not the substring."' + - 'done_when 2, verbatim: "A case-table test drives each of the four inputs above through the real CLI error path and asserts the message names the detected format, or \"model\" when detection failed. It must fail on today''s Display (mutation: restore `\"Invalid APR format: {0}\"`, and the test goes RED)."' + - 'done_when 3, verbatim: "A truncated `.apr` still says APR, so the fix doesn''t just delete the word."' + - 'done_when 4, verbatim: "No double prefix (\"Invalid model format: … Invalid model format: …\") in any of the four messages."' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'Claimed on the issue by aprender-62 (2026-09-21 08:17Z). HOW IT IS MET. New variant `CliError::InvalidModelFile { format, message }`, Display "Invalid {format} file: {message}", exit code 4 (the same arm as InvalidFormat and InvalidInput). It is built only by `CliError::invalid_model_file(path, context, &e)`, which (a) reads the format from the file''s magic bytes through `commands::hex::detected_model_format`, a new 9-byte wrapper over hex.rs''s existing `detect_format`/`format_display_name` that returns "APR", "GGUF", "SafeTensors", or "model" when the bytes match none or the file cannot be read; and (b) takes a core `FormatError`''s message, not its Display, so the core "Invalid model format:" prefix does not appear under the CLI''s own. Three call sites move to it: inspect.rs `run_rosetta_inspect`, tensors.rs `run`, and profile_safetensors.rs (the same `Inspection failed` wrap). `CliError::InvalidFormat` is unchanged, so every other site and the tests that pin "Invalid APR format" (lint_exit_convention_tests, lint_family_guard, beat_run, grad_norm) are untouched. The core double prefix is fixed at its source: aprender-core format/safetensors.rs (include!d into format/tensors.rs) wrapped `GgufReader::from_bytes`''s error as `Failed to parse GGUF: {e}`, and `e` is itself a FormatError whose Display is "Invalid model format: …"; it now takes the inner message. WHAT THE FOUR MESSAGES SAY NOW (the case table renders `error: {e}` exactly as main.rs prints it): inspect truncated GGUF "error: Invalid GGUF file: Inspection failed: Unexpected EOF reading u64 at offset 24"; tensors truncated GGUF "error: Invalid GGUF file: Failed to list tensors: Failed to parse GGUF: Unexpected EOF reading u64 at offset 24"; inspect random .safetensors "error: Invalid model file: Inspection failed: SafeTensors open failed: Invalid SafeTensors file: metadata length 12928470711364830350 exceeds file size"; tensors random .safetensors "error: Invalid model file: Failed to list tensors: Unknown model format: magic bytes 8e44f12f. Supported formats: APR (.apr), GGUF (.gguf), SafeTensors (.safetensors)". A truncated .apr: inspect "error: Invalid APR format: File too small to contain valid APR header (need 64 bytes)" (inspect''s native APR path, unchanged); tensors "error: Invalid APR file: Failed to list tensors: Failed to parse APR v2: Invalid header: file too small". FIXTURES, all built in the test: a GGUF written by aprender''s own `export_tensors_to_gguf`, cut to 28 bytes (4 bytes into the first key''s u64 length, so the reader hits the issue''s own "Unexpected EOF reading u64"); 4 KiB of deterministic noise named .safetensors, opening with the issue''s magic `8e44f12f`; and an `AprV2Writer` file cut to 32 bytes, under its 64-byte header. The case table is `commands/model_file_error_tests_3661.rs`, with 6 rows. It calls the real `inspect::run` / `tensors::run`. For each row it asserts: the rendered prefix; no "Invalid APR" unless the magic is APR''s; no "Invalid model format" anywhere, which is stronger than "not twice"; and `exit_code_value() == 4`. MUTANTS, each RED: M1, the three call sites restored to `CliError::InvalidFormat(format!(…: {e}))` (the pre-fix code at d0771c5c5), fails 14 checks and reproduces the issue''s strings, e.g. "error: Invalid APR format: Failed to list tensors: Invalid model format: Failed to parse GGUF: Unexpected EOF reading u64 at offset 24". M2, the new variant''s Display restored to "Invalid APR format: {message}", fails 9 checks: every non-APR row. M3, the core wrap restored to `Failed to parse GGUF: {e}`, fails 1 check: the tensors-GGUF row''s "Failed to parse GGUF: Invalid model format: …". With all restored: 1 passed. MEASURED at this commit: `cargo test -p apr-cli --lib` 7294 passed, 0 failed, 12 ignored; `cargo test -p aprender-core --lib format::tensors` 131 passed; `cargo clippy -p apr-cli --lib -- -D warnings` rc 0 and `cargo clippy -p aprender-core --lib -- -D warnings` rc 0 (`--tests` on apr-cli fails only in pre-existing tests/falsification_crux_*.rs and nf4_classifier.rs, none in a touched file); `cargo fmt --all -- --check` rc 0; `scripts/check_complexity_ratchet.sh` "none new, none grown"; `bash scripts/guard_tree.sh --no-cargo` 76 checks, 0 failed. NOT DONE: a run of a freshly built `apr` binary. The host was at load 57 on 48 cores and a release build of the facade would have added to it; the table renders the same `error: {e}` line main.rs prints and asserts the exit-code mapping directly. FOLLOW-UP, filed before the PR as #3691: six more sites name GGUF or SafeTensors in their own message and still wrap it in `InvalidFormat` (hex.rs:323/350, safe_tensors_header.rs:180/182, trace_likely_has_repeated.rs:267/332). They are outside this issue''s done_when, which names inspect and tensors.' - id: PMAT-3662 github_issue: 3662 item_type: task @@ -19848,6 +19952,61 @@ roadmap: labels: - kind:code notes: 'ACCEPTANCE — issue #3670 done_when, verbatim in substance. (1) Each fleet-state branch prints UNMEASURED runner=... reason= ..., exits 0, and never prints PASS or SUMMARY PASS. (2) A mutant restoring FAIL on pin-mismatch exits 1, and a mutant restoring the ENV exit on pv-env exits 2; each mutant row refuses to pass if its edit did not apply. (3) A planted violation from a matching pinned binary is still RED, and a control that does not fire is still RED. (4) Real runs: pin forced to 0.65.2 under a 0.68.2 binary gives UNMEASURED reason=pin-mismatch (rc 0); the fleet pv on a tree with migrated parity records gives PASS verdict=Pass. RULING: PASS/FAIL only from a verdict produced by a binary that matches its pin; fleet state is UNMEASURED with a named reason and exit 0. Scope: scripts/check_fleet_pv_shapes_gate.sh only, one commit on top of #3653 (053992448).' +- id: PMAT-3671 + github_issue: 3671 + item_type: task + title: 'check_fleet_pv_shapes_gate.sh: reason=pv-env (exit 0) covers a pinned pv that PANICS — only pv rc 2 is fleet state; any other no-JSON rc must stay RED' + status: in_progress + priority: high + assigned_to: aprender-62 + created: 2026-09-21T10:15:00Z + updated: 2026-09-21T10:15:00Z + spec: null + acceptance_criteria: + - 'Issue #3671 done_when 1, verbatim: "pv''s own exit code is captured before the parser runs. `reason=pv-env`, exit 0, applies **only** when pv''s rc is 2 and it produced no JSON."' + - 'done_when 2, verbatim: "A pinned pv that produces no JSON with any other rc (101 panic, 134/139 signal, 1 with no output) goes RED, not `UNMEASURED`. It exits non-zero, and the row names pv''s rc and its first stderr line."' + - 'done_when 3, verbatim: "Case-table rows: a panic stub (101, no JSON) → RED. The existing rc-2-no-JSON stub stays `UNMEASURED reason=pv-env`. A mutant that removes the pv-rc check sends the panic row back to `UNMEASURED` (the row discriminates)."' + - 'done_when 4, verbatim: "The other fleet-state branches (`no-pin`, `no-binary`, `pin-mismatch`, `incapable`, `judge-env`) are unchanged."' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'Found by aprender-62 cross-inspecting the andon commit 984fc76a9 (PMAT-3670) in the 0.69 batch #3669; the cop ruled it a separate ticket for after the batch. HOW IT IS MET. `judge` captures pv''s own exit as `pvrc` (the parser still receives it, for its message) and the verdict `case` is keyed on the PAIR `:`: `2:2` is `UNMEASURED reason=pv-env` exit 0, unchanged; `2:*` is `FAIL runner=… reason=pv-no-verdict … pv exited … pv stderr: ` to stderr, exit 1. The arms for parser rc 0 (PASS), 1 (FAIL) and other (judge-env) are unchanged, and the no-pin / no-binary / pin-mismatch / incapable branches return before pv runs, so they are untouched. RED is FAIL (exit 1), not ENV (exit 2): a panic caused by a contract in the tree is tree state. SIGNAL ROW. The case table uses SIGKILL (rc 137, `kill -KILL $$` in the stub), a real signal death that writes no core file on a runner; 134/139 are the same `2:*` arm, since the arm matches any pv rc but 2. MEASURED at this commit: `bash scripts/check_fleet_pv_shapes_gate.sh --self-test` 19/19 ok (was 15). The new rows are 13-15 (panic 101, SIGKILL 137, rc 1 with no output: each RED naming pv''s rc and stderr, never UNMEASURED) and 16 (mutant `2:2)` -> `2:*)`, i.e. pv-env keyed on the parser alone: the panic stub goes back to `UNMEASURED reason=pv-env`, exit 0, so the panic row discriminates). BEFORE THE FIX, the same self-test with the new rows: rows 13-15 FAIL with "UNMEASURED runner=probe-runner reason=pv-env … ENV no JSON verdict (rc=101) … panicked", rc 0, reproducing the issue''s table. ALSO FIXED, same rows: the existing pv-env mutant (row 12) was VACUOUS. Its sed replaced the arm''s opening single quote with a double quote, so the mutant script was a bash syntax error (`bash -n` rc 2 at "line 146: syntax error near unexpected token `(`") and "exited non-zero" without reaching the arm it claimed to test. Both mutants now keep the arm''s text through a sed backreference, are refused unless `bash -n` passes, and assert the specific outcome (row 12: exit 2 and `ENV (rc=2)`; row 16: exit 0 and `UNMEASURED … reason=pv-env`). `--help` printed a hardcoded `sed -n 2,44p`, which cut the header off once it grew; it now prints the leading comment block, found by awk. `bashrs lint`: 0 errors (baseline 0). `bash scripts/guard_tree.sh --no-cargo`: 76 checks, 0 failed.' +- id: PMAT-3672 + github_issue: 3672 + item_type: task + title: 'apr run applies the chat template TWICE: the prompt reaches the model as an escaped user turn inside a user turn' + status: planned + priority: high + assigned_to: null + created: 2026-09-21T11:00:21Z + updated: 2026-09-21T11:00:21Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'ACCEPTANCE (hand-entered from issue #3672 done_when, verbatim; minted on the PR branch so the AD-04 quorum can run). (1) apr run -v on Qwen2.5-0.5B-Instruct-f16.gguf with a plain prompt shows exactly one template application and no \u{200B} in formatted_prompt. (2) The prompt token count equals llama.cpp''s for the same rendered conversation (21 for the prompt above, with no system turn). (3) A test fails if the prompt is templated twice (mutation: restore the upstream wrap → RED). ROOT CAUSE (issue comment): crates/apr-cli/src/dispatch_run.rs (GH-638) pre-wrapped the prompt in hard-coded ChatML, and realizar prepare_tokens (GH-278) then applied the model''s own template again. Fix: the prompt stays raw; --chat (or the GH-638 instruct/chat source-name heuristic) reaches realizar as InferenceConfig::force_chat_template, which prepare_tokens_{gguf,safetensors,apr} OR into its single template decision.' +- id: PMAT-3673 + github_issue: 3673 + item_type: task + title: 'Batching is the default merge path: scripts/batch_fold.sh + APR-RELEASE-001 §13 (operator 2026-09-21)' + status: planned + priority: critical + assigned_to: null + created: 2026-09-21T08:23:21Z + updated: 2026-09-21T08:23:21Z + spec: docs/specifications/APR-RELEASE-001-train-and-build-kaizen.md + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'ACCEPTANCE (quoted from issue #3673 "What lands"): 1. scripts/batch_fold.sh — fold receipted PR branches into one release/-batch branch, git merge --no-ff in the given order: a conflict ONLY in the generated set {docs/roadmaps/roadmap.yaml, contracts/census.json, contracts/contracts.nt, contracts/shapes.ttl, the CONTRACT_COUNT blocks of README.md} is taken and marked for regeneration; a conflict ANYWHERE else — including README prose outside a count block — aborts that branch''s merge and reports SKIP with the paths, never resolved by picking a side; --regen regenerates the generated set ONCE with the tree-built pv (scripts/pv_bin.sh), asserts the fixed points (aggregate --check, pv extract --check, readme_sync.sh --check), and commits the regeneration; exit 0 all folded; 1 any SKIP; 2 usage/git/regeneration failure; a folded-but-unregenerated generated set is never silent; --self-test case table, wired into CI. 2. APR-RELEASE-001 §13 — the batching procedure as the default path (cop prompt §7): assembly, generated-set regeneration, left-out rules, what needs a new quorum (only content new to the batch), constituent disarm/close, a red batch, and the review-lane VOID rule while paiml-implement#317 is open. AUTHOR NOTES (not criteria): the case table lives in scripts/check_batch_fold.sh (guard-tree runs it) because a --self-test case inside batch_fold.sh made check_guards_are_wired.sh RED; §13 required amending hard rule 9 ("No hand-squash of batches") and the matching §8 stop, which the new default contradicts.' - id: PMAT-3681 github_issue: 3681 item_type: task @@ -19946,6 +20105,25 @@ roadmap: labels: - kind:code notes: 'ANDON 2, found by aprender-62 as first responder on #3689: workspace-test-shard 1/3 (job 106309701273, runner yoga-build3) failed ont4c3_parity_receipts::the_committed_tree_agrees_with_its_own_denominator with "scripts/parity_receipt_denominator.sh: line 61: python3: command not found" for every receipt. classify() then printed nothing, the loop read that as "other", and the script said "the tree holds 0", exit 1. That is a missing interpreter scored as a count, the class of the git refusal in #3682. The cop ruled (~11:40Z) to fold it into #3689''s push under four conditions, which are done_when 1-5 above. SCOPE: scripts/parity_receipt_denominator.sh, crates/aprender-contracts-cli/tests/ont4c3_parity_receipts.rs, and this fragment. HOW IT IS MET. PY_BIN (seam PARITY_PYTHON, default python3). count_and_check checks `command -v "$PY_BIN"` after listing the tree and before any file: absent -> UNMEASURED line to stderr, return 3, which verify and --print propagate. Each file''s class must now be exactly record|legacy|other; anything else, including nothing, is "ENV the classifier () gave no answer for ", return 2 (the old `*) :` fall-through was the defect). --self-test on a runner with no interpreter prints an UNMEASURED line and exits 0: the classification rows cannot run there. The Rust test accepts exit 3 only when stderr carries both "UNMEASURED runner=" and "reason=no-interpreter interpreter="; any other non-success still fails. MEASURED at this commit: `--self-test` rc 0 with 7 rows, the 5 existing plus "no interpreter is UNMEASURED exit 3 naming it, never a count of 0" and "an interpreter that answers nothing is ENV exit 2, never a count of 0"; a bare run with python3 gives rc 0, "PASS 7 receipt(s) under evidence/parity/**, and evidence/parity/EXPECTED_RECEIPTS says 7." (the T-2 requirement, measured); PARITY_PYTHON=/nonexistent gives rc 3 "UNMEASURED runner=nopy-probe reason=no-interpreter interpreter=/nonexistent/python3 …"; PARITY_PYTHON=/bin/true gives rc 2 "ENV the classifier (/bin/true) gave no answer for evidence/parity/derived_expiries.json". THE YOGA SHAPE, a PATH made of every /usr/bin and /bin executable except python*: the PRE-FIX script (25d2ee264) prints "python3: command not found" 118 times, then "FAIL evidence/parity/EXPECTED_RECEIPTS says 7; the tree holds 0.", rc 1, which is #3689''s failure reproduced; the fix prints "UNMEASURED runner=yoga-shape reason=no-interpreter interpreter=python3 …", rc 3. MUTANT, the silent-classifier arm restored to the old `*) :` fall-through: self-test rc 1, "FAIL a silent interpreter gave exit 1: FAIL evidence/parity/EXPECTED_RECEIPTS says 2; the tree holds 0." THE RUST TEST: `cargo test -p aprender-contracts-cli --test ont4c3_parity_receipts` gives 9 passed with python3; PARITY_PYTHON=/nonexistent/python3 gives 1 passed, printing the UNMEASURED line; PARITY_PYTHON=/bin/true gives FAILED, "exit Some(2)" plus the ENV line, so a crash is still RED and only the named UNMEASURED line buys exit 3. `cargo fmt --all -- --check` rc 0.' +- id: PMAT-3696 + github_issue: 3696 + item_type: task + title: 'publish_strict.sh: a blank line in publish-order.txt stops the cascade naming an EMPTY crate — refuse it by line number' + status: in_progress + priority: low + assigned_to: aprender-62 + created: 2026-09-21T11:39:00Z + updated: 2026-09-21T11:39:00Z + spec: null + acceptance_criteria: + - 'Issue #3696 done_when 1, verbatim: "An empty crate name is refused **by name**, before any set check: `STOP publish: blank line N in publish-order.txt` (every blank or whitespace-only line, each by its number), rc 1, and no `bad array subscript`."' + - 'done_when 2, verbatim: "Measured in a private clone: a blank line at line 36 gives exactly that message; the clean file still plans 74 rows, rc 0."' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'Found by aprender-7a seat-filling #3657; the cop ruled it a small 0.70 row stacked on #3657''s 6fd241645, which is not touched (this commit is its child). SCOPE: scripts/release/publish_strict.sh (4 lines before the mapfile) and this fragment. HOW IT IS MET: awk lists every line of publish-order.txt that is empty or whitespace-only, by number; any hit stops with `die "blank line in publish-order.txt: an empty crate name is not a crate"` before ORDER is built, so no empty string reaches the set checks or bash''s associative-array subscript. publish_strict.sh has no --self-test on purpose (one would make it a guard that must be workflow-invoked), so the proof is end to end. MEASURED in a private clone (its own refs; tag v9.9.9 in the clone only; RELEASE_AP in scratch; --plan reads the sparse index and uploads nothing). BEFORE, at 6fd241645, a blank line at line 36: rc 1, "scripts/release/publish_strict.sh: line 47: EXPECT: bad array subscript" and "STOP publish: in the publish order but not in the universe: " (an empty name). AFTER, at this commit: the clean file gives rc 0 and 74 rows; a blank line at 36 gives rc 1, "STOP publish: blank line 36 in publish-order.txt: an empty crate name is not a crate", with 0 subscript errors; a whitespace-only line at 3 plus a blank line (now at 37) gives rc 1, "STOP publish: blank line 3, 37 in publish-order.txt: an empty crate name is not a crate". `bash scripts/check_release_scripts_derive_identity.sh`: PASS (no literal count, no identity literal).' - id: PMAT-3704 github_issue: 3704 item_type: task diff --git a/docs/specifications/APR-RELEASE-001-train-and-build-kaizen.md b/docs/specifications/APR-RELEASE-001-train-and-build-kaizen.md index b96f30d350..814b114df8 100644 --- a/docs/specifications/APR-RELEASE-001-train-and-build-kaizen.md +++ b/docs/specifications/APR-RELEASE-001-train-and-build-kaizen.md @@ -13,6 +13,10 @@ tag inheritance from the release commit, B2 split by capability (cpu shards + gp derived from `cargo metadata` (TIERS deleted), unattended cascade under standing authorization with receipts, v0.68.0 = GitHub-only / 0.68.1 = crates.io, P0·Fan-out as the 0.69 train's first PR. Earlier revisions kept where still true. *(Merged 2026-09-18 onto the `main` lineage — #3268 §4.1–4.3, §5.1, §6.1–6.4, §10–§12 and #3455 `check_milestone_cut.sh` — which the 09-15/16/17 working copy had been cut before; where the two disagreed the later operator ruling wins and the older text is kept as history.)* +**Revision 2026-09-21** — §13: integration batches are the DEFAULT merge path (operator: "we are +arbitrarily going slow because each PR adds time, but in most cases 80% of PR can be batched"), +assembled by `scripts/batch_fold.sh`. Hard rule 9 and the §8 "hand-squashed batch" stop are amended +to match (§13.9 says what that costs). **Revision 2026-09-16** — adds §1.6 critical path for 0.68, milestone hygiene (§6), the CI-wedge mechanism and its two sanctioned rules (P0·Unwedge), clean-room-on-tag (T-3), reaper budget as a manifest fact, and the capacity-source rule (no runners API, ever). @@ -178,9 +182,15 @@ that decides a gate. 2026-09-13 stands: "T4 is never mine … releases are automated by release train … for ALL releases".)* 8. `pmat work add` from the driver session only. Stop the line on RED — no reruns-as-passes, no `--skip`, no waivers. -9. **No hand-squash of batches.** Batch verification is the merge queue's job (merge_group, - group size 8); each PR lands as its own commit. A red batch fans out one job per PR, never - hand-bisected. +9. **A batch is an integration PR assembled by `scripts/batch_fold.sh`, never by hand (§13, + operator 2026-09-21).** Receipted PRs fold into `release/-batch`; only the generated set is + ever resolved by the tool, every other conflict leaves that PR out, and a red batch is traced to + its constituent with `git log --merges` on the batch branch (§13.7). + *(Superseded 2026-09-21. Until then this rule read: "No hand-squash of batches. Batch + verification is the merge queue's job (merge_group, group size 8); each PR lands as its own + commit. A red batch fans out one job per PR, never hand-bisected." The queue it described was + not the queue we had: mergeMethod SQUASH, maximumEntriesToMerge=1, measured on #3658 — so every + PR paid its own CI run, queue slot and roadmap re-merge.)* 10. **WIP is enforced at the producer.** paiml-implement's pre-push hook refuses to mint a PR when open aprender PRs ≥ 13 (basis: measured drain 8.8/day × 1.5 day, 2026-09-15; ratchet after 20 records) and emits an issue instead. Sweeps emit issues, not PRs. @@ -654,7 +664,7 @@ next: train eligible at - Any queue entry ejected twice on the same cause → stop, five-whys, no third re-queue. - Any measurement whose tree line shows `behind≠0`, or taken outside a per-session worktree → discard and re-measure from `origin/main`. -- Any step would need an invented threshold, a hand-squashed batch, a hand-cancelled run with started +- Any step would need an invented threshold, a batch assembled by hand instead of `scripts/batch_fold.sh` (§13), a hand-cancelled run with started jobs, a label edit to dodge a runner, an env bypass of a prejob gate, `--allow-dirty`, or a host CONFIG change made over SSH instead of through forjar (§3.5 — SSH for measurement and for unclogging is expected) → stop. - The upstream ontology spec, at the version this spec cites, is not reachable at a committed sha @@ -987,3 +997,106 @@ them are report lines. | FX-3 | §12.1.3 | every counter on a §7 line has a named ratchet target and moves one way | move one backward by hand → RED | | FX-4 | §12.2.C | `beats:` says `measured-on published` only when the dogfood ran on a `cargo install aprender` binary (G13: no embedded SHA, `DOGFOOD_ALLOW_UNPINNED=1`) | report `published` from a dev build → RED | | FX-5 | §12.2.C | a beat's status in `docs/BEATS.md` matches its contract | change the status without the contract → `readme_contract` RED | + +## §13 Integration batches — the default merge path (operator 2026-09-21) + +> "we are arbitrarily going slow because each PR adds time, but in most cases 80% of PR can be batched." + +The per-PR queue is the slow path. Every PR paid one CI run (~32 min median wall, ~68 runner-min), +one queue slot (~26 min per merge, measured), and, while `roadmap.yaml` is committed (#3650), about +one re-merge of the generated files per landing. The default is now ONE integration batch PR that +carries every receipted PR that is ready. First run: #3669, 18 PRs plus an andon fix. + +### §13.1 Who does what + +- **An author** stops at the receipt: branch, roadmap fragment, AD-04 quorum, + `docs/audits/quorum-.json` committed on the branch, lanes path to the batch driver. **Never + arms, and never merges main just to stay current.** +- **The batch driver** (the traffic cop) folds, regenerates, pushes and arms the batch. +- **A folded PR** is never pushed to again. Fixes go to the batch branch, or to the author's + branch followed by a re-fold. + +### §13.2 Assembly + +``` +git worktree add -b release/-batch origin/main +cd && bash scripts/batch_fold.sh --regen ... # priority order, cut-blockers first +``` + +Run it under **bash**. zsh does not word-split `$var`, which breaks a hand-rolled loop over branch +names. `batch_fold.sh` merges each branch `--no-ff`. It resolves ONLY the **generated set**: +`docs/roadmaps/roadmap.yaml`, `contracts/census.json`, `contracts/contracts.nt`, +`contracts/shapes.ttl`, and the `CONTRACT_COUNT` blocks of `README.md`. README is merged three-way +with the counts normalised out, so a README conflict outside a count block is a real conflict. Every +other conflict aborts that branch's merge (`SKIP : conflict in `). A clean fold that +touches the generated set still marks it stale, because two PRs that each add a contract merge +textually and the census is then wrong. + +`--regen` regenerates the set ONCE with the pv built from the batch tree (`scripts/pv_bin.sh`): +roadmap aggregate, `pv census`, `pv extract`, `readme_sync.sh --write`. It then asserts the fixed +points (`make roadmap-aggregate-check`, `pv extract --check`, `readme_sync.sh --check`) and commits +the regeneration. Exit codes: + +| Exit | Meaning | +|---|---| +| 0 | all folded, set consistent | +| 1 | a SKIP | +| 2 | a usage, git or regeneration failure, including a failed fixed point | +| 3 | folded, but the generated set is stale (`REGEN REQUIRED`) | + +### §13.3 Left out, and said so + +A PR is left out of the batch for exactly two reasons, and the batch body names which: + +- a real conflict (`SKIP`, e.g. #3606 against #3634); +- a tree-wide gate it fails on its own (e.g. `pv census` refused #3642's YAML). + +Nothing else keeps a receipted PR out. + +### §13.4 What needs a new quorum + +Each constituent's receipt stands, because its diff is unchanged inside the batch. Only content +NEW to the batch needs a quorum: fixes made in the batch branch, run with `--base `. The +regeneration needs none; its fixed points are asserted by `--regen`. While paiml-implement#317 is +open, a review lane whose agy conversation store (`~/.gemini/antigravity-cli/conversations/.db`) +never references the judged worktree is **VOID**. Stale shared scratch clones have produced +`measured` PASSes from another branch with rewritten baselines (#3664). A NON-author's measured +verification in the judged tree, posted on the PR with commands and outputs, fills the seat. + +### §13.5 Push, then disarm + +1. Push the batch first so CI starts. +2. Run `cargo check --workspace --all-targets` locally in parallel. +3. Comment "folded into #N" on each constituent and disarm it. +4. Cancel in-flight PR CI outside the batch; it goes stale when the batch moves main. +5. Dequeue anything the batch supersedes. + +### §13.6 The batch body, arming, closing + +The body has a table of constituents: PR, receipt path, head sha. It carries every constituent's +closing keywords, plus a `keep-open:` line naming the constituent PR numbers, or +`check_pr_closes_issue.sh` fails. Arm the batch. After it merges, close each constituent as "landed +in #N". + +### §13.7 A red batch + +1. Run `git log --merges` on the batch branch. +2. Find the constituent whose merge introduced the red, and reproduce it on that merge. +3. Either fix it in the batch (new content, so §13.4 applies) or drop it: rebuild the batch without + it, then `--regen`. +4. Push once. + +### §13.8 What this costs, stated rather than found + +The queue is SQUASH, so a batch lands on main as ONE commit. `git bisect` on main therefore stops at +the batch, not the constituent. The constituent history survives on the batch branch (the `--no-ff` +merges) and in the body's head-sha table, so the batch branch is kept until the train's next tag. +Deleting it early turns "which PR broke main" into a rebuild. + +### §13.9 Falsifiers + +| # | Assertion | Mutation | +|---|---|---| +| FX-B1 | `batch_fold.sh` resolves only the generated set: a code conflict and a README prose conflict are SKIP, a count-block conflict folds (`check_batch_fold.sh`, run by guard-tree on every PR) | treat README as wholly generated, or take every conflict → `check_batch_fold.sh --self-test` RED | +| FX-B2 | a stale generated set is never silent: a clean fold touching the census exits 3 | never mark a clean fold stale → RED | +| FX-B3 | a failed fixed point stops the batch (exit 2, the check named) | ignore `pv extract --check` → RED | diff --git a/evidence/pmat-3076-f16-bf16-matvec/findings.json b/evidence/pmat-3076-f16-bf16-matvec/findings.json new file mode 100644 index 0000000000..b6be71a302 --- /dev/null +++ b/evidence/pmat-3076-f16-bf16-matvec/findings.json @@ -0,0 +1,210 @@ +{ + "schema": "pmat-3076-findings-v1", + "ticket": "PMAT-3076 (#3076)", + "host": "noah-Lambda-Vector, AMD Ryzen Threadripper 7960X 24-Cores (Zen 4; avx2 fma f16c avx512f avx512_bf16), 48 threads", + "binaries": { + "before": "apr 0.68.2 (00e5d2dde)", + "after": "apr 0.68.2 (e3ceb27f2), pinned to HEAD by scripts/apr_bin.sh", + "note": "crates/aprender-serve/src/gguf/inference is unchanged between 00e5d2dde and this branch's base 79a3af79d except for this diff" + }, + "caveat": "load average 27-42 throughout (other sessions' builds on the host); every comparison interleaved before/after under the same load", + "decode_ab": { + "method": "apr run --no-gpu --temperature 0 --json, prompt 'Write a long story about a dragon who lives in a mountain.'; ms/token = (inference_time_ms at n=64 - at n=8) / 56; 3 alternating reps; default rayon pool (48 threads)", + "models": { + "Qwen2.5-0.5B-Instruct-f16.gguf": { + "sha256": "f1ad9d1174ce6ab47b584d522634c47e411b75bffdffd9a4e106e21e882392e5", + "source": "bartowski/Qwen2.5-0.5B-Instruct-GGUF" + }, + "Qwen3-0.6B-BF16.gguf": { + "sha256": "f9c9f1d3c1e21755b82d4e165f88dbbbd4355646d632fb5d6cef7c66ed4ee04e", + "source": "unsloth/Qwen3-0.6B-GGUF" + } + }, + "runs": { + "Qwen2.5-0.5B-Instruct-f16.gguf": { + "before": [ + { + "rep": "rep1", + "n8_ms": 6983.65, + "n64_ms": 13073.59, + "ms_per_token": 108.75, + "loadavg_1m": 26.67 + }, + { + "rep": "rep2", + "n8_ms": 6987.02, + "n64_ms": 12178.45, + "ms_per_token": 92.7, + "loadavg_1m": 36.42 + }, + { + "rep": "rep3", + "n8_ms": 6905.24, + "n64_ms": 11864.66, + "ms_per_token": 88.56, + "loadavg_1m": 39.06 + } + ], + "after": [ + { + "rep": "rep1", + "n8_ms": 4722.61, + "n64_ms": 7301.97, + "ms_per_token": 46.06, + "loadavg_1m": 29.55 + }, + { + "rep": "rep2", + "n8_ms": 4363.21, + "n64_ms": 7012.17, + "ms_per_token": 47.3, + "loadavg_1m": 37.27 + }, + { + "rep": "rep3", + "n8_ms": 4295.88, + "n64_ms": 6940.17, + "ms_per_token": 47.22, + "loadavg_1m": 41.81 + } + ] + }, + "Qwen3-0.6B-BF16.gguf": { + "before": [ + { + "rep": "rep1", + "n8_ms": 5008.17, + "n64_ms": 11435.03, + "ms_per_token": 114.77, + "loadavg_1m": 37.75 + }, + { + "rep": "rep2", + "n8_ms": 5001.23, + "n64_ms": 12961.04, + "ms_per_token": 142.14, + "loadavg_1m": 40.46 + }, + { + "rep": "rep3", + "n8_ms": 6618.38, + "n64_ms": 15653.82, + "ms_per_token": 161.35, + "loadavg_1m": 42.03 + } + ], + "after": [ + { + "rep": "rep1", + "n8_ms": 3442.39, + "n64_ms": 6717.29, + "ms_per_token": 58.48, + "loadavg_1m": 36.85 + }, + { + "rep": "rep2", + "n8_ms": 4017.9, + "n64_ms": 8384.22, + "ms_per_token": 77.97, + "loadavg_1m": 37.7 + }, + { + "rep": "rep3", + "n8_ms": 4325.96, + "n64_ms": 8894.84, + "ms_per_token": 81.59, + "loadavg_1m": 40.86 + } + ] + } + }, + "median_ms_per_token": { + "Qwen2.5-0.5B-Instruct-f16.gguf": { + "before": 92.7, + "after": 47.22 + }, + "Qwen3-0.6B-BF16.gguf": { + "before": 142.14, + "after": 77.97 + } + } + }, + "brick_profile_us_per_call": { + "note": "BRICK-PROFILE from apr run --trace --trace-level basic, F16 instruct model, 60 forward positions (52 prefill + 8 decode)", + "threads_48": { + "before": { + "total_ms": 5640.86, + "tok_per_sec": 10.6, + "FFN": 1844.4, + "DownProjection": 804.8, + "QkvProjection": 476.7, + "OutputProjection": 423.6, + "LmHead": 9907.0, + "AttentionScore": 13.7, + "RopeEmbedding": 2.2, + "Embedding": 2.7 + }, + "after": { + "total_ms": 2970.81, + "tok_per_sec": 20.2, + "FFN": 979.5, + "DownProjection": 405.3, + "QkvProjection": 288.1, + "OutputProjection": 280.2, + "LmHead": 2961.4, + "AttentionScore": 14.6, + "RopeEmbedding": 2.0, + "Embedding": 2.0 + } + }, + "threads_8": { + "before": { + "total_ms": 9626.07, + "tok_per_sec": 6.2, + "FFN": 2866.2, + "LmHead": 42950.6, + "DownProjection": 1442.8, + "QkvProjection": 381.1, + "OutputProjection": 301.9, + "AttentionScore": 11.0, + "RopeEmbedding": 1.5, + "Embedding": 3.0 + }, + "after": { + "total_ms": 874.55, + "tok_per_sec": 68.6, + "FFN": 252.5, + "DownProjection": 134.1, + "LmHead": 2716.3, + "QkvProjection": 53.4, + "OutputProjection": 49.2, + "AttentionScore": 10.6, + "RopeEmbedding": 1.1, + "Embedding": 1.5 + } + }, + "reading": "at 8 threads the kernel is about an order of magnitude faster per matmul; at the default 48 on a contended host the small projections sit on a ~300-500us per-call floor attributed to pool fan-out (a hypothesis, not isolated here)" + }, + "greedy_parity_before_vs_after": { + "method": "4 prompts x both instruct models, 32 tokens, text and token count compared", + "result": "8/8 identical, all texts non-empty" + }, + "greedy_parity_vs_llama_cpp": { + "gate": "scripts/float16_greedy_parity.sh --baseline ", + "comparator": "llama-completion, llama.cpp commit d1d3c3396 (the scripts/llama_pin.toml build_commit)", + "why_template_free_copies": "apr run double-templates instruct prompts (#3672), so parity models are copies with tokenizer.chat_template removed (gguf_new_metadata.py --remove-metadata); tensors unchanged", + "receipts": [ + "greedy-parity-Qwen2.5-0.5B-f16-notemplate.json", + "greedy-parity-Qwen3-0.6B-bf16-notemplate.json" + ], + "result": "F16 6/6 identical; BF16 5/6 identical + 1 KNOWN (diverges at char 83, reproduced byte-for-byte by the pre-change binary)", + "negative_control": { + "binary": "e3ceb27f2 source with two deliberate mutations (F16 routed to the BF16 kernel; BF16 <<15), a dirty-tree build and NOT the branch", + "receipts": [ + "negative-control-mutant-Qwen2.5-0.5B-f16-notemplate.json", + "negative-control-mutant-Qwen3-0.6B-bf16-notemplate.json" + ], + "result": "12/12 FAIL at char 0, gate rc=1 on both models" + } + } +} \ No newline at end of file diff --git a/evidence/pmat-3076-f16-bf16-matvec/greedy-parity-Qwen2.5-0.5B-f16-notemplate.json b/evidence/pmat-3076-f16-bf16-matvec/greedy-parity-Qwen2.5-0.5B-f16-notemplate.json new file mode 100644 index 0000000000..45f7a7b550 --- /dev/null +++ b/evidence/pmat-3076-f16-bf16-matvec/greedy-parity-Qwen2.5-0.5B-f16-notemplate.json @@ -0,0 +1 @@ +{"schema":"float16-greedy-parity-v1","apr":"apr 0.68.2 (e3ceb27f2)","baseline":"apr 0.68.2 (00e5d2dde)","comparator":"version: 0.4.1-dev (build 10987, commit d1d3c3396)","comparator_pin":"d1d3c3396","model":"Qwen2.5-0.5B-f16-notemplate.gguf","model_sha256":"3af2616a19a4b237ed5173aaf1d5bbdacacd667fefaab10460e3f45fd54e8b0e","host":"noah-Lambda-Vector","cpu":"AMD Ryzen Threadripper 7960X 24-Cores","n":32,"failures":0,"known":0,"prompts":[{"prompt":"The capital of France is Paris. The capital of Germany is","verdict":"PASS","first_diff":-1,"apr_text":"Berlin. The capital of Italy is Rome. The capital of Spain is Madrid. The capital of the United Kingdom is London. The capital of the United States is","llama_text":"Berlin. The capital of Italy is Rome. The capital of Spain is Madrid. The capital of the United Kingdom is London. The capital of the United States is"},{"prompt":"def fibonacci(n):","verdict":"PASS","first_diff":-1,"apr_text":"if n <= 1: \n return n \n else: \n return(fibonacci(n-1) + fibonacci(n-2))","llama_text":"if n <= 1: \n return n \n else: \n return(fibonacci(n-1) + fibonacci(n-2))"},{"prompt":"Water boils at 100 degrees Celsius. Ice melts at","verdict":"PASS","first_diff":-1,"apr_text":"0 degrees Celsius. How many degrees Celsius is the temperature at which water and ice are equally hot? To determine the temperature at which water and ice are equally","llama_text":"0 degrees Celsius. How many degrees Celsius is the temperature at which water and ice are equally hot? To determine the temperature at which water and ice are equally"},{"prompt":"Once upon a time, in a small village by the sea,","verdict":"PASS","first_diff":-1,"apr_text":"there lived a young man named Jack. Jack was a kind-hearted and honest man, and he always tried to help others. One day, he decided to go","llama_text":"there lived a young man named Jack. Jack was a kind-hearted and honest man, and he always tried to help others. One day, he decided to go"},{"prompt":"1, 2, 3, 5, 8, 13,","verdict":"PASS","first_diff":-1,"apr_text":"21, 34, 55, 89, 144, 233, 377,","llama_text":"21, 34, 55, 89, 144, 233, 377,"},{"prompt":"The three primary colors are","verdict":"PASS","first_diff":-1,"apr_text":"red, green, and blue. If you mix these colors in a certain ratio, you can create a new color. If you mix 1/2 of","llama_text":"red, green, and blue. If you mix these colors in a certain ratio, you can create a new color. If you mix 1/2 of"}]} diff --git a/evidence/pmat-3076-f16-bf16-matvec/greedy-parity-Qwen3-0.6B-bf16-notemplate.json b/evidence/pmat-3076-f16-bf16-matvec/greedy-parity-Qwen3-0.6B-bf16-notemplate.json new file mode 100644 index 0000000000..f00cc65866 --- /dev/null +++ b/evidence/pmat-3076-f16-bf16-matvec/greedy-parity-Qwen3-0.6B-bf16-notemplate.json @@ -0,0 +1 @@ +{"schema":"float16-greedy-parity-v1","apr":"apr 0.68.2 (e3ceb27f2)","baseline":"apr 0.68.2 (00e5d2dde)","comparator":"version: 0.4.1-dev (build 10987, commit d1d3c3396)","comparator_pin":"d1d3c3396","model":"Qwen3-0.6B-bf16-notemplate.gguf","model_sha256":"f9cabfa5074bc43ba8063f04bef61646c1ff2afa740d4a404ab481af5a74fbf5","host":"noah-Lambda-Vector","cpu":"AMD Ryzen Threadripper 7960X 24-Cores","n":32,"failures":0,"known":1,"prompts":[{"prompt":"The capital of France is Paris. The capital of Germany is","verdict":"PASS","first_diff":-1,"apr_text":"Berlin. The capital of Italy is Rome. The capital of Japan is Tokyo. The capital of China is Beijing. The capital of India is New Delhi. The","llama_text":"Berlin. The capital of Italy is Rome. The capital of Japan is Tokyo. The capital of China is Beijing. The capital of India is New Delhi. The"},{"prompt":"def fibonacci(n):","verdict":"PASS","first_diff":-1,"apr_text":"if n == 0:\n return 0\n elif n == 1:\n return 1\n else:\n return fibonacci(n","llama_text":"if n == 0:\n return 0\n elif n == 1:\n return 1\n else:\n return fibonacci(n"},{"prompt":"Water boils at 100 degrees Celsius. Ice melts at","verdict":"PASS","first_diff":-1,"apr_text":"0 degrees Celsius. What is the temperature difference between these two points?\n\nA) 100 degrees Celsius\n\nB) 100 degrees Fahrenheit","llama_text":"0 degrees Celsius. What is the temperature difference between these two points?\n\nA) 100 degrees Celsius\n\nB) 100 degrees Fahrenheit"},{"prompt":"Once upon a time, in a small village by the sea,","verdict":"KNOWN","first_diff":83,"apr_text":"there was a young girl named Lila. She lived in a cottage that was surrounded by a forest. One day, she noticed that the sun was setting,","llama_text":"there was a young girl named Lila. She lived in a cottage that was surrounded by a garden. The garden was filled with flowers and trees, and she"},{"prompt":"1, 2, 3, 5, 8, 13,","verdict":"PASS","first_diff":-1,"apr_text":"21, 34, 55, 89, 144, 233, 377,","llama_text":"21, 34, 55, 89, 144, 233, 377,"},{"prompt":"The three primary colors are","verdict":"PASS","first_diff":-1,"apr_text":"red, blue, and yellow. The three secondary colors are red and blue, blue and yellow, and red and yellow. What is the name of the color","llama_text":"red, blue, and yellow. The three secondary colors are red and blue, blue and yellow, and red and yellow. What is the name of the color"}]} diff --git a/evidence/pmat-3076-f16-bf16-matvec/negative-control-mutant-Qwen2.5-0.5B-f16-notemplate.json b/evidence/pmat-3076-f16-bf16-matvec/negative-control-mutant-Qwen2.5-0.5B-f16-notemplate.json new file mode 100644 index 0000000000..3eb859e8b1 --- /dev/null +++ b/evidence/pmat-3076-f16-bf16-matvec/negative-control-mutant-Qwen2.5-0.5B-f16-notemplate.json @@ -0,0 +1 @@ +{"schema":"float16-greedy-parity-v1","apr":"apr 0.68.2 (e3ceb27f2)","baseline":"apr 0.68.2 (00e5d2dde)","comparator":"version: 0.4.1-dev (build 10987, commit d1d3c3396)","comparator_pin":"d1d3c3396","model":"Qwen2.5-0.5B-f16-notemplate.gguf","model_sha256":"3af2616a19a4b237ed5173aaf1d5bbdacacd667fefaab10460e3f45fd54e8b0e","host":"noah-Lambda-Vector","cpu":"AMD Ryzen Threadripper 7960X 24-Cores","n":32,"failures":6,"known":0,"prompts":[{"prompt":"The capital of France is Paris. The capital of Germany is","verdict":"FAIL","first_diff":0,"apr_text":"!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","llama_text":"Berlin. The capital of Italy is Rome. The capital of Spain is Madrid. The capital of the United Kingdom is London. The capital of the United States is"},{"prompt":"def fibonacci(n):","verdict":"FAIL","first_diff":0,"apr_text":"!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","llama_text":"if n <= 1: \n return n \n else: \n return(fibonacci(n-1) + fibonacci(n-2))"},{"prompt":"Water boils at 100 degrees Celsius. Ice melts at","verdict":"FAIL","first_diff":0,"apr_text":"!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","llama_text":"0 degrees Celsius. How many degrees Celsius is the temperature at which water and ice are equally hot? To determine the temperature at which water and ice are equally"},{"prompt":"Once upon a time, in a small village by the sea,","verdict":"FAIL","first_diff":0,"apr_text":"!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","llama_text":"there lived a young man named Jack. Jack was a kind-hearted and honest man, and he always tried to help others. One day, he decided to go"},{"prompt":"1, 2, 3, 5, 8, 13,","verdict":"FAIL","first_diff":0,"apr_text":"!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","llama_text":"21, 34, 55, 89, 144, 233, 377,"},{"prompt":"The three primary colors are","verdict":"FAIL","first_diff":0,"apr_text":"!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","llama_text":"red, green, and blue. If you mix these colors in a certain ratio, you can create a new color. If you mix 1/2 of"}]} diff --git a/evidence/pmat-3076-f16-bf16-matvec/negative-control-mutant-Qwen3-0.6B-bf16-notemplate.json b/evidence/pmat-3076-f16-bf16-matvec/negative-control-mutant-Qwen3-0.6B-bf16-notemplate.json new file mode 100644 index 0000000000..0ab7854f50 --- /dev/null +++ b/evidence/pmat-3076-f16-bf16-matvec/negative-control-mutant-Qwen3-0.6B-bf16-notemplate.json @@ -0,0 +1 @@ +{"schema":"float16-greedy-parity-v1","apr":"apr 0.68.2 (e3ceb27f2)","baseline":"apr 0.68.2 (00e5d2dde)","comparator":"version: 0.4.1-dev (build 10987, commit d1d3c3396)","comparator_pin":"d1d3c3396","model":"Qwen3-0.6B-bf16-notemplate.gguf","model_sha256":"f9cabfa5074bc43ba8063f04bef61646c1ff2afa740d4a404ab481af5a74fbf5","host":"noah-Lambda-Vector","cpu":"AMD Ryzen Threadripper 7960X 24-Cores","n":32,"failures":6,"known":0,"prompts":[{"prompt":"The capital of France is Paris. The capital of Germany is","verdict":"FAIL","first_diff":0,"apr_text":"!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","llama_text":"Berlin. The capital of Italy is Rome. The capital of Japan is Tokyo. The capital of China is Beijing. The capital of India is New Delhi. The"},{"prompt":"def fibonacci(n):","verdict":"FAIL","first_diff":0,"apr_text":"!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","llama_text":"if n == 0:\n return 0\n elif n == 1:\n return 1\n else:\n return fibonacci(n"},{"prompt":"Water boils at 100 degrees Celsius. Ice melts at","verdict":"FAIL","first_diff":0,"apr_text":"!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","llama_text":"0 degrees Celsius. What is the temperature difference between these two points?\n\nA) 100 degrees Celsius\n\nB) 100 degrees Fahrenheit"},{"prompt":"Once upon a time, in a small village by the sea,","verdict":"FAIL","first_diff":0,"apr_text":"!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","llama_text":"there was a young girl named Lila. She lived in a cottage that was surrounded by a garden. The garden was filled with flowers and trees, and she"},{"prompt":"1, 2, 3, 5, 8, 13,","verdict":"FAIL","first_diff":0,"apr_text":"!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","llama_text":"21, 34, 55, 89, 144, 233, 377,"},{"prompt":"The three primary colors are","verdict":"FAIL","first_diff":0,"apr_text":"!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","llama_text":"red, blue, and yellow. The three secondary colors are red and blue, blue and yellow, and red and yellow. What is the name of the color"}]} diff --git a/scripts/batch_fold.sh b/scripts/batch_fold.sh new file mode 100755 index 0000000000..c3b9994b97 --- /dev/null +++ b/scripts/batch_fold.sh @@ -0,0 +1,183 @@ +#!/usr/bin/env bash +# batch_fold.sh -- fold receipted PR branches into ONE integration batch branch +# (APR-RELEASE-001 §13; operator 2026-09-21: "we are arbitrarily going slow +# because each PR adds time, but in most cases 80% of PR can be batched"). +# +# Run INSIDE the batch worktree (release/-batch, cut from origin/main). +# Each branch is merged --no-ff, in the order given (cut-blockers first). +# +# THE GENERATED SET is the only thing this script ever resolves by itself: +# docs/roadmaps/roadmap.yaml aggregate of docs/roadmaps/entries/ +# contracts/census.json pv census +# contracts/contracts.nt pv extract +# contracts/shapes.ttl pv extract (every `shape:` block) +# README.md CONTRACT_COUNT scripts/readme_sync.sh -- the COUNT BLOCKS only +# A conflict in those is taken and marked for regeneration. README.md is merged +# three-way with the counts normalised out, so a README conflict OUTSIDE a count +# block is a real conflict, not a generated one. Any real conflict aborts that +# branch's merge and reports SKIP with the paths -- never resolved by picking a +# side. The branch is left out of the batch and named. +# +# A merge that folds CLEANLY can still leave the generated set wrong: two PRs +# that each add a contract merge textually and the census count is then off by +# one. So every fold that touches a generated file marks the set STALE, and a +# stale set is never silent: --regen regenerates it ONCE, with the pv built from +# THIS tree (scripts/pv_bin.sh), asserts the three fixed points, and commits the +# regeneration; without --regen the run ends `REGEN REQUIRED` and exits 3. +# +# bash scripts/batch_fold.sh [--regen] BRANCH... +# bash scripts/batch_fold.sh --regen regenerate only +# +# Output, one line per branch: +# folded generated=[] merged; were taken for regeneration +# SKIP : conflict in merge aborted, branch left out +# ERROR : git could not merge it at all +# Exit: 0 every branch folded and the generated set is consistent; 1 any SKIP; +# 2 a usage, git or regeneration failure (including a failed fixed-point +# check); 3 folded, but the generated set is stale (run --regen). +# The case table is scripts/check_batch_fold.sh (guard-tree runs it). +# +# Test seam: BATCH_FOLD_PV= skips building pv (scripts/pv_bin.sh) and uses it. +set -uo pipefail + +GENERATED_WHOLE="docs/roadmaps/roadmap.yaml contracts/census.json contracts/contracts.nt contracts/shapes.ttl" +COUNT_RE='()[0-9]+()' + +die() { printf 'ERROR %s\n' "$*" >&2; exit 2; } +rmtree() { case "${1:-}" in ''|/) return 0 ;; *) [ -d "$1" ] && rm -rf -- "$1" ;; esac; return 0; } + +is_generated_whole() { # PATH -> 0 iff the whole file is derived + local g + for g in $GENERATED_WHOLE; do [ "$1" = "$g" ] && return 0; done + return 1 +} + +touches_generated() { # REV_A REV_B -> prints the generated paths that differ + local p + while IFS= read -r p; do + [ -n "$p" ] || continue + if is_generated_whole "$p" || [ "$p" = README.md ]; then printf '%s\n' "$p"; fi + done < <(git diff --no-renames --name-only "$1" "$2") +} + +# readme_counts_only -> 0 iff README.md's conflict is ONLY in CONTRACT_COUNT blocks. +# On 0 the file is written merged (counts left for readme_sync.sh) and staged. +readme_counts_only() { + local t rc=0 + t=$(mktemp -d "${TMPDIR:-/tmp}/batch-fold-readme.XXXXXX") || return 1 + if ! { git show :1:README.md > "$t/base" && git show :2:README.md > "$t/ours" && git show :3:README.md > "$t/theirs"; } 2>/dev/null; then + rmtree "$t"; return 1 # an add/add or delete conflict is not a count conflict + fi + local n + n=$(grep -m1 -oE "$COUNT_RE" "$t/ours" | sed -E 's/[^0-9]//g') + sed -E -i "s/$COUNT_RE/\\1N\\2/g" "$t/base" "$t/ours" "$t/theirs" + git merge-file -p "$t/ours" "$t/base" "$t/theirs" > "$t/merged" 2>/dev/null || rc=$? + if [ "$rc" = 0 ]; then + # put OUR count back so the file stays well-formed; --regen rewrites every block + sed -E -i "s/()N()/\\1${n:-0}\\2/g" "$t/merged" + cp -- "$t/merged" README.md && git add -- README.md || rc=1 + fi + rmtree "$t" + return "$rc" +} + +fold_one() { # BRANCH -> one verdict line; 0 folded, 1 skipped, 2 git failure + local br=$1 f real="" gen="" conf before + before=$(git rev-parse HEAD) || return 2 + if git merge --no-ff --no-edit "$br" >/dev/null 2>&1; then + gen=$(touches_generated "$before" HEAD | tr '\n' ' ') + [ -z "$gen" ] || STALE=1 + printf 'folded %s generated=[%s]\n' "$br" "${gen% }"; return 0 + fi + if ! git rev-parse -q --verify MERGE_HEAD >/dev/null 2>&1; then + printf 'ERROR %s: git merge failed without a conflict state\n' "$br"; return 2 + fi + conf=$(git diff --name-only --diff-filter=U) + while IFS= read -r f; do + [ -n "$f" ] || continue + if is_generated_whole "$f"; then + gen="$gen $f" + elif [ "$f" = README.md ] && readme_counts_only; then + gen="$gen README.md" + else + real="$real $f" + fi + done <<< "$conf" + if [ -n "$real" ]; then + git merge --abort >/dev/null 2>&1 + printf 'SKIP %s: conflict in%s\n' "$br" "$real"; return 1 + fi + for f in $gen; do + [ "$f" = README.md ] && continue # already merged and staged above + { git checkout --theirs -- "$f" && git add -- "$f"; } >/dev/null 2>&1 \ + || { git merge --abort >/dev/null 2>&1; printf 'ERROR %s: cannot take %s\n' "$br" "$f"; return 2; } + done + git -c commit.gpgsign=false commit -q --no-edit >/dev/null 2>&1 \ + || { git merge --abort >/dev/null 2>&1; printf 'ERROR %s: commit failed\n' "$br"; return 2; } + STALE=1 + printf 'folded %s generated=[%s]\n' "$br" "${gen# }"; return 0 +} + +regen() { # regenerate the generated set once, assert the fixed points, commit it + local PV extra + if [ -n "${BATCH_FOLD_PV:-}" ]; then + PV=$BATCH_FOLD_PV + else + # shellcheck source=scripts/pv_bin.sh + . scripts/pv_bin.sh || die "regen: scripts/pv_bin.sh could not build/resolve pv from this tree" + fi + make roadmap-aggregate >/dev/null 2>&1 || die "regen: make roadmap-aggregate failed" + "$PV" census contracts --format json > contracts/census.json 2>/dev/null || die "regen: pv census failed" + "$PV" extract contracts >/dev/null 2>&1 || die "regen: pv extract failed" + bash scripts/readme_sync.sh --write >/dev/null 2>&1 || die "regen: readme_sync.sh --write failed" + # the fixed points: each regenerated artifact must equal a second regeneration + make roadmap-aggregate-check >/dev/null 2>&1 || die "regen: fixed point FAILED -- make roadmap-aggregate-check" + "$PV" extract contracts --check >/dev/null 2>&1 || die "regen: fixed point FAILED -- pv extract contracts --check" + bash scripts/readme_sync.sh --check >/dev/null 2>&1 || die "regen: fixed point FAILED -- readme_sync.sh --check" + # only the generated set may have moved; anything else is a tool writing where it should not + extra=$(git status --porcelain --untracked-files=no | awk '{print $2}' \ + | grep -vxE 'docs/roadmaps/roadmap\.yaml|contracts/census\.json|contracts/contracts\.nt|contracts/shapes\.ttl|README\.md' || true) + [ -z "$extra" ] || die "regen: wrote outside the generated set: $(printf '%s ' $extra)" + if [ -n "$(git status --porcelain --untracked-files=no)" ]; then + git add -- docs/roadmaps/roadmap.yaml contracts/census.json contracts/contracts.nt contracts/shapes.ttl README.md 2>/dev/null + git -c commit.gpgsign=false commit -q -m "batch: regenerate the generated set once (batch_fold.sh --regen) + +roadmap aggregate, pv census, pv extract (contracts.nt + shapes.ttl) and the +README CONTRACT_COUNT blocks, regenerated with the pv built from this tree; +fixed points asserted: make roadmap-aggregate-check, pv extract --check, +readme_sync.sh --check." >/dev/null 2>&1 || die "regen: commit failed" + printf 'regenerated the generated set and committed it (fixed points asserted)\n' + else + printf 'regenerated the generated set: already consistent (fixed points asserted)\n' + fi +} + +case "${1:-}" in -h|--help) sed -n '2,42p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; esac + +REGEN=0; branches=() +while [ $# -gt 0 ]; do + case "$1" in + --regen) REGEN=1; shift ;; + -*) printf 'batch_fold.sh: unknown flag %s\n' "$1" >&2; exit 2 ;; + *) branches+=("$1"); shift ;; + esac +done +[ "${#branches[@]}" -gt 0 ] || [ "$REGEN" = 1 ] \ + || { printf 'usage: bash scripts/batch_fold.sh [--regen] BRANCH... | --regen\n' >&2; exit 2; } +git rev-parse --is-inside-work-tree >/dev/null 2>&1 || die "not inside a git work tree" +[ -z "$(git status --porcelain --untracked-files=no)" ] \ + || die "the batch worktree has uncommitted changes; fold into a clean tree only" + +STALE=0; status=0 +for br in "${branches[@]}"; do + fold_one "$br"; rc=$? + [ "$rc" = 2 ] && exit 2 + [ "$rc" = 1 ] && status=1 +done +if [ "$REGEN" = 1 ]; then + regen +elif [ "$STALE" = 1 ]; then + printf 'REGEN REQUIRED: a folded branch touched the generated set; run: bash scripts/batch_fold.sh --regen\n' + [ "$status" = 0 ] && status=3 +fi +exit "$status" diff --git a/scripts/check_batch_fold.sh b/scripts/check_batch_fold.sh new file mode 100644 index 0000000000..37d980b225 --- /dev/null +++ b/scripts/check_batch_fold.sh @@ -0,0 +1,169 @@ +#!/usr/bin/env bash +# check_batch_fold.sh -- the case table for scripts/batch_fold.sh, the integration-batch +# folder (#3673, APR-RELEASE-001 §13). Batching is the DEFAULT merge path now (operator +# 2026-09-21), so the folder decides what lands with a batch: it may resolve ONLY the +# generated set, and must leave out -- never silently resolve -- every real conflict. +# +# The table runs the real batch_fold.sh against throwaway git repos. Regeneration uses +# stub tools (a stub pv via BATCH_FOLD_PV, a stub Makefile and readme_sync.sh in the +# fixture), so no build and no network are needed. +# +# WHY THIS FILE AND NOT `batch_fold.sh --self-test`: a script that carries its own +# --self-test case claims to be a guard, and check_guards_are_wired.sh requires a +# workflow to run it -- measured: the handed-off draft did exactly that and turned +# the meta-guard RED ("NEW: batch_fold.sh"). A check_*.sh is dispatched by +# guard_tree.sh --no-cargo, so the table runs on every PR with no workflow edit. +# +# check_batch_fold.sh the case table +# check_batch_fold.sh --self-test the mutants: four wrong folders, each must turn a row RED +set -uo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" || exit 2 +SUBJECT="${BATCH_FOLD_SUBJECT:-$ROOT/scripts/batch_fold.sh}" + +rmtree() { case "${1:-}" in ''|/) return 0 ;; *) [ -d "$1" ] && rm -rf -- "$1" ;; esac; return 0; } + +# mkfixture DIR -- a repo with a generated set, README count blocks, code, and branches: +# plain (no generated path) gen1/gen2 (roadmap-only conflict) code1/code2 (real conflict) +# rc1/rc2 (README count-only conflict) rp1/rp2 (README prose conflict) +# cen (clean census edit) plus stub tools for --regen +mkfixture() { + local d=$1 + git init -q -b main "$d" || return 2 + ( + cd "$d" || exit 2 + export GIT_AUTHOR_NAME=t GIT_AUTHOR_EMAIL=t@t GIT_COMMITTER_NAME=t GIT_COMMITTER_EMAIL=t@t + export GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0=core.hooksPath GIT_CONFIG_VALUE_0=/dev/null + mkdir -p docs/roadmaps contracts src scripts + printf 'a\n' > docs/roadmaps/roadmap.yaml + printf '{"n_files": 5}\n' > contracts/census.json + printf 't0\n' > contracts/contracts.nt + printf 's0\n' > contracts/shapes.ttl + printf 'x\n' > src/lib.rs + printf '# T\nprose line one\ncount: 5\nprose line two\n' > README.md + printf 'roadmap-aggregate:\n\t@printf %s > docs/roadmaps/roadmap.yaml\nroadmap-aggregate-check:\n\t@test "$$(cat docs/roadmaps/roadmap.yaml)" = aggregated\n' "'aggregated\\n'" > Makefile + cat > scripts/readme_sync.sh <<'SH' +#!/usr/bin/env bash +case "$1" in + --write) sed -E -i 's/()[0-9]+()/\17\2/g' README.md ;; + --check) grep -q 'CONTRACT_COUNT_START -->7510511[0-9]+7