Skip to content

fix(tokenizer): the unknown token is the model's, not BPE's — Qwen3.5 serves; a model with no vocabulary refuses by name (PMAT-3609) #3678

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
noahgift wants to merge 1 commit into from
Closed

fix(tokenizer): the unknown token is the model's, not BPE's — Qwen3.5 serves; a model with no vocabulary refuses by name (PMAT-3609) #3678

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions crates/apr-cli/src/commands/serve/chat.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,8 +74,14 @@ pub(crate) fn load_safetensors_tokenizer(path: &Path) -> Option<SafeTensorsToken
let mut vocab = vocab;
let (bos_token_id, eos_token_id) = merge_special_tokens_into_vocab(added_tokens, &mut vocab);

// Create BPE tokenizer with vocab and merge rules
let tokenizer = realizar::tokenizer::BPETokenizer::new(vocab.clone(), merges, "<unk>").ok()?;
// Create BPE tokenizer with vocab and merge rules. #3609: the unknown token is the one
// tokenizer.json DECLARES (`model.unk_token`, null for byte-level models like Qwen),
// not a literal "<unk>" every model is assumed to have.
let unk_token = json
.get("model")
.and_then(|m| m.get("unk_token"))
.and_then(|v| v.as_str());
let tokenizer = realizar::tokenizer::BPETokenizer::new(vocab.clone(), merges, unk_token).ok()?;

Some(SafeTensorsTokenizerInfo {
tokenizer: std::sync::Arc::new(tokenizer),
Expand Down
43 changes: 22 additions & 21 deletions crates/apr-cli/src/commands/serve/handler_gpu_completion.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -418,7 +418,7 @@ fn start_gguf_server(model_path: &Path, config: &ServerConfig) -> Result<()> {
.green()
);

let vocab = extract_gguf_vocab(&mapped_model, quantized_model.config().vocab_size);
let vocab = extract_gguf_vocab(&mapped_model)?;

// PERF-021 / N4 / I-2: a request has a RESOLUTION, and it is reported.
//
Expand Down Expand Up @@ -463,26 +463,27 @@ fn start_gguf_server(model_path: &Path, config: &ServerConfig) -> Result<()> {
run_cpu_server(quantized_model, vocab, Some(mapped_model), config, Some(offload))
}

/// Extract vocabulary from GGUF model, falling back to placeholder tokens.
/// Extract the vocabulary from a GGUF model, or refuse to serve it (#3609).
///
/// GH-226: When the GGUF lacks `tokenizer.ggml.tokens` metadata, the placeholder
/// vocabulary must include `<unk>` so that `BPETokenizer::new` can find it.
/// Without this, the serve path fails with "Unknown token '<unk>' not in vocabulary"
/// and the server exits before binding — causing "Server failed to become ready".
fn extract_gguf_vocab(
mapped_model: &realizar::gguf::MappedGGUFModel,
vocab_size: usize,
) -> Vec<String> {
mapped_model.model.vocabulary().unwrap_or_else(|| {
eprintln!("Warning: No vocabulary in GGUF, using placeholder tokens");
let mut vocab: Vec<String> = (0..vocab_size).map(|i| format!("token{i}")).collect();
// GH-226: Ensure <unk> is present for BPETokenizer compatibility.
// Use slot 0 (standard convention for unknown token).
if !vocab.is_empty() {
vocab[0] = "<unk>".to_string();
}
vocab
})
/// GH-226 once filled a missing `tokenizer.ggml.tokens` with `token0..tokenN` and put
/// `<unk>` in slot 0, because `BPETokenizer::new` required one. So a model with NO
/// vocabulary loaded, while a model with a real vocabulary that lacked `<unk>` (Qwen3.5)
/// was refused: the less-specified input was the one accepted. The unknown token is
/// now optional, and a model with no vocabulary refuses by name. Placeholder tokens
/// are not a tokenizer.
fn extract_gguf_vocab(mapped_model: &realizar::gguf::MappedGGUFModel) -> Result<Vec<String>> {
mapped_model
.model
.vocabulary()
.ok_or_else(|| no_vocabulary("the GGUF (no tokenizer.ggml.tokens)"))
}

/// #3609: the refusal every serve path gives for a model with no vocabulary.
fn no_vocabulary(what: &str) -> CliError {
CliError::ModelLoadFailed(format!(
"{what} has no vocabulary to tokenize with, so there is nothing to serve; \
refusing to substitute placeholder tokens (#3609)"
))
}

/// #2762: resolve the KV-cache context length for the GGUF + CUDA serve path.
Expand Down Expand Up @@ -651,7 +652,7 @@ fn start_gguf_server_cuda(
let quantized_model = OwnedQuantizedModel::from_mapped(&mapped_model).map_err(|e| {
CliError::ModelLoadFailed(format!("Failed to rebuild quantized model: {e}"))
})?;
let vocab = extract_gguf_vocab(&mapped_model, quantized_model.config().vocab_size);
let vocab = extract_gguf_vocab(&mapped_model)?;
// CUDA init failed and this process fell back to CPU. The offload
// report travels with it UNCHANGED, so `/v1/effective-config` shows
// `gpu_layers_resolved` beside `backend_loaded: ["cpu"]` — which is
Expand Down
27 changes: 8 additions & 19 deletions crates/apr-cli/src/commands/serve/handlers.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -594,16 +594,13 @@ fn wgpu_smoke_test(
}

/// PMAT-340: Extract the real vocab from the GGUF for tokenize/detokenize.
/// #3609: a GGUF with no vocabulary refuses by name (see `extract_gguf_vocab`).
#[cfg(feature = "wgpu")]
fn wgpu_vocabulary(mapped: &realizar::gguf::MappedGGUFModel, vocab_size: usize) -> Vec<String> {
mapped.model.vocabulary().unwrap_or_else(|| {
eprintln!("Warning: No vocabulary in GGUF, using placeholder");
let mut v: Vec<String> = (0..vocab_size).map(|i| format!("token{i}")).collect();
if !v.is_empty() {
v[0] = "<unk>".to_string();
}
v
})
fn wgpu_vocabulary(mapped: &realizar::gguf::MappedGGUFModel) -> Result<Vec<String>> {
mapped
.model
.vocabulary()
.ok_or_else(|| no_vocabulary("the GGUF (no tokenizer.ggml.tokens)"))
}

/// PMAT-341: Build a BPE tokenizer from the GGUF merge rules.
Expand Down Expand Up @@ -822,7 +819,7 @@ fn serve_wgpu_backend(
println!("{}", "Starting WGPU inference server...".cyan());

// PMAT-340: Extract real vocab from GGUF for tokenization/detokenization
let vocab = wgpu_vocabulary(mapped, vocab_size);
let vocab = wgpu_vocabulary(mapped)?;
// PMAT-341: Extract BPE merge rules for proper tokenization
let merges = mapped.model.merge_rules().unwrap_or_default();
println!(
Expand Down Expand Up @@ -1443,15 +1440,7 @@ fn try_apr_quantized_cpu(model_path: &Path, config: &ServerConfig) -> Result<()>
let vocab = mapped
.metadata
.get_embedded_vocabulary()
.unwrap_or_else(|| {
let vocab_size = mapped.metadata.vocab_size.unwrap_or(32000);
eprintln!("Warning: No embedded vocabulary in APR, using placeholder tokens");
let mut v: Vec<String> = (0..vocab_size).map(|i| format!("token{i}")).collect();
if !v.is_empty() {
v[0] = "<unk>".to_string();
}
v
});
.ok_or_else(|| no_vocabulary("the APR file (no embedded vocabulary)"))?;

println!("{}", "Q4K CPU inference ready".green());

Expand Down
49 changes: 15 additions & 34 deletions crates/apr-cli/src/commands/serve/handlers_include_01.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,15 +50,10 @@ fn start_apr_server_gpu(
);

// Extract vocabulary from embedded APR metadata
let vocab = mapped.metadata.get_embedded_vocabulary().unwrap_or_else(|| {
let vocab_size = mapped.metadata.vocab_size.unwrap_or(32000);
eprintln!("Warning: No embedded vocabulary in APR, using placeholder tokens");
let mut v: Vec<String> = (0..vocab_size).map(|i| format!("token{i}")).collect();
if !v.is_empty() {
v[0] = "<unk>".to_string();
}
v
});
let vocab = mapped
.metadata
.get_embedded_vocabulary()
.ok_or_else(|| no_vocabulary("the APR file (no embedded vocabulary)"))?;

// GH-88: Extract merge rules for proper BPE tokenization (HuggingFace models)
let merges = mapped.metadata.get_embedded_merges();
Expand Down Expand Up @@ -126,13 +121,9 @@ fn start_apr_q4k_server_gpu(
.and_then(|m| m.load_embedded_tokenizer())
.map(|t| (t.id_to_token.clone(), None))
})
.unwrap_or_else(|| {
println!(
"{}",
"Warning: No vocabulary found, using placeholder tokens".yellow()
);
((0..151936).map(|i| format!("token{i}")).collect(), None)
});
.ok_or_else(|| {
no_vocabulary("the APR model (no sibling tokenizer.json, no embedded tokenizer)")
})?;

println!(" Vocab: {} tokens", vocab.len());
if let Some(eos) = eos_id {
Expand Down Expand Up @@ -200,15 +191,10 @@ fn start_safetensors_server_gpu(
);

// Extract vocabulary from embedded APR metadata
let vocab = mapped.metadata.get_embedded_vocabulary().unwrap_or_else(|| {
let vocab_size = mapped.metadata.vocab_size.unwrap_or(32000);
eprintln!("Warning: No embedded vocabulary in APR, using placeholder tokens");
let mut v: Vec<String> = (0..vocab_size).map(|i| format!("token{i}")).collect();
if !v.is_empty() {
v[0] = "<unk>".to_string();
}
v
});
let vocab = mapped
.metadata
.get_embedded_vocabulary()
.ok_or_else(|| no_vocabulary("the APR file (no embedded vocabulary)"))?;

// GH-88: Extract merge rules for proper BPE tokenization (HuggingFace models)
let merges = mapped.metadata.get_embedded_merges();
Expand Down Expand Up @@ -281,15 +267,10 @@ fn start_safetensors_server_cpu_quantized(
);

// Extract vocabulary from embedded APR metadata
let vocab = mapped.metadata.get_embedded_vocabulary().unwrap_or_else(|| {
let vocab_size = mapped.metadata.vocab_size.unwrap_or(32000);
eprintln!("Warning: No embedded vocabulary in APR, using placeholder tokens");
let mut v: Vec<String> = (0..vocab_size).map(|i| format!("token{i}")).collect();
if !v.is_empty() {
v[0] = "<unk>".to_string();
}
v
});
let vocab = mapped
.metadata
.get_embedded_vocabulary()
.ok_or_else(|| no_vocabulary("the APR file (no embedded vocabulary)"))?;

let _ = std::fs::remove_file(&tmp_apr);

Expand Down
5 changes: 4 additions & 1 deletion crates/aprender-serve/book/src/examples/api-server.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,10 @@ let config = ModelConfig {
eps: 1e-5,
};
let model = Model::new(config)?;
let tokenizer = BPETokenizer::new(vocab, merges, "<unk>")?;
// The unknown token is the model's own, if it has one: LLaMA-style vocabularies declare
// `<unk>`, byte-level ones (GPT-2, Qwen) have none. BPETokenizer requires neither (#3609).
let unk = realizar::tokenizer::vocabulary_unk_token(&vocab); // Some("<unk>") or None
let tokenizer = BPETokenizer::new(vocab, merges, unk)?;

// Create server
let state = AppState::new(model, tokenizer);
Expand Down
5 changes: 4 additions & 1 deletion crates/aprender-serve/examples/model_cache.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,10 @@ fn create_demo_model(vocab_size: usize) -> (Model, BPETokenizer) {
}
})
.collect();
let tokenizer = BPETokenizer::new(vocab, vec![], "<unk>").expect("Failed to create tokenizer");
// This demo vocabulary declares `<unk>` at id 0, so it passes it. The unknown token is
// the model's own: a byte-level vocabulary (GPT-2, Qwen) has none and passes `None` (#3609).
let tokenizer =
BPETokenizer::new(vocab, vec![], Some("<unk>")).expect("Failed to create tokenizer");

(model, tokenizer)
}
Expand Down
24 changes: 16 additions & 8 deletions crates/aprender-serve/src/api/mod_app_state_gpu.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,8 @@ impl AppState {
}
})
.collect();
let tokenizer = BPETokenizer::new(vocab, vec![], "<unk>")?;
let unk = crate::tokenizer::vocabulary_unk_token(&vocab); // #3609: never a literal
let tokenizer = BPETokenizer::new(vocab, vec![], unk)?;

let (audit_logger, audit_sink) = create_audit_state();
Ok(Self {
Expand Down Expand Up @@ -85,7 +86,8 @@ impl AppState {
cached_model: crate::gguf::OwnedQuantizedModelCachedSync,
vocab: Vec<String>,
) -> Result<Self, RealizarError> {
let tokenizer = BPETokenizer::new(vocab, vec![], "<unk>")?;
let unk = crate::tokenizer::vocabulary_unk_token(&vocab); // #3609: never a literal
let tokenizer = BPETokenizer::new(vocab, vec![], unk)?;

let (audit_logger, audit_sink) = create_audit_state();
Ok(Self {
Expand Down Expand Up @@ -140,7 +142,8 @@ impl AppState {
quantized_model: crate::gguf::OwnedQuantizedModel,
vocab: Vec<String>,
) -> Result<Self, RealizarError> {
let tokenizer = BPETokenizer::new(vocab, vec![], "<unk>")?;
let unk = crate::tokenizer::vocabulary_unk_token(&vocab); // #3609: never a literal
let tokenizer = BPETokenizer::new(vocab, vec![], unk)?;

// PMAT-181: Cache architecture for chat template auto-detection.
// Qwen3 models get Qwen3NoThinkTemplate (disables thinking mode).
Expand Down Expand Up @@ -208,7 +211,8 @@ impl AppState {
cuda_model: crate::gguf::OwnedQuantizedModelCuda,
vocab: Vec<String>,
) -> Result<Self, RealizarError> {
let tokenizer = BPETokenizer::new(vocab, vec![], "<unk>")?;
let unk = crate::tokenizer::vocabulary_unk_token(&vocab); // #3609: never a literal
let tokenizer = BPETokenizer::new(vocab, vec![], unk)?;
// PMAT-073: Cache architecture at construction to avoid RwLock in hot path.
// model_architecture() was blocking HTTP handlers for ~2s due to read lock
// contention with the batch scheduler's write lock.
Expand Down Expand Up @@ -265,7 +269,8 @@ impl AppState {
vocab: Vec<String>,
merges: Vec<(String, String)>,
) -> Result<Self, RealizarError> {
let tokenizer = BPETokenizer::with_merges(vocab, merges, "<unk>")?;
let unk = crate::tokenizer::vocabulary_unk_token(&vocab); // #3609: never a literal
let tokenizer = BPETokenizer::with_merges(vocab, merges, unk)?;
let arch = Some(cuda_model.model().config.architecture.clone());
let eos = cuda_model.model().config.eos_token_id;

Expand Down Expand Up @@ -328,7 +333,8 @@ impl AppState {
transformer: crate::apr_transformer::AprTransformer,
vocab: Vec<String>,
) -> Result<Self, RealizarError> {
let tokenizer = BPETokenizer::new(vocab, vec![], "<unk>")?;
let unk = crate::tokenizer::vocabulary_unk_token(&vocab); // #3609: never a literal
let tokenizer = BPETokenizer::new(vocab, vec![], unk)?;

let (audit_logger, audit_sink) = create_audit_state();
Ok(Self {
Expand Down Expand Up @@ -555,7 +561,8 @@ impl AppState {
vocab: Vec<String>,
eos_id: Option<u32>,
) -> Result<Self, RealizarError> {
let tokenizer = BPETokenizer::new(vocab, vec![], "<unk>")?;
let unk = crate::tokenizer::vocabulary_unk_token(&vocab); // #3609: never a literal
let tokenizer = BPETokenizer::new(vocab, vec![], unk)?;
let (audit_logger, audit_sink) = create_audit_state();
Ok(Self {
model: None,
Expand Down Expand Up @@ -601,7 +608,8 @@ impl AppState {
model: crate::safetensors_cuda::SafeTensorsCudaModel,
vocab: Vec<String>,
) -> Result<Self, RealizarError> {
let tokenizer = BPETokenizer::new(vocab, vec![], "<unk>")?;
let unk = crate::tokenizer::vocabulary_unk_token(&vocab); // #3609: never a literal
let tokenizer = BPETokenizer::new(vocab, vec![], unk)?;
let metrics = Arc::new(MetricsCollector::new());
let (audit_logger, audit_sink) = create_audit_state();

Expand Down
15 changes: 10 additions & 5 deletions crates/aprender-serve/src/api/mod_app_state_new.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -227,8 +227,9 @@ impl AppState {
}
})
.collect();
let unk = crate::tokenizer::vocabulary_unk_token(&vocab); // #3609: never a literal
let tokenizer =
BPETokenizer::new(vocab, vec![], "<unk>").expect("Failed to create tokenizer");
BPETokenizer::new(vocab, vec![], unk).expect("Failed to create tokenizer");

let (audit_logger, audit_sink) = create_audit_state();
Self {
Expand Down Expand Up @@ -298,7 +299,8 @@ impl AppState {
}
})
.collect();
let tokenizer = BPETokenizer::new(vocab, vec![], "<unk>")?;
let unk = crate::tokenizer::vocabulary_unk_token(&vocab); // #3609: never a literal
let tokenizer = BPETokenizer::new(vocab, vec![], unk)?;

// Create demo APR model (real inference, not mock)
// Simple model: sum of inputs with bias
Expand Down Expand Up @@ -424,7 +426,8 @@ impl AppState {
}
})
.collect();
let tokenizer = BPETokenizer::new(vocab, vec![], "<unk>")?;
let unk = crate::tokenizer::vocabulary_unk_token(&vocab); // #3609: never a literal
let tokenizer = BPETokenizer::new(vocab, vec![], unk)?;

let (audit_logger, audit_sink) = create_audit_state();
Ok(Self {
Expand Down Expand Up @@ -480,7 +483,8 @@ impl AppState {
gpu_model: crate::gpu::GpuModel,
vocab: Vec<String>,
) -> Result<Self, RealizarError> {
let tokenizer = BPETokenizer::new(vocab, vec![], "<unk>")?;
let unk = crate::tokenizer::vocabulary_unk_token(&vocab); // #3609: never a literal
let tokenizer = BPETokenizer::new(vocab, vec![], unk)?;

let (audit_logger, audit_sink) = create_audit_state();
Ok(Self {
Expand Down Expand Up @@ -545,7 +549,8 @@ impl AppState {
}
})
.collect();
let tokenizer = BPETokenizer::new(vocab, vec![], "<unk>")?;
let unk = crate::tokenizer::vocabulary_unk_token(&vocab); // #3609: never a literal
let tokenizer = BPETokenizer::new(vocab, vec![], unk)?;

let (audit_logger, audit_sink) = create_audit_state();
Ok(Self {
Expand Down
Loading
Loading