From 75b6057224b7b8feaadb7cdffe8a62aa0071a64c Mon Sep 17 00:00:00 2001 From: Dominic Rubas <1042243+rubas@users.noreply.github.com> Date: Fri, 16 Oct 2026 16:00:58 +0200 Subject: [PATCH 1/3] fix: detect the language on the offset window without VAD whisper.cpp v1.9.4 auto-detects the language at offset 0 before it reads offset_ms, so transcribe/3 with :offset_ms and no VAD decoded the window in the language of the first 30 s of the buffer. Cut the window from the samples in Rust on both paths, stop passing offset_ms/duration_ms to whisper.cpp, and shift segment and word times back by the window start. --- native/whisper_cpp_native/src/transcribe.rs | 119 +++++++++++--------- native/whisper_cpp_native/src/vad.rs | 28 ----- 2 files changed, 67 insertions(+), 80 deletions(-) diff --git a/native/whisper_cpp_native/src/transcribe.rs b/native/whisper_cpp_native/src/transcribe.rs index 5d208a5..17dc452 100644 --- a/native/whisper_cpp_native/src/transcribe.rs +++ b/native/whisper_cpp_native/src/transcribe.rs @@ -150,17 +150,6 @@ fn build_params<'a>(req: &'a TranscribeRequest, language: Option<&'a str>) -> Fu if let Some(c) = req.n_max_text_ctx { params.set_n_max_text_ctx(u32_to_i32(c)); } - // With VAD active the window is cut from the original buffer before - // detection (see `transcribe_one`); whisper must then see the whole - // filtered buffer, not a window of it. - if req.vad_model_path.is_none() { - if let Some(o) = req.offset_ms { - params.set_offset_ms(u32_to_i32(o)); - } - if let Some(d) = req.duration_ms { - params.set_duration_ms(u32_to_i32(d)); - } - } if let Some(t) = req.temperature { params.set_temperature(t); } @@ -197,6 +186,20 @@ fn build_params<'a>(req: &'a TranscribeRequest, language: Option<&'a str>) -> Fu params } +/// Select the sample range that `:offset_ms`/`:duration_ms` describe on +/// the buffer, clamped to its length. +fn window(offset_ms: Option, duration_ms: Option, n_samples: usize) -> (usize, usize) { + const SAMPLES_PER_MS: usize = 16; + let start = (offset_ms.unwrap_or(0) as usize * SAMPLES_PER_MS).min(n_samples); + let end = match duration_ms { + Some(d) => start + .saturating_add(d as usize * SAMPLES_PER_MS) + .min(n_samples), + None => n_samples, + }; + (start, end) +} + /// Wire optional cooperative-cancellation and progress callbacks onto /// the `FullParams`. Both hooks are no-ops when the caller omits them. /// @@ -249,7 +252,7 @@ fn install_callbacks( /// model do not serialise. pub(crate) fn transcribe_one( model: &crate::WhisperResource, - samples: Vec, + mut samples: Vec, request: &TranscribeRequest, abort_flag: Option>, progress_pid: Option, @@ -271,40 +274,34 @@ pub(crate) fn transcribe_one( return Err(invalid_request("initial_prompt must not contain NUL bytes")); } - // VAD runs before any whisper state exists: silence-only audio - // returns the documented empty transcription without paying for an - // encoder pass. `:offset_ms`/`:duration_ms` are applied here, on - // the original timeline - whisper.cpp would otherwise window the - // silence-stripped buffer instead of the audio the caller meant. - let vad_outcome = match request.vad_model_path { - Some(ref path) => { - let (win_start, win_end) = - vad::window(request.offset_ms, request.duration_ms, samples.len()); - Some(vad::filter_speech( - path, - request, - &samples[win_start..win_end], - win_start, - )?) - } - None => None, - }; - #[allow(clippy::cast_precision_loss)] let duration_s = samples.len() as f32 / 16_000.0_f32; - // With VAD, whisper only sees the filtered copy: free the original - // buffer now instead of holding both through the whole inference. - let (inference_samples, vad_ranges) = match vad_outcome { - Some(VadOutcome::NoSpeech) => { + // `:offset_ms`/`:duration_ms` cut the window here, on the original + // timeline. whisper.cpp's own offset windows the decode but not the + // language detection, which always reads the start of the buffer, and + // with VAD it would window the silence-stripped buffer. + let (win_start, win_end) = window(request.offset_ms, request.duration_ms, samples.len()); + + // VAD runs before any whisper state exists: silence-only audio + // returns the documented empty transcription without paying for an + // encoder pass. Whisper only sees the filtered copy, so the original + // buffer is freed instead of held through the whole inference. + let (inference_samples, vad_ranges) = if let Some(ref path) = request.vad_model_path { + match vad::filter_speech(path, request, &samples[win_start..win_end], win_start)? { // No detection ran; report the resolved request or none. - return Ok(TranscriptionResult::empty(language, duration_s)); - } - Some(VadOutcome::Speech { filtered, ranges }) => { - drop(samples); - (filtered, Some(ranges)) + VadOutcome::NoSpeech => { + return Ok(TranscriptionResult::empty(language, duration_s)); + } + VadOutcome::Speech { filtered, ranges } => { + drop(samples); + (filtered, Some(ranges)) + } } - None => (samples, None), + } else { + samples.truncate(win_end); + samples.drain(..win_start); + (samples, None) }; // whisper.cpp polls the abort callback only inside `full()`; honour @@ -320,8 +317,10 @@ pub(crate) fn transcribe_one( // whisper.cpp detects the language before it checks the length. Up to // 40 samples the mel spectrogram has no frame (`n_len_org = 1 + // (n - 200) / 160` is 0), so detection and `full()` fail. With a pinned - // language the same audio decodes to nothing; return that result. - if language.is_none() && inference_samples.len() <= 40 { + // language the same audio decodes to nothing; return that result. An + // empty window never reaches whisper: the mel reflect pad reads out of + // bounds on it. + if inference_samples.is_empty() || (language.is_none() && inference_samples.len() <= 40) { return Ok(TranscriptionResult::empty(language, duration_s)); } @@ -369,15 +368,21 @@ pub(crate) fn transcribe_one( segments.push(extract_segment(&seg, words, token_eot)?); } - if let Some(ref ranges) = vad_ranges { - for seg in &mut segments { - seg.start = vad::remap_seconds(ranges, seg.start); - seg.end = vad::remap_seconds(ranges, seg.end); - if let Some(ref mut words) = seg.words { - for word in words { - word.start = vad::remap_seconds(ranges, word.start); - word.end = vad::remap_seconds(ranges, word.end); - } + // Report times on the original timeline. The VAD ranges carry the + // window origin; without VAD whisper saw the window from its start. + #[allow(clippy::cast_precision_loss)] + let win_start_s = win_start as f32 / 16_000.0_f32; + let to_original = |t: f32| match vad_ranges { + Some(ref ranges) => vad::remap_seconds(ranges, t), + None => t + win_start_s, + }; + for seg in &mut segments { + seg.start = to_original(seg.start); + seg.end = to_original(seg.end); + if let Some(ref mut words) = seg.words { + for word in words { + word.start = to_original(word.start); + word.end = to_original(word.end); } } } @@ -586,6 +591,16 @@ mod tests { use super::*; use crate::errors::kind_of; + #[test] + fn window_selects_offset_and_duration_in_samples() { + const N: usize = 16_000 * 10; + assert_eq!(window(None, None, N), (0, N)); + assert_eq!(window(Some(2_000), None, N), (32_000, N)); + assert_eq!(window(Some(2_000), Some(4_000), N), (32_000, 96_000)); + // clamped to the buffer + assert_eq!(window(Some(20_000), Some(1_000), N), (N, N)); + } + #[test] fn resolve_language_returns_the_iso_code_for_codes_and_full_names() { for (requested, multilingual, code) in [ diff --git a/native/whisper_cpp_native/src/vad.rs b/native/whisper_cpp_native/src/vad.rs index c738903..6af6336 100644 --- a/native/whisper_cpp_native/src/vad.rs +++ b/native/whisper_cpp_native/src/vad.rs @@ -41,25 +41,6 @@ pub(crate) enum VadOutcome { }, } -/// Select the sample window that `:offset_ms`/`:duration_ms` describe -/// on the original buffer. whisper.cpp cannot apply them itself when -/// VAD is active - it would window the silence-stripped timeline. -pub(crate) fn window( - offset_ms: Option, - duration_ms: Option, - n_samples: usize, -) -> (usize, usize) { - const SAMPLES_PER_MS: usize = SAMPLE_RATE / 1000; - let start = (offset_ms.unwrap_or(0) as usize * SAMPLES_PER_MS).min(n_samples); - let end = match duration_ms { - Some(d) => start - .saturating_add(d as usize * SAMPLES_PER_MS) - .min(n_samples), - None => n_samples, - }; - (start, end) -} - /// Run silero VAD over `samples` (a window starting at /// `origin_offset` samples into the original buffer) and stitch the /// speech segments into a filtered buffer. Ranges carry original-buffer @@ -246,15 +227,6 @@ mod tests { // 16k samples = 1 s. Spans in centiseconds. const N: usize = 16_000 * 10; - #[test] - fn window_selects_offset_and_duration_in_original_samples() { - assert_eq!(window(None, None, N), (0, N)); - assert_eq!(window(Some(2_000), None, N), (32_000, N)); - assert_eq!(window(Some(2_000), Some(4_000), N), (32_000, 96_000)); - // clamped to the buffer - assert_eq!(window(Some(20_000), Some(1_000), N), (N, N)); - } - #[test] fn build_ranges_bakes_in_the_window_origin() { let ranges = build_ranges(&[(100.0, 200.0)], N, 32_000); From 19ea631ed0c32fd9d536feb7caea18331e6a7784 Mon Sep 17 00:00:00 2001 From: Dominic Rubas <1042243+rubas@users.noreply.github.com> Date: Fri, 16 Oct 2026 16:00:58 +0200 Subject: [PATCH 2/3] test: cover offset and duration windows without VAD --- test/integration_test.exs | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/test/integration_test.exs b/test/integration_test.exs index da708ad..30e177f 100644 --- a/test/integration_test.exs +++ b/test/integration_test.exs @@ -53,6 +53,41 @@ defmodule WhisperCpp.IntegrationTest do assert first.start >= 2.0 end + test "offset and duration window the timeline without VAD", %{model_path: model, pcm: pcm} do + {:ok, model_ref} = WhisperCpp.load_model(model) + + assert {:ok, %WhisperCpp.Transcription{segments: [first | _] = segs}} = + WhisperCpp.transcribe(model_ref, {:pcm_f32, pcm}, + language: "en", + n_threads: 4, + offset_ms: 2_000, + duration_ms: 6_000 + ) + + assert first.start >= 2.0 + assert List.last(segs).end <= 8.5 + end + + test "offset_ms detects the language on the window, not on the start of the buffer", %{pcm: pcm} do + {:ok, model_ref} = WhisperCpp.load_model(Fixtures.ensure_multilingual_model!()) + + # 30 s of white noise before the English speech. tiny detects "nn" on + # this noise, and "en" on silence, so silence would not show a + # detection on the start of the buffer. + :rand.seed(:exsss, {1, 2, 3}) + noise = for _ <- 1..(16_000 * 30), into: <<>>, do: <<(:rand.uniform() - 0.5) * 0.2::little-float-32>> + buf = noise <> pcm + buf_s = byte_size(buf) / 4 / 16_000 + + assert {:ok, %WhisperCpp.Transcription{language: "en", text: text} = windowed} = + WhisperCpp.transcribe(model_ref, {:pcm_f32, buf}, offset_ms: 30_000, n_threads: 4) + + assert {:ok, %WhisperCpp.Transcription{language: "en", text: ^text}} = + WhisperCpp.transcribe_slice(model_ref, buf, {30.0, buf_s}, n_threads: 4) + + assert hd(windowed.segments).start >= 30.0 + end + test "word_timestamps attaches per-word timings", %{model_path: model, pcm: pcm} do {:ok, model_ref} = WhisperCpp.load_model(model) From 0475fd1d4558a3467987350e92c488d83afbe9b9 Mon Sep 17 00:00:00 2001 From: Dominic Rubas <1042243+rubas@users.noreply.github.com> Date: Fri, 16 Oct 2026 16:00:58 +0200 Subject: [PATCH 3/3] docs: document that the offset window drives language detection --- CHANGELOG.md | 9 +++++++++ lib/whisper_cpp.ex | 4 +++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 196d039..2449e4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,15 @@ All notable changes to `whisper_cpp` will be documented in this file. The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Fixed +- `transcribe/3` with `:offset_ms` or `:duration_ms` and no + `:vad_model_path` detects the language on the window. Before, whisper.cpp + detected it on the first 30 s of the whole buffer, so the window could + decode in the wrong language. Segments also end inside the window now: + before, a `:duration_ms` window could return text from after its end. + ## [0.5.0] - 2026-09-23 Several changes are caller-visible. Items that need an action from the diff --git a/lib/whisper_cpp.ex b/lib/whisper_cpp.ex index bff0f4e..0e8c4df 100644 --- a/lib/whisper_cpp.ex +++ b/lib/whisper_cpp.ex @@ -198,7 +198,9 @@ defmodule WhisperCpp do picks `min(4, cores)`. - `:n_max_text_ctx` - cap decoder context tokens. - `:offset_ms`, `:duration_ms` - clip the audio window; `:duration_ms` - must be at least 1. + must be at least 1. Language detection and decoding see only the + window, as in `transcribe_slice/4`. Timestamps stay absolute on the + whole buffer. - `:no_speech_thold` - silence detection threshold. Default `0.6`. - `:logprob_thold` - reject segments with `avg_logprob` below this. - `:suppress_blank`, `:suppress_non_speech_tokens` - decoder suppressions.