fix: detect the language on the offset window without VAD

CHANGELOG.md

@@ -4,6 +4,15 @@ All notable changes to `whisper_cpp` will be documented in this file. The
 format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/);
 this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [Unreleased]
+
+### Fixed
+- `transcribe/3` with `:offset_ms` or `:duration_ms` and no
+  `:vad_model_path` detects the language on the window. Before, whisper.cpp
+  detected it on the first 30 s of the whole buffer, so the window could
+  decode in the wrong language. Segments also end inside the window now:
+  before, a `:duration_ms` window could return text from after its end.
+
 ## [0.5.0] - 2026-09-23
 
 Several changes are caller-visible. Items that need an action from the

lib/whisper_cpp.ex

@@ -198,7 +198,9 @@ defmodule WhisperCpp do
     picks `min(4, cores)`.
   - `:n_max_text_ctx` - cap decoder context tokens.
   - `:offset_ms`, `:duration_ms` - clip the audio window; `:duration_ms`
-    must be at least 1.
+    must be at least 1. Language detection and decoding see only the
+    window, as in `transcribe_slice/4`. Timestamps stay absolute on the
+    whole buffer.
   - `:no_speech_thold` - silence detection threshold. Default `0.6`.
   - `:logprob_thold` - reject segments with `avg_logprob` below this.
   - `:suppress_blank`, `:suppress_non_speech_tokens` - decoder suppressions.

native/whisper_cpp_native/src/transcribe.rs

@@ -150,17 +150,6 @@ fn build_params<'a>(req: &'a TranscribeRequest, language: Option<&'a str>) -> Fu
     if let Some(c) = req.n_max_text_ctx {
         params.set_n_max_text_ctx(u32_to_i32(c));
     }
-    // With VAD active the window is cut from the original buffer before
-    // detection (see `transcribe_one`); whisper must then see the whole
-    // filtered buffer, not a window of it.
-    if req.vad_model_path.is_none() {
-        if let Some(o) = req.offset_ms {
-            params.set_offset_ms(u32_to_i32(o));
-        }
-        if let Some(d) = req.duration_ms {
-            params.set_duration_ms(u32_to_i32(d));
-        }
-    }
     if let Some(t) = req.temperature {
         params.set_temperature(t);
     }
@@ -197,6 +186,20 @@ fn build_params<'a>(req: &'a TranscribeRequest, language: Option<&'a str>) -> Fu
     params
 }
 
+/// Select the sample range that `:offset_ms`/`:duration_ms` describe on
+/// the buffer, clamped to its length.
+fn window(offset_ms: Option<u32>, duration_ms: Option<u32>, n_samples: usize) -> (usize, usize) {
+    const SAMPLES_PER_MS: usize = 16;
+    let start = (offset_ms.unwrap_or(0) as usize * SAMPLES_PER_MS).min(n_samples);
+    let end = match duration_ms {
+        Some(d) => start
+            .saturating_add(d as usize * SAMPLES_PER_MS)
+            .min(n_samples),
+        None => n_samples,
+    };
+    (start, end)
+}
+
 /// Wire optional cooperative-cancellation and progress callbacks onto
 /// the `FullParams`. Both hooks are no-ops when the caller omits them.
 ///
@@ -249,7 +252,7 @@ fn install_callbacks(
 /// model do not serialise.
 pub(crate) fn transcribe_one(
     model: &crate::WhisperResource,
-    samples: Vec<f32>,
+    mut samples: Vec<f32>,
     request: &TranscribeRequest,
     abort_flag: Option<Arc<AtomicBool>>,
     progress_pid: Option<LocalPid>,
@@ -271,40 +274,34 @@ pub(crate) fn transcribe_one(
         return Err(invalid_request("initial_prompt must not contain NUL bytes"));
     }
 
-    // VAD runs before any whisper state exists: silence-only audio
-    // returns the documented empty transcription without paying for an
-    // encoder pass. `:offset_ms`/`:duration_ms` are applied here, on
-    // the original timeline - whisper.cpp would otherwise window the
-    // silence-stripped buffer instead of the audio the caller meant.
-    let vad_outcome = match request.vad_model_path {
-        Some(ref path) => {
-            let (win_start, win_end) =
-                vad::window(request.offset_ms, request.duration_ms, samples.len());
-            Some(vad::filter_speech(
-                path,
-                request,
-                &samples[win_start..win_end],
-                win_start,
-            )?)
-        }
-        None => None,
-    };
-
     #[allow(clippy::cast_precision_loss)]
     let duration_s = samples.len() as f32 / 16_000.0_f32;
 
-    // With VAD, whisper only sees the filtered copy: free the original
-    // buffer now instead of holding both through the whole inference.
-    let (inference_samples, vad_ranges) = match vad_outcome {
-        Some(VadOutcome::NoSpeech) => {
+    // `:offset_ms`/`:duration_ms` cut the window here, on the original
+    // timeline. whisper.cpp's own offset windows the decode but not the
+    // language detection, which always reads the start of the buffer, and
+    // with VAD it would window the silence-stripped buffer.
+    let (win_start, win_end) = window(request.offset_ms, request.duration_ms, samples.len());
+
+    // VAD runs before any whisper state exists: silence-only audio
+    // returns the documented empty transcription without paying for an
+    // encoder pass. Whisper only sees the filtered copy, so the original
+    // buffer is freed instead of held through the whole inference.
+    let (inference_samples, vad_ranges) = if let Some(ref path) = request.vad_model_path {
+        match vad::filter_speech(path, request, &samples[win_start..win_end], win_start)? {
             // No detection ran; report the resolved request or none.
-            return Ok(TranscriptionResult::empty(language, duration_s));
-        }
-        Some(VadOutcome::Speech { filtered, ranges }) => {
-            drop(samples);
-            (filtered, Some(ranges))
+            VadOutcome::NoSpeech => {
+                return Ok(TranscriptionResult::empty(language, duration_s));
+            }
+            VadOutcome::Speech { filtered, ranges } => {
+                drop(samples);
+                (filtered, Some(ranges))
+            }
         }
-        None => (samples, None),
+    } else {
+        samples.truncate(win_end);
+        samples.drain(..win_start);
+        (samples, None)
     };
 
     // whisper.cpp polls the abort callback only inside `full()`; honour
@@ -320,8 +317,10 @@ pub(crate) fn transcribe_one(
     // whisper.cpp detects the language before it checks the length. Up to
     // 40 samples the mel spectrogram has no frame (`n_len_org = 1 +
     // (n - 200) / 160` is 0), so detection and `full()` fail. With a pinned
-    // language the same audio decodes to nothing; return that result.
-    if language.is_none() && inference_samples.len() <= 40 {
+    // language the same audio decodes to nothing; return that result. An
+    // empty window never reaches whisper: the mel reflect pad reads out of
+    // bounds on it.
+    if inference_samples.is_empty() || (language.is_none() && inference_samples.len() <= 40) {
         return Ok(TranscriptionResult::empty(language, duration_s));
     }
 
@@ -369,15 +368,21 @@ pub(crate) fn transcribe_one(
         segments.push(extract_segment(&seg, words, token_eot)?);
     }
 
-    if let Some(ref ranges) = vad_ranges {
-        for seg in &mut segments {
-            seg.start = vad::remap_seconds(ranges, seg.start);
-            seg.end = vad::remap_seconds(ranges, seg.end);
-            if let Some(ref mut words) = seg.words {
-                for word in words {
-                    word.start = vad::remap_seconds(ranges, word.start);
-                    word.end = vad::remap_seconds(ranges, word.end);
-                }
+    // Report times on the original timeline. The VAD ranges carry the
+    // window origin; without VAD whisper saw the window from its start.
+    #[allow(clippy::cast_precision_loss)]
+    let win_start_s = win_start as f32 / 16_000.0_f32;
+    let to_original = |t: f32| match vad_ranges {
+        Some(ref ranges) => vad::remap_seconds(ranges, t),
+        None => t + win_start_s,
+    };
+    for seg in &mut segments {
+        seg.start = to_original(seg.start);
+        seg.end = to_original(seg.end);
+        if let Some(ref mut words) = seg.words {
+            for word in words {
+                word.start = to_original(word.start);
+                word.end = to_original(word.end);
             }
         }
     }
@@ -586,6 +591,16 @@ mod tests {
     use super::*;
     use crate::errors::kind_of;
 
+    #[test]
+    fn window_selects_offset_and_duration_in_samples() {
+        const N: usize = 16_000 * 10;
+        assert_eq!(window(None, None, N), (0, N));
+        assert_eq!(window(Some(2_000), None, N), (32_000, N));
+        assert_eq!(window(Some(2_000), Some(4_000), N), (32_000, 96_000));
+        // clamped to the buffer
+        assert_eq!(window(Some(20_000), Some(1_000), N), (N, N));
+    }
+
     #[test]
     fn resolve_language_returns_the_iso_code_for_codes_and_full_names() {
         for (requested, multilingual, code) in [

native/whisper_cpp_native/src/vad.rs

@@ -41,25 +41,6 @@ pub(crate) enum VadOutcome {
     },
 }
 
-/// Select the sample window that `:offset_ms`/`:duration_ms` describe
-/// on the original buffer. whisper.cpp cannot apply them itself when
-/// VAD is active - it would window the silence-stripped timeline.
-pub(crate) fn window(
-    offset_ms: Option<u32>,
-    duration_ms: Option<u32>,
-    n_samples: usize,
-) -> (usize, usize) {
-    const SAMPLES_PER_MS: usize = SAMPLE_RATE / 1000;
-    let start = (offset_ms.unwrap_or(0) as usize * SAMPLES_PER_MS).min(n_samples);
-    let end = match duration_ms {
-        Some(d) => start
-            .saturating_add(d as usize * SAMPLES_PER_MS)
-            .min(n_samples),
-        None => n_samples,
-    };
-    (start, end)
-}
-
 /// Run silero VAD over `samples` (a window starting at
 /// `origin_offset` samples into the original buffer) and stitch the
 /// speech segments into a filtered buffer. Ranges carry original-buffer
@@ -246,15 +227,6 @@ mod tests {
     // 16k samples = 1 s. Spans in centiseconds.
     const N: usize = 16_000 * 10;
 
-    #[test]
-    fn window_selects_offset_and_duration_in_original_samples() {
-        assert_eq!(window(None, None, N), (0, N));
-        assert_eq!(window(Some(2_000), None, N), (32_000, N));
-        assert_eq!(window(Some(2_000), Some(4_000), N), (32_000, 96_000));
-        // clamped to the buffer
-        assert_eq!(window(Some(20_000), Some(1_000), N), (N, N));
-    }
-
     #[test]
     fn build_ranges_bakes_in_the_window_origin() {
         let ranges = build_ranges(&[(100.0, 200.0)], N, 32_000);

test/integration_test.exs

@@ -53,6 +53,41 @@ defmodule WhisperCpp.IntegrationTest do
     assert first.start >= 2.0
   end
 
+  test "offset and duration window the timeline without VAD", %{model_path: model, pcm: pcm} do
+    {:ok, model_ref} = WhisperCpp.load_model(model)
+
+    assert {:ok, %WhisperCpp.Transcription{segments: [first | _] = segs}} =
+             WhisperCpp.transcribe(model_ref, {:pcm_f32, pcm},
+               language: "en",
+               n_threads: 4,
+               offset_ms: 2_000,
+               duration_ms: 6_000
+             )
+
+    assert first.start >= 2.0
+    assert List.last(segs).end <= 8.5
+  end
+
+  test "offset_ms detects the language on the window, not on the start of the buffer", %{pcm: pcm} do
+    {:ok, model_ref} = WhisperCpp.load_model(Fixtures.ensure_multilingual_model!())
+
+    # 30 s of white noise before the English speech. tiny detects "nn" on
+    # this noise, and "en" on silence, so silence would not show a
+    # detection on the start of the buffer.
+    :rand.seed(:exsss, {1, 2, 3})
+    noise = for _ <- 1..(16_000 * 30), into: <<>>, do: <<(:rand.uniform() - 0.5) * 0.2::little-float-32>>
+    buf = noise <> pcm
+    buf_s = byte_size(buf) / 4 / 16_000
+
+    assert {:ok, %WhisperCpp.Transcription{language: "en", text: text} = windowed} =
+             WhisperCpp.transcribe(model_ref, {:pcm_f32, buf}, offset_ms: 30_000, n_threads: 4)
+
+    assert {:ok, %WhisperCpp.Transcription{language: "en", text: ^text}} =
+             WhisperCpp.transcribe_slice(model_ref, buf, {30.0, buf_s}, n_threads: 4)
+
+    assert hd(windowed.segments).start >= 30.0
+  end
+
   test "word_timestamps attaches per-word timings", %{model_path: model, pcm: pcm} do
     {:ok, model_ref} = WhisperCpp.load_model(model)