fix: enforce whisper long-form invariants

CONFIGURATION.md

@@ -109,7 +109,7 @@ This turns off the normal segmentation rules as far as practical. The result is
 | `output_path` | Required | Replaced if it already exists |
 | `model_name` | Required | A Hugging Face model ID downloaded on first use, or a local directory; the web interface uses `openai/whisper-large-v3` |
 | `device` | `cpu` | Use `auto` for the best available device, `cuda` for NVIDIA, or `mps` for Apple Silicon |
-| `language` | `en` | Language code passed to Whisper |
+| `language` | `en` | Language code passed to Whisper; use `null` for automatic detection with multilingual models |
 | `embedding_model` | `all-MiniLM-L6-v2` | Model used to compare segment meaning |
 | `log_level` | `INFO` | Also accepts `DEBUG`, `WARNING`, and `ERROR` |
 
@@ -127,14 +127,14 @@ Most users can leave these alone.
 
 ```yaml
 performance:
-  whisper_batch_size: 4
+  whisper_batch_size: 1
   similarity_batch_size: 32
   chunk_size: 1000
 ```
 
-`whisper_batch_size` controls how many VAD speech chunks Whisper transcribes together. The default is `4` on CUDA and
-`1` on CPU or MPS. Lower it if Whisper runs out of accelerator memory; a failed batch automatically retries one chunk
-at a time.
+`whisper_batch_size` controls how many VAD speech chunks Whisper transcribes together. The default is `1` on every
+device because output parity for larger real-model batches is not yet established. An explicitly configured batch that
+exhausts accelerator memory automatically retries one chunk at a time.
 
 `similarity_batch_size` controls how many text embeddings are created at once. Lower it if the embedding step runs out of memory.
 

README.md

@@ -61,7 +61,7 @@ TextPlease writes tab-separated columns. The example config uses a `.csv` name,
 
 ## Quality status
 
-The English tests cover silence, rain, music, speech over music, short speech, and clean read speech at three speeds. They also cover continuous 30 and 60 minute meetings. The active non-speech and acceptance short-text gates pass. The timestamp gate still fails and is a known issue.
+The English tests cover silence, rain, music, speech over music, short speech, and clean read speech at three speeds. They also cover continuous 30 and 60 minute meetings. The enabled non-speech, acceptance short-text, and timestamp gates pass. WER, boundary, resource, and broader quality thresholds remain disabled while evidence grows.
 
 The [baseline](evaluation/BASELINE.md) has all results and audio credits. The [protocol](evaluation/protocol.json) defines the gates. The [manifest](evaluation/manifest.jsonl) records the source, license, credit, changes, and hash for each file. Use `uv run --locked python scripts/evaluate_audio_quality.py --help` to find the inference and scoring commands.
 

evaluation/manifest.jsonl

@@ -1,5 +1,5 @@
 {"id":"silence-5s","audio":"fixtures/silence-5s.wav","sha256":"6611051942c29e6d244d71236bb78e90c38e30952c3656f9bb435e3c69f96628","duration_ms":5000,"language":"en","split":"acceptance","strata":["non_speech","silence","pcm_wav"],"reference":{"text":"","speech_intervals_ms":[]},"source":{"url":"https://ffmpeg.org/ffmpeg-filters.html#anullsrc","item":"FFmpeg anullsrc, mono 16 kHz PCM16, 5 seconds","revision":"ffmpeg-9.0"},"license":{"id":"CC0-1.0","url":"https://creativecommons.org/publicdomain/zero/1.0/","attribution":"Generated for TextPlease with FFmpeg anullsrc."}}
-{"id":"rain-10s","audio":"fixtures/Rain.ogg","sha256":"c34d12243125c28f09426c076019f8321b602347a8564c38be771cb997c1a152","duration_ms":10331,"language":"en","split":"tuning","strata":["non_speech","background_noise","stereo_44khz_ogg"],"reference":{"text":"","speech_intervals_ms":[]},"source":{"url":"https://commons.wikimedia.org/w/index.php?title=File:Rain.ogg&oldid=597184901","item":"Rain.ogg","revision":"597184901"},"license":{"id":"Public-Domain","url":"https://commons.wikimedia.org/w/index.php?title=File:Rain.ogg&oldid=597184901#Licensing","attribution":"Recorded by Wikimedia Commons user ジダネ."}}
+{"id":"rain-10s","audio":"fixtures/Rain.ogg","sha256":"c34d12243125c28f09426c076019f8321b602347a8564c38be771cb997c1a152","duration_ms":10331,"language":"en","split":"tuning","strata":["non_speech","background_noise","mono_44khz_ogg"],"reference":{"text":"","speech_intervals_ms":[]},"source":{"url":"https://commons.wikimedia.org/w/index.php?title=File:Rain.ogg&oldid=597184901","item":"Rain.ogg","revision":"597184901"},"license":{"id":"Public-Domain","url":"https://commons.wikimedia.org/w/index.php?title=File:Rain.ogg&oldid=597184901#Licensing","attribution":"Recorded by Wikimedia Commons user ジダネ."}}
 {"id":"music-jazz-sax-24s","audio":"fixtures/Jazz-Sax.ogg","sha256":"312a88774e7dd9d18e20420c2a1f3721156709032d55ba1e8cb630f433255a5b","duration_ms":24000,"language":"en","split":"tuning","strata":["non_speech","music","saxophone","stereo_44khz_ogg"],"reference":{"text":"","speech_intervals_ms":[]},"source":{"url":"https://commons.wikimedia.org/w/index.php?title=File:Jazz-Sax.ogg&oldid=971661804","item":"Jazz Tenor Sax","revision":"971661804"},"license":{"id":"CC-BY-2.5","url":"https://creativecommons.org/licenses/by/2.5/","attribution":"Jazz Tenor Sax by Wikimedia Commons user Serolillo."}}
 {"id":"music-36s","audio":"fixtures/Greensleaves.ogg","sha256":"1c31668ade22bb83f067ce178a2282ebfdf320bbca9b86b4413fedba64025ec9","duration_ms":36409,"language":"en","split":"acceptance","strata":["non_speech","music","stereo_44khz_ogg"],"reference":{"text":"","speech_intervals_ms":[]},"source":{"url":"https://commons.wikimedia.org/w/index.php?title=File:Greensleaves.ogg&oldid=845754359","item":"Greensleaves.ogg","revision":"845754359"},"license":{"id":"Public-Domain","url":"https://commons.wikimedia.org/w/index.php?title=File:Greensleaves.ogg&oldid=845754359#Licensing","attribution":"Performed and recorded by Wikimedia Commons user Rv87."}}
 {"id":"speech-over-music-ear","audio":"fixtures/ear-over-jazz.flac","sha256":"021cfdc6737d763cafa4ca828f6c77da712fed661fe41a94fac3987e910510e8","duration_ms":24000,"language":"en","split":"tuning","strata":["speech","music","speech_over_music","short_utterance","mono_16khz_flac"],"reference":{"text":"ear","speech_intervals_ms":[[9782,10219]]},"source":{"url":"https://commons.wikimedia.org/w/index.php?title=File:Jazz-Sax.ogg&oldid=971661804","item":"TextPlease speech-over-music mix of Jazz Tenor Sax and En-uk-ear.ogg","revision":"components-sha256:312a88774e7dd9d18e20420c2a1f3721156709032d55ba1e8cb630f433255a5b+34d9e3db4cac7a9091362cbabd59983cf4cd6d5d463a19bf481254a32799b356","transformation":"FFmpeg 9.0: attenuate music to 0.5, delay speech by 9782 ms so the word crosses 10 seconds, mix without normalization, and encode mono 16 kHz FLAC"},"license":{"id":"CC-BY-2.5","url":"https://creativecommons.org/licenses/by/2.5/","attribution":"Jazz Tenor Sax by Wikimedia Commons user Serolillo, mixed with the public-domain En-uk-ear recording by Wikimedia Commons user Chris Melville."}}

examples/config_example.yaml

@@ -19,7 +19,7 @@ embedding_model: "all-MiniLM-L6-v2"
 
 # Performance
 performance:
-  whisper_batch_size: 1     # set to 4 on CUDA; lower it if accelerator memory is limited
+  whisper_batch_size: 1      # deterministic default, larger batches need model-specific parity checks
   similarity_batch_size: 32  # batch size for pre-encoding segment embeddings
   chunk_size: 1000           # process segments in chunks of this size (0 = no chunking)
 

tests/test_device_utils.py

@@ -61,7 +61,7 @@ def test_pipeline_uses_resolved_device(monkeypatch, tmp_path):
     assert transcribe_audio.call_args.args[0] == str(input_path)
     assert transcribe_audio.call_args.args[2] == "cuda"
     assert "temporary_directory" in transcribe_audio.call_args.kwargs
-    assert transcribe_audio.call_args.kwargs["batch_size"] == 4
+    assert transcribe_audio.call_args.kwargs["batch_size"] == 1
     assert sentence_transformer.call_args.kwargs["device"] == "cuda"
     assert segment_transcript.call_args.kwargs["preferred_device"] == "cuda"
     assert [entry[0] for entry in calls.mock_calls] == ["transcription", "embedding"]

tests/test_whisper_batching.py

@@ -14,7 +14,7 @@ class FakeTokenizer:
     def __init__(
         self,
         include_offsets: bool = True,
-        timestamp: tuple[float, float] = (0.0, 0.5),
+        timestamp: tuple[float | None, float | None] = (0.0, 0.5),
     ):
         self.include_offsets = include_offsets
         self.timestamp = timestamp
@@ -39,7 +39,7 @@ class FakeProcessor:
     def __init__(
         self,
         include_offsets: bool = True,
-        timestamp: tuple[float, float] = (0.0, 0.5),
+        timestamp: tuple[float | None, float | None] = (0.0, 0.5),
     ):
         self.tokenizer = FakeTokenizer(include_offsets, timestamp)
         self.audio_batches = []
@@ -60,12 +60,21 @@ def __call__(self, audio, **kwargs):
 class FakeModel:
     config = SimpleNamespace(max_source_positions=1500)
 
-    def __init__(self, fail_batched: bool = False, error: Exception | None = None):
+    def __init__(
+        self,
+        fail_batched: bool = False,
+        error: Exception | None = None,
+        *,
+        is_multilingual: bool = True,
+    ):
         self.batch_sizes = []
         self.fail_batched = fail_batched
         self.error = error
+        self.generation_config = SimpleNamespace(is_multilingual=is_multilingual)
+        self.generation_calls = []
 
     def generate(self, **kwargs):
+        self.generation_calls.append(kwargs)
         input_features = kwargs["input_features"]
         batch_size = len(input_features)
         self.batch_sizes.append(batch_size)
@@ -108,7 +117,14 @@ def audio_classifier(monkeypatch):
     )
 
 
-def _run_transcription(monkeypatch, batch_size, *, fail_batched=False):
+def _run_transcription(
+    monkeypatch,
+    batch_size,
+    *,
+    fail_batched=False,
+    is_multilingual=True,
+    language="en",
+):
     audio = np.concatenate(
         [np.full(TARGET_SAMPLE_RATE, segment_number, dtype=np.float32) for segment_number in (1, 2, 3)]
     )
@@ -117,7 +133,7 @@ def _run_transcription(monkeypatch, batch_size, *, fail_batched=False):
         {"start": TARGET_SAMPLE_RATE, "end": 2 * TARGET_SAMPLE_RATE},
         {"start": 2 * TARGET_SAMPLE_RATE, "end": 3 * TARGET_SAMPLE_RATE},
     ]
-    model = FakeModel(fail_batched=fail_batched)
+    model = FakeModel(fail_batched=fail_batched, is_multilingual=is_multilingual)
     processor = FakeProcessor()
 
     monkeypatch.setattr(transformers_pipeline, "_load_model_and_processor", lambda *args: (model, processor))
@@ -134,24 +150,87 @@ def _run_transcription(monkeypatch, batch_size, *, fail_batched=False):
         "test-model",
         "cpu",
         batch_size=batch_size,
+        language=language,
     )
-    return segments, model.batch_sizes
+    return segments, model
 
 
 def test_transcribe_batches_without_changing_segments(monkeypatch):
-    sequential, sequential_batch_sizes = _run_transcription(monkeypatch, 1)
-    batched, batched_batch_sizes = _run_transcription(monkeypatch, 2)
+    sequential, sequential_model = _run_transcription(monkeypatch, 1)
+    batched, batched_model = _run_transcription(monkeypatch, 2)
 
     assert batched == sequential
-    assert sequential_batch_sizes == [1, 1, 1]
-    assert batched_batch_sizes == [2, 1]
+    assert sequential_model.batch_sizes == [1, 1, 1]
+    assert batched_model.batch_sizes == [2, 1]
 
 
 def test_transcribe_retries_batch_after_out_of_memory(monkeypatch):
-    segments, batch_sizes = _run_transcription(monkeypatch, 2, fail_batched=True)
+    segments, model = _run_transcription(monkeypatch, 2, fail_batched=True)
 
     assert [segment["text"] for segment in segments] == ["Segment 1", "Segment 2", "Segment 3"]
-    assert batch_sizes == [2, 1, 1, 1]
+    assert model.batch_sizes == [2, 1, 1, 1]
+
+
+def test_transcribe_clamps_offsets_and_preserves_terminal_text(monkeypatch):
+    interval_start = TARGET_SAMPLE_RATE // 4
+    interval_end = 3 * TARGET_SAMPLE_RATE // 4
+    audio = np.ones(TARGET_SAMPLE_RATE, dtype=np.float32)
+    offsets = [
+        {"text": "Leading", "timestamp": (-1.0, 0.2)},
+        {"text": " overlap", "timestamp": (0.1, 0.3)},
+        {"text": " terminal", "timestamp": (0.4, None)},
+    ]
+    monkeypatch.setattr(transformers_pipeline, "load_pcm_wav", lambda path: audio)
+    monkeypatch.setattr(
+        transformers_pipeline,
+        "_get_speech_segments",
+        lambda *args: (
+            [{"start": interval_start, "end": interval_end}],
+            [(interval_start, interval_end)],
+        ),
+    )
+    monkeypatch.setattr(
+        transformers_pipeline,
+        "_load_model_and_processor",
+        lambda *args: (FakeModel(), FakeProcessor()),
+    )
+    monkeypatch.setattr(
+        transformers_pipeline,
+        "_transcribe_speech_segments",
+        lambda *args: [offsets],
+    )
+    monkeypatch.setattr(transformers_pipeline.torch.cuda, "is_available", lambda: False)
+
+    segments = transformers_pipeline.transcribe(
+        "input.wav",
+        "test-model",
+        "cpu",
+    )
+
+    assert segments == [
+        {"text": "Leading", "start_time": "00:00:00.250", "end_time": "00:00:00.450"},
+        {"text": "overlap", "start_time": "00:00:00.450", "end_time": "00:00:00.550"},
+        {"text": "terminal", "start_time": "00:00:00.650", "end_time": "00:00:00.750"},
+    ]
+
+
+def test_english_only_model_does_not_request_language_or_task(monkeypatch):
+    _, model = _run_transcription(monkeypatch, 1, is_multilingual=False)
+
+    assert model.generation_calls[0]["language"] is None
+    assert model.generation_calls[0]["task"] is None
+
+
+@pytest.mark.parametrize(
+    ("language", "expected_language"),
+    [("fr", "fr"), (None, None)],
+)
+def test_multilingual_model_supports_explicit_or_detected_language(monkeypatch, language, expected_language):
+    _, model = _run_transcription(monkeypatch, 1, language=language)
+
+    generation_call = model.generation_calls[0]
+    assert generation_call["task"] == "transcribe"
+    assert generation_call.get("language") == expected_language
 
 
 def test_no_speech_skips_whisper(monkeypatch):

textplease/backends/transformers_pipeline.py

@@ -3,7 +3,6 @@
 import gc
 import re
 import logging
-import warnings
 from typing import TypedDict
 from functools import lru_cache
 
@@ -23,9 +22,6 @@
 
 logger = logging.getLogger(__name__)
 
-warnings.filterwarnings("ignore", message=".*Whisper did not predict an ending timestamp.*")
-warnings.filterwarnings("ignore", message=".*attention mask is not set.*")
-
 
 class _WhisperOffset(TypedDict):
     text: str
@@ -142,7 +138,7 @@ def _transcribe_speech_segments(
     processor: WhisperProcessor,
     audio_chunks: list[np.ndarray],
     device: str,
-    language: str,
+    language: str | None,
 ) -> list[list[_WhisperOffset]]:
     """Transcribe speech chunks and return decoded offsets for each chunk."""
     torch_dtype = torch.float16 if device not in ("cpu", "mps") else torch.float32
@@ -162,12 +158,14 @@ def _transcribe_speech_segments(
     else:
         attention_mask = None
 
+    is_multilingual = getattr(model.generation_config, "is_multilingual", True)
+
     with torch.no_grad():
         generated_ids = model.generate(
             input_features=input_features,
             attention_mask=attention_mask,
-            language=language,
-            task="transcribe",
+            language=language if is_multilingual else None,
+            task="transcribe" if is_multilingual else None,
             return_timestamps=True,
             temperature=(0.0, 0.2, 0.4, 0.6, 0.8, 1.0),
             compression_ratio_threshold=1.35,
@@ -217,9 +215,10 @@ def _transcribe_chunks(
     chunks: list[tuple[int, int, np.ndarray]],
     batch_size: int,
     device: str,
-    language: str,
+    language: str | None,
 ) -> list[_WhisperOffset]:
     all_offsets: list[_WhisperOffset] = []
+    previous_end_s = 0.0
     for batch_start in range(0, len(chunks), batch_size):
         batch = chunks[batch_start : batch_start + batch_size]
         first_start, _, _ = batch[0]
@@ -244,13 +243,28 @@ def _transcribe_chunks(
                 _transcribe_speech_segments(model, processor, [chunk], device, language)[0] for chunk in audio_chunks
             ]
 
-        for (start, _, _), offsets in zip(batch, batch_offsets, strict=True):
+        for (start, end, _), offsets in zip(batch, batch_offsets, strict=True):
             start_s = start / TARGET_SAMPLE_RATE
-            for offset in offsets:
-                ts = offset.get("timestamp", (0.0, 0.0))
-                if len(ts) == 2 and ts[0] is not None and ts[1] is not None:
-                    offset["timestamp"] = (ts[0] + start_s, ts[1] + start_s)
-            all_offsets.extend(offsets)
+            end_s = end / TARGET_SAMPLE_RATE
+            for offset_index, offset in enumerate(offsets):
+                local_start, local_end = offset["timestamp"]
+                if local_start is None:
+                    continue
+                if local_end is None and offset_index != len(offsets) - 1:
+                    continue
+
+                bounded_start = max(start_s, local_start + start_s, previous_end_s)
+                bounded_end = end_s if local_end is None else min(end_s, local_end + start_s)
+                if bounded_end <= bounded_start:
+                    continue
+
+                all_offsets.append(
+                    {
+                        "text": offset["text"],
+                        "timestamp": (bounded_start, bounded_end),
+                    }
+                )
+                previous_end_s = bounded_end
 
     return all_offsets
 
@@ -272,7 +286,7 @@ def transcribe(
     model_name: str,
     device: str,
     *,
-    language: str = "en",
+    language: str | None = "en",
     batch_size: int = 1,
 ) -> list[dict[str, str]]:
     """Transcribe a normalized mono 16 kHz PCM16 WAV."""

textplease/pipeline.py

@@ -121,7 +121,7 @@ def _extract_config_params(config: dict) -> dict:
         "min_segment_chars": config.get("min_segment_chars", 15),
         "max_segment_words": config.get("max_segment_words", 100),
         "language": config.get("language", "en"),
-        "whisper_batch_size": config.get("performance", {}).get("whisper_batch_size"),
+        "whisper_batch_size": config.get("performance", {}).get("whisper_batch_size", 1),
         "similarity_batch_size": config.get("performance", {}).get("similarity_batch_size", 32),
         "chunk_size": config.get("performance", {}).get("chunk_size", 1000),
     }
@@ -220,8 +220,6 @@ def run_transcription_pipeline(config: dict) -> None:
     _validate_pipeline_config(config)
     params = _extract_config_params(config)
     params["device"] = detect_device(params["device"])
-    if params["whisper_batch_size"] is None:
-        params["whisper_batch_size"] = 4 if params["device"] == "cuda" else 1
 
     logger.info(f"Input: {params['input_path']} → Output: {params['output_path']}")
     logger.info(f"ASR: {params['model_name']} | Device: {params['device']}")

textplease/transcriber.py

@@ -24,6 +24,6 @@ def transcribe_audio(
         audio_path=normalized_audio_path,
         model_name=model_name,
         device=device,
-        language=language or "en",
+        language=language,
         batch_size=batch_size,
     )