diff --git a/CONFIGURATION.md b/CONFIGURATION.md index 7215564..df3a7f6 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -109,7 +109,7 @@ This turns off the normal segmentation rules as far as practical. The result is | `output_path` | Required | Replaced if it already exists | | `model_name` | Required | A Hugging Face model ID downloaded on first use, or a local directory; the web interface uses `openai/whisper-large-v3` | | `device` | `cpu` | Use `auto` for the best available device, `cuda` for NVIDIA, or `mps` for Apple Silicon | -| `language` | `en` | Language code passed to Whisper | +| `language` | `en` | Language code passed to Whisper; use `null` for automatic detection with multilingual models | | `embedding_model` | `all-MiniLM-L6-v2` | Model used to compare segment meaning | | `log_level` | `INFO` | Also accepts `DEBUG`, `WARNING`, and `ERROR` | @@ -127,14 +127,14 @@ Most users can leave these alone. ```yaml performance: - whisper_batch_size: 4 + whisper_batch_size: 1 similarity_batch_size: 32 chunk_size: 1000 ``` -`whisper_batch_size` controls how many VAD speech chunks Whisper transcribes together. The default is `4` on CUDA and -`1` on CPU or MPS. Lower it if Whisper runs out of accelerator memory; a failed batch automatically retries one chunk -at a time. +`whisper_batch_size` controls how many VAD speech chunks Whisper transcribes together. The default is `1` on every +device because output parity for larger real-model batches is not yet established. An explicitly configured batch that +exhausts accelerator memory automatically retries one chunk at a time. `similarity_batch_size` controls how many text embeddings are created at once. Lower it if the embedding step runs out of memory. diff --git a/README.md b/README.md index d07ec7b..2120c4c 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ TextPlease writes tab-separated columns. The example config uses a `.csv` name, ## Quality status -The English tests cover silence, rain, music, speech over music, short speech, and clean read speech at three speeds. They also cover continuous 30 and 60 minute meetings. The active non-speech and acceptance short-text gates pass. The timestamp gate still fails and is a known issue. +The English tests cover silence, rain, music, speech over music, short speech, and clean read speech at three speeds. They also cover continuous 30 and 60 minute meetings. The enabled non-speech, acceptance short-text, and timestamp gates pass. WER, boundary, resource, and broader quality thresholds remain disabled while evidence grows. The [baseline](evaluation/BASELINE.md) has all results and audio credits. The [protocol](evaluation/protocol.json) defines the gates. The [manifest](evaluation/manifest.jsonl) records the source, license, credit, changes, and hash for each file. Use `uv run --locked python scripts/evaluate_audio_quality.py --help` to find the inference and scoring commands. diff --git a/evaluation/BASELINE.md b/evaluation/BASELINE.md index a96a6df..549d32d 100644 --- a/evaluation/BASELINE.md +++ b/evaluation/BASELINE.md @@ -6,14 +6,14 @@ This report scores the configured public `textplease` pipeline against the versi | Field | Value | |---|---| -| Manifest SHA-256 | `d172a05be7283bf43c496f9686262764ef23c449f4b6a22e01e3b4c8c0483d65` | +| Manifest SHA-256 | `58a4e462e4c2fcbbd61615d6506328ca524e43485615bc1d696283ef4704be9c` | | Protocol SHA-256 | `84a14380f14e18c5912d557018f6f942d940e0caa4182e86097481e2404e6048` | -| Inference evaluator SHA-256 | `4475d7dbc7340f795e872245b2539f29443b2df6e5609734d17bb7d0432f796e` | +| Inference evaluator SHA-256 | `7a508a172bfcf7f9a6f6d844a5d9286d16d2a43b2d9e816d7ba2481756f8be89` | | Scorer SHA-256 | `7a508a172bfcf7f9a6f6d844a5d9286d16d2a43b2d9e816d7ba2481756f8be89` | | Scorer JiWER | `4.0.0` | | Scorer RapidFuzz | `3.14.5` | | Random seed | `0` | -| Source revision | `a4bf9773a2ad618d82ebe2a87c5bf417e7cf2c34` | +| Source revision | `74a4054d227183954cd7eb6c295c37ad7efa6e69` | | Source dirty | `True` | | Device | `mps` | | Whisper batch size | `1` | @@ -69,59 +69,60 @@ This report scores the configured public `textplease` pipeline against the versi | Non-speech error cases | 0 | | Prediction error cases | 0 | | Reference speech (ms) | 5026202 | -| Missed speech (ms) | 468652 | -| Missed speech rate | 0.0932 | +| Missed speech (ms) | 471550 | +| Missed speech rate | 0.0938 | | Reference non-speech (ms) | 539150 | -| False alarm (ms) | 115947 | -| False-alarm rate | 0.2151 | -| Boundary precision | 0.0934 | -| Boundary recall | 0.5186 | -| Boundary median error (ms) | 98.0000 | +| False alarm (ms) | 102493 | +| False-alarm rate | 0.1901 | +| Boundary precision | 0.0951 | +| Boundary recall | 0.5279 | +| Boundary median error (ms) | 99.0000 | | Boundary p95 error (ms) | 234.0000 | | Onset median error (ms) | 58.0000 | | Onset p95 error (ms) | 230.0000 | | Offset median error (ms) | 142.0000 | | Offset p95 error (ms) | 234.0000 | -| Timestamp violation cases | 3 | -| Timestamp violations | 3 | +| Timestamp violation cases | 0 | +| Timestamp violations | 0 | | Parity mismatch cases | 0 | -| Median RTF | 0.1331 | -| p95 RTF | 4.6037 | -| Peak RSS (MiB) | 8059.9062 | +| Median RTF | 0.1416 | +| p95 RTF | 5.3647 | +| Peak RSS (MiB) | 9451.6250 | | Peak CUDA allocation (MiB) | — | ## Per stratum | Group | Cases | WER | CER | Short exact | Non-speech nonempty | Miss (ms) | Miss rate | False alarm (ms) | False-alarm rate | Boundary P | Boundary R | Timestamp violations | RTF | |---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| -| split=tuning | 9 | 0.2607 | 0.1942 | 0.0000 | 0 | 231408 | 0.1325 | 5730 | 0.0456 | 0.0381 | 0.4706 | 1 | 0.1313 | -| split=acceptance | 9 | 0.2222 | 0.1504 | 1.0000 | 0 | 237244 | 0.0723 | 110217 | 0.2666 | 0.1234 | 0.5276 | 2 | 0.1349 | -| language=en | 18 | 0.2359 | 0.1659 | 0.5000 | 0 | 468652 | 0.0932 | 115947 | 0.2151 | 0.0934 | 0.5186 | 3 | 0.1331 | -| stratum=30_minute | 1 | 0.2733 | 0.2055 | — | 0 | 231289 | 0.1335 | 5450 | 0.0805 | 0.0335 | 0.4375 | 0 | 0.1299 | -| stratum=60_minute | 1 | 0.2280 | 0.1555 | — | 0 | 237128 | 0.0726 | 92921 | 0.2777 | 0.1224 | 0.5279 | 1 | 0.1355 | -| stratum=background_noise | 1 | — | — | — | 0 | 0 | — | 0 | 0.0000 | — | — | 0 | 0.0102 | -| stratum=clean | 8 | 0.0358 | 0.0136 | — | 0 | 92 | 0.0034 | 732 | 0.6393 | 0.7500 | 0.7500 | 1 | 0.1495 | -| stratum=long_form | 2 | 0.2438 | 0.1729 | — | 0 | 468417 | 0.0937 | 98371 | 0.2445 | 0.0912 | 0.5142 | 1 | 0.1327 | -| stratum=meeting | 2 | 0.2438 | 0.1729 | — | 0 | 468417 | 0.0937 | 98371 | 0.2445 | 0.0912 | 0.5142 | 1 | 0.1327 | -| stratum=mono_16khz_flac | 12 | 0.2358 | 0.1659 | 0.5000 | 0 | 468536 | 0.0932 | 115892 | 0.2503 | 0.0924 | 0.5156 | 2 | 0.1352 | -| stratum=multi_speaker | 2 | 0.2438 | 0.1729 | — | 0 | 468417 | 0.0937 | 98371 | 0.2445 | 0.0912 | 0.5142 | 1 | 0.1327 | -| stratum=music | 4 | 0.5000 | 0.2857 | 0.5000 | 0 | 27 | 0.0315 | 16789 | 0.1400 | 0.5000 | 0.5000 | 0 | 0.0306 | -| stratum=name | 1 | 0.0000 | 0.0000 | 1.0000 | 0 | 116 | 0.2755 | 55 | 0.1250 | 1.0000 | 1.0000 | 0 | 1.3946 | -| stratum=non_speech | 4 | — | — | — | 0 | 0 | — | 0 | 0.0000 | — | — | 0 | 0.0155 | -| stratum=pace_fast | 2 | 0.0403 | 0.0094 | — | 0 | — | — | — | — | — | — | 0 | 0.1631 | -| stratum=pace_normal | 2 | 0.0232 | 0.0203 | — | 0 | — | — | — | — | — | — | 1 | 0.1362 | -| stratum=pace_slow | 2 | 0.0545 | 0.0139 | — | 0 | — | — | — | — | — | — | 0 | 0.1301 | -| stratum=pcm_wav | 1 | — | — | — | 0 | 0 | — | 0 | 0.0000 | — | — | 0 | 0.0193 | -| stratum=read_speech | 8 | 0.0358 | 0.0136 | — | 0 | 92 | 0.0034 | 732 | 0.6393 | 0.7500 | 0.7500 | 1 | 0.1495 | -| stratum=saxophone | 1 | — | — | — | 0 | 0 | — | 0 | 0.0000 | — | — | 0 | 0.0222 | -| stratum=short_utterance | 4 | 0.5000 | 0.2857 | 0.5000 | 0 | 143 | 0.0833 | 16844 | 0.2808 | 0.7500 | 0.7500 | 1 | 0.7624 | -| stratum=silence | 1 | — | — | — | 0 | 0 | — | 0 | 0.0000 | — | — | 0 | 0.0193 | -| stratum=single_speaker | 6 | 0.0372 | 0.0149 | — | 0 | — | — | — | — | — | — | 1 | 0.1362 | -| stratum=speech | 14 | 0.2359 | 0.1659 | 0.5000 | 0 | 468652 | 0.0932 | 115947 | 0.2502 | 0.0934 | 0.5186 | 3 | 0.1365 | -| stratum=speech_over_music | 2 | 0.5000 | 0.2857 | 0.5000 | 0 | 27 | 0.0315 | 16789 | 0.2819 | 0.5000 | 0.5000 | 0 | 0.0846 | -| stratum=spontaneous_speech | 2 | 0.2438 | 0.1729 | — | 0 | 468417 | 0.0937 | 98371 | 0.2445 | 0.0912 | 0.5142 | 1 | 0.1327 | -| stratum=stereo_44khz_ogg | 5 | 0.5000 | 0.2857 | 0.5000 | 0 | 116 | 0.1352 | 55 | 0.0008 | 1.0000 | 1.0000 | 1 | 0.0222 | -| stratum=word | 1 | 1.0000 | 0.6667 | 0.0000 | 0 | 0 | 0.0000 | 0 | — | 1.0000 | 1.0000 | 1 | 4.6037 | +| split=tuning | 9 | 0.2607 | 0.1942 | 0.0000 | 0 | 232022 | 0.1328 | 4696 | 0.0373 | 0.0389 | 0.4804 | 0 | 0.1431 | +| split=acceptance | 9 | 0.2222 | 0.1504 | 1.0000 | 0 | 239528 | 0.0730 | 97797 | 0.2366 | 0.1255 | 0.5368 | 0 | 0.1394 | +| language=en | 18 | 0.2359 | 0.1659 | 0.5000 | 0 | 471550 | 0.0938 | 102493 | 0.1901 | 0.0951 | 0.5279 | 0 | 0.1416 | +| stratum=30_minute | 1 | 0.2733 | 0.2055 | — | 0 | 231903 | 0.1339 | 4416 | 0.0652 | 0.0343 | 0.4479 | 0 | 0.1355 | +| stratum=60_minute | 1 | 0.2280 | 0.1555 | — | 0 | 239412 | 0.0733 | 80501 | 0.2406 | 0.1246 | 0.5372 | 0 | 0.1369 | +| stratum=background_noise | 1 | — | — | — | 0 | 0 | — | 0 | 0.0000 | — | — | 0 | 0.0104 | +| stratum=clean | 8 | 0.0358 | 0.0136 | — | 0 | 92 | 0.0034 | 732 | 0.6393 | 0.7500 | 0.7500 | 0 | 0.1615 | +| stratum=long_form | 2 | 0.2438 | 0.1729 | — | 0 | 471315 | 0.0943 | 84917 | 0.2111 | 0.0929 | 0.5237 | 0 | 0.1362 | +| stratum=meeting | 2 | 0.2438 | 0.1729 | — | 0 | 471315 | 0.0943 | 84917 | 0.2111 | 0.0929 | 0.5237 | 0 | 0.1362 | +| stratum=mono_16khz_flac | 12 | 0.2358 | 0.1659 | 0.5000 | 0 | 471434 | 0.0938 | 102438 | 0.2213 | 0.0941 | 0.5249 | 0 | 0.1444 | +| stratum=mono_44khz_ogg | 1 | — | — | — | 0 | 0 | — | 0 | 0.0000 | — | — | 0 | 0.0104 | +| stratum=multi_speaker | 2 | 0.2438 | 0.1729 | — | 0 | 471315 | 0.0943 | 84917 | 0.2111 | 0.0929 | 0.5237 | 0 | 0.1362 | +| stratum=music | 4 | 0.5000 | 0.2857 | 0.5000 | 0 | 27 | 0.0315 | 16789 | 0.1400 | 0.5000 | 0.5000 | 0 | 0.0434 | +| stratum=name | 1 | 0.0000 | 0.0000 | 1.0000 | 0 | 116 | 0.2755 | 55 | 0.1250 | 1.0000 | 1.0000 | 0 | 1.6527 | +| stratum=non_speech | 4 | — | — | — | 0 | 0 | — | 0 | 0.0000 | — | — | 0 | 0.0187 | +| stratum=pace_fast | 2 | 0.0403 | 0.0094 | — | 0 | — | — | — | — | — | — | 0 | 0.1856 | +| stratum=pace_normal | 2 | 0.0232 | 0.0203 | — | 0 | — | — | — | — | — | — | 0 | 0.1472 | +| stratum=pace_slow | 2 | 0.0545 | 0.0139 | — | 0 | — | — | — | — | — | — | 0 | 0.1412 | +| stratum=pcm_wav | 1 | — | — | — | 0 | 0 | — | 0 | 0.0000 | — | — | 0 | 0.0197 | +| stratum=read_speech | 8 | 0.0358 | 0.0136 | — | 0 | 92 | 0.0034 | 732 | 0.6393 | 0.7500 | 0.7500 | 0 | 0.1615 | +| stratum=saxophone | 1 | — | — | — | 0 | 0 | — | 0 | 0.0000 | — | — | 0 | 0.0410 | +| stratum=short_utterance | 4 | 0.5000 | 0.2857 | 0.5000 | 0 | 143 | 0.0833 | 16844 | 0.2808 | 0.7500 | 0.7500 | 0 | 0.8964 | +| stratum=silence | 1 | — | — | — | 0 | 0 | — | 0 | 0.0000 | — | — | 0 | 0.0197 | +| stratum=single_speaker | 6 | 0.0372 | 0.0149 | — | 0 | — | — | — | — | — | — | 0 | 0.1472 | +| stratum=speech | 14 | 0.2359 | 0.1659 | 0.5000 | 0 | 471550 | 0.0938 | 102493 | 0.2212 | 0.0951 | 0.5279 | 0 | 0.1472 | +| stratum=speech_over_music | 2 | 0.5000 | 0.2857 | 0.5000 | 0 | 27 | 0.0315 | 16789 | 0.2819 | 0.5000 | 0.5000 | 0 | 0.0930 | +| stratum=spontaneous_speech | 2 | 0.2438 | 0.1729 | — | 0 | 471315 | 0.0943 | 84917 | 0.2111 | 0.0929 | 0.5237 | 0 | 0.1362 | +| stratum=stereo_44khz_ogg | 4 | 0.5000 | 0.2857 | 0.5000 | 0 | 116 | 0.1352 | 55 | 0.0009 | 1.0000 | 1.0000 | 0 | 0.8468 | +| stratum=word | 1 | 1.0000 | 0.6667 | 0.0000 | 0 | 0 | 0.0000 | 0 | — | 1.0000 | 1.0000 | 0 | 5.3647 | ## Gates @@ -129,43 +130,43 @@ Gates evaluate only manifest rows with `split=acceptance`. | Gate | Rule | Actual | Status | |---|---:|---:|---| -| `boundary_precision` | min 0.9500 | 0.1234 | DISABLED | -| `boundary_recall` | min 0.9500 | 0.5276 | DISABLED | +| `boundary_precision` | min 0.9500 | 0.1255 | DISABLED | +| `boundary_recall` | min 0.9500 | 0.5368 | DISABLED | | `cer` | max 0.0500 | 0.1504 | DISABLED | -| `false_alarm_rate` | max 0.0500 | 0.2666 | DISABLED | -| `missed_speech_rate` | max 0.0500 | 0.0723 | DISABLED | +| `false_alarm_rate` | max 0.0500 | 0.2366 | DISABLED | +| `missed_speech_rate` | max 0.0500 | 0.0730 | DISABLED | | `non_speech_error_cases` | max 0.0000 | 0 | PASS | | `non_speech_nonempty_cases` | max 0.0000 | 0 | PASS | | `parity_mismatch_cases` | max 0.0000 | 0 | DISABLED | | `peak_cuda_mb_max` | max 16384.0000 | — | DISABLED | -| `peak_rss_mb_max` | max 16384.0000 | 2700.9062 | DISABLED | -| `rtf_median` | max 1.0000 | 0.1349 | DISABLED | +| `peak_rss_mb_max` | max 16384.0000 | 2702.6406 | DISABLED | +| `rtf_median` | max 1.0000 | 0.1394 | DISABLED | | `short_exact_match_rate` | min 1.0000 | 1.0000 | PASS | -| `timestamp_violation_cases` | max 0.0000 | 2 | FAIL | +| `timestamp_violation_cases` | max 0.0000 | 0 | PASS | | `wer` | max 0.1000 | 0.2222 | DISABLED | ## Cases | Case | Split | Strata | Duration (s) | Inference (s) | RTF | Peak RSS (MiB) | Peak CUDA (MiB) | WER | CER | Miss (ms) | Miss rate | False alarm (ms) | False-alarm rate | Timestamp violations | Error | |---|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---| -| silence-5s | acceptance | non_speech, silence, pcm_wav | 5.0000 | 0.0965 | 0.0193 | 427.4219 | — | — | — | 0 | — | 0 | 0.0000 | 0 | — | -| rain-10s | tuning | non_speech, background_noise, stereo_44khz_ogg | 10.3310 | 0.1054 | 0.0102 | 433.9375 | — | — | — | 0 | — | 0 | 0.0000 | 0 | — | -| music-jazz-sax-24s | tuning | non_speech, music, saxophone, stereo_44khz_ogg | 24.0000 | 0.5318 | 0.0222 | 558.6875 | — | — | — | 0 | — | 0 | 0.0000 | 0 | — | -| music-36s | acceptance | non_speech, music, stereo_44khz_ogg | 36.4090 | 0.4277 | 0.0117 | 568.1562 | — | — | — | 0 | — | 0 | 0.0000 | 0 | — | -| speech-over-music-ear | tuning | speech, music, speech_over_music, short_utterance, mono_16khz_flac | 24.0000 | 3.1219 | 0.1301 | 8059.9062 | — | 1.0000 | 0.6667 | 27 | 0.0618 | 90 | 0.0038 | 0 | — | -| speech-over-music-john | acceptance | speech, music, speech_over_music, short_utterance, mono_16khz_flac | 36.4090 | 1.4224 | 0.0391 | 1363.5156 | — | 0.0000 | 0.0000 | 0 | 0.0000 | 16699 | 0.4640 | 0 | — | -| short-word-ear | tuning | speech, short_utterance, word, stereo_44khz_ogg | 0.4370 | 2.0118 | 4.6037 | 2040.9062 | — | 1.0000 | 0.6667 | 0 | 0.0000 | 0 | — | 1 | — | -| short-name-john | acceptance | speech, short_utterance, name, stereo_44khz_ogg | 0.8610 | 1.2008 | 1.3946 | 1206.2656 | — | 0.0000 | 0.0000 | 116 | 0.2755 | 55 | 0.1250 | 0 | — | -| librispeech-sample-1 | tuning | speech, read_speech, clean, mono_16khz_flac | 13.6900 | 2.4534 | 0.1792 | 1586.2969 | — | 0.0455 | 0.0041 | 92 | 0.0068 | 190 | 0.9135 | 0 | — | -| librispeech-sample-2 | acceptance | speech, read_speech, clean, mono_16khz_flac | 14.2150 | 2.3456 | 0.1650 | 1491.2188 | — | 0.0000 | 0.0000 | 0 | 0.0000 | 542 | 0.5784 | 0 | — | -| pace-slow-tuning | tuning | speech, read_speech, clean, single_speaker, pace_slow, mono_16khz_flac | 34.9400 | 4.5868 | 0.1313 | 1170.0938 | — | 0.0127 | 0.0049 | — | — | — | — | 0 | — | -| pace-normal-tuning | tuning | speech, read_speech, clean, single_speaker, pace_normal, mono_16khz_flac | 39.7550 | 5.4665 | 0.1375 | 1159.1562 | — | 0.0273 | 0.0265 | — | — | — | — | 0 | — | -| pace-fast-tuning | tuning | speech, read_speech, clean, single_speaker, pace_fast, mono_16khz_flac | 36.4550 | 5.8908 | 0.1616 | 1176.4531 | — | 0.0441 | 0.0088 | — | — | — | — | 0 | — | -| pace-slow-acceptance | acceptance | speech, read_speech, clean, single_speaker, pace_slow, mono_16khz_flac | 35.5900 | 4.5872 | 0.1289 | 1170.1875 | — | 0.0930 | 0.0208 | — | — | — | — | 0 | — | -| pace-normal-acceptance | acceptance | speech, read_speech, clean, single_speaker, pace_normal, mono_16khz_flac | 53.6300 | 7.2369 | 0.1349 | 1175.0469 | — | 0.0201 | 0.0155 | — | — | — | — | 1 | — | -| pace-fast-acceptance | acceptance | speech, read_speech, clean, single_speaker, pace_fast, mono_16khz_flac | 31.7750 | 5.2293 | 0.1646 | 1163.7969 | — | 0.0357 | 0.0100 | — | — | — | — | 0 | — | -| ami-meeting-30m | tuning | speech, meeting, multi_speaker, spontaneous_speech, long_form, 30_minute, mono_16khz_flac | 1800.0000 | 233.7446 | 0.1299 | 2495.3750 | — | 0.2733 | 0.2055 | 231289 | 0.1335 | 5450 | 0.0805 | 0 | — | -| ami-meeting-60m | acceptance | speech, meeting, multi_speaker, spontaneous_speech, long_form, 60_minute, mono_16khz_flac | 3600.0000 | 487.6979 | 0.1355 | 2700.9062 | — | 0.2280 | 0.1555 | 237128 | 0.0726 | 92921 | 0.2777 | 1 | — | +| silence-5s | acceptance | non_speech, silence, pcm_wav | 5.0000 | 0.0987 | 0.0197 | 428.8594 | — | — | — | 0 | — | 0 | 0.0000 | 0 | — | +| rain-10s | tuning | non_speech, background_noise, mono_44khz_ogg | 10.3310 | 0.1076 | 0.0104 | 436.1719 | — | — | — | 0 | — | 0 | 0.0000 | 0 | — | +| music-jazz-sax-24s | tuning | non_speech, music, saxophone, stereo_44khz_ogg | 24.0000 | 0.9837 | 0.0410 | 559.3125 | — | — | — | 0 | — | 0 | 0.0000 | 0 | — | +| music-36s | acceptance | non_speech, music, stereo_44khz_ogg | 36.4090 | 0.6445 | 0.0177 | 571.8281 | — | — | — | 0 | — | 0 | 0.0000 | 0 | — | +| speech-over-music-ear | tuning | speech, music, speech_over_music, short_utterance, mono_16khz_flac | 24.0000 | 3.3622 | 0.1401 | 9451.6250 | — | 1.0000 | 0.6667 | 27 | 0.0618 | 90 | 0.0038 | 0 | — | +| speech-over-music-john | acceptance | speech, music, speech_over_music, short_utterance, mono_16khz_flac | 36.4090 | 1.6717 | 0.0459 | 1297.3750 | — | 0.0000 | 0.0000 | 0 | 0.0000 | 16699 | 0.4640 | 0 | — | +| short-word-ear | tuning | speech, short_utterance, word, stereo_44khz_ogg | 0.4370 | 2.3444 | 5.3647 | 2026.0469 | — | 1.0000 | 0.6667 | 0 | 0.0000 | 0 | — | 0 | — | +| short-name-john | acceptance | speech, short_utterance, name, stereo_44khz_ogg | 0.8610 | 1.4229 | 1.6527 | 1222.7969 | — | 0.0000 | 0.0000 | 116 | 0.2755 | 55 | 0.1250 | 0 | — | +| librispeech-sample-1 | tuning | speech, read_speech, clean, mono_16khz_flac | 13.6900 | 2.7604 | 0.2016 | 1631.3906 | — | 0.0455 | 0.0041 | 92 | 0.0068 | 190 | 0.9135 | 0 | — | +| librispeech-sample-2 | acceptance | speech, read_speech, clean, mono_16khz_flac | 14.2150 | 2.6495 | 0.1864 | 1599.9062 | — | 0.0000 | 0.0000 | 0 | 0.0000 | 542 | 0.5784 | 0 | — | +| pace-slow-tuning | tuning | speech, read_speech, clean, single_speaker, pace_slow, mono_16khz_flac | 34.9400 | 5.0002 | 0.1431 | 1205.1875 | — | 0.0127 | 0.0049 | — | — | — | — | 0 | — | +| pace-normal-tuning | tuning | speech, read_speech, clean, single_speaker, pace_normal, mono_16khz_flac | 39.7550 | 5.9147 | 0.1488 | 1181.9844 | — | 0.0273 | 0.0265 | — | — | — | — | 0 | — | +| pace-fast-tuning | tuning | speech, read_speech, clean, single_speaker, pace_fast, mono_16khz_flac | 36.4550 | 7.1773 | 0.1969 | 1200.1094 | — | 0.0441 | 0.0088 | — | — | — | — | 0 | — | +| pace-slow-acceptance | acceptance | speech, read_speech, clean, single_speaker, pace_slow, mono_16khz_flac | 35.5900 | 4.9602 | 0.1394 | 1193.7031 | — | 0.0930 | 0.0208 | — | — | — | — | 0 | — | +| pace-normal-acceptance | acceptance | speech, read_speech, clean, single_speaker, pace_normal, mono_16khz_flac | 53.6300 | 7.8137 | 0.1457 | 1197.7969 | — | 0.0201 | 0.0155 | — | — | — | — | 0 | — | +| pace-fast-acceptance | acceptance | speech, read_speech, clean, single_speaker, pace_fast, mono_16khz_flac | 31.7750 | 5.5379 | 0.1743 | 1176.5625 | — | 0.0357 | 0.0100 | — | — | — | — | 0 | — | +| ami-meeting-30m | tuning | speech, meeting, multi_speaker, spontaneous_speech, long_form, 30_minute, mono_16khz_flac | 1800.0000 | 243.9746 | 0.1355 | 2499.2031 | — | 0.2733 | 0.2055 | 231903 | 0.1339 | 4416 | 0.0652 | 0 | — | +| ami-meeting-60m | acceptance | speech, meeting, multi_speaker, spontaneous_speech, long_form, 60_minute, mono_16khz_flac | 3600.0000 | 492.9831 | 0.1369 | 2702.6406 | — | 0.2280 | 0.1555 | 239412 | 0.0733 | 80501 | 0.2406 | 0 | — | ## Interpretation limits diff --git a/evaluation/manifest.jsonl b/evaluation/manifest.jsonl index 044971e..392e932 100644 --- a/evaluation/manifest.jsonl +++ b/evaluation/manifest.jsonl @@ -1,5 +1,5 @@ {"id":"silence-5s","audio":"fixtures/silence-5s.wav","sha256":"6611051942c29e6d244d71236bb78e90c38e30952c3656f9bb435e3c69f96628","duration_ms":5000,"language":"en","split":"acceptance","strata":["non_speech","silence","pcm_wav"],"reference":{"text":"","speech_intervals_ms":[]},"source":{"url":"https://ffmpeg.org/ffmpeg-filters.html#anullsrc","item":"FFmpeg anullsrc, mono 16 kHz PCM16, 5 seconds","revision":"ffmpeg-9.0"},"license":{"id":"CC0-1.0","url":"https://creativecommons.org/publicdomain/zero/1.0/","attribution":"Generated for TextPlease with FFmpeg anullsrc."}} -{"id":"rain-10s","audio":"fixtures/Rain.ogg","sha256":"c34d12243125c28f09426c076019f8321b602347a8564c38be771cb997c1a152","duration_ms":10331,"language":"en","split":"tuning","strata":["non_speech","background_noise","stereo_44khz_ogg"],"reference":{"text":"","speech_intervals_ms":[]},"source":{"url":"https://commons.wikimedia.org/w/index.php?title=File:Rain.ogg&oldid=597184901","item":"Rain.ogg","revision":"597184901"},"license":{"id":"Public-Domain","url":"https://commons.wikimedia.org/w/index.php?title=File:Rain.ogg&oldid=597184901#Licensing","attribution":"Recorded by Wikimedia Commons user ジダネ."}} +{"id":"rain-10s","audio":"fixtures/Rain.ogg","sha256":"c34d12243125c28f09426c076019f8321b602347a8564c38be771cb997c1a152","duration_ms":10331,"language":"en","split":"tuning","strata":["non_speech","background_noise","mono_44khz_ogg"],"reference":{"text":"","speech_intervals_ms":[]},"source":{"url":"https://commons.wikimedia.org/w/index.php?title=File:Rain.ogg&oldid=597184901","item":"Rain.ogg","revision":"597184901"},"license":{"id":"Public-Domain","url":"https://commons.wikimedia.org/w/index.php?title=File:Rain.ogg&oldid=597184901#Licensing","attribution":"Recorded by Wikimedia Commons user ジダネ."}} {"id":"music-jazz-sax-24s","audio":"fixtures/Jazz-Sax.ogg","sha256":"312a88774e7dd9d18e20420c2a1f3721156709032d55ba1e8cb630f433255a5b","duration_ms":24000,"language":"en","split":"tuning","strata":["non_speech","music","saxophone","stereo_44khz_ogg"],"reference":{"text":"","speech_intervals_ms":[]},"source":{"url":"https://commons.wikimedia.org/w/index.php?title=File:Jazz-Sax.ogg&oldid=971661804","item":"Jazz Tenor Sax","revision":"971661804"},"license":{"id":"CC-BY-2.5","url":"https://creativecommons.org/licenses/by/2.5/","attribution":"Jazz Tenor Sax by Wikimedia Commons user Serolillo."}} {"id":"music-36s","audio":"fixtures/Greensleaves.ogg","sha256":"1c31668ade22bb83f067ce178a2282ebfdf320bbca9b86b4413fedba64025ec9","duration_ms":36409,"language":"en","split":"acceptance","strata":["non_speech","music","stereo_44khz_ogg"],"reference":{"text":"","speech_intervals_ms":[]},"source":{"url":"https://commons.wikimedia.org/w/index.php?title=File:Greensleaves.ogg&oldid=845754359","item":"Greensleaves.ogg","revision":"845754359"},"license":{"id":"Public-Domain","url":"https://commons.wikimedia.org/w/index.php?title=File:Greensleaves.ogg&oldid=845754359#Licensing","attribution":"Performed and recorded by Wikimedia Commons user Rv87."}} {"id":"speech-over-music-ear","audio":"fixtures/ear-over-jazz.flac","sha256":"021cfdc6737d763cafa4ca828f6c77da712fed661fe41a94fac3987e910510e8","duration_ms":24000,"language":"en","split":"tuning","strata":["speech","music","speech_over_music","short_utterance","mono_16khz_flac"],"reference":{"text":"ear","speech_intervals_ms":[[9782,10219]]},"source":{"url":"https://commons.wikimedia.org/w/index.php?title=File:Jazz-Sax.ogg&oldid=971661804","item":"TextPlease speech-over-music mix of Jazz Tenor Sax and En-uk-ear.ogg","revision":"components-sha256:312a88774e7dd9d18e20420c2a1f3721156709032d55ba1e8cb630f433255a5b+34d9e3db4cac7a9091362cbabd59983cf4cd6d5d463a19bf481254a32799b356","transformation":"FFmpeg 9.0: attenuate music to 0.5, delay speech by 9782 ms so the word crosses 10 seconds, mix without normalization, and encode mono 16 kHz FLAC"},"license":{"id":"CC-BY-2.5","url":"https://creativecommons.org/licenses/by/2.5/","attribution":"Jazz Tenor Sax by Wikimedia Commons user Serolillo, mixed with the public-domain En-uk-ear recording by Wikimedia Commons user Chris Melville."}} diff --git a/examples/config_example.yaml b/examples/config_example.yaml index 4834408..5b17760 100644 --- a/examples/config_example.yaml +++ b/examples/config_example.yaml @@ -19,7 +19,7 @@ embedding_model: "all-MiniLM-L6-v2" # Performance performance: - whisper_batch_size: 1 # set to 4 on CUDA; lower it if accelerator memory is limited + whisper_batch_size: 1 # deterministic default, larger batches need model-specific parity checks similarity_batch_size: 32 # batch size for pre-encoding segment embeddings chunk_size: 1000 # process segments in chunks of this size (0 = no chunking) diff --git a/tests/test_device_utils.py b/tests/test_device_utils.py index a786467..431fc9a 100644 --- a/tests/test_device_utils.py +++ b/tests/test_device_utils.py @@ -61,7 +61,7 @@ def test_pipeline_uses_resolved_device(monkeypatch, tmp_path): assert transcribe_audio.call_args.args[0] == str(input_path) assert transcribe_audio.call_args.args[2] == "cuda" assert "temporary_directory" in transcribe_audio.call_args.kwargs - assert transcribe_audio.call_args.kwargs["batch_size"] == 4 + assert transcribe_audio.call_args.kwargs["batch_size"] == 1 assert sentence_transformer.call_args.kwargs["device"] == "cuda" assert segment_transcript.call_args.kwargs["preferred_device"] == "cuda" assert [entry[0] for entry in calls.mock_calls] == ["transcription", "embedding"] diff --git a/tests/test_whisper_batching.py b/tests/test_whisper_batching.py index fbcefa4..abeb863 100644 --- a/tests/test_whisper_batching.py +++ b/tests/test_whisper_batching.py @@ -14,7 +14,7 @@ class FakeTokenizer: def __init__( self, include_offsets: bool = True, - timestamp: tuple[float, float] = (0.0, 0.5), + timestamp: tuple[float | None, float | None] = (0.0, 0.5), ): self.include_offsets = include_offsets self.timestamp = timestamp @@ -39,7 +39,7 @@ class FakeProcessor: def __init__( self, include_offsets: bool = True, - timestamp: tuple[float, float] = (0.0, 0.5), + timestamp: tuple[float | None, float | None] = (0.0, 0.5), ): self.tokenizer = FakeTokenizer(include_offsets, timestamp) self.audio_batches = [] @@ -60,12 +60,21 @@ def __call__(self, audio, **kwargs): class FakeModel: config = SimpleNamespace(max_source_positions=1500) - def __init__(self, fail_batched: bool = False, error: Exception | None = None): + def __init__( + self, + fail_batched: bool = False, + error: Exception | None = None, + *, + is_multilingual: bool = True, + ): self.batch_sizes = [] self.fail_batched = fail_batched self.error = error + self.generation_config = SimpleNamespace(is_multilingual=is_multilingual) + self.generation_calls = [] def generate(self, **kwargs): + self.generation_calls.append(kwargs) input_features = kwargs["input_features"] batch_size = len(input_features) self.batch_sizes.append(batch_size) @@ -108,7 +117,14 @@ def audio_classifier(monkeypatch): ) -def _run_transcription(monkeypatch, batch_size, *, fail_batched=False): +def _run_transcription( + monkeypatch, + batch_size, + *, + fail_batched=False, + is_multilingual=True, + language="en", +): audio = np.concatenate( [np.full(TARGET_SAMPLE_RATE, segment_number, dtype=np.float32) for segment_number in (1, 2, 3)] ) @@ -117,7 +133,7 @@ def _run_transcription(monkeypatch, batch_size, *, fail_batched=False): {"start": TARGET_SAMPLE_RATE, "end": 2 * TARGET_SAMPLE_RATE}, {"start": 2 * TARGET_SAMPLE_RATE, "end": 3 * TARGET_SAMPLE_RATE}, ] - model = FakeModel(fail_batched=fail_batched) + model = FakeModel(fail_batched=fail_batched, is_multilingual=is_multilingual) processor = FakeProcessor() monkeypatch.setattr(transformers_pipeline, "_load_model_and_processor", lambda *args: (model, processor)) @@ -134,24 +150,87 @@ def _run_transcription(monkeypatch, batch_size, *, fail_batched=False): "test-model", "cpu", batch_size=batch_size, + language=language, ) - return segments, model.batch_sizes + return segments, model def test_transcribe_batches_without_changing_segments(monkeypatch): - sequential, sequential_batch_sizes = _run_transcription(monkeypatch, 1) - batched, batched_batch_sizes = _run_transcription(monkeypatch, 2) + sequential, sequential_model = _run_transcription(monkeypatch, 1) + batched, batched_model = _run_transcription(monkeypatch, 2) assert batched == sequential - assert sequential_batch_sizes == [1, 1, 1] - assert batched_batch_sizes == [2, 1] + assert sequential_model.batch_sizes == [1, 1, 1] + assert batched_model.batch_sizes == [2, 1] def test_transcribe_retries_batch_after_out_of_memory(monkeypatch): - segments, batch_sizes = _run_transcription(monkeypatch, 2, fail_batched=True) + segments, model = _run_transcription(monkeypatch, 2, fail_batched=True) assert [segment["text"] for segment in segments] == ["Segment 1", "Segment 2", "Segment 3"] - assert batch_sizes == [2, 1, 1, 1] + assert model.batch_sizes == [2, 1, 1, 1] + + +def test_transcribe_clamps_offsets_and_preserves_terminal_text(monkeypatch): + interval_start = TARGET_SAMPLE_RATE // 4 + interval_end = 3 * TARGET_SAMPLE_RATE // 4 + audio = np.ones(TARGET_SAMPLE_RATE, dtype=np.float32) + offsets = [ + {"text": "Leading", "timestamp": (-1.0, 0.2)}, + {"text": " overlap", "timestamp": (0.1, 0.3)}, + {"text": " terminal", "timestamp": (0.4, None)}, + ] + monkeypatch.setattr(transformers_pipeline, "load_pcm_wav", lambda path: audio) + monkeypatch.setattr( + transformers_pipeline, + "_get_speech_segments", + lambda *args: ( + [{"start": interval_start, "end": interval_end}], + [(interval_start, interval_end)], + ), + ) + monkeypatch.setattr( + transformers_pipeline, + "_load_model_and_processor", + lambda *args: (FakeModel(), FakeProcessor()), + ) + monkeypatch.setattr( + transformers_pipeline, + "_transcribe_speech_segments", + lambda *args: [offsets], + ) + monkeypatch.setattr(transformers_pipeline.torch.cuda, "is_available", lambda: False) + + segments = transformers_pipeline.transcribe( + "input.wav", + "test-model", + "cpu", + ) + + assert segments == [ + {"text": "Leading", "start_time": "00:00:00.250", "end_time": "00:00:00.450"}, + {"text": "overlap", "start_time": "00:00:00.450", "end_time": "00:00:00.550"}, + {"text": "terminal", "start_time": "00:00:00.650", "end_time": "00:00:00.750"}, + ] + + +def test_english_only_model_does_not_request_language_or_task(monkeypatch): + _, model = _run_transcription(monkeypatch, 1, is_multilingual=False) + + assert model.generation_calls[0]["language"] is None + assert model.generation_calls[0]["task"] is None + + +@pytest.mark.parametrize( + ("language", "expected_language"), + [("fr", "fr"), (None, None)], +) +def test_multilingual_model_supports_explicit_or_detected_language(monkeypatch, language, expected_language): + _, model = _run_transcription(monkeypatch, 1, language=language) + + generation_call = model.generation_calls[0] + assert generation_call["task"] == "transcribe" + assert generation_call.get("language") == expected_language def test_no_speech_skips_whisper(monkeypatch): diff --git a/textplease/backends/transformers_pipeline.py b/textplease/backends/transformers_pipeline.py index 33b3df8..105d536 100644 --- a/textplease/backends/transformers_pipeline.py +++ b/textplease/backends/transformers_pipeline.py @@ -3,7 +3,6 @@ import gc import re import logging -import warnings from typing import TypedDict from functools import lru_cache @@ -23,9 +22,6 @@ logger = logging.getLogger(__name__) -warnings.filterwarnings("ignore", message=".*Whisper did not predict an ending timestamp.*") -warnings.filterwarnings("ignore", message=".*attention mask is not set.*") - class _WhisperOffset(TypedDict): text: str @@ -142,7 +138,7 @@ def _transcribe_speech_segments( processor: WhisperProcessor, audio_chunks: list[np.ndarray], device: str, - language: str, + language: str | None, ) -> list[list[_WhisperOffset]]: """Transcribe speech chunks and return decoded offsets for each chunk.""" torch_dtype = torch.float16 if device not in ("cpu", "mps") else torch.float32 @@ -162,12 +158,14 @@ def _transcribe_speech_segments( else: attention_mask = None + is_multilingual = getattr(model.generation_config, "is_multilingual", True) + with torch.no_grad(): generated_ids = model.generate( input_features=input_features, attention_mask=attention_mask, - language=language, - task="transcribe", + language=language if is_multilingual else None, + task="transcribe" if is_multilingual else None, return_timestamps=True, temperature=(0.0, 0.2, 0.4, 0.6, 0.8, 1.0), compression_ratio_threshold=1.35, @@ -217,9 +215,10 @@ def _transcribe_chunks( chunks: list[tuple[int, int, np.ndarray]], batch_size: int, device: str, - language: str, + language: str | None, ) -> list[_WhisperOffset]: all_offsets: list[_WhisperOffset] = [] + previous_end_s = 0.0 for batch_start in range(0, len(chunks), batch_size): batch = chunks[batch_start : batch_start + batch_size] first_start, _, _ = batch[0] @@ -244,13 +243,28 @@ def _transcribe_chunks( _transcribe_speech_segments(model, processor, [chunk], device, language)[0] for chunk in audio_chunks ] - for (start, _, _), offsets in zip(batch, batch_offsets, strict=True): + for (start, end, _), offsets in zip(batch, batch_offsets, strict=True): start_s = start / TARGET_SAMPLE_RATE - for offset in offsets: - ts = offset.get("timestamp", (0.0, 0.0)) - if len(ts) == 2 and ts[0] is not None and ts[1] is not None: - offset["timestamp"] = (ts[0] + start_s, ts[1] + start_s) - all_offsets.extend(offsets) + end_s = end / TARGET_SAMPLE_RATE + for offset_index, offset in enumerate(offsets): + local_start, local_end = offset["timestamp"] + if local_start is None: + continue + if local_end is None and offset_index != len(offsets) - 1: + continue + + bounded_start = max(start_s, local_start + start_s, previous_end_s) + bounded_end = end_s if local_end is None else min(end_s, local_end + start_s) + if bounded_end <= bounded_start: + continue + + all_offsets.append( + { + "text": offset["text"], + "timestamp": (bounded_start, bounded_end), + } + ) + previous_end_s = bounded_end return all_offsets @@ -272,7 +286,7 @@ def transcribe( model_name: str, device: str, *, - language: str = "en", + language: str | None = "en", batch_size: int = 1, ) -> list[dict[str, str]]: """Transcribe a normalized mono 16 kHz PCM16 WAV.""" diff --git a/textplease/pipeline.py b/textplease/pipeline.py index 406619c..bcaf4b4 100644 --- a/textplease/pipeline.py +++ b/textplease/pipeline.py @@ -121,7 +121,7 @@ def _extract_config_params(config: dict) -> dict: "min_segment_chars": config.get("min_segment_chars", 15), "max_segment_words": config.get("max_segment_words", 100), "language": config.get("language", "en"), - "whisper_batch_size": config.get("performance", {}).get("whisper_batch_size"), + "whisper_batch_size": config.get("performance", {}).get("whisper_batch_size", 1), "similarity_batch_size": config.get("performance", {}).get("similarity_batch_size", 32), "chunk_size": config.get("performance", {}).get("chunk_size", 1000), } @@ -220,8 +220,6 @@ def run_transcription_pipeline(config: dict) -> None: _validate_pipeline_config(config) params = _extract_config_params(config) params["device"] = detect_device(params["device"]) - if params["whisper_batch_size"] is None: - params["whisper_batch_size"] = 4 if params["device"] == "cuda" else 1 logger.info(f"Input: {params['input_path']} → Output: {params['output_path']}") logger.info(f"ASR: {params['model_name']} | Device: {params['device']}") diff --git a/textplease/transcriber.py b/textplease/transcriber.py index 99882b3..c029394 100644 --- a/textplease/transcriber.py +++ b/textplease/transcriber.py @@ -24,6 +24,6 @@ def transcribe_audio( audio_path=normalized_audio_path, model_name=model_name, device=device, - language=language or "en", + language=language, batch_size=batch_size, )