From cd13ea7b72f483c3dac9b4a0784d1241929760cc Mon Sep 17 00:00:00 2001 From: Vikhyat Korrapati Date: Wed, 26 Aug 2026 14:10:03 -0700 Subject: [PATCH] Document Photon speech transcription --- docs/overview.md | 6 +- docs/quickstart.md | 4 +- docs/running-locally.md | 17 +++- docs/transcription.md | 205 ++++++++++++++++++++++++++++++++++++++++ sidebars.ts | 1 + 5 files changed, 227 insertions(+), 6 deletions(-) create mode 100644 docs/transcription.md diff --git a/docs/overview.md b/docs/overview.md index f5e54d5..e9ca195 100644 --- a/docs/overview.md +++ b/docs/overview.md @@ -8,9 +8,9 @@ title: "Overview" Moondream is an open-weight family of Vision Language Models (VLMs) built for powerful, efficient visual reasoning. The Moondream 3 family uses a mixture-of-experts architecture with grounded visual reasoning, a 32k context window, and native support for multiple vision skills—like pointing, counting, and object detection—all designed with a deployment-friendly ethos. -Moondream 3 Preview remains the default for local inference with -[Photon](/running-locally). For hosted inference, choose a model through the -[Cloud API quickstart](./quickstart). +For local inference, [Photon](/running-locally) exposes Moondream, Qwen, Gemma, +and Whisper models through the Moondream Python package. For hosted inference, +choose a model through the [Cloud API quickstart](./quickstart). ### Key stats diff --git a/docs/quickstart.md b/docs/quickstart.md index 466a92b..eba1080 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -342,7 +342,9 @@ console.log(result.caption); ## Running Locally -Want to run Moondream on your own hardware instead of using the Cloud API? Check out our [guide to running locally](/running-locally). +Want to run models on your own hardware instead of using the Cloud API? Start +with the [Photon local inference guide](/running-locally), or use +[Whisper speech transcription](/transcription) on a supported NVIDIA GPU. ## Next Steps diff --git a/docs/running-locally.md b/docs/running-locally.md index 1e7b8ac..44f23ee 100644 --- a/docs/running-locally.md +++ b/docs/running-locally.md @@ -9,8 +9,9 @@ description: High-performance local inference with Photon on NVIDIA GPUs and App Photon is Moondream's high-performance local inference engine for NVIDIA GPUs (Linux x86_64 / aarch64 or Windows AMD64) and Apple Silicon Macs. It supports -Moondream, Qwen, and Gemma models with custom CUDA and Metal kernels, automatic -batching, paged KV caching, and prefix caching. +Moondream, Qwen, and Gemma vision-language models, plus Whisper speech +transcription on NVIDIA GPUs. Photon provides custom CUDA and Metal kernels, +automatic batching, paged KV caching, and prefix caching. ## Requirements @@ -93,6 +94,9 @@ qwen = md.photon("Qwen/Qwen3.5-4B") # Gemma 4 E2B gemma = md.photon("google/gemma-4-E2B-it") + +# Whisper large-v3-turbo (NVIDIA GPU) +speech = md.photon("openai/whisper-large-v3-turbo") ``` | Family | Supported models | @@ -101,12 +105,17 @@ gemma = md.photon("google/gemma-4-E2B-it") | Qwen 3.5 | 0.8B, 2B, 4B, 9B, 27B, and 35B-A3B; base variants where published | | Qwen 3.6 | 27B and 35B-A3B; BF16 and FP8 checkpoints | | Gemma 4 | E2B, E4B, and 31B base and instruction variants | +| Whisper | Whisper large-v3-turbo transcription and English translation | Use `md.photon_models()` to list the exact identifiers registered by the installed release. Models expose `model.tasks` and `model.supports(task)` so applications can discover their capabilities; not every model implements every Moondream-specific skill. +Whisper currently runs through Photon's CUDA path on supported NVIDIA GPUs. +See [Speech Transcription](/transcription) for files, progressive results, live +PCM, translation, and timestamps. + Model weights are automatically downloaded from Hugging Face on first run and cached locally. Photon clients with identical configurations share an engine. Close clients @@ -132,6 +141,10 @@ for chunk in model.query(image, "Describe this scene in detail.", stream=True)[" print(chunk, end="", flush=True) ``` +Speech models return replaceable transcript snapshots rather than token deltas. +See [Progressive transcription](/transcription#progressive-transcription) and +[Live PCM](/transcription#live-pcm). + ## Using Finetunes If you've created a finetuned model through the [Moondream finetuning API](/finetuning), you can use it locally with Photon: diff --git a/docs/transcription.md b/docs/transcription.md new file mode 100644 index 0000000..0a0a678 --- /dev/null +++ b/docs/transcription.md @@ -0,0 +1,205 @@ +--- +id: transcription +slug: transcription +title: Speech Transcription +description: Transcribe and translate files or live PCM with Photon and Whisper +--- + +# Speech Transcription + +Photon serves `openai/whisper-large-v3-turbo` through the Moondream Python +package on supported NVIDIA GPUs. It handles long-form files, progressive +transcripts, live mono PCM, English translation, and segment or word +timestamps. + +## Installation + +Install Moondream 2.1 or newer: + +```bash +pip install --upgrade "moondream>=2.1" +``` + +Photon downloads the Whisper checkpoint from Hugging Face on first use and +caches it locally. Local Photon models do not require a Moondream API key. + +## Transcribe a file + +```python +from pathlib import Path + +import moondream as md + + +with md.photon("openai/whisper-large-v3-turbo") as speech: + result = speech.transcribe( + audio=Path("meeting.m4a"), + timestamps="word", + ) + +print(result["text"]) +print("Detected language:", result["language"]) + +for segment in result["segments"]: + for word in segment.get("words", []): + print(word["start"], word["end"], word["word"]) +``` + +Encoded input can be a path, bytes, or a bounded binary stream. Supported +containers include WAV/WAVEX, FLAC, MP3, Ogg Vorbis, Opus, M4A, MP4, MOV, and +WebM. Multichannel files are downmixed to mono from their container metadata. + +Raw audio can be a one-dimensional NumPy array or CPU Torch tensor containing +mono PCM. Pass its sample rate explicitly: + +```python +import moondream as md + + +with md.photon("openai/whisper-large-v3-turbo") as speech: + result = speech.transcribe( + audio=pcm, + sample_rate=48_000, + timestamps="segment", + ) +``` + +## Translate to English + +Set `task="translate"` to translate non-English speech into English. Use +`language` when the source language is known, or omit it for automatic language +detection. + +```python +from pathlib import Path + +import moondream as md + + +with md.photon("openai/whisper-large-v3-turbo") as speech: + result = speech.transcribe( + audio=Path("interview-es.mp3"), + task="translate", + language="es", + ) +print(result["text"]) +``` + +## Progressive transcription + +Set `stream=True` for progress while Photon processes a long file. Each update +is the newest complete transcript snapshot, not a token delta, so replace the +displayed text instead of appending it. + +```python +from pathlib import Path + +import moondream as md + + +with md.photon("openai/whisper-large-v3-turbo") as speech: + updates = speech.transcribe( + audio=Path("meeting.m4a"), + timestamps="segment", + stream=True, + ) + + for update in updates: + print(update["text"]) + + result = updates.result() +``` + +Progress snapshots are bounded and may be coalesced when the producer is +faster than the consumer. `result()` always returns the final transcript. + +## Live PCM + +Live input is an asynchronous iterator of nonempty, one-dimensional NumPy +arrays or CPU Torch tensors containing mono PCM. Every chunk uses the same +sample rate and contains at most 1,048,576 samples. + +```python +import asyncio + +import moondream as md + + +async def microphone_chunks(): + while (chunk := await microphone.read()) is not None: + yield chunk + + +async def main(): + with md.photon("openai/whisper-large-v3-turbo") as speech: + updates = await speech.atranscribe( + audio=microphone_chunks(), + sample_rate=48_000, + timestamps="segment", + stream=True, + ) + + async for update in updates: + print(update["text"]) + + result = await updates.aresult() + print("Final:", result["text"]) + + +asyncio.run(main()) +``` + +For a final result without progress snapshots, pass `stream=False` to +`atranscribe` and await the returned transcript: + +```python +import moondream as md + + +async def final_transcript(audio_chunks): + with md.photon("openai/whisper-large-v3-turbo") as speech: + return await speech.atranscribe( + audio=audio_chunks, + sample_rate=48_000, + stream=False, + ) +``` + +## Options + +Arguments are passed directly to the selected Photon model: + +| Argument | Default | Description | +|----------|---------|-------------| +| `audio` | Required | Encoded audio, raw PCM, or an asynchronous live PCM iterator | +| `sample_rate` | None | Required for raw or live PCM; do not set for encoded audio | +| `language` | Automatic | Source language code such as `"en"` or `"es"` | +| `task` | `"transcribe"` | Use `"translate"` for English translation | +| `timestamps` | `"segment"` | `"none"`, `"segment"`, or `"word"` | +| `initial_prompt` | None | Text context for names or domain-specific vocabulary | +| `condition_on_previous_text` | `True` | Carry bounded text context between long-form windows | +| `clip_start_seconds` | `0.0` | Start offset for an encoded source | +| `clip_end_seconds` | Source end | Exclusive end offset for an encoded source | +| `stream` | `False` | Return progressive transcript snapshots | +| `settings` | None | Sampling settings such as `temperature` and `max_tokens` | + +Live PCM does not accept clip ranges. For encoded bytes and binary streams, +Photon snapshots at most 64 MiB before incremental decoding. Use an asynchronous +PCM iterator for raw audio longer than 30 seconds. Encoded files and live +sessions are limited to 24 hours. + +## Output + +The final dictionary includes: + +- `text`: the complete transcript or English translation; +- `language`: the detected or requested source language; +- `task`: `"transcribe"` or `"translate"`; +- `segments`: timestamped segments, with `words` when requested; +- source, clip, and decode diagnostic fields. + +Word entries include `word`, `start`, `end`, and `probability`. All timestamps +are in seconds relative to the source audio. + +For vision-language models and supported hardware, see +[Run Moondream Locally](/running-locally). diff --git a/sidebars.ts b/sidebars.ts index 5438c56..591fa27 100644 --- a/sidebars.ts +++ b/sidebars.ts @@ -11,6 +11,7 @@ const sidebars: SidebarsConfig = { 'reasoning', 'rate-limits', 'running-locally', + 'transcription', { type: 'category', label: 'Skills',