Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 28 additions & 6 deletions apps/speech/app/audio-file-transcription/index.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,12 @@ import {
TouchableOpacity,
TextInput,
} from 'react-native';
import { useSpeechToText, models, WHISPER_SAMPLE_RATE_HZ } from 'react-native-executorch';
import {
useSpeechToText,
models,
QWEN3_ASR_LANGUAGES,
WHISPER_SAMPLE_RATE_HZ,
} from 'react-native-executorch';
import { decodeAudioData } from 'react-native-audio-api';

import ScreenWrapper from '../../components/ScreenWrapper';
Expand Down Expand Up @@ -126,8 +131,24 @@ const MODELS = [
config: models.speechToText.WHISPER.SMALL.MLX_INT8,
disabled: Platform.OS !== 'ios',
},
{
name: 'Qwen3-ASR 0.6B (CPU)',
config: models.speechToText.QWEN3_ASR.SIZE_0_6B.XNNPACK_8DA4W,
},
{
name: 'Qwen3-ASR 0.6B (MLX + Core ML)',
config: models.speechToText.QWEN3_ASR.SIZE_0_6B.MLX_COREML_INT4,
disabled: Platform.OS !== 'ios',
},
];

// Qwen3-ASR detects the language itself; 'auto' leaves it to the model.
const AUTO = 'auto';
const languagesOf = (config: (typeof MODELS)[number]['config']): readonly string[] =>
'supportedLanguages' in config
? config.supportedLanguages
: [AUTO, ...Object.keys(QWEN3_ASR_LANGUAGES)];

function STTAudioContent() {
const [selectedModel, setSelectedModel] = useState(MODELS[0]!);
const [selectedLanguage, setSelectedLanguage] = useState<string>('en');
Expand All @@ -145,7 +166,7 @@ function STTAudioContent() {
error: modelError,
} = useSpeechToText(selectedModel.config);

const supportedLanguages = selectedModel.config.supportedLanguages;
const supportedLanguages = languagesOf(selectedModel.config);

useEffect(() => {
return () => {
Expand All @@ -167,7 +188,8 @@ function STTAudioContent() {
setStatus('Transcribing...');

let currentText = '';
const text = await transcribe(samples, { language: selectedLanguage as any }, (token) => {
const language = selectedLanguage === AUTO ? undefined : selectedLanguage;
const text = await transcribe(samples, { language: language as any }, (token) => {
currentText += token;
setAudioText(currentText);
});
Expand Down Expand Up @@ -196,7 +218,7 @@ function STTAudioContent() {
<ScrollView style={styles.container} contentContainerStyle={styles.content}>
<View style={styles.card}>
<ModelPicker
label="Whisper Model"
label="Speech-to-Text Model"
options={MODELS.map((m) => ({
label: m.name,
value: m,
Expand All @@ -205,14 +227,14 @@ function STTAudioContent() {
selectedValue={selectedModel}
onValueChange={(m) => {
setSelectedModel(m);
setSelectedLanguage(m.config.supportedLanguages[0]!);
setSelectedLanguage(languagesOf(m.config)[0]!);
}}
/>
<ModelStatus
isReady={isSttReady}
downloadProgress={downloadProgress}
error={modelError ? modelError.message : null}
modelTypeLabel="Whisper model"
modelTypeLabel="Speech-to-text model"
/>
{supportedLanguages.length > 1 && (
<ModelPicker
Expand Down
34 changes: 28 additions & 6 deletions apps/speech/app/microphone-transcription/index.tsx
Original file line number Diff line number Diff line change
@@ -1,6 +1,11 @@
import React, { useEffect, useRef, useState } from 'react';
import { Platform, View, Text, StyleSheet, ScrollView, TouchableOpacity } from 'react-native';
import { useSpeechToText, models, WHISPER_SAMPLE_RATE_HZ } from 'react-native-executorch';
import {
useSpeechToText,
models,
QWEN3_ASR_LANGUAGES,
WHISPER_SAMPLE_RATE_HZ,
} from 'react-native-executorch';
import { AudioManager, AudioRecorder } from 'react-native-audio-api';
import DeviceInfo from 'react-native-device-info';

Expand Down Expand Up @@ -119,8 +124,24 @@ const MODELS = [
config: models.speechToText.WHISPER.SMALL.MLX_INT8,
disabled: Platform.OS !== 'ios',
},
{
name: 'Qwen3-ASR 0.6B (CPU)',
config: models.speechToText.QWEN3_ASR.SIZE_0_6B.XNNPACK_8DA4W,
},
{
name: 'Qwen3-ASR 0.6B (MLX + Core ML)',
config: models.speechToText.QWEN3_ASR.SIZE_0_6B.MLX_COREML_INT4,
disabled: Platform.OS !== 'ios',
},
];

// Qwen3-ASR detects the language itself; 'auto' leaves it to the model.
const AUTO = 'auto';
const languagesOf = (config: (typeof MODELS)[number]['config']): readonly string[] =>
'supportedLanguages' in config
? config.supportedLanguages
: [AUTO, ...Object.keys(QWEN3_ASR_LANGUAGES)];

const isSimulator = DeviceInfo.isEmulatorSync();

function STTContent() {
Expand All @@ -141,7 +162,7 @@ function STTContent() {
error: modelError,
} = useSpeechToText(selectedModel.config);

const supportedLanguages = selectedModel.config.supportedLanguages;
const supportedLanguages = languagesOf(selectedModel.config);

const recorderRef = useRef<AudioRecorder | null>(null);

Expand All @@ -168,7 +189,8 @@ function STTContent() {

(async () => {
try {
const textStream = stream({ language: selectedLanguage as any });
const language = selectedLanguage === AUTO ? undefined : selectedLanguage;
const textStream = stream({ language: language as any });
for await (const result of textStream) {
setCommittedText(result.committed);
setNonCommittedText(result.nonCommitted);
Expand Down Expand Up @@ -216,7 +238,7 @@ function STTContent() {
<ScrollView style={styles.container} contentContainerStyle={styles.content}>
<View style={styles.card}>
<ModelPicker
label="Whisper Model"
label="Speech-to-Text Model"
options={MODELS.map((m) => ({
label: m.name,
value: m,
Expand All @@ -225,14 +247,14 @@ function STTContent() {
selectedValue={selectedModel}
onValueChange={(m) => {
setSelectedModel(m);
setSelectedLanguage(m.config.supportedLanguages[0]!);
setSelectedLanguage(languagesOf(m.config)[0]!);
}}
/>
<ModelStatus
isReady={isSttReady}
downloadProgress={downloadProgress}
error={modelError ? modelError.message : null}
modelTypeLabel="Whisper model"
modelTypeLabel="Speech-to-text model"
/>
{supportedLanguages.length > 1 && (
<ModelPicker
Expand Down
31 changes: 24 additions & 7 deletions docs/docs/02-extensions/speech/03-speech-to-text.md
Original file line number Diff line number Diff line change
@@ -1,14 +1,15 @@
---
title: Speech-to-Text (STT)
slug: /extensions/speech-to-text
description: 'Transcribe spoken audio in real time on-device in React Native using OpenAI Whisper with integrated Voice Activity Detection.'
description: 'Transcribe spoken audio in real time on-device in React Native using OpenAI Whisper or Qwen3-ASR with integrated Voice Activity Detection.'
keywords:
[
react native,
speech to text,
stt,
asr,
whisper,
qwen3-asr,
voice recognition,
transcription,
vad,
Expand All @@ -20,7 +21,7 @@ keywords:

# Speech-to-Text (STT)

The Speech-to-Text extension transcribes spoken audio into text directly on-device using OpenAI's Whisper model paired with an integrated FSMN Voice Activity Detector (VAD).
The Speech-to-Text extension transcribes spoken audio into text directly on-device using OpenAI's Whisper or Alibaba's [Qwen3-ASR](#qwen3-asr) paired with an integrated FSMN Voice Activity Detector (VAD).

The pipeline supports two primary workflows:

Expand Down Expand Up @@ -202,22 +203,35 @@ const transcript = stt.transcribeWorklet(audioData, { language: 'en' });

See [Worklets & Threading](../../03-core-and-advanced/06-worklets-and-threading.md) for details on worklet execution contexts and zero-copy host objects.

## Qwen3-ASR

[Qwen3-ASR](https://huggingface.co/Qwen/Qwen3-ASR-0.6B) works with the same hook and the same `transcribe` / `stream` API. It detects the language by itself; pass `language` (one of [`QWEN3_ASR_LANGUAGES`](../../06-api-reference/variables/QWEN3_ASR_LANGUAGES.md)) only to force one:

```typescript
const stt = useSpeechToText(models.speechToText.QWEN3_ASR.SIZE_0_6B.DEFAULT);

const transcript = await stt.transcribe(audioData); // auto-detect
const french = await stt.transcribe(audioData, { language: 'fr' });
```

## Available Models

The library provides ready-to-use Whisper models from the [Software Mansion HuggingFace Whisper Collection](https://huggingface.co/collections/software-mansion/whisper), available in [`models.speechToText`](../../06-api-reference/variables/models.md#speechtotext):

| Model Family | Variants | Size Range | Supported Backends | Languages | Notes |
| :---------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | :----------------- | :------------------------------------------------------------ | :----------------------------------------------------------------------------------------------------- | :-------------------------------------------------------------- |
| **Whisper Tiny** | [`Multilingual`](../../06-api-reference/variables/models.md#speechtotextwhispertiny), [`English`](../../06-api-reference/variables/models.md#speechtotextwhisperentiny) | 57.1 MB – 221.8 MB | XNNPACK (CPU), Core ML (Apple), MLX (Apple), Vulkan (Android) | English / [`WHISPER_LANGUAGES`](../../06-api-reference/variables/WHISPER_LANGUAGES.md) (99+ languages) | Ultra-fast transcription with minimal RAM usage. |
| **Whisper Base** | [`Multilingual`](../../06-api-reference/variables/models.md#speechtotextwhisperbase), [`English`](../../06-api-reference/variables/models.md#speechtotextwhisperenbase) | 97.8 MB – 380.2 MB | XNNPACK (CPU), Core ML (Apple), MLX (Apple), Vulkan (Android) | English / [`WHISPER_LANGUAGES`](../../06-api-reference/variables/WHISPER_LANGUAGES.md) (99+ languages) | Balanced accuracy and speed for general voice dictation. |
| **Whisper Small** | [`Multilingual`](../../06-api-reference/variables/models.md#speechtotextwhispersmall), [`English`](../../06-api-reference/variables/models.md#speechtotextwhisperensmall) | 276.0 MB – 1.05 GB | XNNPACK (CPU), Core ML (Apple), MLX (Apple), Vulkan (Android) | English / [`WHISPER_LANGUAGES`](../../06-api-reference/variables/WHISPER_LANGUAGES.md) (99+ languages) | High-capacity model for complex, noisy, or multi-speaker audio. |
| Model Family | Variants | Size Range | Supported Backends | Languages | Notes |
| :----------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | :------------------ | :------------------------------------------------------------ | :----------------------------------------------------------------------------------------------------- | :-------------------------------------------------------------- |
| **Whisper Tiny** | [`Multilingual`](../../06-api-reference/variables/models.md#speechtotextwhispertiny), [`English`](../../06-api-reference/variables/models.md#speechtotextwhisperentiny) | 57.1 MB – 221.8 MB | XNNPACK (CPU), Core ML (Apple), MLX (Apple), Vulkan (Android) | English / [`WHISPER_LANGUAGES`](../../06-api-reference/variables/WHISPER_LANGUAGES.md) (99+ languages) | Ultra-fast transcription with minimal RAM usage. |
| **Whisper Base** | [`Multilingual`](../../06-api-reference/variables/models.md#speechtotextwhisperbase), [`English`](../../06-api-reference/variables/models.md#speechtotextwhisperenbase) | 97.8 MB – 380.2 MB | XNNPACK (CPU), Core ML (Apple), MLX (Apple), Vulkan (Android) | English / [`WHISPER_LANGUAGES`](../../06-api-reference/variables/WHISPER_LANGUAGES.md) (99+ languages) | Balanced accuracy and speed for general voice dictation. |
| **Whisper Small** | [`Multilingual`](../../06-api-reference/variables/models.md#speechtotextwhispersmall), [`English`](../../06-api-reference/variables/models.md#speechtotextwhisperensmall) | 276.0 MB – 1.05 GB | XNNPACK (CPU), Core ML (Apple), MLX (Apple), Vulkan (Android) | English / [`WHISPER_LANGUAGES`](../../06-api-reference/variables/WHISPER_LANGUAGES.md) (99+ languages) | High-capacity model for complex, noisy, or multi-speaker audio. |
| **Qwen3-ASR 0.6B** | [`0.6B`](../../06-api-reference/variables/models.md#speechtotextqwen3_asr) | 707.7 MB – 808.1 MB | XNNPACK (CPU), Core ML + MLX (Apple) | [`QWEN3_ASR_LANGUAGES`](../../06-api-reference/variables/QWEN3_ASR_LANGUAGES.md) (30 languages) | Automatic language detection. |

## API Reference

### Hooks & Pipelines

- [`useSpeechToText()`](../../06-api-reference/functions/useSpeechToText.md) — React hook for Whisper model loading, downloading, and live transcription state.
- [`createWhisperSpeechToText()`](../../06-api-reference/functions/createWhisperSpeechToText.md) — Imperative factory for Whisper Speech-to-Text pipelines.
- [`createQwen3AsrSpeechToText()`](../../06-api-reference/functions/createQwen3AsrSpeechToText.md) — Imperative factory for Qwen3-ASR Speech-to-Text pipelines.

### Types & Options

Expand All @@ -226,15 +240,18 @@ The library provides ready-to-use Whisper models from the [Software Mansion Hugg
- [`WhisperSttOptions`](../../06-api-reference/type-aliases/WhisperSttOptions.md) — Per-call transcription options (`language`).
- [`WhisperStreamOptions`](../../06-api-reference/type-aliases/WhisperStreamOptions.md) — Live microphone streaming options (`language`, `vadOptions`).
- [`WhisperLanguage`](../../06-api-reference/type-aliases/WhisperLanguage.md) — Union of supported Whisper language codes.
- [`Qwen3AsrSttModel`](../../06-api-reference/type-aliases/Qwen3AsrSttModel.md), [`Qwen3AsrSttOptions`](../../06-api-reference/type-aliases/Qwen3AsrSttOptions.md), [`Qwen3AsrLanguage`](../../06-api-reference/type-aliases/Qwen3AsrLanguage.md) — Qwen3-ASR counterparts.

### Constants & Model Presets

- [`WHISPER_SAMPLE_RATE_HZ`](../../06-api-reference/variables/WHISPER_SAMPLE_RATE_HZ.md) — Target audio sample rate expected by Whisper models (16000 Hz).
- [`WHISPER_LANGUAGES`](../../06-api-reference/variables/WHISPER_LANGUAGES.md) — Array of 99+ supported language codes.
- [`QWEN3_ASR_LANGUAGES`](../../06-api-reference/variables/QWEN3_ASR_LANGUAGES.md) — Language codes Qwen3-ASR can be forced to.
- [`models.speechToText`](../../06-api-reference/variables/models.md#speechtotext) — Pre-configured Whisper models registry.

:::info Source Code
View the implementation on GitHub:

- [`src/extensions/speech/tasks/whisperSpeechToText.ts` ↗](https://github.com/software-mansion/react-native-executorch/blob/main/packages/react-native-executorch/src/extensions/speech/tasks/whisperSpeechToText.ts)
- [`src/extensions/speech/tasks/qwen3AsrSpeechToText.ts` ↗](https://github.com/software-mansion/react-native-executorch/blob/main/packages/react-native-executorch/src/extensions/speech/tasks/qwen3AsrSpeechToText.ts)
:::
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,8 @@ exports[`public API surface matches the recorded export list 1`] = `
"PASCAL_VOC_LABELS",
"PRIVACY_FILTER_NEMOTRON_LABELS",
"PRIVACY_FILTER_OPENAI_LABELS",
"QWEN3_ASR_LANGUAGES",
"QWEN3_ASR_SAMPLE_RATE_HZ",
"RnExecuTorchError",
"SUPERTONIC_DEFAULT_VOICE_NAMES",
"SUPERTONIC_SAMPLE_RATE",
Expand All @@ -142,6 +144,7 @@ exports[`public API surface matches the recorded export list 1`] = `
"createObjectDetector",
"createPaddleOcr",
"createPrivacyFilter",
"createQwen3AsrSpeechToText",
"createResourceScope",
"createSdxsTextToImage",
"createSemanticSegmenter",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -320,6 +320,17 @@ describe('variant selection rules', () => {
expect(defaultKeyOf(distiluse(registry))).toBe('XNNPACK_8DA4W');
});

it('picks a variant naming two backends only when both are linked', () => {
// Qwen3-ASR's MLX file runs its audio tower on Core ML.
const qwen3Asr = (registry: Node): Node =>
((registry.speechToText as Node).QWEN3_ASR as Node).SIZE_0_6B as Node;

expect(defaultKeyOf(qwen3Asr(registryFor({ os: 'ios' })))).toBe('MLX_COREML_INT4');
expect(
defaultKeyOf(qwen3Asr(registryFor({ os: 'ios', backends: ['XnnpackBackend', 'MLXBackend'] })))
).toBe('XNNPACK_8DA4W');
});

it('applies a pin only on the platform it names', () => {
// The same group pins MLX for iOS only; Android resolves by its own order.
expect(defaultKeyOf(distiluse(registryFor({ os: 'android' })))).toBe('VULKAN_FP16');
Expand Down Expand Up @@ -363,6 +374,13 @@ describe('feature map', () => {
if (typeof value === 'string') {
const match = value.match(/\/(xnnpack|coreml|mlx|vulkan)\//);
if (match) found.add(match[1]!);
// A file can need a second backend, which only its name records:
// `mlx/qwen3_asr_0_6b_coreml_mlx_4w.pte` runs its audio tower on Core ML.
if (value.endsWith('.pte')) {
for (const part of value.split('/').pop()!.split('_')) {
if (/^(xnnpack|coreml|mlx|vulkan)$/.test(part)) found.add(part);
}
}
} else if (Array.isArray(value)) value.forEach(walk);
else if (isObject(value)) Object.values(value).forEach(walk);
};
Expand Down
Loading
Loading