From 55a6d4c98ae31d34ddcebe44df2cf111ccfaeeda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20S=C5=82uszniak?= Date: Wed, 21 Oct 2026 12:36:48 +0200 Subject: [PATCH 1/5] feat: add Qwen3-ASR speech-to-text --- .../app/audio-file-transcription/index.tsx | 29 +- .../app/microphone-transcription/index.tsx | 29 +- .../02-extensions/speech/03-speech-to-text.md | 31 +- .../api/__snapshots__/apiSurface.test.ts.snap | 3 + .../tasks/qwen3AsrSpeechToText.test.ts | 212 ++++++ .../speech/tasks/qwen3AsrSpeechToText.ts | 629 ++++++++++++++++++ .../src/hooks/useSpeechToText.ts | 103 ++- packages/react-native-executorch/src/index.ts | 1 + .../react-native-executorch/src/models.ts | 21 + 9 files changed, 1038 insertions(+), 20 deletions(-) create mode 100644 packages/react-native-executorch/__tests__/tasks/qwen3AsrSpeechToText.test.ts create mode 100644 packages/react-native-executorch/src/extensions/speech/tasks/qwen3AsrSpeechToText.ts diff --git a/apps/speech/app/audio-file-transcription/index.tsx b/apps/speech/app/audio-file-transcription/index.tsx index 1bba02bab5..a4c1085eb2 100644 --- a/apps/speech/app/audio-file-transcription/index.tsx +++ b/apps/speech/app/audio-file-transcription/index.tsx @@ -8,7 +8,12 @@ import { TouchableOpacity, TextInput, } from 'react-native'; -import { useSpeechToText, models, WHISPER_SAMPLE_RATE_HZ } from 'react-native-executorch'; +import { + useSpeechToText, + models, + QWEN3_ASR_LANGUAGES, + WHISPER_SAMPLE_RATE_HZ, +} from 'react-native-executorch'; import { decodeAudioData } from 'react-native-audio-api'; import ScreenWrapper from '../../components/ScreenWrapper'; @@ -126,8 +131,19 @@ const MODELS = [ config: models.speechToText.WHISPER.SMALL.MLX_INT8, disabled: Platform.OS !== 'ios', }, + { + name: 'Qwen3-ASR 0.6B (CPU)', + config: models.speechToText.QWEN3_ASR.SIZE_0_6B.XNNPACK_8DA4W, + }, ]; +// Qwen3-ASR detects the language itself; 'auto' leaves it to the model. +const AUTO = 'auto'; +const languagesOf = (config: (typeof MODELS)[number]['config']): readonly string[] => + 'supportedLanguages' in config + ? config.supportedLanguages + : [AUTO, ...Object.keys(QWEN3_ASR_LANGUAGES)]; + function STTAudioContent() { const [selectedModel, setSelectedModel] = useState(MODELS[0]!); const [selectedLanguage, setSelectedLanguage] = useState('en'); @@ -145,7 +161,7 @@ function STTAudioContent() { error: modelError, } = useSpeechToText(selectedModel.config); - const supportedLanguages = selectedModel.config.supportedLanguages; + const supportedLanguages = languagesOf(selectedModel.config); useEffect(() => { return () => { @@ -167,7 +183,8 @@ function STTAudioContent() { setStatus('Transcribing...'); let currentText = ''; - const text = await transcribe(samples, { language: selectedLanguage as any }, (token) => { + const language = selectedLanguage === AUTO ? undefined : selectedLanguage; + const text = await transcribe(samples, { language: language as any }, (token) => { currentText += token; setAudioText(currentText); }); @@ -196,7 +213,7 @@ function STTAudioContent() { ({ label: m.name, value: m, @@ -205,14 +222,14 @@ function STTAudioContent() { selectedValue={selectedModel} onValueChange={(m) => { setSelectedModel(m); - setSelectedLanguage(m.config.supportedLanguages[0]!); + setSelectedLanguage(languagesOf(m.config)[0]!); }} /> {supportedLanguages.length > 1 && ( + 'supportedLanguages' in config + ? config.supportedLanguages + : [AUTO, ...Object.keys(QWEN3_ASR_LANGUAGES)]; + const isSimulator = DeviceInfo.isEmulatorSync(); function STTContent() { @@ -141,7 +157,7 @@ function STTContent() { error: modelError, } = useSpeechToText(selectedModel.config); - const supportedLanguages = selectedModel.config.supportedLanguages; + const supportedLanguages = languagesOf(selectedModel.config); const recorderRef = useRef(null); @@ -168,7 +184,8 @@ function STTContent() { (async () => { try { - const textStream = stream({ language: selectedLanguage as any }); + const language = selectedLanguage === AUTO ? undefined : selectedLanguage; + const textStream = stream({ language: language as any }); for await (const result of textStream) { setCommittedText(result.committed); setNonCommittedText(result.nonCommitted); @@ -216,7 +233,7 @@ function STTContent() { ({ label: m.name, value: m, @@ -225,14 +242,14 @@ function STTContent() { selectedValue={selectedModel} onValueChange={(m) => { setSelectedModel(m); - setSelectedLanguage(m.config.supportedLanguages[0]!); + setSelectedLanguage(languagesOf(m.config)[0]!); }} /> {supportedLanguages.length > 1 && ( ` header is stripped from the transcript. + * + * The fake decoder replays a scripted token sequence, so these tests pin the + * TypeScript around the model rather than the model itself. + */ +import { ConstantDim, RangeDim, f32, i64, method } from '../../src/core/schema'; +import { createQwen3AsrSpeechToText } from '../../src/extensions/speech/tasks/qwen3AsrSpeechToText'; +import { fakeJsi } from '../support/fakeJsi'; +import type { FakeTensor } from '../support/fakeTensor'; +import { exported } from '../support/fixtures'; + +const MODEL_PATH = '/models/qwen3-asr.pte'; +const TOKENIZER_PATH = '/models/tokenizer.json'; +const VAD_PATH = '/models/vad.pte'; + +const VAD_FRAMES = RangeDim(1, 500); +const VAD_SCHEMA = exported(method('forward', [f32(VAD_FRAMES, 400)], [f32(1, VAD_FRAMES, 2)])); +const VAD_CONFIG = { + modelPath: VAD_PATH, + defaultOptions: { + speechThreshold: 0.5, + minSpeechDurationMs: 250, + minSilenceDurationMs: 100, + speechPadMs: 30, + mergeGapMs: 100, + }, +} as const; + +const config = { + name: 'qwen3-asr', + modelPath: MODEL_PATH, + tokenizerPath: TOKENIZER_PATH, + vadModel: VAD_CONFIG, +} as const; + +// prettier-ignore +const TOKENS = [ + '<|endoftext|>', '<|im_start|>', '<|im_end|>', '<|audio_start|>', '<|audio_end|>', + '', 'system', 'user', 'assistant', 'language', 'English', 'hello', 'world', +]; +const ID = Object.fromEntries(TOKENS.map((t, i) => [t, i])); +const DIM = 4; +const VOCAB = TOKENS.length; + +const spec = (melFrames = RangeDim(100, 800, 100), audioTokens = RangeDim(13, 104, 13)) => + exported({ + ...method('preprocess', [f32(RangeDim(401, 480000))], [f32(1, 128, RangeDim(2, 3000))]), + ...method('audio_encoder', [f32(1, 128, melFrames), i64(1)], [f32(1, audioTokens, DIM)]), + ...method('token_embedding', [i64(1, RangeDim(1, 1024))], [f32(1, RangeDim(1, 1024), DIM)]), + ...method('text_decoder', [f32(1, RangeDim(1, 1023), DIM), i64(1)], [f32(1, VOCAB)]), + }); + +type Calls = { + encoder: { width: number; valid: number }[]; + decoder: { length: number; position: number }[]; +}; + +/** Registers the model with a decoder that replays `script`, one token per call. */ +const register = (script: string[], schema = spec()) => { + const calls: Calls = { encoder: [], decoder: [] }; + let step = 0; + fakeJsi.registerModel(MODEL_PATH, { + schema, + execute: (name, inputs, outputs) => { + const [a, b] = inputs as FakeTensor[]; + if (name === 'audio_encoder') { + calls.encoder.push({ width: a!.shape[2]!, valid: Number(b!.getElement(0)) }); + } else if (name === 'text_decoder') { + calls.decoder.push({ length: a!.shape[1]!, position: Number(b!.getElement(0)) }); + const logits = outputs[0]!; + for (let i = 0; i < VOCAB; i++) logits.setElement(i, 0); + logits.setElement(ID[script[Math.min(step++, script.length - 1)]!]!, 1); + } + }, + }); + fakeJsi.registerModel(VAD_PATH, { schema: VAD_SCHEMA }); + fakeJsi.registerTokenizer(TOKENIZER_PATH, { tokens: TOKENS, specialIds: [0, 1, 2, 3, 4, 5] }); + return calls; +}; + +const seconds = (s: number) => new Float32Array(s * 16000); + +describe('createQwen3AsrSpeechToText', () => { + it('strips the detected-language header from the transcript', async () => { + register(['language', 'English', '', 'hello', 'world', '<|im_end|>']); + const stt = await createQwen3AsrSpeechToText(config); + + const streamed: string[] = []; + const text = await stt.transcribe(seconds(3), undefined, (t) => streamed.push(t)); + + expect(text).toBe('hello world'); + // Tokens reach the RN thread through scheduleOnRN, a macrotask. + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(streamed).toEqual(['hello', 'world']); + stt.dispose(); + }); + + it('prefills the prompt around the audio, then decodes one position at a time', async () => { + const calls = register(['language', 'English', '', 'hello', '<|im_end|>']); + const stt = await createQwen3AsrSpeechToText(config); + + await stt.transcribe(seconds(3)); + + // 3 s = 300 mel frames = 39 audio tokens, between a 6-token prefix + // (<|im_start|> system <|im_end|> <|im_start|> user <|audio_start|>) and a + // 4-token suffix (<|audio_end|> <|im_end|> <|im_start|> assistant). + const promptLength = 6 + 39 + 4; + expect(calls.decoder[0]).toEqual({ length: promptLength, position: 0 }); + expect(calls.decoder.slice(1)).toEqual( + [0, 1, 2, 3].map((i) => ({ length: 1, position: promptLength + i })) + ); + stt.dispose(); + }); + + it('forces the language through the prompt and keeps every token', async () => { + const calls = register(['hello', 'world', '<|im_end|>']); + const stt = await createQwen3AsrSpeechToText(config); + + const text = await stt.transcribe(seconds(3), { language: 'en' }); + + expect(text).toBe('hello world'); + // The suffix grows by `language English `. + expect(calls.decoder[0]!.length).toBe(6 + 39 + 4 + 3); + stt.dispose(); + }); + + it('returns nothing for audio the model marks as speechless', async () => { + register(['language', '', '<|im_end|>']); + const stt = await createQwen3AsrSpeechToText(config); + + expect(await stt.transcribe(seconds(2))).toBe(''); + stt.dispose(); + }); + + it('encodes 8 s windows, the last one cut to whole chunks', async () => { + const calls = register(['', '<|im_end|>']); + const stt = await createQwen3AsrSpeechToText(config); + + // 10.5 s = 1050 frames: one full window, then 250 frames padded to 300. + await stt.transcribe(new Float32Array(10.5 * 16000)); + + expect(calls.encoder).toEqual([ + { width: 800, valid: 104 }, + { width: 300, valid: 33 }, + ]); + stt.dispose(); + }); + + it('pads every window to 8 s for a static encoder', async () => { + const calls = register(['', '<|im_end|>'], spec(ConstantDim(800), ConstantDim(104))); + const stt = await createQwen3AsrSpeechToText(config); + + await stt.transcribe(new Float32Array(10.5 * 16000)); + + expect(calls.encoder).toEqual([ + { width: 800, valid: 104 }, + { width: 800, valid: 33 }, + ]); + stt.dispose(); + }); + + it('zero-pads audio shorter than half a second', async () => { + const calls = register(['', '<|im_end|>']); + const stt = await createQwen3AsrSpeechToText(config); + + await stt.transcribe(new Float32Array(1000)); + + // 0.5 s = 50 frames = 7 tokens. + expect(calls.encoder).toEqual([{ width: 100, valid: 7 }]); + stt.dispose(); + }); + + it('rejects an unsupported language', async () => { + register(['<|im_end|>']); + const stt = await createQwen3AsrSpeechToText(config); + + await expect(stt.transcribe(seconds(1), { language: 'xx' as never })).rejects.toThrow( + /not supported/ + ); + stt.dispose(); + }); + + it('rejects a decoder whose hidden size differs from the encoder', async () => { + register(['<|im_end|>']); + fakeJsi.registerModel(MODEL_PATH, { + schema: exported({ + ...spec(), + ...method('text_decoder', [f32(1, RangeDim(1, 1023), DIM * 2), i64(1)], [f32(1, VOCAB)]), + }), + }); + + await expect(createQwen3AsrSpeechToText(config)).rejects.toThrow( + /doesn't match any of the provided variants/ + ); + }); + + it('releases the model, the tokenizer and the nested VAD pipeline on dispose', async () => { + register(['<|im_end|>']); + const stt = await createQwen3AsrSpeechToText(config); + expect(fakeJsi.liveModels()).toEqual([MODEL_PATH, VAD_PATH]); + + stt.dispose(); + + expect(fakeJsi.liveModels()).toEqual([]); + expect(fakeJsi.liveTokenizers()).toEqual([]); + expect(fakeJsi.liveTensors()).toBe(0); + }); +}); diff --git a/packages/react-native-executorch/src/extensions/speech/tasks/qwen3AsrSpeechToText.ts b/packages/react-native-executorch/src/extensions/speech/tasks/qwen3AsrSpeechToText.ts new file mode 100644 index 0000000000..5a25cde3c7 --- /dev/null +++ b/packages/react-native-executorch/src/extensions/speech/tasks/qwen3AsrSpeechToText.ts @@ -0,0 +1,629 @@ +/** + * Qwen3-ASR Speech-to-Text (STT) transcription and live-streaming task pipeline. + */ + +import type { WorkletRuntime } from 'react-native-worklets'; +import { scheduleOnRN, createSynchronizable } from 'react-native-worklets'; + +import { tensor, type Tensor } from '../../../core/tensor'; +import { loadModel } from '../../../core/model'; +import { wrapAsync } from '../../../core/runtime'; +import { + f32, + i64, + method, + validateSpec, + DynamicDim as Dyn, + StaticDim as Static, + type SymbolicDim, +} from '../../../core/schema'; + +import { argmax } from '../../../extensions/math'; +import { loadTokenizer } from '../../nlp/tokenizer'; +import { + createFsmnVoiceActivityDetector, + type FsmnVadModel, + type VadStreamOptions, +} from './fsmnVoiceActivityDetection'; +import { RnExecuTorchError } from '../../../core/error'; +import { createResourceScope } from '../../../core/lifetime'; + +/** + * Sample rate (Hz) Qwen3-ASR models expect their input waveform to be at. + * @category Speech / Constants + */ +export const QWEN3_ASR_SAMPLE_RATE_HZ = 16000; + +const MEL_BINS = 128; +const HOP_LENGTH = 160; // waveform samples per mel frame +const CHUNK_FRAMES = 100; // the encoder convolves 1 s chunks independently +const TOKENS_PER_CHUNK = 13; // audio tokens per full chunk +const WINDOW_FRAMES = 800; // the encoder attends within 8 s windows +const MIN_SAMPLES = QWEN3_ASR_SAMPLE_RATE_HZ / 2; // shorter audio is zero padded, like the reference +const SPLIT_SEARCH_SAMPLES = 5 * QWEN3_ASR_SAMPLE_RATE_HZ; // how far back to look for a quiet cut +const ENERGY_WINDOW_SAMPLES = QWEN3_ASR_SAMPLE_RATE_HZ / 10; // 100 ms energy window for the cut +const STRIDE_SIZE = 1 * QWEN3_ASR_SAMPLE_RATE_HZ; // streaming: new samples before re-transcribing + +/** + * Language codes supported by Qwen3-ASR, mapped to the language names the + * model was trained to emit and accept. + * @category Speech / Constants + */ +export const QWEN3_ASR_LANGUAGES = { + zh: 'Chinese', + en: 'English', + yue: 'Cantonese', + ar: 'Arabic', + de: 'German', + fr: 'French', + es: 'Spanish', + pt: 'Portuguese', + id: 'Indonesian', + it: 'Italian', + ko: 'Korean', + ru: 'Russian', + th: 'Thai', + vi: 'Vietnamese', + ja: 'Japanese', + tr: 'Turkish', + hi: 'Hindi', + ms: 'Malay', + nl: 'Dutch', + sv: 'Swedish', + da: 'Danish', + fi: 'Finnish', + pl: 'Polish', + cs: 'Czech', + fil: 'Filipino', + fa: 'Persian', + el: 'Greek', + ro: 'Romanian', + hu: 'Hungarian', + mk: 'Macedonian', +} as const; + +/** + * Union type of all language codes supported by Qwen3-ASR. Derived from + * {@link QWEN3_ASR_LANGUAGES}. + * @category Speech / Types + */ +export type Qwen3AsrLanguage = keyof typeof QWEN3_ASR_LANGUAGES; + +/** + * Options passed to a single transcription call. + * @category Speech / Types + */ +export type Qwen3AsrSttOptions = { + /** + * Language of the spoken audio. Omit to let the model detect it; set it to + * skip detection and force a transcript in that language. + */ + readonly language?: Qwen3AsrLanguage; +}; + +/** + * Options for the live-streaming transcription API. + * Extends {@link Qwen3AsrSttOptions} with optional VAD tuning. + * @category Speech / Types + */ +export type Qwen3AsrStreamOptions = Qwen3AsrSttOptions & { + /** + * Fine-tuning knobs forwarded to the voice-activity detector. Omit to use + * built-in defaults. + */ + readonly vadOptions?: VadStreamOptions; +}; + +/** + * Paths required to instantiate a Qwen3-ASR speech-to-text model. + * @category Speech / Types + */ +export type Qwen3AsrSttModel = { + /** Pipeline tag, which tells {@link useSpeechToText} which runner to build. */ + readonly name: 'qwen3-asr'; + /** Local path or remote URL of the `.pte` model. */ + readonly modelPath: string; + /** Local path or remote URL of the `tokenizer.json` file. */ + readonly tokenizerPath: string; + /** VAD model configuration used for speech segmentation while streaming. */ + readonly vadModel: FsmnVadModel; +}; + +/** + * Qwen3-ASR speech-to-text task runner. + * @category Speech / Types + */ +export type Qwen3AsrSpeechToText = { + /** + * Releases all allocated native resources. + */ + readonly dispose: () => void; + + /** + * Asynchronously transcribes a pre-recorded mono waveform sampled at + * {@link QWEN3_ASR_SAMPLE_RATE_HZ}. Audio of any length is accepted; long + * recordings are split at quiet points into pieces the model can take in + * one pass. + * @param audio Raw 16 kHz mono PCM audio samples (Float32Array). + * @param options Transcription options. See {@link Qwen3AsrSttOptions}. + * @param onToken Optional callback fired on the RN thread for each decoded + * transcript token. + * @returns A promise resolving to the full transcript string. + * @throws {RnExecuTorchError} With code `INVALID_ARGUMENT` if the language is + * unsupported, `RESOURCE_BUSY` if the model is in use, or `RESOURCE_DISPOSED` + * if disposed. + */ + readonly transcribe: ( + audio: Float32Array, + options?: Qwen3AsrSttOptions, + onToken?: (token: string) => void + ) => Promise; + + /** + * Synchronous version of {@link transcribe} to be executed directly on the + * caller or worklet thread. + */ + readonly transcribeWorklet: ( + audio: Float32Array, + options?: Qwen3AsrSttOptions, + onToken?: (token: string) => void + ) => string; + + /** + * Interrupts and stops any active transcription call. + */ + readonly transcribeStop: () => void; + + /** + * Async generator for real-time microphone transcription. Feed audio with + * {@link streamInsert} and stop with {@link streamStop}. Yields `{ committed, + * nonCommitted }` on every VAD or transcription event: `committed` is the + * finalized transcript so far; `nonCommitted` is the in-progress text that + * may still change. + * @param options Stream options (language and optional VAD tuning). + * See {@link Qwen3AsrStreamOptions}. + * @returns An AsyncGenerator yielding transcript updates. + * @throws {RnExecuTorchError} With code `INVALID_STATE` if streaming is + * already in progress, or `INVALID_ARGUMENT` if the language is unsupported. + */ + readonly stream: ( + options?: Qwen3AsrStreamOptions + ) => AsyncGenerator<{ committed: string; nonCommitted: string }>; + + /** + * Signals the {@link stream} generator to finalize the current segment and + * return. Safe to call even when streaming is not active. + */ + readonly streamStop: () => void; + + /** + * Appends a new PCM chunk to the live streaming buffer consumed by + * {@link stream}. Ignored when streaming is not active. + * @param audioChunk The newly captured audio samples (16 kHz mono Float32 + * PCM). + */ + readonly streamInsert: (audioChunk: Float32Array) => void; +}; + +/** + * Number of audio tokens the encoder emits for `frames` mel frames. + * @param frames Mel frames. + * @returns Audio token count. + */ +function audioTokenCount(frames: number): number { + 'worklet'; + const rem = frames % CHUNK_FRAMES; + let tail = 0; + if (rem > 0) { + const a = Math.floor((rem - 1) / 2) + 1; + const b = Math.floor((a - 1) / 2) + 1; + tail = Math.floor((b - 1) / 2) + 1; + } + return Math.floor(frames / CHUNK_FRAMES) * TOKENS_PER_CHUNK + tail; +} + +/** + * Splits `audio` into pieces of at most `maxSamples`, cutting each at the + * quietest 100 ms within the last 5 s before the limit, so a word is rarely + * cut in half. + * @param audio Input waveform. + * @param maxSamples Longest piece the model accepts. + * @returns `[start, end)` sample ranges covering `audio` without gaps. + */ +function splitAtQuietPoints(audio: Float32Array, maxSamples: number): [number, number][] { + 'worklet'; + const pieces: [number, number][] = []; + let start = 0; + while (audio.length - start > maxSamples) { + const limit = start + maxSamples; + const from = Math.max(start + ENERGY_WINDOW_SAMPLES, limit - SPLIT_SEARCH_SAMPLES); + let energy = 0; + for (let i = from - ENERGY_WINDOW_SAMPLES; i < from; i++) energy += audio[i]! * audio[i]!; + let best = from; + let bestEnergy = energy; + for (let end = from + 1; end <= limit; end++) { + const enter = audio[end - 1]!; + const leave = audio[end - 1 - ENERGY_WINDOW_SAMPLES]!; + energy += enter * enter - leave * leave; + if (energy < bestEnergy) { + bestEnergy = energy; + best = end; + } + } + // Cut in the middle of the quietest window. + const cut = best - ENERGY_WINDOW_SAMPLES / 2; + pieces.push([start, cut]); + start = cut; + } + pieces.push([start, audio.length]); + return pieces; +} + +const qwen3AsrSpec = (melFrames: SymbolicDim, audioTokens: SymbolicDim) => ({ + ...method('preprocess', [f32(Dyn('Samples'))], [f32(1, MEL_BINS, Dyn('Frames'))]), + ...method( + 'audio_encoder', // One 8 s attention window: whole 100-frame chunks, plus the valid token count. + [f32(1, MEL_BINS, melFrames), i64(1)], + [f32(1, audioTokens, 'Dim')] + ), + ...method('token_embedding', [i64(1, Dyn('Tokens'))], [f32(1, Dyn('Tokens'), 'Dim')]), + ...method('text_decoder', [f32(1, Dyn('PrefillLen'), 'Dim'), i64(1)], [f32(1, 'VocabSize')]), +}); + +/** + * Loads a Qwen3-ASR model and returns a set of transcription helpers. + * @category Speech / Tasks + * @param config Model paths and VAD configuration. See {@link Qwen3AsrSttModel}. + * @param runtime Optional worklet runtime thread on which to run the model + * execution. + * @returns A promise resolving to the instantiated {@link Qwen3AsrSpeechToText} + * runner. + * @throws {RnExecuTorchError} With code `LOAD_FAILED` if the model, tokenizer, + * or VAD fails to load, or `SCHEMA_MISMATCH` if model schemas do not match the + * Qwen3-ASR specification. + */ +export async function createQwen3AsrSpeechToText( + config: Qwen3AsrSttModel, + runtime?: WorkletRuntime +): Promise { + const scope = createResourceScope(); + const dispose = scope.dispose; + + try { + const { modelPath, tokenizerPath, vadModel } = config; + const model = scope.track(await wrapAsync(loadModel, runtime)(modelPath)); + const tokenizer = scope.track(await wrapAsync(loadTokenizer, runtime)(tokenizerPath)); + const voiceDetector = scope.track(await createFsmnVoiceActivityDetector(vadModel, runtime)); + + const match = validateSpec(model.schema, { + dynamic: qwen3AsrSpec(Dyn('MelFrames'), Dyn('AudioTokens')), + static: qwen3AsrSpec(Static('MelFrames'), Static('AudioTokens')), + }); + const isEncoderDynamic = match.variant === 'dynamic'; + const [dim, vocabSize] = match.dims.constant('Dim', 'VocabSize'); + const [samples, tokens, prefill] = match.dims.range('Samples', 'Tokens', 'PrefillLen'); + // The KV cache holds as many positions as token_embedding accepts. + const maxContext = tokens.max; + const maxPrefill = prefill.max; + // Longest piece transcribed in one pass: the preprocess bound, capped so + // its audio tokens leave at least half of the context for the transcript. + const maxSamples = Math.min( + samples.max, + Math.floor((maxContext / 2 / TOKENS_PER_CHUNK) * CHUNK_FRAMES) * HOP_LENGTH + ); + + const id = (token: string) => tokenizer.tokenToId(token); + const text = (s: string) => Array.from(tokenizer.encode(s)); + const asrTextToken = id(''); + const eosTokens = [id('<|im_end|>'), id('<|endoftext|>')]; + const promptPrefix = [ + id('<|im_start|>'), + ...text('system\n'), + id('<|im_end|>'), + ...text('\n'), + id('<|im_start|>'), + ...text('user\n'), + id('<|audio_start|>'), + ]; + const promptSuffix = [ + id('<|audio_end|>'), + id('<|im_end|>'), + ...text('\n'), + id('<|im_start|>'), + ...text('assistant\n'), + ]; + + const tensors = [ + tensor('int64', [1]), // tPosition + tensor('int64', [1]), // tValidTokens + tensor('int64', [1, 1]), // tToken + tensor('float32', [1, 1, dim]), // tTokenEmbedding + tensor('float32', [1, vocabSize]), // tLogits + tensor('int32', [1, 1]), // tArgmax + ] as const; + tensors.forEach(scope.track); + const [tPosition, tValidTokens, tToken, tTokenEmbedding, tLogits, tArgmax] = tensors; + + const isCancelled = createSynchronizable(false); + + // Runs `fn` with freshly allocated tensors, disposing them afterwards. + const withTensors = (created: Tensor[], fn: () => R): R => { + 'worklet'; + try { + return fn(); + } finally { + created.forEach((t) => t.dispose()); + } + }; + + const embedTokens = (ids: number[]): Float32Array => { + 'worklet'; + const tIds = tensor('int64', [1, ids.length], BigInt64Array.from(ids.map(BigInt))); + const tOut = tensor('float32', [1, ids.length, dim]); + return withTensors([tIds, tOut], () => { + model.execute('token_embedding', [tIds], [tOut]); + return tOut.getData(new Float32Array(ids.length * dim)); + }); + }; + + const encodeAudio = (audio: Float32Array): Float32Array => { + 'worklet'; + const frames = Math.floor(audio.length / HOP_LENGTH); + const tWave = tensor('float32', [audio.length], audio); + const tMel = tensor('float32', [1, MEL_BINS, frames]); + const mel = withTensors([tWave, tMel], () => { + model.execute('preprocess', [tWave], [tMel]); + return tMel.getData(new Float32Array(MEL_BINS * frames)); + }); + + const embeddings = new Float32Array(audioTokenCount(frames) * dim); + let written = 0; + for (let w0 = 0; w0 < frames; w0 += WINDOW_FRAMES) { + const f = Math.min(WINDOW_FRAMES, frames - w0); + // The partial last chunk is zero padded, as the reference pads it. + const width = isEncoderDynamic ? Math.ceil(f / CHUNK_FRAMES) * CHUNK_FRAMES : WINDOW_FRAMES; + const window = new Float32Array(MEL_BINS * width); + for (let bin = 0; bin < MEL_BINS; bin++) { + window.set(mel.subarray(bin * frames + w0, bin * frames + w0 + f), bin * width); + } + const valid = audioTokenCount(f); + const outTokens = (width / CHUNK_FRAMES) * TOKENS_PER_CHUNK; + const tWindow = tensor('float32', [1, MEL_BINS, width], window); + const tOut = tensor('float32', [1, outTokens, dim]); + withTensors([tWindow, tOut], () => { + tValidTokens.setData(new BigInt64Array([BigInt(valid)])); + model.execute('audio_encoder', [tWindow, tValidTokens], [tOut]); + const out = tOut.getData(new Float32Array(outTokens * dim)); + embeddings.set(out.subarray(0, valid * dim), written); + }); + written += valid * dim; + } + return embeddings; + }; + + // Feeds `embeddings` into the KV cache from `start`; returns the next token. + const prefillEmbeddings = (embeddings: Float32Array, start: number): number => { + 'worklet'; + const length = embeddings.length / dim; + for (let offset = 0; offset < length; offset += maxPrefill) { + const n = Math.min(maxPrefill, length - offset); + const tChunk = tensor( + 'float32', + [1, n, dim], + embeddings.subarray(offset * dim, (offset + n) * dim).slice() + ); + withTensors([tChunk], () => { + tPosition.setData(new BigInt64Array([BigInt(start + offset)])); + model.execute('text_decoder', [tChunk, tPosition], [tLogits]); + }); + } + return tLogits.through(argmax, tArgmax).getData(new Int32Array(1))[0]!; + }; + + const decodeStep = (token: number, position: number): number => { + 'worklet'; + tToken.setData(new BigInt64Array([BigInt(token)])); + model.execute('token_embedding', [tToken], [tTokenEmbedding]); + tPosition.setData(new BigInt64Array([BigInt(position)])); + model.execute('text_decoder', [tTokenEmbedding, tPosition], [tLogits]); + return tLogits.through(argmax, tArgmax).getData(new Int32Array(1))[0]!; + }; + + const transcribePiece = ( + audio: Float32Array, + forcedLanguage: string | undefined, + onToken?: (token: string) => void + ): string => { + 'worklet'; + if (audio.length < MIN_SAMPLES) { + const padded = new Float32Array(MIN_SAMPLES); + padded.set(audio); + audio = padded; + } + const audioEmbeddings = encodeAudio(audio); + const suffix = forcedLanguage + ? [...promptSuffix, ...text(`language ${forcedLanguage}`), asrTextToken] + : promptSuffix; + const prefix = embedTokens(promptPrefix); + const tail = embedTokens(suffix); + const prompt = new Float32Array(prefix.length + audioEmbeddings.length + tail.length); + prompt.set(prefix); + prompt.set(audioEmbeddings, prefix.length); + prompt.set(tail, prefix.length + audioEmbeddings.length); + + let position = prompt.length / dim; + let nextToken = prefillEmbeddings(prompt, 0); + + // Without a forced language the model first writes `language `, + // then ``, then the transcript; only the transcript is kept. + let inTranscript = forcedLanguage !== undefined; + const generated: number[] = []; + while (position < maxContext && !eosTokens.includes(nextToken)) { + if (isCancelled.getBlocking()) break; + if (inTranscript) { + generated.push(nextToken); + if (onToken) scheduleOnRN(onToken, tokenizer.decode(Int32Array.of(nextToken))); + } else if (nextToken === asrTextToken) { + inTranscript = true; + generated.length = 0; // drop the `language ` header + } else { + generated.push(nextToken); + } + nextToken = decodeStep(nextToken, position); + position++; + } + // No `` tag at all: the whole output is the transcript, which + // is how the reference parses it. + if (!inTranscript && onToken && generated.length > 0) { + scheduleOnRN(onToken, tokenizer.decode(Int32Array.from(generated))); + } + const transcript = tokenizer.decode(Int32Array.from(generated)).trim(); + // `language None` marks audio with no speech in it. + return !inTranscript && transcript.startsWith('language ') ? '' : transcript; + }; + + const resolveLanguage = (options?: Qwen3AsrSttOptions): string | undefined => { + 'worklet'; + if (options?.language === undefined) return undefined; + const name = (QWEN3_ASR_LANGUAGES as Record)[options.language]; + if (name === undefined) { + throw RnExecuTorchError( + 'INVALID_ARGUMENT', + `Language "${options.language}" is not supported by Qwen3-ASR.` + ); + } + return name; + }; + + const transcribeWorklet = ( + audio: Float32Array, + options?: Qwen3AsrSttOptions, + onToken?: (token: string) => void + ): string => { + 'worklet'; + isCancelled.setBlocking(false); + const language = resolveLanguage(options); + + const parts: string[] = []; + for (const [start, end] of splitAtQuietPoints(audio, maxSamples)) { + if (isCancelled.getBlocking()) break; + const part = transcribePiece(audio.subarray(start, end), language, onToken); + if (part !== '') parts.push(part); + } + return parts.join(' ').trim(); + }; + + const transcribe = wrapAsync(transcribeWorklet, runtime); + const transcribeStop = () => isCancelled.setBlocking(true); + + let isStreaming = false; + let audioBuffer = new Float32Array(0); + let signal: (() => void) | null = null; + + const streamInsert = (audioChunk: Float32Array): void => { + if (!isStreaming) return; + + const next = new Float32Array(audioBuffer.length + audioChunk.length); + next.set(audioBuffer); + next.set(audioChunk, audioBuffer.length); + audioBuffer = next; + signal?.(); + signal = null; + }; + + const streamStop = (): void => { + if (!isStreaming) return; + + isStreaming = false; + signal?.(); + signal = null; + }; + + async function* stream( + options?: Qwen3AsrStreamOptions + ): AsyncGenerator<{ committed: string; nonCommitted: string }> { + if (isStreaming) { + throw RnExecuTorchError('INVALID_STATE', 'Streaming is already in progress'); + } + resolveLanguage(options); + isStreaming = true; + audioBuffer = new Float32Array(0); + + voiceDetector.resetStream(); + + let isSpeaking = false; + let currentText = ''; + let committedText = ''; + let processedLength = 0; + + const commit = () => { + if (currentText !== '') { + committedText += (committedText ? ' ' : '') + currentText; + currentText = ''; + } + audioBuffer = audioBuffer.slice(processedLength); + processedLength = 0; + }; + + try { + while (isStreaming) { + if (audioBuffer.length - processedLength < STRIDE_SIZE) { + await new Promise((resolve) => (signal = resolve)); + continue; + } + + const newSamples = audioBuffer.slice(processedLength); + processedLength = audioBuffer.length; + + const event = voiceDetector.detectVoiceOnStream(newSamples, options?.vadOptions); + switch (event) { + case 'speechStart': + isSpeaking = true; + break; + case 'speechEnd': + isSpeaking = false; + currentText = await transcribe(audioBuffer.slice(0, processedLength), options); + commit(); + yield { committed: committedText, nonCommitted: '' }; + continue; + } + + if (isSpeaking) { + currentText = await transcribe(audioBuffer.slice(0, processedLength), options); + if (processedLength >= maxSamples) commit(); + yield { committed: committedText, nonCommitted: currentText }; + } else { + const retainSamples = Math.min(audioBuffer.length, STRIDE_SIZE); + audioBuffer = audioBuffer.slice(audioBuffer.length - retainSamples); + processedLength = audioBuffer.length; + yield { committed: committedText, nonCommitted: '' }; + } + } + + if (isSpeaking && audioBuffer.length > 0) { + currentText = await transcribe(audioBuffer, options); + commit(); + yield { committed: committedText, nonCommitted: '' }; + } + } finally { + signal = null; + isStreaming = false; + voiceDetector.resetStream(); + audioBuffer = new Float32Array(0); + } + } + + return { + dispose, + transcribe, + transcribeWorklet, + transcribeStop, + stream, + streamStop, + streamInsert, + }; + } catch (error) { + dispose(); + throw error; + } +} diff --git a/packages/react-native-executorch/src/hooks/useSpeechToText.ts b/packages/react-native-executorch/src/hooks/useSpeechToText.ts index e9c2e5b217..6c279b8388 100644 --- a/packages/react-native-executorch/src/hooks/useSpeechToText.ts +++ b/packages/react-native-executorch/src/hooks/useSpeechToText.ts @@ -4,7 +4,13 @@ import { createWhisperSpeechToText, type WhisperSttModel, type WhisperLanguage, + type WhisperSpeechToText, } from '../extensions/speech/tasks/whisperSpeechToText'; +import { + createQwen3AsrSpeechToText, + type Qwen3AsrSttModel, + type Qwen3AsrSpeechToText, +} from '../extensions/speech/tasks/qwen3AsrSpeechToText'; /** * React hook to load and run a Whisper speech-to-text (ASR) model. @@ -26,11 +32,106 @@ import { export function useSpeechToText( config: WhisperSttModel, options?: ResourceOptions +): { + /** Whether the model is loaded and ready to transcribe. */ + isReady: boolean; + /** The download or load error, if any. */ + error: Error | undefined; + /** Download progress across every asset, in percent. */ + downloadProgress: number; + /** The config with every remote URL resolved to a local path. */ + resource: WhisperSttModel | undefined; + transcribe: WhisperSpeechToText['transcribe'] | undefined; + transcribeWorklet: WhisperSpeechToText['transcribeWorklet'] | undefined; + transcribeStop: WhisperSpeechToText['transcribeStop'] | undefined; + stream: WhisperSpeechToText['stream'] | undefined; + streamInsert: WhisperSpeechToText['streamInsert'] | undefined; + streamStop: WhisperSpeechToText['streamStop'] | undefined; +}; + +/** + * React hook to load and run a Qwen3-ASR speech-to-text model. + * + * This hook manages downloading (if remote URLs are provided) and loading the + * model assets, tokenizer, and voice activity detector, tracking download + * progress and load errors, and releasing native memory when the component + * unmounts or the configuration changes. + * + * For imperative usage, see {@link createQwen3AsrSpeechToText}. + * @category Hooks + * @param config The Qwen3-ASR model configuration. See {@link Qwen3AsrSttModel}. + * @param options Load and caching options. See {@link ResourceOptions}. + * @returns The same object as {@link Qwen3AsrSpeechToText} (without `dispose`), + * combined with loading state and download progress. + * @see {@link Qwen3AsrSpeechToText} + */ +export function useSpeechToText( + config: Qwen3AsrSttModel, + options?: ResourceOptions +): { + /** Whether the model is loaded and ready to transcribe. */ + isReady: boolean; + /** The download or load error, if any. */ + error: Error | undefined; + /** Download progress across every asset, in percent. */ + downloadProgress: number; + /** The config with every remote URL resolved to a local path. */ + resource: Qwen3AsrSttModel | undefined; + transcribe: Qwen3AsrSpeechToText['transcribe'] | undefined; + transcribeWorklet: Qwen3AsrSpeechToText['transcribeWorklet'] | undefined; + transcribeStop: Qwen3AsrSpeechToText['transcribeStop'] | undefined; + stream: Qwen3AsrSpeechToText['stream'] | undefined; + streamInsert: Qwen3AsrSpeechToText['streamInsert'] | undefined; + streamStop: Qwen3AsrSpeechToText['streamStop'] | undefined; +}; + +/** + * React hook to load and run either speech-to-text pipeline, for callers that + * pick the model at runtime (e.g. from a list mixing Whisper and Qwen3-ASR). + * @category Hooks + * @param config A Whisper or Qwen3-ASR model configuration. + * @param options Load and caching options. See {@link ResourceOptions}. + * @returns The runner of whichever pipeline `config` names, combined with + * loading state and download progress. + */ +export function useSpeechToText( + config: WhisperSttModel | Qwen3AsrSttModel, + options?: ResourceOptions +): { + /** Whether the model is loaded and ready to transcribe. */ + isReady: boolean; + /** The download or load error, if any. */ + error: Error | undefined; + /** Download progress across every asset, in percent. */ + downloadProgress: number; + /** The config with every remote URL resolved to a local path. */ + resource: WhisperSttModel | Qwen3AsrSttModel | undefined; + transcribe: (WhisperSpeechToText | Qwen3AsrSpeechToText)['transcribe'] | undefined; + transcribeWorklet: + | (WhisperSpeechToText | Qwen3AsrSpeechToText)['transcribeWorklet'] + | undefined; + transcribeStop: (WhisperSpeechToText | Qwen3AsrSpeechToText)['transcribeStop'] | undefined; + stream: (WhisperSpeechToText | Qwen3AsrSpeechToText)['stream'] | undefined; + streamInsert: (WhisperSpeechToText | Qwen3AsrSpeechToText)['streamInsert'] | undefined; + streamStop: (WhisperSpeechToText | Qwen3AsrSpeechToText)['streamStop'] | undefined; +}; + +export function useSpeechToText( + config: WhisperSttModel | Qwen3AsrSttModel, + options?: ResourceOptions ) { + // Qwen3-ASR configs carry a pipeline tag; Whisper configs predate it. + const factory = ( + sttConfig: WhisperSttModel | Qwen3AsrSttModel + ): Promise | Qwen3AsrSpeechToText> => + 'name' in sttConfig && sttConfig.name === 'qwen3-asr' + ? createQwen3AsrSpeechToText(sttConfig) + : createWhisperSpeechToText(sttConfig as WhisperSttModel); + // Resolves the model, the tokenizer and the nested VAD model in one pass, // with progress weighted across all three. const { resource, downloadProgress, downloadError } = useResourceDownload(config, options); - const { model, error } = useModel(createWhisperSpeechToText, resource); + const { model, error } = useModel(factory, resource); return { isReady: !!model, diff --git a/packages/react-native-executorch/src/index.ts b/packages/react-native-executorch/src/index.ts index b9fff6b01a..10e7d65e90 100644 --- a/packages/react-native-executorch/src/index.ts +++ b/packages/react-native-executorch/src/index.ts @@ -88,6 +88,7 @@ export * from './extensions/nlp/tasks/colbertEmbedding'; export * from './extensions/nlp/tasks/privacyFilter'; export * from './extensions/speech/tasks/fsmnVoiceActivityDetection'; export * from './extensions/speech/tasks/whisperSpeechToText'; +export * from './extensions/speech/tasks/qwen3AsrSpeechToText'; export * from './extensions/speech/tasks/supertonicTextToSpeech'; export * from './extensions/speech/tasks/kokoroTextToSpeech'; diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index de37ec051d..d237eb273d 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -35,6 +35,7 @@ import { type WhisperSttModel, WHISPER_LANGUAGES, } from './extensions/speech/tasks/whisperSpeechToText'; +import type { Qwen3AsrSttModel } from './extensions/speech/tasks/qwen3AsrSpeechToText'; import type { PaddleOcrModel } from './extensions/cv/tasks/paddleOcr'; import type { LLMModel } from './extensions/llm/tasks/llmChatSession'; import { @@ -1626,6 +1627,14 @@ const WHISPER_SMALL_VULKAN_INT8: WhisperSttModel = { vadModel: FSMN_VAD_XNNPACK_FP32, }; +const QWEN3_ASR_0_6B_TOKENIZER = `${BASE_URL}-qwen-3-asr/${NEXT_VERSION_TAG}/0_6b/tokenizer.json`; +const QWEN3_ASR_0_6B_XNNPACK_8DA4W: Qwen3AsrSttModel = { + name: 'qwen3-asr', + modelPath: `${BASE_URL}-qwen-3-asr/${NEXT_VERSION_TAG}/0_6b/xnnpack/qwen3_asr_0_6b_xnnpack_8da4w.pte`, + tokenizerPath: QWEN3_ASR_0_6B_TOKENIZER, + vadModel: FSMN_VAD_XNNPACK_FP32, +}; + // ============================================================================= // Text to Image // ============================================================================= @@ -2981,6 +2990,18 @@ export const models = { ), }, }, + + /** + * Qwen3-ASR speech recognition with integrated Voice Activity Detection. + * Detects the spoken language automatically across 30 languages, or + * transcribes in a language you pass. + */ + QWEN3_ASR: family({ + /** Qwen3-ASR 0.6B: a 0.18B audio encoder in front of a Qwen3-0.6B decoder. */ + SIZE_0_6B: variants({ + XNNPACK_8DA4W: QWEN3_ASR_0_6B_XNNPACK_8DA4W, + }), + }), }, /** From 10b4d7ef05ae90b1140ddc692ce7e362bbf67db5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20S=C5=82uszniak?= Date: Wed, 21 Oct 2026 12:51:58 +0200 Subject: [PATCH 2/5] fix: encode forced Qwen3-ASR language on the worklet runtime --- .../src/extensions/speech/tasks/qwen3AsrSpeechToText.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/react-native-executorch/src/extensions/speech/tasks/qwen3AsrSpeechToText.ts b/packages/react-native-executorch/src/extensions/speech/tasks/qwen3AsrSpeechToText.ts index 5a25cde3c7..7914ae39f4 100644 --- a/packages/react-native-executorch/src/extensions/speech/tasks/qwen3AsrSpeechToText.ts +++ b/packages/react-native-executorch/src/extensions/speech/tasks/qwen3AsrSpeechToText.ts @@ -313,7 +313,10 @@ export async function createQwen3AsrSpeechToText( ); const id = (token: string) => tokenizer.tokenToId(token); - const text = (s: string) => Array.from(tokenizer.encode(s)); + const text = (s: string) => { + 'worklet'; + return Array.from(tokenizer.encode(s)); + }; const asrTextToken = id(''); const eosTokens = [id('<|im_end|>'), id('<|endoftext|>')]; const promptPrefix = [ From 4367e532579b64eae471a2e5d803ed06b7485e2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20S=C5=82uszniak?= Date: Wed, 21 Oct 2026 15:01:13 +0200 Subject: [PATCH 3/5] feat: add Qwen3-ASR Core ML + MLX variant for iOS Audio tower on Core ML, decoder on MLX. A variant key naming two backends resolves only when both are linked. --- .../app/audio-file-transcription/index.tsx | 5 +++++ .../app/microphone-transcription/index.tsx | 5 +++++ .../02-extensions/speech/03-speech-to-text.md | 12 +++++----- .../__tests__/api/modelVariants.test.ts | 11 ++++++++++ .../react-native-executorch/src/models.ts | 22 ++++++++++++++++--- 5 files changed, 46 insertions(+), 9 deletions(-) diff --git a/apps/speech/app/audio-file-transcription/index.tsx b/apps/speech/app/audio-file-transcription/index.tsx index a4c1085eb2..3f9901e3fd 100644 --- a/apps/speech/app/audio-file-transcription/index.tsx +++ b/apps/speech/app/audio-file-transcription/index.tsx @@ -135,6 +135,11 @@ const MODELS = [ name: 'Qwen3-ASR 0.6B (CPU)', config: models.speechToText.QWEN3_ASR.SIZE_0_6B.XNNPACK_8DA4W, }, + { + name: 'Qwen3-ASR 0.6B (MLX + Core ML)', + config: models.speechToText.QWEN3_ASR.SIZE_0_6B.MLX_COREML_INT4, + disabled: Platform.OS !== 'ios', + }, ]; // Qwen3-ASR detects the language itself; 'auto' leaves it to the model. diff --git a/apps/speech/app/microphone-transcription/index.tsx b/apps/speech/app/microphone-transcription/index.tsx index f4f0a58368..e81ddf54bb 100644 --- a/apps/speech/app/microphone-transcription/index.tsx +++ b/apps/speech/app/microphone-transcription/index.tsx @@ -128,6 +128,11 @@ const MODELS = [ name: 'Qwen3-ASR 0.6B (CPU)', config: models.speechToText.QWEN3_ASR.SIZE_0_6B.XNNPACK_8DA4W, }, + { + name: 'Qwen3-ASR 0.6B (MLX + Core ML)', + config: models.speechToText.QWEN3_ASR.SIZE_0_6B.MLX_COREML_INT4, + disabled: Platform.OS !== 'ios', + }, ]; // Qwen3-ASR detects the language itself; 'auto' leaves it to the model. diff --git a/docs/docs/02-extensions/speech/03-speech-to-text.md b/docs/docs/02-extensions/speech/03-speech-to-text.md index b873b75213..e0088bbb4b 100644 --- a/docs/docs/02-extensions/speech/03-speech-to-text.md +++ b/docs/docs/02-extensions/speech/03-speech-to-text.md @@ -218,12 +218,12 @@ const french = await stt.transcribe(audioData, { language: 'fr' }); The library provides ready-to-use Whisper models from the [Software Mansion HuggingFace Whisper Collection](https://huggingface.co/collections/software-mansion/whisper), available in [`models.speechToText`](../../06-api-reference/variables/models.md#speechtotext): -| Model Family | Variants | Size Range | Supported Backends | Languages | Notes | -| :----------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | :----------------- | :------------------------------------------------------------ | :----------------------------------------------------------------------------------------------------- | :-------------------------------------------------------------- | -| **Whisper Tiny** | [`Multilingual`](../../06-api-reference/variables/models.md#speechtotextwhispertiny), [`English`](../../06-api-reference/variables/models.md#speechtotextwhisperentiny) | 57.1 MB – 221.8 MB | XNNPACK (CPU), Core ML (Apple), MLX (Apple), Vulkan (Android) | English / [`WHISPER_LANGUAGES`](../../06-api-reference/variables/WHISPER_LANGUAGES.md) (99+ languages) | Ultra-fast transcription with minimal RAM usage. | -| **Whisper Base** | [`Multilingual`](../../06-api-reference/variables/models.md#speechtotextwhisperbase), [`English`](../../06-api-reference/variables/models.md#speechtotextwhisperenbase) | 97.8 MB – 380.2 MB | XNNPACK (CPU), Core ML (Apple), MLX (Apple), Vulkan (Android) | English / [`WHISPER_LANGUAGES`](../../06-api-reference/variables/WHISPER_LANGUAGES.md) (99+ languages) | Balanced accuracy and speed for general voice dictation. | -| **Whisper Small** | [`Multilingual`](../../06-api-reference/variables/models.md#speechtotextwhispersmall), [`English`](../../06-api-reference/variables/models.md#speechtotextwhisperensmall) | 276.0 MB – 1.05 GB | XNNPACK (CPU), Core ML (Apple), MLX (Apple), Vulkan (Android) | English / [`WHISPER_LANGUAGES`](../../06-api-reference/variables/WHISPER_LANGUAGES.md) (99+ languages) | High-capacity model for complex, noisy, or multi-speaker audio. | -| **Qwen3-ASR 0.6B** | [`0.6B`](../../06-api-reference/variables/models.md#speechtotextqwen3asr) | 707.7 MB | XNNPACK (CPU) | [`QWEN3_ASR_LANGUAGES`](../../06-api-reference/variables/QWEN3_ASR_LANGUAGES.md) (30 languages) | Automatic language detection. | +| Model Family | Variants | Size Range | Supported Backends | Languages | Notes | +| :----------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | :------------------ | :------------------------------------------------------------ | :----------------------------------------------------------------------------------------------------- | :-------------------------------------------------------------- | +| **Whisper Tiny** | [`Multilingual`](../../06-api-reference/variables/models.md#speechtotextwhispertiny), [`English`](../../06-api-reference/variables/models.md#speechtotextwhisperentiny) | 57.1 MB – 221.8 MB | XNNPACK (CPU), Core ML (Apple), MLX (Apple), Vulkan (Android) | English / [`WHISPER_LANGUAGES`](../../06-api-reference/variables/WHISPER_LANGUAGES.md) (99+ languages) | Ultra-fast transcription with minimal RAM usage. | +| **Whisper Base** | [`Multilingual`](../../06-api-reference/variables/models.md#speechtotextwhisperbase), [`English`](../../06-api-reference/variables/models.md#speechtotextwhisperenbase) | 97.8 MB – 380.2 MB | XNNPACK (CPU), Core ML (Apple), MLX (Apple), Vulkan (Android) | English / [`WHISPER_LANGUAGES`](../../06-api-reference/variables/WHISPER_LANGUAGES.md) (99+ languages) | Balanced accuracy and speed for general voice dictation. | +| **Whisper Small** | [`Multilingual`](../../06-api-reference/variables/models.md#speechtotextwhispersmall), [`English`](../../06-api-reference/variables/models.md#speechtotextwhisperensmall) | 276.0 MB – 1.05 GB | XNNPACK (CPU), Core ML (Apple), MLX (Apple), Vulkan (Android) | English / [`WHISPER_LANGUAGES`](../../06-api-reference/variables/WHISPER_LANGUAGES.md) (99+ languages) | High-capacity model for complex, noisy, or multi-speaker audio. | +| **Qwen3-ASR 0.6B** | [`0.6B`](../../06-api-reference/variables/models.md#speechtotextqwen3asr) | 707.7 MB – 808.1 MB | XNNPACK (CPU), Core ML + MLX (Apple) | [`QWEN3_ASR_LANGUAGES`](../../06-api-reference/variables/QWEN3_ASR_LANGUAGES.md) (30 languages) | Automatic language detection. | ## API Reference diff --git a/packages/react-native-executorch/__tests__/api/modelVariants.test.ts b/packages/react-native-executorch/__tests__/api/modelVariants.test.ts index ef303485c6..bcc59c0e11 100644 --- a/packages/react-native-executorch/__tests__/api/modelVariants.test.ts +++ b/packages/react-native-executorch/__tests__/api/modelVariants.test.ts @@ -320,6 +320,17 @@ describe('variant selection rules', () => { expect(defaultKeyOf(distiluse(registry))).toBe('XNNPACK_8DA4W'); }); + it('picks a variant naming two backends only when both are linked', () => { + // Qwen3-ASR's MLX file runs its audio tower on Core ML. + const qwen3Asr = (registry: Node): Node => + ((registry.speechToText as Node).QWEN3_ASR as Node).SIZE_0_6B as Node; + + expect(defaultKeyOf(qwen3Asr(registryFor({ os: 'ios' })))).toBe('MLX_COREML_INT4'); + expect( + defaultKeyOf(qwen3Asr(registryFor({ os: 'ios', backends: ['XnnpackBackend', 'MLXBackend'] }))) + ).toBe('XNNPACK_8DA4W'); + }); + it('applies a pin only on the platform it names', () => { // The same group pins MLX for iOS only; Android resolves by its own order. expect(defaultKeyOf(distiluse(registryFor({ os: 'android' })))).toBe('VULKAN_FP16'); diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index d237eb273d..cc3a1db2f8 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -122,13 +122,21 @@ const CANDIDATE_BACKENDS = getCandidateBackends(); */ function pickVariant(keys: readonly string[], pin?: string): string { const backendOf = (key: string) => key.toLowerCase().split('_')[0] as BackendTag; - - if (pin !== undefined && keys.includes(pin) && CANDIDATE_BACKENDS.includes(backendOf(pin))) { + // A key may name a second backend the file also needs (`MLX_COREML_INT4` + // runs its audio tower on Core ML); every one of them has to be linked. + const isLinked = (key: string) => + key + .toLowerCase() + .split('_') + .filter((part): part is BackendTag => (ALL_BACKENDS as readonly string[]).includes(part)) + .every((tag) => CANDIDATE_BACKENDS.includes(tag)); + + if (pin !== undefined && keys.includes(pin) && isLinked(pin)) { return pin; } for (const tag of CANDIDATE_BACKENDS) { - const match = keys.find((key) => backendOf(key) === tag); + const match = keys.find((key) => backendOf(key) === tag && isLinked(key)); if (match !== undefined) return match; } @@ -1634,6 +1642,12 @@ const QWEN3_ASR_0_6B_XNNPACK_8DA4W: Qwen3AsrSttModel = { tokenizerPath: QWEN3_ASR_0_6B_TOKENIZER, vadModel: FSMN_VAD_XNNPACK_FP32, }; +const QWEN3_ASR_0_6B_MLX_COREML_INT4: Qwen3AsrSttModel = { + name: 'qwen3-asr', + modelPath: `${BASE_URL}-qwen-3-asr/${NEXT_VERSION_TAG}/0_6b/mlx/qwen3_asr_0_6b_coreml_mlx_4w.pte`, + tokenizerPath: QWEN3_ASR_0_6B_TOKENIZER, + vadModel: FSMN_VAD_XNNPACK_FP32, +}; // ============================================================================= // Text to Image @@ -2999,6 +3013,8 @@ export const models = { QWEN3_ASR: family({ /** Qwen3-ASR 0.6B: a 0.18B audio encoder in front of a Qwen3-0.6B decoder. */ SIZE_0_6B: variants({ + /** Audio tower on Core ML, decoder on MLX: needs both backends linked. */ + MLX_COREML_INT4: QWEN3_ASR_0_6B_MLX_COREML_INT4, XNNPACK_8DA4W: QWEN3_ASR_0_6B_XNNPACK_8DA4W, }), }), From 9091e63231a8a38d79f83503b1e8de037d755b79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20S=C5=82uszniak?= Date: Wed, 21 Oct 2026 15:47:09 +0200 Subject: [PATCH 4/5] fix: name an unlinked backend when a model method loads Fixes the Qwen3-ASR docs anchor and lets the feature-map test read backends from file names. --- .../02-extensions/speech/03-speech-to-text.md | 2 +- .../__tests__/api/modelVariants.test.ts | 7 +++ .../cpp/core/model.cpp | 63 +++++++++++++------ .../cpp/tests/core/ModelTest.cpp | 10 +++ .../scripts/download-libs.js | 3 +- 5 files changed, 64 insertions(+), 21 deletions(-) diff --git a/docs/docs/02-extensions/speech/03-speech-to-text.md b/docs/docs/02-extensions/speech/03-speech-to-text.md index e0088bbb4b..393a7b93ad 100644 --- a/docs/docs/02-extensions/speech/03-speech-to-text.md +++ b/docs/docs/02-extensions/speech/03-speech-to-text.md @@ -223,7 +223,7 @@ The library provides ready-to-use Whisper models from the [Software Mansion Hugg | **Whisper Tiny** | [`Multilingual`](../../06-api-reference/variables/models.md#speechtotextwhispertiny), [`English`](../../06-api-reference/variables/models.md#speechtotextwhisperentiny) | 57.1 MB – 221.8 MB | XNNPACK (CPU), Core ML (Apple), MLX (Apple), Vulkan (Android) | English / [`WHISPER_LANGUAGES`](../../06-api-reference/variables/WHISPER_LANGUAGES.md) (99+ languages) | Ultra-fast transcription with minimal RAM usage. | | **Whisper Base** | [`Multilingual`](../../06-api-reference/variables/models.md#speechtotextwhisperbase), [`English`](../../06-api-reference/variables/models.md#speechtotextwhisperenbase) | 97.8 MB – 380.2 MB | XNNPACK (CPU), Core ML (Apple), MLX (Apple), Vulkan (Android) | English / [`WHISPER_LANGUAGES`](../../06-api-reference/variables/WHISPER_LANGUAGES.md) (99+ languages) | Balanced accuracy and speed for general voice dictation. | | **Whisper Small** | [`Multilingual`](../../06-api-reference/variables/models.md#speechtotextwhispersmall), [`English`](../../06-api-reference/variables/models.md#speechtotextwhisperensmall) | 276.0 MB – 1.05 GB | XNNPACK (CPU), Core ML (Apple), MLX (Apple), Vulkan (Android) | English / [`WHISPER_LANGUAGES`](../../06-api-reference/variables/WHISPER_LANGUAGES.md) (99+ languages) | High-capacity model for complex, noisy, or multi-speaker audio. | -| **Qwen3-ASR 0.6B** | [`0.6B`](../../06-api-reference/variables/models.md#speechtotextqwen3asr) | 707.7 MB – 808.1 MB | XNNPACK (CPU), Core ML + MLX (Apple) | [`QWEN3_ASR_LANGUAGES`](../../06-api-reference/variables/QWEN3_ASR_LANGUAGES.md) (30 languages) | Automatic language detection. | +| **Qwen3-ASR 0.6B** | [`0.6B`](../../06-api-reference/variables/models.md#speechtotextqwen3_asr) | 707.7 MB – 808.1 MB | XNNPACK (CPU), Core ML + MLX (Apple) | [`QWEN3_ASR_LANGUAGES`](../../06-api-reference/variables/QWEN3_ASR_LANGUAGES.md) (30 languages) | Automatic language detection. | ## API Reference diff --git a/packages/react-native-executorch/__tests__/api/modelVariants.test.ts b/packages/react-native-executorch/__tests__/api/modelVariants.test.ts index bcc59c0e11..fbf5d11af1 100644 --- a/packages/react-native-executorch/__tests__/api/modelVariants.test.ts +++ b/packages/react-native-executorch/__tests__/api/modelVariants.test.ts @@ -374,6 +374,13 @@ describe('feature map', () => { if (typeof value === 'string') { const match = value.match(/\/(xnnpack|coreml|mlx|vulkan)\//); if (match) found.add(match[1]!); + // A file can need a second backend, which only its name records: + // `mlx/qwen3_asr_0_6b_coreml_mlx_4w.pte` runs its audio tower on Core ML. + if (value.endsWith('.pte')) { + for (const part of value.split('/').pop()!.split('_')) { + if (/^(xnnpack|coreml|mlx|vulkan)$/.test(part)) found.add(part); + } + } } else if (Array.isArray(value)) value.forEach(walk); else if (isObject(value)) Object.values(value).forEach(walk); }; diff --git a/packages/react-native-executorch/cpp/core/model.cpp b/packages/react-native-executorch/cpp/core/model.cpp index 648fb62629..2740f6e1b5 100644 --- a/packages/react-native-executorch/cpp/core/model.cpp +++ b/packages/react-native-executorch/cpp/core/model.cpp @@ -1,5 +1,7 @@ #include "model.h" +#include +#include #include #include #include @@ -42,6 +44,25 @@ void unwrap(RnExecuTorchErrorCode code, const std::string &ctx, executorch::runt } } +// A file can mix backends across methods: Qwen3-ASR's MLX file runs its audio +// tower on Core ML. Name the backend an app did not link instead of leaving +// load_method to fail with a generic error. +void requireRegisteredBackends(const std::string &modelPath, const std::string &methodName, + const std::vector &backends) { + for (const auto &name : backends) { + if (executorch::runtime::get_backend_class(name.c_str()) != nullptr) { + continue; + } + auto configName = name.substr(0, name.rfind("Backend")); + std::ranges::transform(configName, configName.begin(), [](unsigned char c) { return static_cast(std::tolower(c)); }); + throw rnexecutorch::core::error::LoadFailed( + std::format("Model '{}': method '{}' runs on {}, which this app does not link. Add \"{}\" to " + "\"react-native-executorch\".backends in the app's package.json.", + modelPath, methodName, name, configName), + executorch::runtime::Error::NotFound); + } +} + } // namespace namespace rnexecutorch::core::model { @@ -77,29 +98,30 @@ ModelHostObject::ModelHostObject(const std::string &modelPath, const bool eagerL } for (const auto &methodName : methodNames) { - if (eagerLoadMethods && methodName != kGetModelSchemaMethod) { - auto ctx = std::format("Load method '{}' failed.\n" - "\n" - "Common causes:\n" - " 1. Backend not registered\n" - " Ensure backends from `model.backends` are registered\n" - " in the ExecuTorch runtime\n" - " (use `getRegisteredBackends()` to check registered backends).\n" - "\n" - " 2. Bad model export\n" - " The model export itself might be broken or incompatible\n" - " with the currently linked ExecuTorch runtime version.\n" - "\n" - "Error", - methodName); - unwrap(RnExecuTorchErrorCode::LoadFailed, ctx, etModule_->load_method(methodName)); - } - auto ctx = std::format("Method '{}'", methodName); auto methodMeta = unwrap(RnExecuTorchErrorCode::LoadFailed, ctx, etModule_->method_meta(methodName)); + backends_[methodName] = schema::getUsedBackends(methodMeta); + + if (eagerLoadMethods && methodName != kGetModelSchemaMethod) { + requireRegisteredBackends(modelPath_, methodName, backends_[methodName]); + auto loadCtx = std::format("Load method '{}' failed.\n" + "\n" + "Common causes:\n" + " 1. Backend not registered\n" + " Ensure backends from `model.backends` are registered\n" + " in the ExecuTorch runtime\n" + " (use `getRegisteredBackends()` to check registered backends).\n" + "\n" + " 2. Bad model export\n" + " The model export itself might be broken or incompatible\n" + " with the currently linked ExecuTorch runtime version.\n" + "\n" + "Error", + methodName); + unwrap(RnExecuTorchErrorCode::LoadFailed, loadCtx, etModule_->load_method(methodName)); + } spec_[methodName] = schema::methodSpecFromMetadata(methodMeta); - backends_[methodName] = schema::getUsedBackends(methodMeta); if (overrideSpec.contains(methodName)) { spec_[methodName] = std::move(overrideSpec[methodName]); @@ -198,6 +220,9 @@ jsi::Value ModelHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) { schema::validateRuntimeConstraints(rt, methodSpec.runtimeConstraints, inputShapes, std::format("execute: method '{}'", methodName)); + if (!self->etModule_->is_method_loaded(methodName)) { + requireRegisteredBackends(self->modelPath_, methodName, self->backends_.at(methodName)); + } auto startTime = std::chrono::high_resolution_clock::now(); auto executeResult = self->etModule_->execute(methodName, inputs); diff --git a/packages/react-native-executorch/cpp/tests/core/ModelTest.cpp b/packages/react-native-executorch/cpp/tests/core/ModelTest.cpp index afdcd7f7fd..bd163ae0d5 100644 --- a/packages/react-native-executorch/cpp/tests/core/ModelTest.cpp +++ b/packages/react-native-executorch/cpp/tests/core/ModelTest.cpp @@ -177,6 +177,16 @@ TEST_F(ModelTest, ExposesBackendsToJs) { "XnnpackBackend"); } +// The host build links no delegate, so loading the XNNPACK fixture eagerly is +// exactly the case of an app that left out a backend its model needs. The +// error has to name it and the config key that brings it in. +TEST_F(ModelTest, NamesAnUnlinkedBackendOnEagerLoad) { + auto thrown = evalThrowing(std::format("__rnexecutorch_jsi__.loadModel('{}');", kFixture)); + + EXPECT_TRUE(isCodedError(thrown, "LOAD_FAILED", "method 'forward' runs on XnnpackBackend")); + EXPECT_THAT(thrown.message, HasSubstr("Add \"xnnpack\" to \"react-native-executorch\".backends")); +} + TEST_F(ModelTest, ReportsAMissingFileAsLoadFailed) { auto thrown = evalThrowing("__rnexecutorch_jsi__.loadModel('/definitely/not/a/model.pte');"); diff --git a/packages/react-native-executorch/scripts/download-libs.js b/packages/react-native-executorch/scripts/download-libs.js index 45732133b8..185c50fd03 100644 --- a/packages/react-native-executorch/scripts/download-libs.js +++ b/packages/react-native-executorch/scripts/download-libs.js @@ -132,7 +132,8 @@ const FEATURE_MAP = { multimodalLLM: { backends: ['xnnpack', 'mlx', 'vulkan'], libs: ['opencv'] }, // Privacy filter classifiers ship xnnpack + an MLX iOS export. privacyFilter: { backends: ['xnnpack', 'mlx'], libs: [] }, - // Whisper ships xnnpack, coreml, an MLX iOS export and a Vulkan Android one. + // Whisper ships xnnpack, coreml, an MLX iOS export and a Vulkan Android one; + // Qwen3-ASR's MLX file also needs coreml, which runs its audio tower. speechToText: { backends: ['xnnpack', 'coreml', 'mlx', 'vulkan'], libs: [] }, // Kokoro ships xnnpack + coreml; Supertonic adds an MLX iOS export and a // Vulkan Android one. From e315b13bcfc31531c52370d44c092a4f1233441a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20S=C5=82uszniak?= Date: Fri, 23 Oct 2026 00:09:46 +0200 Subject: [PATCH 5/5] fix(libs): bump native libs to v0.10.7-libs --- packages/react-native-executorch/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/react-native-executorch/package.json b/packages/react-native-executorch/package.json index 881fe1e5cd..b20d6c0925 100644 --- a/packages/react-native-executorch/package.json +++ b/packages/react-native-executorch/package.json @@ -1,7 +1,7 @@ { "name": "react-native-executorch", "version": "0.11.0", - "nativeLibsVersion": "0.10.6", + "nativeLibsVersion": "0.10.7", "description": "An easy way to run AI models in React Native with ExecuTorch", "main": "./lib/module/index.js", "module": "./lib/module/index.js",