From c540f56176b2ca2b09f377940acc77b1d774afd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20S=C5=82uszniak?= Date: Wed, 21 Oct 2026 15:53:51 +0200 Subject: [PATCH 1/2] feat(tts): add Chinese to Kokoro Bumps phonemis to v0.3.2, which adds a Mandarin G2P matching misaki's Kokoro v1.0, and adds models.textToSpeech.KOKORO.ZH with the 8 upstream zh voices. --- .cspell-wordlist.txt | 8 ++++ apps/benchmarks/scripts/variant-sizes.json | 12 ++++- .../app/kokoro-text-to-speech/index.tsx | 2 + .../02-extensions/speech/02-text-to-speech.md | 2 +- .../cpp/tests/extensions/PhonemizerTest.cpp | 2 +- .../src/extensions/speech/utils/phonemizer.ts | 19 ++++++-- .../react-native-executorch/src/models.ts | 48 +++++++++++++++++-- .../third-party/common/phonemis | 2 +- 8 files changed, 84 insertions(+), 11 deletions(-) diff --git a/.cspell-wordlist.txt b/.cspell-wordlist.txt index 2e3e47e2a5..29bdec7737 100644 --- a/.cspell-wordlist.txt +++ b/.cspell-wordlist.txt @@ -424,3 +424,11 @@ denoiser imageembeddings unparseable zipdepth +xiaobei +xiaoni +xiaoxiao +xiaoyi +yunjian +yunxi +yunxia +yunyang diff --git a/apps/benchmarks/scripts/variant-sizes.json b/apps/benchmarks/scripts/variant-sizes.json index 73da00d94a..95038cfb04 100644 --- a/apps/benchmarks/scripts/variant-sizes.json +++ b/apps/benchmarks/scripts/variant-sizes.json @@ -418,5 +418,15 @@ "https://huggingface.co/software-mansion/react-native-executorch-clip-vit-base-patch32/resolve/v0.11.0/coreml/clip_vit_base_patch32_text_coreml_int8.pte": 63921048, "https://huggingface.co/software-mansion/react-native-executorch-blazeface/resolve/v0.11.0/vulkan/blazeface_vulkan_fp16.pte": 372866, "https://huggingface.co/software-mansion/react-native-executorch-all-MiniLM-L6-v2/resolve/v0.11.0/tokenizer.json": 711544, - "https://huggingface.co/software-mansion/react-native-executorch-all-MiniLM-L6-v2/resolve/v0.11.0/coreml/all_minilm_l6_v2_coreml_int8.pte": 22817740 + "https://huggingface.co/software-mansion/react-native-executorch-all-MiniLM-L6-v2/resolve/v0.11.0/coreml/all_minilm_l6_v2_coreml_int8.pte": 22817740, + "https://huggingface.co/software-mansion/react-native-executorch-kokoro/resolve/v0.11.0/phonemizer/zh/lexicon.json": 2334963, + "https://huggingface.co/software-mansion/react-native-executorch-kokoro/resolve/v0.11.0/phonemizer/zh/segmenter.json": 8288194, + "https://huggingface.co/software-mansion/react-native-executorch-kokoro/resolve/v0.11.0/voices/zf_xiaobei.bin": 522240, + "https://huggingface.co/software-mansion/react-native-executorch-kokoro/resolve/v0.11.0/voices/zf_xiaoni.bin": 522240, + "https://huggingface.co/software-mansion/react-native-executorch-kokoro/resolve/v0.11.0/voices/zf_xiaoxiao.bin": 522240, + "https://huggingface.co/software-mansion/react-native-executorch-kokoro/resolve/v0.11.0/voices/zf_xiaoyi.bin": 522240, + "https://huggingface.co/software-mansion/react-native-executorch-kokoro/resolve/v0.11.0/voices/zm_yunjian.bin": 522240, + "https://huggingface.co/software-mansion/react-native-executorch-kokoro/resolve/v0.11.0/voices/zm_yunxi.bin": 522240, + "https://huggingface.co/software-mansion/react-native-executorch-kokoro/resolve/v0.11.0/voices/zm_yunxia.bin": 522240, + "https://huggingface.co/software-mansion/react-native-executorch-kokoro/resolve/v0.11.0/voices/zm_yunyang.bin": 522240 } diff --git a/apps/speech/app/kokoro-text-to-speech/index.tsx b/apps/speech/app/kokoro-text-to-speech/index.tsx index 678ee5e4fb..b990b0b4cc 100644 --- a/apps/speech/app/kokoro-text-to-speech/index.tsx +++ b/apps/speech/app/kokoro-text-to-speech/index.tsx @@ -25,6 +25,7 @@ const LANGUAGE_OPTIONS = [ { label: 'Hindi', value: 'HI' as const }, { label: 'Polish', value: 'PL' as const }, { label: 'German', value: 'DE' as const }, + { label: 'Chinese', value: 'ZH' as const }, ]; type KokoroLanguage = (typeof LANGUAGE_OPTIONS)[number]['value']; @@ -56,6 +57,7 @@ const SAMPLE_TEXTS: Record = { HI: 'कोकोरो एक छोटा टेक्स्ट-टू-स्पीच मॉडल है जो पूरी तरह से आपके डिवाइस पर चलता है।', PL: 'Kokoro to niewielki model syntezy mowy, który działa w całości na twoim urządzeniu, bez połączenia z internetem.', DE: 'Kokoro ist ein kompaktes Sprachsynthesemodell, das vollständig auf deinem Gerät läuft, ganz ohne Internetverbindung.', + ZH: 'Kokoro 是一个小巧的语音合成模型，完全在你的设备上运行，无需联网。', }; // cspell:enable diff --git a/docs/docs/02-extensions/speech/02-text-to-speech.md b/docs/docs/02-extensions/speech/02-text-to-speech.md index 5877b6c655..fc4a8a2906 100644 --- a/docs/docs/02-extensions/speech/02-text-to-speech.md +++ b/docs/docs/02-extensions/speech/02-text-to-speech.md @@ -175,7 +175,7 @@ The library provides ready-to-use Text-to-Speech models from the [Software Mansi | Model | Variants | Sub-Models & Assets | Sample Rate | Size Range | Supported Backends | Supported Languages | Notes | | :----------------------------- | :----------------------------------------------------------------------- | :--------------------------------------------------------------------------- | :---------- | :---------- | :------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | **Supertonic 3** | [See](../../06-api-reference/variables/models.md#texttospeechsupertonic) | Text Encoder, Duration Predictor, Vector Estimator, Vocoder, Voice Styles | 44.1 kHz | 398 MB | XNNPACK (CPU), MLX (Apple), Vulkan (Android) | [English, Spanish, French, German, Korean, Japanese & more](../../06-api-reference/react-native-executorch/namespaces/speech/variables/SUPERTONIC_SUPPORTED_LANGUAGES.md) | Faster generation and broad multilingual coverage across 10 bundled [speaker styles](../../06-api-reference/variables/SUPERTONIC_DEFAULT_VOICE_NAMES.md), with slightly lower voice naturalness than Kokoro. | -| **Kokoro (Language Packages)** | [See](../../06-api-reference/variables/models.md#texttospeechkokoro) | Duration Predictor, Synthesizer, G2P Lexicon / Neural Model, Language Voices | 24.0 kHz | 332 MB each | XNNPACK (CPU), Core ML (Apple) | English (US/GB), Spanish, French, Italian, Portuguese, Hindi, Polish, German | Exceptional voice naturalness and intonation per language package, but heavier compute per synthesized chunk. | +| **Kokoro (Language Packages)** | [See](../../06-api-reference/variables/models.md#texttospeechkokoro) | Duration Predictor, Synthesizer, G2P Lexicon / Neural Model, Language Voices | 24.0 kHz | 332 MB each | XNNPACK (CPU), Core ML (Apple) | English (US/GB), Spanish, French, Italian, Portuguese, Hindi, Polish, German, Chinese | Exceptional voice naturalness and intonation per language package, but heavier compute per synthesized chunk. | :::note Model-Specific Pipelines Because Text-to-Speech architectures require distinct multi-model orchestration in TypeScript (coordinating phonemizers, duration predictors, flow-matching loops, and neural vocoders), TTS pipelines are model-specific. To use custom voices or fine-tuned weights, provide your modified `.pte` models or custom voice JSON/BIN files matching the [`SupertonicTtsModel`](../../06-api-reference/type-aliases/SupertonicTtsModel.md) or [`KokoroTtsModel`](../../06-api-reference/type-aliases/KokoroTtsModel.md) specifications. diff --git a/packages/react-native-executorch/cpp/tests/extensions/PhonemizerTest.cpp b/packages/react-native-executorch/cpp/tests/extensions/PhonemizerTest.cpp index f81da2a910..011aea9a6f 100644 --- a/packages/react-native-executorch/cpp/tests/extensions/PhonemizerTest.cpp +++ b/packages/react-native-executorch/cpp/tests/extensions/PhonemizerTest.cpp @@ -25,7 +25,7 @@ TEST_F(PhonemizerTest, AcceptsEverySupportedLanguage) { // The language string is passed straight through to phonemis, which picks // the pipeline from it. A profile it does not know throws (below), so this // is what catches a name drifting apart from the TS side. - for (const auto *lang : {"en-us", "en-gb", "de", "fr", "es", "it", "pl", "pt", "hi"}) { + for (const auto *lang : {"en-us", "en-gb", "de", "fr", "es", "it", "pl", "pt", "hi", "zh"}) { EXPECT_TRUE(evalBool(std::string("__rnexecutorch_jsi__.speech.createPhonemizer({ lang: '") + lang + "' }); return true;")) << "rejected language: " << lang; diff --git a/packages/react-native-executorch/src/extensions/speech/utils/phonemizer.ts b/packages/react-native-executorch/src/extensions/speech/utils/phonemizer.ts index 5c19125d8d..c55ba3e2a2 100644 --- a/packages/react-native-executorch/src/extensions/speech/utils/phonemizer.ts +++ b/packages/react-native-executorch/src/extensions/speech/utils/phonemizer.ts @@ -11,7 +11,17 @@ declare const phonemizerBrand: unique symbol; * Union of all supported language codes in the G2P pipeline. * @category Speech / Types */ -export type PhonemizerLanguage = 'en-us' | 'en-gb' | 'fr' | 'es' | 'it' | 'pt' | 'de' | 'pl' | 'hi'; +export type PhonemizerLanguage = + | 'en-us' + | 'en-gb' + | 'fr' + | 'es' + | 'it' + | 'pt' + | 'de' + | 'pl' + | 'hi' + | 'zh'; /** * Configuration options and asset paths for initializing a {@link Phonemizer}. @@ -51,9 +61,10 @@ export type PhonemizedText = { /** IPA transcription of the whole text, the same as `phonemize(text)` returns. */ readonly phonemes: string; /** - * The whitespace-separated words of the input text, in order. Each maps onto - * one space-separated phoneme group, except numbers, which map onto as many as - * they were spelled out into. Empty if the words could not be matched. + * The whitespace-separated words of the input text (segmented words for + * Chinese), in order. Each maps onto one space-separated phoneme group, + * except numbers, which map onto as many as they were spelled out into. + * Empty if the words could not be matched. */ readonly words: readonly PhonemizedWord[]; }; diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index f889717b01..7d8f6f3c67 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -1848,9 +1848,9 @@ const kokoroModelPaths = ( synthesizer: `${KOKORO_ROOT}/${backend}/${dir}/synthesizer_${variant}_${backend}_fp32.pte`, }); -const kokoroVoices = (names: readonly N[]) => +const kokoroVoices = (names: readonly N[], root = KOKORO_ROOT) => names.reduce( - (acc, name) => ({ ...acc, [name]: `${KOKORO_ROOT}/voices/${name}.bin` }), + (acc, name) => ({ ...acc, [name]: `${root}/voices/${name}.bin` }), {} as Record ); @@ -1863,13 +1863,39 @@ const kokoroEnglishPhonemizer = (lang: 'en-us' | 'en-gb') => ({ neuralModelSource: `${KOKORO_PHONEMIZER_ROOT}/${lang}/phonemizer_${lang.replace('-', '_')}.pte`, }); -const kokoroNeuralPhonemizer = >( +const kokoroNeuralPhonemizer = < + const L extends Exclude, +>( lang: L ) => ({ lang, neuralModelSource: `${KOKORO_PHONEMIZER_ROOT}/${lang}/phonemizer_${lang}.pte`, }); +// Chinese is published from the next tag. Its phonemizer segments the text +// into words, with the segmenter data in the tagger slot, and reads each word +// from the lexicon. +const KOKORO_NEXT_ROOT = `${BASE_URL}-kokoro/${NEXT_VERSION_TAG}`; +const KOKORO_ZH_PHONEMIZER = { + lang: 'zh' as const, + taggerSource: `${KOKORO_NEXT_ROOT}/phonemizer/zh/segmenter.json`, + lexiconSource: `${KOKORO_NEXT_ROOT}/phonemizer/zh/lexicon.json`, +}; +const KOKORO_ZH_VOICES = kokoroVoices( + [ + 'zf_xiaobei', + 'zf_xiaoni', + 'zf_xiaoxiao', + 'zf_xiaoyi', + 'zm_yunjian', + 'zm_yunxi', + 'zm_yunxia', + 'zm_yunyang', + ], + KOKORO_NEXT_ROOT +); +type KokoroZhVoice = keyof typeof KOKORO_ZH_VOICES; + const KOKORO_EN_US_XNNPACK_FP32: KokoroTtsModel< 'af_heart' | 'af_river' | 'af_sarah' | 'am_adam' | 'am_michael' | 'am_santa' > = { @@ -1926,6 +1952,12 @@ const KOKORO_DE_XNNPACK_FP32: KokoroTtsModel<'df_anna'> = { phonemizer: kokoroNeuralPhonemizer('de'), voices: kokoroVoices(['df_anna']), }; +const KOKORO_ZH_XNNPACK_FP32: KokoroTtsModel = { + name: 'kokoro', + modelPaths: kokoroModelPaths('xnnpack', 'std', 'standard'), + phonemizer: KOKORO_ZH_PHONEMIZER, + voices: KOKORO_ZH_VOICES, +}; // Core ML counterparts: the same weights, phonemizer and voices as the XNNPACK // presets, only the `.pte` files differ. Their token axis is fixed at 128, so @@ -1987,6 +2019,12 @@ const KOKORO_DE_COREML_FP32: KokoroTtsModel<'df_anna'> = { phonemizer: kokoroNeuralPhonemizer('de'), voices: kokoroVoices(['df_anna']), }; +const KOKORO_ZH_COREML_FP32: KokoroTtsModel = { + name: 'kokoro', + modelPaths: kokoroModelPaths('coreml', 'std', 'standard'), + phonemizer: KOKORO_ZH_PHONEMIZER, + voices: KOKORO_ZH_VOICES, +}; // ============================================================================= // Privacy Filter @@ -3622,6 +3660,10 @@ export const models = { XNNPACK_FP32: KOKORO_DE_XNNPACK_FP32, COREML_FP32: KOKORO_DE_COREML_FP32, }), + ZH: variants({ + XNNPACK_FP32: KOKORO_ZH_XNNPACK_FP32, + COREML_FP32: KOKORO_ZH_COREML_FP32, + }), }, }, diff --git a/packages/react-native-executorch/third-party/common/phonemis b/packages/react-native-executorch/third-party/common/phonemis index 68c81c61b4..d2595d8136 160000 --- a/packages/react-native-executorch/third-party/common/phonemis +++ b/packages/react-native-executorch/third-party/common/phonemis @@ -1 +1 @@ -Subproject commit 68c81c61b423e997aeefb65fbb4854d9a0b919fe +Subproject commit d2595d8136aa51fd6a55b8bfe2e940a6cd2f063d From 8ea788ade04671972c8071b7f1814bfb69aeefa1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20S=C5=82uszniak?= Date: Wed, 21 Oct 2026 16:19:25 +0200 Subject: [PATCH 2/2] fix(tts): drop zf_xiaoni from Kokoro Chinese It is unintelligible in the reference PyTorch model too (Whisper CER ~0.53 vs <=0.03 for the other voices). --- .cspell-wordlist.txt | 1 - apps/benchmarks/scripts/variant-sizes.json | 1 - packages/react-native-executorch/src/models.ts | 14 +++----------- 3 files changed, 3 insertions(+), 13 deletions(-) diff --git a/.cspell-wordlist.txt b/.cspell-wordlist.txt index 29bdec7737..930b641acd 100644 --- a/.cspell-wordlist.txt +++ b/.cspell-wordlist.txt @@ -425,7 +425,6 @@ imageembeddings unparseable zipdepth xiaobei -xiaoni xiaoxiao xiaoyi yunjian diff --git a/apps/benchmarks/scripts/variant-sizes.json b/apps/benchmarks/scripts/variant-sizes.json index 95038cfb04..ab7ff4b88f 100644 --- a/apps/benchmarks/scripts/variant-sizes.json +++ b/apps/benchmarks/scripts/variant-sizes.json @@ -422,7 +422,6 @@ "https://huggingface.co/software-mansion/react-native-executorch-kokoro/resolve/v0.11.0/phonemizer/zh/lexicon.json": 2334963, "https://huggingface.co/software-mansion/react-native-executorch-kokoro/resolve/v0.11.0/phonemizer/zh/segmenter.json": 8288194, "https://huggingface.co/software-mansion/react-native-executorch-kokoro/resolve/v0.11.0/voices/zf_xiaobei.bin": 522240, - "https://huggingface.co/software-mansion/react-native-executorch-kokoro/resolve/v0.11.0/voices/zf_xiaoni.bin": 522240, "https://huggingface.co/software-mansion/react-native-executorch-kokoro/resolve/v0.11.0/voices/zf_xiaoxiao.bin": 522240, "https://huggingface.co/software-mansion/react-native-executorch-kokoro/resolve/v0.11.0/voices/zf_xiaoyi.bin": 522240, "https://huggingface.co/software-mansion/react-native-executorch-kokoro/resolve/v0.11.0/voices/zm_yunjian.bin": 522240, diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index 7d8f6f3c67..f2aae05b9c 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -1874,7 +1874,8 @@ const kokoroNeuralPhonemizer = < // Chinese is published from the next tag. Its phonemizer segments the text // into words, with the segmenter data in the tagger slot, and reads each word -// from the lexicon. +// from the lexicon. Upstream's zf_xiaoni is left out: it is unintelligible even +// in the reference PyTorch model. const KOKORO_NEXT_ROOT = `${BASE_URL}-kokoro/${NEXT_VERSION_TAG}`; const KOKORO_ZH_PHONEMIZER = { lang: 'zh' as const, @@ -1882,16 +1883,7 @@ const KOKORO_ZH_PHONEMIZER = { lexiconSource: `${KOKORO_NEXT_ROOT}/phonemizer/zh/lexicon.json`, }; const KOKORO_ZH_VOICES = kokoroVoices( - [ - 'zf_xiaobei', - 'zf_xiaoni', - 'zf_xiaoxiao', - 'zf_xiaoyi', - 'zm_yunjian', - 'zm_yunxi', - 'zm_yunxia', - 'zm_yunyang', - ], + ['zf_xiaobei', 'zf_xiaoxiao', 'zf_xiaoyi', 'zm_yunjian', 'zm_yunxi', 'zm_yunxia', 'zm_yunyang'], KOKORO_NEXT_ROOT ); type KokoroZhVoice = keyof typeof KOKORO_ZH_VOICES;