diff --git a/.cspell-wordlist.txt b/.cspell-wordlist.txt index 2e3e47e2a5..930b641acd 100644 --- a/.cspell-wordlist.txt +++ b/.cspell-wordlist.txt @@ -424,3 +424,10 @@ denoiser imageembeddings unparseable zipdepth +xiaobei +xiaoxiao +xiaoyi +yunjian +yunxi +yunxia +yunyang diff --git a/apps/benchmarks/scripts/variant-sizes.json b/apps/benchmarks/scripts/variant-sizes.json index 73da00d94a..ab7ff4b88f 100644 --- a/apps/benchmarks/scripts/variant-sizes.json +++ b/apps/benchmarks/scripts/variant-sizes.json @@ -418,5 +418,14 @@ "https://huggingface.co/software-mansion/react-native-executorch-clip-vit-base-patch32/resolve/v0.11.0/coreml/clip_vit_base_patch32_text_coreml_int8.pte": 63921048, "https://huggingface.co/software-mansion/react-native-executorch-blazeface/resolve/v0.11.0/vulkan/blazeface_vulkan_fp16.pte": 372866, "https://huggingface.co/software-mansion/react-native-executorch-all-MiniLM-L6-v2/resolve/v0.11.0/tokenizer.json": 711544, - "https://huggingface.co/software-mansion/react-native-executorch-all-MiniLM-L6-v2/resolve/v0.11.0/coreml/all_minilm_l6_v2_coreml_int8.pte": 22817740 + "https://huggingface.co/software-mansion/react-native-executorch-all-MiniLM-L6-v2/resolve/v0.11.0/coreml/all_minilm_l6_v2_coreml_int8.pte": 22817740, + "https://huggingface.co/software-mansion/react-native-executorch-kokoro/resolve/v0.11.0/phonemizer/zh/lexicon.json": 2334963, + "https://huggingface.co/software-mansion/react-native-executorch-kokoro/resolve/v0.11.0/phonemizer/zh/segmenter.json": 8288194, + "https://huggingface.co/software-mansion/react-native-executorch-kokoro/resolve/v0.11.0/voices/zf_xiaobei.bin": 522240, + "https://huggingface.co/software-mansion/react-native-executorch-kokoro/resolve/v0.11.0/voices/zf_xiaoxiao.bin": 522240, + "https://huggingface.co/software-mansion/react-native-executorch-kokoro/resolve/v0.11.0/voices/zf_xiaoyi.bin": 522240, + "https://huggingface.co/software-mansion/react-native-executorch-kokoro/resolve/v0.11.0/voices/zm_yunjian.bin": 522240, + "https://huggingface.co/software-mansion/react-native-executorch-kokoro/resolve/v0.11.0/voices/zm_yunxi.bin": 522240, + "https://huggingface.co/software-mansion/react-native-executorch-kokoro/resolve/v0.11.0/voices/zm_yunxia.bin": 522240, + "https://huggingface.co/software-mansion/react-native-executorch-kokoro/resolve/v0.11.0/voices/zm_yunyang.bin": 522240 } diff --git a/apps/speech/app/kokoro-text-to-speech/index.tsx b/apps/speech/app/kokoro-text-to-speech/index.tsx index 678ee5e4fb..b990b0b4cc 100644 --- a/apps/speech/app/kokoro-text-to-speech/index.tsx +++ b/apps/speech/app/kokoro-text-to-speech/index.tsx @@ -25,6 +25,7 @@ const LANGUAGE_OPTIONS = [ { label: 'Hindi', value: 'HI' as const }, { label: 'Polish', value: 'PL' as const }, { label: 'German', value: 'DE' as const }, + { label: 'Chinese', value: 'ZH' as const }, ]; type KokoroLanguage = (typeof LANGUAGE_OPTIONS)[number]['value']; @@ -56,6 +57,7 @@ const SAMPLE_TEXTS: Record = { HI: 'कोकोरो एक छोटा टेक्स्ट-टू-स्पीच मॉडल है जो पूरी तरह से आपके डिवाइस पर चलता है।', PL: 'Kokoro to niewielki model syntezy mowy, który działa w całości na twoim urządzeniu, bez połączenia z internetem.', DE: 'Kokoro ist ein kompaktes Sprachsynthesemodell, das vollständig auf deinem Gerät läuft, ganz ohne Internetverbindung.', + ZH: 'Kokoro 是一个小巧的语音合成模型，完全在你的设备上运行，无需联网。', }; // cspell:enable diff --git a/docs/docs/02-extensions/speech/02-text-to-speech.md b/docs/docs/02-extensions/speech/02-text-to-speech.md index 5877b6c655..fc4a8a2906 100644 --- a/docs/docs/02-extensions/speech/02-text-to-speech.md +++ b/docs/docs/02-extensions/speech/02-text-to-speech.md @@ -175,7 +175,7 @@ The library provides ready-to-use Text-to-Speech models from the [Software Mansi | Model | Variants | Sub-Models & Assets | Sample Rate | Size Range | Supported Backends | Supported Languages | Notes | | :----------------------------- | :----------------------------------------------------------------------- | :--------------------------------------------------------------------------- | :---------- | :---------- | :------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | **Supertonic 3** | [See](../../06-api-reference/variables/models.md#texttospeechsupertonic) | Text Encoder, Duration Predictor, Vector Estimator, Vocoder, Voice Styles | 44.1 kHz | 398 MB | XNNPACK (CPU), MLX (Apple), Vulkan (Android) | [English, Spanish, French, German, Korean, Japanese & more](../../06-api-reference/react-native-executorch/namespaces/speech/variables/SUPERTONIC_SUPPORTED_LANGUAGES.md) | Faster generation and broad multilingual coverage across 10 bundled [speaker styles](../../06-api-reference/variables/SUPERTONIC_DEFAULT_VOICE_NAMES.md), with slightly lower voice naturalness than Kokoro. | -| **Kokoro (Language Packages)** | [See](../../06-api-reference/variables/models.md#texttospeechkokoro) | Duration Predictor, Synthesizer, G2P Lexicon / Neural Model, Language Voices | 24.0 kHz | 332 MB each | XNNPACK (CPU), Core ML (Apple) | English (US/GB), Spanish, French, Italian, Portuguese, Hindi, Polish, German | Exceptional voice naturalness and intonation per language package, but heavier compute per synthesized chunk. | +| **Kokoro (Language Packages)** | [See](../../06-api-reference/variables/models.md#texttospeechkokoro) | Duration Predictor, Synthesizer, G2P Lexicon / Neural Model, Language Voices | 24.0 kHz | 332 MB each | XNNPACK (CPU), Core ML (Apple) | English (US/GB), Spanish, French, Italian, Portuguese, Hindi, Polish, German, Chinese | Exceptional voice naturalness and intonation per language package, but heavier compute per synthesized chunk. | :::note Model-Specific Pipelines Because Text-to-Speech architectures require distinct multi-model orchestration in TypeScript (coordinating phonemizers, duration predictors, flow-matching loops, and neural vocoders), TTS pipelines are model-specific. To use custom voices or fine-tuned weights, provide your modified `.pte` models or custom voice JSON/BIN files matching the [`SupertonicTtsModel`](../../06-api-reference/type-aliases/SupertonicTtsModel.md) or [`KokoroTtsModel`](../../06-api-reference/type-aliases/KokoroTtsModel.md) specifications. diff --git a/packages/react-native-executorch/cpp/tests/extensions/PhonemizerTest.cpp b/packages/react-native-executorch/cpp/tests/extensions/PhonemizerTest.cpp index f81da2a910..011aea9a6f 100644 --- a/packages/react-native-executorch/cpp/tests/extensions/PhonemizerTest.cpp +++ b/packages/react-native-executorch/cpp/tests/extensions/PhonemizerTest.cpp @@ -25,7 +25,7 @@ TEST_F(PhonemizerTest, AcceptsEverySupportedLanguage) { // The language string is passed straight through to phonemis, which picks // the pipeline from it. A profile it does not know throws (below), so this // is what catches a name drifting apart from the TS side. - for (const auto *lang : {"en-us", "en-gb", "de", "fr", "es", "it", "pl", "pt", "hi"}) { + for (const auto *lang : {"en-us", "en-gb", "de", "fr", "es", "it", "pl", "pt", "hi", "zh"}) { EXPECT_TRUE(evalBool(std::string("__rnexecutorch_jsi__.speech.createPhonemizer({ lang: '") + lang + "' }); return true;")) << "rejected language: " << lang; diff --git a/packages/react-native-executorch/src/extensions/speech/utils/phonemizer.ts b/packages/react-native-executorch/src/extensions/speech/utils/phonemizer.ts index 5c19125d8d..c55ba3e2a2 100644 --- a/packages/react-native-executorch/src/extensions/speech/utils/phonemizer.ts +++ b/packages/react-native-executorch/src/extensions/speech/utils/phonemizer.ts @@ -11,7 +11,17 @@ declare const phonemizerBrand: unique symbol; * Union of all supported language codes in the G2P pipeline. * @category Speech / Types */ -export type PhonemizerLanguage = 'en-us' | 'en-gb' | 'fr' | 'es' | 'it' | 'pt' | 'de' | 'pl' | 'hi'; +export type PhonemizerLanguage = + | 'en-us' + | 'en-gb' + | 'fr' + | 'es' + | 'it' + | 'pt' + | 'de' + | 'pl' + | 'hi' + | 'zh'; /** * Configuration options and asset paths for initializing a {@link Phonemizer}. @@ -51,9 +61,10 @@ export type PhonemizedText = { /** IPA transcription of the whole text, the same as `phonemize(text)` returns. */ readonly phonemes: string; /** - * The whitespace-separated words of the input text, in order. Each maps onto - * one space-separated phoneme group, except numbers, which map onto as many as - * they were spelled out into. Empty if the words could not be matched. + * The whitespace-separated words of the input text (segmented words for + * Chinese), in order. Each maps onto one space-separated phoneme group, + * except numbers, which map onto as many as they were spelled out into. + * Empty if the words could not be matched. */ readonly words: readonly PhonemizedWord[]; }; diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index f889717b01..f2aae05b9c 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -1848,9 +1848,9 @@ const kokoroModelPaths = ( synthesizer: `${KOKORO_ROOT}/${backend}/${dir}/synthesizer_${variant}_${backend}_fp32.pte`, }); -const kokoroVoices = (names: readonly N[]) => +const kokoroVoices = (names: readonly N[], root = KOKORO_ROOT) => names.reduce( - (acc, name) => ({ ...acc, [name]: `${KOKORO_ROOT}/voices/${name}.bin` }), + (acc, name) => ({ ...acc, [name]: `${root}/voices/${name}.bin` }), {} as Record ); @@ -1863,13 +1863,31 @@ const kokoroEnglishPhonemizer = (lang: 'en-us' | 'en-gb') => ({ neuralModelSource: `${KOKORO_PHONEMIZER_ROOT}/${lang}/phonemizer_${lang.replace('-', '_')}.pte`, }); -const kokoroNeuralPhonemizer = >( +const kokoroNeuralPhonemizer = < + const L extends Exclude, +>( lang: L ) => ({ lang, neuralModelSource: `${KOKORO_PHONEMIZER_ROOT}/${lang}/phonemizer_${lang}.pte`, }); +// Chinese is published from the next tag. Its phonemizer segments the text +// into words, with the segmenter data in the tagger slot, and reads each word +// from the lexicon. Upstream's zf_xiaoni is left out: it is unintelligible even +// in the reference PyTorch model. +const KOKORO_NEXT_ROOT = `${BASE_URL}-kokoro/${NEXT_VERSION_TAG}`; +const KOKORO_ZH_PHONEMIZER = { + lang: 'zh' as const, + taggerSource: `${KOKORO_NEXT_ROOT}/phonemizer/zh/segmenter.json`, + lexiconSource: `${KOKORO_NEXT_ROOT}/phonemizer/zh/lexicon.json`, +}; +const KOKORO_ZH_VOICES = kokoroVoices( + ['zf_xiaobei', 'zf_xiaoxiao', 'zf_xiaoyi', 'zm_yunjian', 'zm_yunxi', 'zm_yunxia', 'zm_yunyang'], + KOKORO_NEXT_ROOT +); +type KokoroZhVoice = keyof typeof KOKORO_ZH_VOICES; + const KOKORO_EN_US_XNNPACK_FP32: KokoroTtsModel< 'af_heart' | 'af_river' | 'af_sarah' | 'am_adam' | 'am_michael' | 'am_santa' > = { @@ -1926,6 +1944,12 @@ const KOKORO_DE_XNNPACK_FP32: KokoroTtsModel<'df_anna'> = { phonemizer: kokoroNeuralPhonemizer('de'), voices: kokoroVoices(['df_anna']), }; +const KOKORO_ZH_XNNPACK_FP32: KokoroTtsModel = { + name: 'kokoro', + modelPaths: kokoroModelPaths('xnnpack', 'std', 'standard'), + phonemizer: KOKORO_ZH_PHONEMIZER, + voices: KOKORO_ZH_VOICES, +}; // Core ML counterparts: the same weights, phonemizer and voices as the XNNPACK // presets, only the `.pte` files differ. Their token axis is fixed at 128, so @@ -1987,6 +2011,12 @@ const KOKORO_DE_COREML_FP32: KokoroTtsModel<'df_anna'> = { phonemizer: kokoroNeuralPhonemizer('de'), voices: kokoroVoices(['df_anna']), }; +const KOKORO_ZH_COREML_FP32: KokoroTtsModel = { + name: 'kokoro', + modelPaths: kokoroModelPaths('coreml', 'std', 'standard'), + phonemizer: KOKORO_ZH_PHONEMIZER, + voices: KOKORO_ZH_VOICES, +}; // ============================================================================= // Privacy Filter @@ -3622,6 +3652,10 @@ export const models = { XNNPACK_FP32: KOKORO_DE_XNNPACK_FP32, COREML_FP32: KOKORO_DE_COREML_FP32, }), + ZH: variants({ + XNNPACK_FP32: KOKORO_ZH_XNNPACK_FP32, + COREML_FP32: KOKORO_ZH_COREML_FP32, + }), }, }, diff --git a/packages/react-native-executorch/third-party/common/phonemis b/packages/react-native-executorch/third-party/common/phonemis index 68c81c61b4..d2595d8136 160000 --- a/packages/react-native-executorch/third-party/common/phonemis +++ b/packages/react-native-executorch/third-party/common/phonemis @@ -1 +1 @@ -Subproject commit 68c81c61b423e997aeefb65fbb4854d9a0b919fe +Subproject commit d2595d8136aa51fd6a55b8bfe2e940a6cd2f063d