From 992c77cf0a8ed53042cc55b3d6d32c50f7511b48 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Fri, 2 Oct 2026 14:05:58 +0200 Subject: [PATCH 01/83] refactor(llm): remove native C++ LLM runner and tests --- .../android/CMakeLists.txt | 2 - .../cpp/RnExecutorch.cpp | 2 - .../cpp/extensions/llm/install.cpp | 14 - .../cpp/extensions/llm/install.h | 7 - .../cpp/extensions/llm/llm_runner.cpp | 658 ------------------ .../cpp/extensions/llm/llm_runner.h | 90 --- .../cpp/tests/CMakeLists.txt | 17 +- .../cpp/tests/core/UtilsTest.cpp | 2 +- .../cpp/tests/extensions/LlmRunnerTest.cpp | 99 --- 9 files changed, 6 insertions(+), 885 deletions(-) delete mode 100644 packages/react-native-executorch/cpp/extensions/llm/install.cpp delete mode 100644 packages/react-native-executorch/cpp/extensions/llm/install.h delete mode 100644 packages/react-native-executorch/cpp/extensions/llm/llm_runner.cpp delete mode 100644 packages/react-native-executorch/cpp/extensions/llm/llm_runner.h delete mode 100644 packages/react-native-executorch/cpp/tests/extensions/LlmRunnerTest.cpp diff --git a/packages/react-native-executorch/android/CMakeLists.txt b/packages/react-native-executorch/android/CMakeLists.txt index 56793ecd17..1dc7f2f6bb 100644 --- a/packages/react-native-executorch/android/CMakeLists.txt +++ b/packages/react-native-executorch/android/CMakeLists.txt @@ -41,7 +41,6 @@ file(GLOB CORE_SOURCES ${CPP_DIR}/core/*.cpp) file(GLOB MATH_SOURCES ${CPP_DIR}/extensions/math/*.cpp) file(GLOB NLP_SOURCES ${CPP_DIR}/extensions/nlp/*.cpp) file(GLOB SPEECH_SOURCES ${CPP_DIR}/extensions/speech/*.cpp) -file(GLOB LLM_SOURCES ${CPP_DIR}/extensions/llm/*.cpp) file(GLOB OPENCV_SOURCES ${CPP_DIR}/extensions/cv/*.cpp) # ============================================================================== @@ -80,7 +79,6 @@ set(RNE_SOURCES ${MATH_SOURCES} ${NLP_SOURCES} ${SPEECH_SOURCES} - ${LLM_SOURCES} cpp-adapter.cpp ) diff --git a/packages/react-native-executorch/cpp/RnExecutorch.cpp b/packages/react-native-executorch/cpp/RnExecutorch.cpp index 98fb8c97ff..df1bcb9d4d 100644 --- a/packages/react-native-executorch/cpp/RnExecutorch.cpp +++ b/packages/react-native-executorch/cpp/RnExecutorch.cpp @@ -1,7 +1,6 @@ #include "RnExecutorch.h" #include "core/install.h" -#include "extensions/llm/install.h" #include "extensions/math/install.h" #include "extensions/nlp/install.h" #include "extensions/speech/install.h" @@ -23,7 +22,6 @@ void install(jsi::Runtime &jsiRuntime) { rnexecutorch::extensions::math::install(jsiRuntime, module); rnexecutorch::extensions::nlp::install(jsiRuntime, module); rnexecutorch::extensions::speech::install(jsiRuntime, module); - rnexecutorch::extensions::llm::install(jsiRuntime, module); jsiRuntime.global().setProperty(jsiRuntime, "__rnexecutorch_jsi__", std::move(module)); } diff --git a/packages/react-native-executorch/cpp/extensions/llm/install.cpp b/packages/react-native-executorch/cpp/extensions/llm/install.cpp deleted file mode 100644 index f30a2bd80a..0000000000 --- a/packages/react-native-executorch/cpp/extensions/llm/install.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "install.h" -#include "llm_runner.h" - -namespace rnexecutorch::extensions::llm { -namespace jsi = facebook::jsi; - -void install(facebook::jsi::Runtime &rt, facebook::jsi::Object &module) { - jsi::Object llmModule = jsi::Object(rt); - - install_createLLMRunner(rt, llmModule); - - module.setProperty(rt, "llm", llmModule); -} -} // namespace rnexecutorch::extensions::llm diff --git a/packages/react-native-executorch/cpp/extensions/llm/install.h b/packages/react-native-executorch/cpp/extensions/llm/install.h deleted file mode 100644 index 89d3de5e6b..0000000000 --- a/packages/react-native-executorch/cpp/extensions/llm/install.h +++ /dev/null @@ -1,7 +0,0 @@ -#pragma once - -#include - -namespace rnexecutorch::extensions::llm { -void install(facebook::jsi::Runtime &rt, facebook::jsi::Object &module); -} // namespace rnexecutorch::extensions::llm diff --git a/packages/react-native-executorch/cpp/extensions/llm/llm_runner.cpp b/packages/react-native-executorch/cpp/extensions/llm/llm_runner.cpp deleted file mode 100644 index 56e10584e4..0000000000 --- a/packages/react-native-executorch/cpp/extensions/llm/llm_runner.cpp +++ /dev/null @@ -1,658 +0,0 @@ -// Upstream ExecuTorch annotates experimental LLM APIs (MultimodalRunner, Stats, -// load_tokenizer, create_multimodal_runner) with `ET_EXPERIMENTAL`, which -// expands to `[[deprecated("...")]]`. We suppress -Wdeprecated-declarations so -// Clang does not fail builds on ExecuTorch's experimental API tags. -#ifdef __clang__ -#pragma clang diagnostic ignored "-Wdeprecated-declarations" -#endif - -#include "llm_runner.h" - -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "core/conversions.h" -#include "core/error.h" -#include "core/tensor_helpers.h" - -namespace rnexecutorch::extensions::llm { -namespace jsi = facebook::jsi; -namespace error = rnexecutorch::core::error; -namespace tensor = rnexecutorch::core::tensor; -namespace conversions = rnexecutorch::core::conversions; - -using rnexecutorch::core::types::DType; - -namespace { - -// ============================================================================ -// NOTE & ARCHITECTURAL DISCLAIMER: -// Upstream ExecuTorch's `TextLLMRunner` and `MultimodalRunner` do not expose -// public getters or setters for: -// 1. `pos_` (the current write head index in the preallocated KV cache) -// 2. `prefill_next_token_` (cached token from prefill phase) -// 3. `metadata_` (unordered map holding context size / max sequence length) -// -// Furthermore, the upstream `reset()` method unconditionally clears the KV -// cache position back to 0. In multi-turn chat sessions and tool-calling flows, -// we need the ability to rewind the KV cache back to an exact token position -// (to discard raw/truncated tool calls and re-prefill formatted responses) -// and inspect context cache utilization without clearing entire buffers. -// -// Below, we use standard ISO C++ friend injection via explicit template -// instantiation ([temp.explicit] / Herb Sutter GotW #76) to define injected -// member-pointer accessors for private runner variables in standard-compliant -// C++ without undefined behavior. -// -// COUPLING / MAINTENANCE WARNING: -// This creates a direct compile-time coupling with the internal implementation -// details of ExecuTorch's `TextLLMRunner` and `MultimodalRunner`. If upstream -// ExecuTorch renames or alters any member accessed below, this code will fail -// to compile and must be adjusted accordingly. -// ============================================================================ - -template -struct PrivateMemberAccessor { - friend constexpr auto getPrivateMember(Tag /*tag*/) { return MemberPtr; } -}; - -struct TextRunnerPosTag {}; -struct TextRunnerPrefillTag {}; -struct TextRunnerMetadataTag {}; - -using executorch::extension::llm::TextLLMRunner; - -template struct PrivateMemberAccessor; -constexpr auto getPrivateMember(TextRunnerPosTag /*tag*/); - -template struct PrivateMemberAccessor; -constexpr auto getPrivateMember(TextRunnerPrefillTag /*tag*/); - -template struct PrivateMemberAccessor; -constexpr auto getPrivateMember(TextRunnerMetadataTag /*tag*/); - -struct MMRunnerPosTag {}; -struct MMRunnerPrefillTag {}; -struct MMRunnerMetadataTag {}; - -using executorch::extension::llm::MultimodalRunner; - -template struct PrivateMemberAccessor; -constexpr auto getPrivateMember(MMRunnerPosTag /*tag*/); - -template struct PrivateMemberAccessor; -constexpr auto getPrivateMember(MMRunnerPrefillTag /*tag*/); - -template struct PrivateMemberAccessor; -constexpr auto getPrivateMember(MMRunnerMetadataTag /*tag*/); - -// Members that decide how a prompt is split into prefill chunks. -using executorch::extension::llm::TextPrefiller; - -struct TextRunnerModuleTag {}; -struct TextRunnerPrefillerTag {}; -struct PrefillerMaxSeqLenTag {}; -struct MMRunnerModuleTag {}; -struct MMRunnerTokenizerTag {}; - -template struct PrivateMemberAccessor; -constexpr auto getPrivateMember(TextRunnerModuleTag /*tag*/); - -template struct PrivateMemberAccessor; -constexpr auto getPrivateMember(TextRunnerPrefillerTag /*tag*/); - -template struct PrivateMemberAccessor; -constexpr auto getPrivateMember(PrefillerMaxSeqLenTag /*tag*/); - -template struct PrivateMemberAccessor; -constexpr auto getPrivateMember(MMRunnerModuleTag /*tag*/); - -template struct PrivateMemberAccessor; -constexpr auto getPrivateMember(MMRunnerTokenizerTag /*tag*/); - -// Upstream sizes prefill chunks from `get_max_seq_len`, which is the KV budget, -// not the widest token input the graph accepts (e.g. gemma4 MLX: 2048 vs 511). -// Prompts past the real bound then fail with `Error::NotSupported`. Returns the -// bound declared by the method's first input, or nullopt if it is not a -// rank>=2 integer tensor. -std::optional readPrefillBound(executorch::extension::Module *module, const std::string &methodName) { - if (module == nullptr) { - return std::nullopt; - } - auto meta = module->method_meta(methodName); - if (!meta.ok() || meta->num_inputs() == 0) { - return std::nullopt; - } - auto inputMeta = meta->input_tensor_meta(0); - if (!inputMeta.ok()) { - return std::nullopt; - } - const auto dtype = inputMeta->scalar_type(); - if (dtype != executorch::aten::ScalarType::Long && dtype != executorch::aten::ScalarType::Int) { - return std::nullopt; - } - const auto sizes = inputMeta->sizes(); - if (sizes.size() < 2) { - return std::nullopt; - } - const auto bound = static_cast(sizes[sizes.size() - 1]); - if (bound <= 0) { - return std::nullopt; - } - return bound; -} - -::tokenizers::Tokenizer *getRunnerTokenizer(executorch::extension::llm::IRunner *runner) { - auto *r = dynamic_cast(runner); - if (r == nullptr) { - return nullptr; - } - return (r->*getPrivateMember(MMRunnerTokenizerTag{})).get(); -} - -// `MultimodalPrefiller` does not chunk, so split text into token inputs that fit -// the bound; `MultimodalRunner` prefills them in order. -void appendTextInput(std::vector &inputs, - std::string text, - ::tokenizers::Tokenizer *tokenizer, - int64_t prefillBound) { - if (tokenizer == nullptr || prefillBound <= 0) { - inputs.emplace_back(std::move(text)); - return; - } - - auto encoded = tokenizer->encode(text, 0, 0); - if (!encoded.ok()) { - inputs.emplace_back(std::move(text)); - return; - } - - const auto &tokens = encoded.get(); - const auto chunk = static_cast(prefillBound); - for (size_t offset = 0; offset < tokens.size(); offset += chunk) { - const auto end = std::min(offset + chunk, tokens.size()); - inputs.emplace_back(std::vector(tokens.begin() + static_cast(offset), - tokens.begin() + static_cast(end))); - } -} - -int64_t getRunnerPos(executorch::extension::llm::IRunner *runner, bool isMultimodal) { - if (runner == nullptr) { - return 0; - } - if (isMultimodal) { - auto *r = dynamic_cast(runner); - if (r != nullptr) { - return r->*getPrivateMember(MMRunnerPosTag{}); - } - } else { - auto *r = dynamic_cast(runner); - if (r != nullptr) { - return r->*getPrivateMember(TextRunnerPosTag{}); - } - } - return 0; -} - -int64_t getRunnerMaxSeqLen(executorch::extension::llm::IRunner *runner, bool isMultimodal) { - if (runner == nullptr) { - return 0; - } - const std::unordered_map *meta = nullptr; - if (isMultimodal) { - auto *r = dynamic_cast(runner); - if (r != nullptr) { - meta = &(r->*getPrivateMember(MMRunnerMetadataTag{})); - } - } else { - auto *r = dynamic_cast(runner); - if (r != nullptr) { - meta = &(r->*getPrivateMember(TextRunnerMetadataTag{})); - } - } - if (meta == nullptr) { - return 0; - } - auto it = meta->find(executorch::extension::llm::kMaxContextLen); - if (it != meta->end()) { - return it->second; - } - it = meta->find(executorch::extension::llm::kMaxSeqLen); - if (it != meta->end()) { - return it->second; - } - return 0; -} - -void setRunnerPos(executorch::extension::llm::IRunner *runner, bool isMultimodal, int64_t targetPos) { - if (runner == nullptr) { - return; - } - if (isMultimodal) { - auto *r = dynamic_cast(runner); - if (r != nullptr) { - r->*getPrivateMember(MMRunnerPosTag{}) = targetPos; - r->*getPrivateMember(MMRunnerPrefillTag{}) = std::nullopt; - } - } else { - auto *r = dynamic_cast(runner); - if (r != nullptr) { - r->*getPrivateMember(TextRunnerPosTag{}) = targetPos; - r->*getPrivateMember(TextRunnerPrefillTag{}) = std::nullopt; - } - } -} - -jsi::Object statsToJs(jsi::Runtime &rt, const executorch::extension::llm::Stats &stats) { - jsi::Object obj(rt); - obj.setProperty(rt, "numPromptTokens", static_cast(stats.num_prompt_tokens)); - obj.setProperty(rt, "numGeneratedTokens", static_cast(stats.num_generated_tokens)); - obj.setProperty(rt, "firstTokenMs", static_cast(stats.first_token_ms)); - obj.setProperty(rt, "inferenceStartMs", static_cast(stats.inference_start_ms)); - obj.setProperty(rt, "inferenceEndMs", static_cast(stats.inference_end_ms)); - obj.setProperty(rt, "modelLoadStartMs", static_cast(stats.model_load_start_ms)); - obj.setProperty(rt, "modelLoadEndMs", static_cast(stats.model_load_end_ms)); - return obj; -} - -std::vector parsePrompt( - jsi::Runtime &rt, - const std::string &ctx, - const jsi::Value &value, - const std::vector &supportedModalities, - ::tokenizers::Tokenizer *tokenizer, - int64_t prefillBound) { - - if (value.isString()) { - std::vector inputs; - appendTextInput(inputs, conversions::asType(rt, ctx, value), tokenizer, prefillBound); - return inputs; - } - - auto arr = conversions::asType(rt, ctx, value); - size_t len = arr.length(rt); - - std::vector inputs; - inputs.reserve(len); - - for (size_t i = 0; i < len; ++i) { - auto elem = arr.getValueAtIndex(rt, i); - std::string itemCtx = std::format("{}[{}]", ctx, i); - - if (elem.isString()) { - appendTextInput(inputs, conversions::asType(rt, itemCtx, elem), tokenizer, prefillBound); - continue; - } - auto mediaObj = conversions::asType(rt, itemCtx, elem); - auto kind = conversions::getRequiredProperty(rt, itemCtx, mediaObj, "kind"); - - if (std::ranges::find(supportedModalities, kind) == supportedModalities.end()) { - throw error::InvalidArgument(std::format("{}: Modality '{}' is not supported " - "by this runner instance", - itemCtx, kind)); - } - - if (kind == "image") { - auto tensorJs = conversions::getRequiredProperty(rt, itemCtx, mediaObj, "image"); - auto tensorHost = tensor::fromJs(rt, itemCtx, tensorJs, DType::float32, {"C", "H", "W"}); - auto tensorLock = tensor::tryLockShared(rt, itemCtx, tensorHost); - - const auto &shape = tensorHost->shape_; - const auto C = shape[0]; - const auto H = shape[1]; - const auto W = shape[2]; - - std::vector data(tensorHost->numel_); - std::memcpy(data.data(), tensorHost->data_.get(), tensorHost->numel_ * sizeof(float)); - inputs.emplace_back(executorch::extension::llm::Image(std::move(data), W, H, C)); - } else if (kind == "audio") { - auto tensorJs = conversions::getRequiredProperty(rt, itemCtx, mediaObj, "audio"); - auto tensorHost = tensor::fromJs(rt, itemCtx, tensorJs, DType::float32, {"batch", "n_bins", "n_frames"}); - auto tensorLock = tensor::tryLockShared(rt, itemCtx, tensorHost); - - const auto &shape = tensorHost->shape_; - const auto batchSize = shape[0]; - const auto nBins = shape[1]; - const auto nFrames = shape[2]; - - std::vector data(tensorHost->numel_); - std::memcpy(data.data(), tensorHost->data_.get(), tensorHost->numel_ * sizeof(float)); - inputs.emplace_back(executorch::extension::llm::Audio(std::move(data), batchSize, nBins, nFrames)); - } else { - throw error::InvalidArgument(std::format("{}: Unsupported media kind '{}'", itemCtx, kind)); - } - } - - return inputs; -} -} // namespace - -LLMRunnerHostObject::LLMRunnerHostObject(const std::string &modelPath, - const std::string &tokenizerPath, - const std::vector &modalities) - : modelPath_(modelPath), - tokenizerPath_(tokenizerPath), - modalities_(modalities) { - - auto tokenizer = executorch::extension::llm::load_tokenizer(tokenizerPath); - if (!tokenizer) { - throw error::LoadFailed(std::format("LLMRunner: Failed to load runner tokenizer at path: {}", tokenizerPath)); - } - - // Plain mmap leaves the weights as clean, file-backed pages the kernel - // can evict, which is what the legacy binding has always passed. - constexpr auto kLoadMode = executorch::extension::Module::LoadMode::Mmap; - - if (modalities_.empty()) { - runner_ = executorch::extension::llm::create_text_llm_runner( - modelPath, std::move(tokenizer), std::nullopt, -1.0f, "forward", kLoadMode); - } else { - runner_ = executorch::extension::llm::create_multimodal_runner( - modelPath, std::move(tokenizer), std::nullopt, kLoadMode); - } - - if (!runner_) { - throw error::LoadFailed("LLMRunner: Failed to create llm runner"); - } - - auto loadStatus = runner_->load(); - if (loadStatus != executorch::runtime::Error::Ok) { - std::string errorMsg = executorch::runtime::to_string(loadStatus); - throw error::LoadFailed(std::format("LLMRunner: Failed to load model: {}", errorMsg), loadStatus); - } - - // Clamp prefill chunks to the graph's real bound (see `readPrefillBound`). - // Text models take tokens on `forward`, multimodal ones on `token_embedding`. - if (modalities_.empty()) { - auto *textRunner = dynamic_cast(runner_.get()); - if (textRunner != nullptr) { - const auto &module = textRunner->*getPrivateMember(TextRunnerModuleTag{}); - const auto bound = readPrefillBound(module.get(), "forward"); - auto &prefiller = textRunner->*getPrivateMember(TextRunnerPrefillerTag{}); - if (bound.has_value() && prefiller) { - // Upstream allows max_seq_len < max_context_len and chunks itself. - auto &maxSeqLen = (*prefiller).*getPrivateMember(PrefillerMaxSeqLenTag{}); - maxSeqLen = std::min(maxSeqLen, *bound); - prefillBound_ = maxSeqLen; - } - } - } else { - auto *mmRunner = dynamic_cast(runner_.get()); - if (mmRunner != nullptr) { - const auto &module = mmRunner->*getPrivateMember(MMRunnerModuleTag{}); - const auto bound = - readPrefillBound(module.get(), executorch::extension::llm::kTokenEmbeddingMethod); - if (bound.has_value()) { - prefillBound_ = *bound; - } - } - } -} - -std::unique_lock LLMRunnerHostObject::tryLockUnique(std::string_view ctx) { - std::unique_lock lock(mutex_, std::try_to_lock); - if (!lock.owns_lock()) { - throw error::ResourceBusy(std::format("{}: Runner is already in use", ctx)); - } - if (!runner_) { - throw error::ResourceDisposed(std::format("{}: Runner has been disposed", ctx)); - } - return lock; -} - -jsi::Value LLMRunnerHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) { - auto nameStr = name.utf8(rt); - - if (nameStr == "modelPath") { - return jsi::String::createFromUtf8(rt, modelPath_); - } - - if (nameStr == "tokenizerPath") { - return jsi::String::createFromUtf8(rt, tokenizerPath_); - } - - if (nameStr == "modalities") { - return conversions::toJsiArray(rt, modalities_); - } - - if (nameStr == "generate") { - auto self = shared_from_this(); - auto fnBody = [self](jsi::Runtime &rt, const jsi::Value & /*thisVal*/, const jsi::Value *args, size_t count) -> jsi::Value { - if (count < 1) { - throw error::InvalidArgument("LLMRunner.generate: Usage: generate(prompt, config?, onToken?)"); - } - - executorch::extension::llm::GenerationConfig config; - config.echo = false; - if (count > 1 && !args[1].isUndefined() && !args[1].isNull()) { - auto configObj = conversions::asType(rt, "LLMRunner.generate: config", args[1]); - if (auto ignoreEosOpt = conversions::getOptionalProperty(rt, "LLMRunner.generate: config", configObj, "ignoreEos")) { - config.ignore_eos = *ignoreEosOpt; - } - if (auto maxNewTokensOpt = conversions::getOptionalProperty(rt, "LLMRunner.generate: config", configObj, "maxNewTokens")) { - config.max_new_tokens = *maxNewTokensOpt; - } - if (auto tempOpt = conversions::getOptionalProperty(rt, "LLMRunner.generate: config", configObj, "temperature")) { - config.temperature = *tempOpt; - } - } - - std::function tokenCallback; - if (count > 2 && !args[2].isUndefined() && !args[2].isNull()) { - auto tokenFn = std::make_shared(conversions::asType(rt, "LLMRunner.generate: onToken", args[2])); - tokenCallback = [&rt, tokenFn](const std::string &token) { - tokenFn->call(rt, jsi::String::createFromUtf8(rt, token)); - }; - } - - auto finalStats = std::make_shared(); - auto statsCallback = [finalStats](const executorch::extension::llm::Stats &stats) { - finalStats->num_prompt_tokens = stats.num_prompt_tokens; - finalStats->num_generated_tokens = stats.num_generated_tokens; - finalStats->first_token_ms = stats.first_token_ms; - finalStats->inference_start_ms = stats.inference_start_ms; - finalStats->inference_end_ms = stats.inference_end_ms; - finalStats->model_load_start_ms = stats.model_load_start_ms; - finalStats->model_load_end_ms = stats.model_load_end_ms; - finalStats->aggregate_sampling_time_ms = stats.aggregate_sampling_time_ms; - }; - - auto lock = self->tryLockUnique("LLMRunner.generate"); - - auto genStatus = executorch::runtime::Error::Ok; - if (self->modalities_.empty()) { - auto prompt = conversions::asType(rt, "LLMRunner.generate: prompt", args[0]); - genStatus = self->runner_->generate(prompt, config, tokenCallback, statsCallback); - } else { - auto inputs = parsePrompt(rt, "LLMRunner.generate", args[0], self->modalities_, - getRunnerTokenizer(self->runner_.get()), self->prefillBound_); - auto *multimodalRunner = dynamic_cast(self->runner_.get()); - if (multimodalRunner == nullptr) { - throw error::InvalidArgument("LLMRunner.generate: Runner instance is not a multimodal model"); - } - genStatus = multimodalRunner->generate(inputs, config, tokenCallback, statsCallback); - } - - if (genStatus != executorch::runtime::Error::Ok) { - std::string errorMsg = executorch::runtime::to_string(genStatus); - throw error::ExecutionFailed(std::format("LLMRunner.generate: Failed to generate: {}", errorMsg), genStatus); - } - - return statsToJs(rt, *finalStats); - }; - return jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, "generate"), 1, error::guarded(fnBody)); - } - - if (nameStr == "prefill") { - auto self = shared_from_this(); - auto fnBody = [self](jsi::Runtime &rt, const jsi::Value & /*thisVal*/, const jsi::Value *args, size_t count) -> jsi::Value { - if (count != 1) { - throw error::InvalidArgument("LLMRunner.prefill: Usage: prefill(prompt)"); - } - - auto lock = self->tryLockUnique("LLMRunner.prefill"); - - auto inputs = parsePrompt(rt, "LLMRunner.prefill", args[0], self->modalities_, - getRunnerTokenizer(self->runner_.get()), self->prefillBound_); - auto result = self->runner_->prefill(inputs); - - if (!result.ok()) { - std::string errorMsg = executorch::runtime::to_string(result.error()); - throw error::ExecutionFailed(std::format("LLMRunner.prefill: Failed: {}", errorMsg), result.error()); - } - - return jsi::Value::undefined(); - }; - return jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, "prefill"), 1, error::guarded(fnBody)); - } - - if (nameStr == "stop") { - auto self = shared_from_this(); - auto fnBody = [self](jsi::Runtime & /*rt*/, const jsi::Value & /*thisVal*/, const jsi::Value * /*args*/, size_t count) -> jsi::Value { - if (count != 0) { - throw error::InvalidArgument("LLMRunner.stop: Usage: stop()"); - } - - // Intentionally no mutex here: stop() is designed to be called - // concurrently to interrupt an in-progress generate(). Taking the - // lock would block until generate() finishes, defeating the point. - // runner_ is only cleared by dispose() on this same (JS) thread, - // so reading it lock-free here is safe. - if (!self->runner_) { - throw error::ResourceDisposed("LLMRunner.stop: Runner has been disposed"); - } - self->runner_->stop(); - return jsi::Value::undefined(); - }; - return jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, "stop"), 0, error::guarded(fnBody)); - } - - if (nameStr == "getKVCacheState") { - auto self = shared_from_this(); - auto fnBody = [self](jsi::Runtime &rt, const jsi::Value & /*thisVal*/, const jsi::Value * /*args*/, size_t count) -> jsi::Value { - if (count != 0) { - throw error::InvalidArgument("LLMRunner.getKVCacheState: Usage: getKVCacheState()"); - } - - auto lock = self->tryLockUnique("LLMRunner.getKVCacheState"); - - bool isMultimodal = !self->modalities_.empty(); - int64_t pos = getRunnerPos(self->runner_.get(), isMultimodal); - int64_t maxSeqLen = getRunnerMaxSeqLen(self->runner_.get(), isMultimodal); - int64_t remaining = std::max(0, maxSeqLen - pos); - double usageRatio = maxSeqLen > 0 ? static_cast(pos) / static_cast(maxSeqLen) : 0.0; - - jsi::Object obj(rt); - obj.setProperty(rt, "pos", static_cast(pos)); - obj.setProperty(rt, "maxSeqLen", static_cast(maxSeqLen)); - obj.setProperty(rt, "remainingTokens", static_cast(remaining)); - obj.setProperty(rt, "usageRatio", usageRatio); - return obj; - }; - return jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, "getKVCacheState"), 0, error::guarded(fnBody)); - } - - if (nameStr == "reset") { - auto self = shared_from_this(); - auto fnBody = [self](jsi::Runtime &rt, const jsi::Value & /*thisVal*/, const jsi::Value *args, size_t count) -> jsi::Value { - if (count > 1) { - throw error::InvalidArgument("LLMRunner.reset: Usage: reset(targetPos?)"); - } - - auto lock = self->tryLockUnique("LLMRunner.reset"); - if (count == 0 || args[0].isUndefined()) { - (*self->runner_).reset(); - } else { - auto targetPos = static_cast(conversions::asType(rt, "LLMRunner.reset", args[0])); - bool isMultimodal = !self->modalities_.empty(); - int64_t currentPos = getRunnerPos(self->runner_.get(), isMultimodal); - if (targetPos < 0 || targetPos > currentPos) { - throw error::InvalidArgument(std::format("LLMRunner.reset: targetPos must be in range [0, {}], but got {}", - currentPos, targetPos)); - } - if (targetPos == 0) { - (*self->runner_).reset(); - } else { - setRunnerPos(self->runner_.get(), isMultimodal, targetPos); - } - } - - return jsi::Value::undefined(); - }; - return jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, "reset"), 1, error::guarded(fnBody)); - } - - if (nameStr == "dispose") { - auto self = shared_from_this(); - auto fnBody = [self](jsi::Runtime & /*rt*/, const jsi::Value & /*thisVal*/, const jsi::Value * /*args*/, size_t count) -> jsi::Value { - if (count != 0) { - throw error::InvalidArgument("LLMRunner.dispose: Usage: dispose()"); - } - - // Signal stop before locking so any in-progress generate() exits - // quickly; we then block on the lock until it returns and clear - // the runner, which frees the model. Idempotent: a second - // dispose() finds a null runner_ and is a no-op. - if (self->runner_) { - self->runner_->stop(); - } - - std::unique_lock lock(self->mutex_); - self->runner_ = nullptr; - - return jsi::Value::undefined(); - }; - return jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, "dispose"), 0, error::guarded(fnBody)); - } - - return jsi::Value::undefined(); -} - -std::vector LLMRunnerHostObject::getPropertyNames(jsi::Runtime &rt) { - std::vector properties; - properties.push_back(jsi::PropNameID::forAscii(rt, "modelPath")); - properties.push_back(jsi::PropNameID::forAscii(rt, "tokenizerPath")); - properties.push_back(jsi::PropNameID::forAscii(rt, "modalities")); - properties.push_back(jsi::PropNameID::forAscii(rt, "prefill")); - properties.push_back(jsi::PropNameID::forAscii(rt, "generate")); - properties.push_back(jsi::PropNameID::forAscii(rt, "stop")); - properties.push_back(jsi::PropNameID::forAscii(rt, "getKVCacheState")); - properties.push_back(jsi::PropNameID::forAscii(rt, "reset")); - properties.push_back(jsi::PropNameID::forAscii(rt, "dispose")); - return properties; -} - -void install_createLLMRunner(jsi::Runtime &rt, jsi::Object &module) { - const auto *name = "createLLMRunner"; - auto fnBody = [](jsi::Runtime &rt, const jsi::Value & /*thisVal*/, const jsi::Value *args, size_t count) -> jsi::Value { - if (count < 2 || count > 3) { - throw error::InvalidArgument("createLLMRunner: Usage: createLLMRunner(modelPath, tokenizerPath, modalities?)"); - } - - auto modelPath = conversions::asType(rt, "createLLMRunner: modelPath", args[0]); - auto tokenizerPath = conversions::asType(rt, "createLLMRunner: tokenizerPath", args[1]); - - std::vector modalities; - if (count > 2 && !args[2].isNull() && !args[2].isUndefined()) { - modalities = conversions::asVector(rt, "createLLMRunner: modalities", args[2]); - } - - auto runnerInstance = std::make_shared(modelPath, tokenizerPath, std::move(modalities)); - return jsi::Object::createFromHostObject(rt, runnerInstance); - }; - - module.setProperty(rt, name, jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, name), 2, error::guarded(fnBody))); -} -} // namespace rnexecutorch::extensions::llm diff --git a/packages/react-native-executorch/cpp/extensions/llm/llm_runner.h b/packages/react-native-executorch/cpp/extensions/llm/llm_runner.h deleted file mode 100644 index 841e36f977..0000000000 --- a/packages/react-native-executorch/cpp/extensions/llm/llm_runner.h +++ /dev/null @@ -1,90 +0,0 @@ -#pragma once - -// Upstream ExecuTorch annotates experimental LLM APIs (TextLLMRunner, Stats, -// etc.) with `ET_EXPERIMENTAL`, which expands to `[[deprecated("This API is -// experimental...")]]`. We suppress -Wdeprecated-declarations so Clang does not -// fail builds on ExecuTorch's experimental API tags. -#ifdef __clang__ -#pragma clang diagnostic ignored "-Wdeprecated-declarations" -#endif - -#include -#include -#include -#include -#include -#include - -#include - -#include - -namespace rnexecutorch::extensions::llm { -/** - * [EXPERIMENTAL] JSI HostObject wrapping an ExecuTorch LLM runner instance - * (`executorch::extension::llm::IRunner`). - * - * @note This implementation is EXPERIMENTAL. It builds upon upstream ExecuTorch - * experimental LLM runtime APIs (`ET_EXPERIMENTAL`) and leverages friend injection - * via explicit template instantiation to manage KV cache write heads and context metrics. - * The underlying interfaces and behavior might change across ExecuTorch releases. - * - * Exposes methods to JavaScript for prefilling prompt context, generating token - * stream continuations, interrupting generation, resetting the KV cache context, - * and releasing native model memory. - */ -class LLMRunnerHostObject final : public facebook::jsi::HostObject, - public std::enable_shared_from_this { -public: - /** - * Constructs an LLMRunnerHostObject by loading an ExecuTorch model binary - * and tokenizer. - * - * @param modelPath Absolute file system path to the `.pte` LLM model - * binary. - * @param tokenizerPath Absolute file system path to the local tokenizer - * configuration file (e.g. `tokenizer.json`). - * @param modalities Vector of supported input modality names (e.g. - * `{"image"}`). - * @throws core::error::RnExecuTorchException with code LoadFailed if - * loading the model or tokenizer fails. - */ - explicit LLMRunnerHostObject(const std::string &modelPath, - const std::string &tokenizerPath, - const std::vector &modalities); - - facebook::jsi::Value get(facebook::jsi::Runtime &rt, const facebook::jsi::PropNameID &name) override; - std::vector getPropertyNames(facebook::jsi::Runtime &rt) override; - -private: - /** - * Tries to acquire a unique lock on the runner's mutex. - * - * @param ctx Context description used to generate helpful error messages. - * @return A unique lock protecting the runner. - * @throws core::error::RnExecuTorchException with code ResourceBusy if the - * lock is currently held by another thread, or ResourceDisposed if the - * runner has already been disposed. - */ - [[nodiscard]] std::unique_lock tryLockUnique(std::string_view ctx); - - /** Owning pointer to the underlying ExecuTorch IRunner instance. */ - std::unique_ptr runner_; - /** Mutex guarding concurrent access to prefill and generation operations. */ - std::mutex mutex_; - /** File system path to the loaded `.pte` model binary. */ - std::string modelPath_; - /** File system path to the loaded tokenizer configuration file. */ - std::string tokenizerPath_; - /** List of supported non-text input modalities. */ - std::vector modalities_; - /** - * Longest token sequence the loaded graph accepts in one prefill step, read - * off the model's own method metadata. 0 when the model does not declare - * one, in which case prompts are passed through unsplit. - */ - int64_t prefillBound_ = 0; -}; - -void install_createLLMRunner(facebook::jsi::Runtime &rt, facebook::jsi::Object &module); -} // namespace rnexecutorch::extensions::llm diff --git a/packages/react-native-executorch/cpp/tests/CMakeLists.txt b/packages/react-native-executorch/cpp/tests/CMakeLists.txt index 8de778f7d2..28ba81983b 100644 --- a/packages/react-native-executorch/cpp/tests/CMakeLists.txt +++ b/packages/react-native-executorch/cpp/tests/CMakeLists.txt @@ -100,18 +100,13 @@ rne_import_et_lib(et_flat_tensor "extension/flat_tensor" extension_flat_tensor) rne_import_et_lib(et_named_data_map "extension/named_data_map" extension_named_data_map) rne_import_et_lib(et_tokenizers "extension/llm/tokenizers" tokenizers) -# The llm extension wraps ExecuTorch's LLM runner, which pulls in the sampler, -# the memory allocator and the portable-kernel utilities it shares with them. -rne_import_et_lib(et_llm_runner "extension/llm/runner" extension_llm_runner) -rne_import_et_lib(et_llm_sampler "extension/llm/sampler" extension_llm_sampler) +# The memory allocator and the portable-kernel utilities. rne_import_et_lib(et_memory_allocator "extension/memory_allocator" extension_memory_allocator) rne_import_et_lib(et_kernels_util "kernels/portable/cpu/util" kernels_util_all_deps) -# Listed consumer-first: GNU ld resolves static archives left to right, so the -# runner has to precede the sampler/allocator/kernel archives it pulls from, and -# the ExecuTorch core has to follow all of them. +# Listed consumer-first: GNU ld resolves static archives left to right. set(ET_LIBS - et_llm_runner et_llm_sampler et_memory_allocator et_kernels_util + et_memory_allocator et_kernels_util et_module et_tensor et_flat_tensor et_named_data_map et_data_loader et_full et_core et_tokenizers) @@ -152,7 +147,6 @@ file(GLOB CORE_SOURCES "${CPP_DIR}/core/*.cpp") file(GLOB MATH_SOURCES "${CPP_DIR}/extensions/math/*.cpp") file(GLOB NLP_SOURCES "${CPP_DIR}/extensions/nlp/*.cpp") file(GLOB SPEECH_SOURCES "${CPP_DIR}/extensions/speech/*.cpp") -file(GLOB LLM_SOURCES "${CPP_DIR}/extensions/llm/*.cpp") file(GLOB OPENCV_SOURCES "${CPP_DIR}/extensions/cv/*.cpp") # phonemizer.cpp is the one speech source gated on phonemis, exactly as in @@ -162,7 +156,7 @@ list(FILTER SPEECH_SOURCES EXCLUDE REGEX "/phonemizer\\.cpp$") set(RNE_SOURCES "${CPP_DIR}/RnExecutorch.cpp" - ${CORE_SOURCES} ${MATH_SOURCES} ${NLP_SOURCES} ${SPEECH_SOURCES} ${LLM_SOURCES}) + ${CORE_SOURCES} ${MATH_SOURCES} ${NLP_SOURCES} ${SPEECH_SOURCES}) # The cv extension uses only core and imgproc. Prefer OpenCV's CMake package # when it is installed (Homebrew, or a full distro OpenCV), but fall back to @@ -356,8 +350,7 @@ set(TEST_SUITES core/SchemaTest.cpp core/UtilsTest.cpp extensions/MathOpsTest.cpp - extensions/SpeechOpsTest.cpp - extensions/LlmRunnerTest.cpp) + extensions/SpeechOpsTest.cpp) if(RNE_TESTS_ENABLE_OPENCV) list(APPEND TEST_SUITES extensions/CvOpsTest.cpp extensions/OcrOpsTest.cpp) diff --git a/packages/react-native-executorch/cpp/tests/core/UtilsTest.cpp b/packages/react-native-executorch/cpp/tests/core/UtilsTest.cpp index d198a82fb2..6093b34904 100644 --- a/packages/react-native-executorch/cpp/tests/core/UtilsTest.cpp +++ b/packages/react-native-executorch/cpp/tests/core/UtilsTest.cpp @@ -53,7 +53,7 @@ TEST_F(UtilsTest, InstallsTheModuleUnderItsProductionName) { EXPECT_TRUE(evalBool("return __rnexecutorch_jsi__.cv === undefined;")); #endif - for (const auto *name : {"math", "nlp", "speech", "llm"}) { + for (const auto *name : {"math", "nlp", "speech"}) { EXPECT_TRUE(evalBool(std::string("return typeof __rnexecutorch_jsi__.") + name + " === 'object';")) << "missing extension namespace: " << name; } diff --git a/packages/react-native-executorch/cpp/tests/extensions/LlmRunnerTest.cpp b/packages/react-native-executorch/cpp/tests/extensions/LlmRunnerTest.cpp deleted file mode 100644 index 6fdb55268c..0000000000 --- a/packages/react-native-executorch/cpp/tests/extensions/LlmRunnerTest.cpp +++ /dev/null @@ -1,99 +0,0 @@ -#include - -#include -#include - -#include "support/JsiTestEnv.h" - -namespace rnexecutorch::tests { -namespace { - -using ::testing::HasSubstr; - -// The llm extension wraps ExecuTorch's LLM runner. Creating one loads a real -// model and a real tokenizer, and running one needs the delegate the model was -// exported against — the same limit ModelTest documents for execute(). What is -// host-testable is everything up to that: the argument contract of -// createLLMRunner and how each failure to load is classified. -class LlmRunnerTest : public JsiTestEnv {}; - -TEST_F(LlmRunnerTest, IsInstalledUnderTheLlmNamespace) { - EXPECT_TRUE(evalBool("return typeof __rnexecutorch_jsi__.llm.createLLMRunner === 'function';")); - // Arity is what the TS layer's `.length` checks and what JS engines report; - // the two optional arguments are not counted. - EXPECT_EQ(evalNumber("return __rnexecutorch_jsi__.llm.createLLMRunner.length;"), 2); -} - -TEST_F(LlmRunnerTest, RejectsWrongArgumentCounts) { - const std::string usage = "Usage: createLLMRunner(modelPath, tokenizerPath, modalities?)"; - - EXPECT_TRUE(isCodedError(evalThrowing("__rnexecutorch_jsi__.llm.createLLMRunner();"), - "INVALID_ARGUMENT", usage)); - EXPECT_TRUE(isCodedError(evalThrowing("__rnexecutorch_jsi__.llm.createLLMRunner('model.pte');"), - "INVALID_ARGUMENT", usage)); - // modalities is the last accepted argument, so a fourth is a mistake rather - // than something to ignore. - EXPECT_TRUE(isCodedError( - evalThrowing("__rnexecutorch_jsi__.llm.createLLMRunner('a.pte', 'b.json', [], 'extra');"), - "INVALID_ARGUMENT", usage)); -} - -TEST_F(LlmRunnerTest, RejectsWronglyTypedArguments) { - EXPECT_TRUE(isCodedError(evalThrowing("__rnexecutorch_jsi__.llm.createLLMRunner(1, 'b.json');"), - "INVALID_ARGUMENT", "modelPath must be a string")); - EXPECT_TRUE(isCodedError(evalThrowing("__rnexecutorch_jsi__.llm.createLLMRunner('a.pte', 2);"), - "INVALID_ARGUMENT", "tokenizerPath must be a string")); - EXPECT_TRUE(isCodedError( - evalThrowing("__rnexecutorch_jsi__.llm.createLLMRunner('a.pte', 'b.json', 'image');"), - "INVALID_ARGUMENT", "modalities must be an Array")); - EXPECT_TRUE(isCodedError( - evalThrowing("__rnexecutorch_jsi__.llm.createLLMRunner('a.pte', 'b.json', [7]);"), - "INVALID_ARGUMENT", "must be a string")); -} - -// null and undefined mean "text only", so they must reach the tokenizer load -// rather than being rejected as a bad modalities list. -TEST_F(LlmRunnerTest, TreatsAbsentModalitiesAsTextOnly) { - for (const auto *modalities : {"null", "undefined"}) { - auto thrown = evalThrowing( - std::string("__rnexecutorch_jsi__.llm.createLLMRunner('/no/model.pte', '/no/tokenizer.json', ") + - modalities + ");"); - EXPECT_TRUE(isCodedError(thrown, "LOAD_FAILED", "/no/tokenizer.json")) << "modalities: " << modalities; - } -} - -// The tokenizer is loaded before the model, so a missing one is reported as -// such instead of as a confusing model failure. -TEST_F(LlmRunnerTest, ReportsAMissingTokenizerAsLoadFailed) { - auto thrown = evalThrowing( - "__rnexecutorch_jsi__.llm.createLLMRunner('/no/model.pte', '/no/tokenizer.json');"); - - EXPECT_TRUE(isCodedError(thrown, "LOAD_FAILED", "Failed to load runner tokenizer")); - EXPECT_THAT(thrown.message, HasSubstr("/no/tokenizer.json")); -} - -#ifdef RNE_TOKENIZER_FIXTURE -// With a loadable tokenizer the failure moves on to the model, which is the -// point: the two load steps are classified separately rather than collapsed. -TEST_F(LlmRunnerTest, ReportsAMissingModelAsLoadFailed) { - auto thrown = evalThrowing( - std::string("__rnexecutorch_jsi__.llm.createLLMRunner('/no/model.pte', '") + - RNE_TOKENIZER_FIXTURE + "');"); - - EXPECT_EQ(thrown.code, "LOAD_FAILED"); - EXPECT_THAT(thrown.message, HasSubstr("LLMRunner:")); - EXPECT_THAT(thrown.message, ::testing::Not(HasSubstr("tokenizer"))); -} - -TEST_F(LlmRunnerTest, ReportsAMissingMultimodalModelAsLoadFailed) { - auto thrown = evalThrowing( - std::string("__rnexecutorch_jsi__.llm.createLLMRunner('/no/model.pte', '") + - RNE_TOKENIZER_FIXTURE + "', ['image']);"); - - EXPECT_EQ(thrown.code, "LOAD_FAILED"); - EXPECT_THAT(thrown.message, HasSubstr("LLMRunner:")); -} -#endif - -} // namespace -} // namespace rnexecutorch::tests From e8a47f63a5a8eda7240c68ddad9c7051d63339b7 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Fri, 2 Oct 2026 14:08:33 +0200 Subject: [PATCH 02/83] refactor(bridge): remove llm namespace from rnexecutorchJsi docstring --- packages/react-native-executorch/src/native/bridge.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/react-native-executorch/src/native/bridge.ts b/packages/react-native-executorch/src/native/bridge.ts index 3d17ab8fc0..998e3c644e 100644 --- a/packages/react-native-executorch/src/native/bridge.ts +++ b/packages/react-native-executorch/src/native/bridge.ts @@ -13,7 +13,7 @@ if (!globalObj.__rnexecutorch_jsi__) { * Provides low-level, synchronous bindings for ExecuTorch core runtime * operations (model loading, tensor allocation and manipulation, registered * backends inspection, emulator detection) and native extension namespaces - * (`cv`, `llm`, `math`, `nlp`, `speech`). + * (`cv`, `math`, `nlp`, `speech`). * @internal */ export const rnexecutorchJsi = globalObj.__rnexecutorch_jsi__; From 9fe43d37d6baf3f10fc3faee165d19d57be040e4 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Fri, 2 Oct 2026 14:56:17 +0200 Subject: [PATCH 03/83] style --- .../src/extensions/llm/tasks/llmChatSession.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts b/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts index aab9b0f9fa..137e5d3d3e 100644 --- a/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts +++ b/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts @@ -203,9 +203,8 @@ export async function createLLMChatSession( const chatPreprocessor = scope.track(createChatPreprocessor(chatPreprocessorConfig)); // Prepare runner - const runner = scope.track( - await wrapAsync(createLLMRunner, runtime)(modelPath, tokenizerPath, modalities) - ); + const load = wrapAsync(createLLMRunner, runtime); + const runner = scope.track(await load(modelPath, tokenizerPath, modalities)); const prefill = wrapAsync(runner.prefill, runtime); const history: ChatMessage[] = []; From 2ae8fa406237d6426e82f610696a97346f78c3e3 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Fri, 2 Oct 2026 16:40:16 +0200 Subject: [PATCH 04/83] feat(model): support returning primitive lists from execute --- .../cpp/core/conversions.h | 25 ++++++++++--------- .../cpp/core/model.cpp | 9 +++++++ .../react-native-executorch/src/core/model.ts | 9 ++++++- 3 files changed, 30 insertions(+), 13 deletions(-) diff --git a/packages/react-native-executorch/cpp/core/conversions.h b/packages/react-native-executorch/cpp/core/conversions.h index d894e066c0..cfaba990d1 100644 --- a/packages/react-native-executorch/cpp/core/conversions.h +++ b/packages/react-native-executorch/cpp/core/conversions.h @@ -164,26 +164,27 @@ template inline constexpr bool kAlwaysFalse = false; /** - * Converts a std::vector of values to a new facebook::jsi::Array. + * Converts a container (e.g. std::vector, ArrayRef) of values to a new facebook::jsi::Array. * Handles strings, booleans, and numeric types appropriately. * - * @tparam T The element type in the vector. + * @tparam Container The container type (must support .size() and operator[]). * @param rt The JSI runtime instance. - * @param vec The source vector to convert. - * @return A new facebook::jsi::Array containing the elements from the vector. + * @param items The source container to convert. + * @return A new facebook::jsi::Array containing the elements from the container. */ -template -jsi::Array toJsiArray(jsi::Runtime &rt, const std::vector &vec) { - jsi::Array arr(rt, vec.size()); - for (size_t i = 0; i < vec.size(); ++i) { +template +jsi::Array toJsiArray(jsi::Runtime &rt, const Container &items) { + jsi::Array arr(rt, items.size()); + for (size_t i = 0; i < items.size(); ++i) { + using T = std::decay_t; if constexpr (std::is_same_v) { - arr.setValueAtIndex(rt, i, jsi::String::createFromUtf8(rt, vec[i])); + arr.setValueAtIndex(rt, i, jsi::String::createFromUtf8(rt, items[i])); } else if constexpr (std::is_same_v) { - arr.setValueAtIndex(rt, i, jsi::Value(static_cast(vec[i]))); + arr.setValueAtIndex(rt, i, jsi::Value(static_cast(items[i]))); } else if constexpr (std::is_arithmetic_v) { - arr.setValueAtIndex(rt, i, jsi::Value(static_cast(vec[i]))); + arr.setValueAtIndex(rt, i, jsi::Value(static_cast(items[i]))); } else { - static_assert(kAlwaysFalse, "Unsupported vector element type for toJsiArray"); + static_assert(kAlwaysFalse, "Unsupported element type for toJsiArray"); } } return arr; diff --git a/packages/react-native-executorch/cpp/core/model.cpp b/packages/react-native-executorch/cpp/core/model.cpp index 648fb62629..745b2bd6f7 100644 --- a/packages/react-native-executorch/cpp/core/model.cpp +++ b/packages/react-native-executorch/cpp/core/model.cpp @@ -287,6 +287,15 @@ jsi::Value ModelHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) { case executorch::runtime::Tag::String: jsOutputArray.setValueAtIndex(rt, outputIdx, jsi::String::createFromUtf8(rt, std::string(output.toString()))); break; + case executorch::runtime::Tag::ListInt: + jsOutputArray.setValueAtIndex(rt, outputIdx, conversions::toJsiArray(rt, output.toIntList())); + break; + case executorch::runtime::Tag::ListDouble: + jsOutputArray.setValueAtIndex(rt, outputIdx, conversions::toJsiArray(rt, output.toDoubleList())); + break; + case executorch::runtime::Tag::ListBool: + jsOutputArray.setValueAtIndex(rt, outputIdx, conversions::toJsiArray(rt, output.toBoolList())); + break; default: throw error::ExecutionFailed(std::format("execute: Unsupported return type: {}", executorch::runtime::tag_to_string(output.tag))); diff --git a/packages/react-native-executorch/src/core/model.ts b/packages/react-native-executorch/src/core/model.ts index da42f818eb..14219aa08b 100644 --- a/packages/react-native-executorch/src/core/model.ts +++ b/packages/react-native-executorch/src/core/model.ts @@ -22,7 +22,14 @@ export type ModelInput = Tensor | number | boolean | null; * A value returned from a model's `execute` method. * @category Core / Types */ -export type ModelOutput = Tensor | number | boolean | string | null; +export type ModelOutput = + | Tensor + | number + | boolean + | string + | null + | readonly number[] + | readonly boolean[]; /** * A compiled, ready-to-run ExecuTorch model loaded into native memory. From ec532a61dfd707f01fe0f71706db21883d87a312 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Fri, 2 Oct 2026 16:53:42 +0200 Subject: [PATCH 05/83] refactor(llm): split runner into runners/ directory and update stats types - Extract createTextRunner into runners/textRunner.ts - Add stubs runners/multimodalRunner.ts, runners/gemmaRunner.ts - Add runners/index.ts re-exporting all three - Add LLMPrefillStats type; replace old timing fields in LLMGenerationStats with numTokens / durationMs / tokensPerSecond / prefill: LLMPrefillStats - Rename LLMKVCacheState.maxSeqLen to maxContextLen - prefill() now returns LLMPrefillStats - Add EOS stopping via eosTokenIds from get_eos_ids metadata method - llmChatSession: capture prefillStats from prefill(), use it for turn 0 --- .../src/extensions/llm/llmRunner.ts | 154 +++++++++++++++--- .../extensions/llm/tasks/llmChatSession.ts | 11 +- 2 files changed, 134 insertions(+), 31 deletions(-) diff --git a/packages/react-native-executorch/src/extensions/llm/llmRunner.ts b/packages/react-native-executorch/src/extensions/llm/llmRunner.ts index ce3867fc7a..b58943fe21 100644 --- a/packages/react-native-executorch/src/extensions/llm/llmRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/llmRunner.ts @@ -1,9 +1,15 @@ /** * Low-level native ExecuTorch LLM runner types and factory. */ - import type { Tensor } from '../../core/tensor'; -import { rnexecutorchJsi } from '../../native/bridge'; +import { wrapAsync } from '../../core/runtime'; +import { loadModel } from '../../core/model'; +import { createResourceScope } from '../../core/lifetime'; +import { f32, i64, method, validateSpec } from '../../core/schema'; + +import { loadTokenizer } from '../nlp'; +import { RnExecuTorchError } from '../../core/error'; +import { createTextRunner } from './runners/textRunner'; declare const llmRunnerBrand: unique symbol; @@ -21,28 +27,34 @@ export type LLMGenerationConfig = { readonly temperature?: number; }; +/** + * Execution and performance statistics for a prefill phase. + * @experimental This API is experimental and might change in future releases. + * @category LLM / Types + */ +export type LLMPrefillStats = { + /** Number of tokens processed during prefill. */ + readonly numTokens: number; + /** Duration in milliseconds spent in prefill. */ + readonly durationMs: number; + /** Prefill throughput in tokens per second. */ + readonly tokensPerSecond: number; +}; + /** * Execution and performance statistics for a generation call. * @experimental This API is experimental and might change in future releases. * @category LLM / Types */ export type LLMGenerationStats = { - /** Number of tokens in the input prompt. */ - readonly numPromptTokens: number; /** Number of newly generated tokens. */ - readonly numGeneratedTokens: number; - /** Timestamp in milliseconds when the first token was generated. */ - readonly firstTokenMs: number; - /** Duration in milliseconds spent in the separate prefill phase (if any). */ - readonly prefillDurationMs?: number; - /** Timestamp in milliseconds when inference started. */ - readonly inferenceStartMs: number; - /** Timestamp in milliseconds when inference completed. */ - readonly inferenceEndMs: number; - /** Timestamp in milliseconds when model loading started. */ - readonly modelLoadStartMs: number; - /** Timestamp in milliseconds when model loading completed. */ - readonly modelLoadEndMs: number; + readonly numTokens: number; + /** Duration in milliseconds spent in decode generation. */ + readonly durationMs: number; + /** Generation throughput in tokens per second. */ + readonly tokensPerSecond: number; + /** Performance statistics of the prefill phase. */ + readonly prefill: LLMPrefillStats; }; /** @@ -77,7 +89,7 @@ export type LLMKVCacheState = { /** Current token position index / number of occupied tokens in the KV cache. */ readonly pos: number; /** Maximum token capacity (context window) supported by the model. */ - readonly maxSeqLen: number; + readonly maxContextLen: number; /** Remaining token capacity before the context window is full. */ readonly remainingTokens: number; /** Fraction of the context window currently occupied (0.0 to 1.0). */ @@ -124,8 +136,9 @@ export type LLMRunner = { /** * Prefills the runner with a prompt to build up the KV cache. * @param prompt The prefill text or multimodal prompt. + * @returns Prefill performance statistics. */ - prefill(prompt: Prompt): void; + prefill(prompt: Prompt): LLMPrefillStats; /** * Generates text continuation from a prompt. @@ -159,11 +172,106 @@ export type LLMRunner = { * `['image']`). When omitted, defaults to text-only. * @returns A native {@link LLMRunner} instance. */ -export function createLLMRunner( +export async function createLLMRunner( modelPath: string, tokenizerPath: string, modalities?: readonly Modality[] -): LLMRunner { - 'worklet'; - return rnexecutorchJsi.llm.createLLMRunner(modelPath, tokenizerPath, modalities ?? []); +): Promise { + const scope = createResourceScope(); + try { + const model = scope.track(await wrapAsync(loadModel)(modelPath)); + const tokenizer = scope.track(await wrapAsync(loadTokenizer)(tokenizerPath)); + + // TODO(@bh): update validate spec after reexporting LLMs with schema const method + const metadata = { + ...method('get_max_seq_len', [], [{ kind: 'Int' }]), + ...method('get_max_context_len', [], [{ kind: 'Int' }]), + ...method('get_vocab_size', [], [{ kind: 'Int' }]), + ...method('use_kv_cache', [], [{ kind: 'Bool' }]), + ...method('enable_dynamic_shape', [], [{ kind: 'Bool' }]), + ...method('get_eos_ids', [], [{ kind: 'ListInt' }]), + }; + + const { variant, dims } = validateSpec(model.schema, { + llm: { + ...method( + 'forward', + // Input: ... + [i64(1, 'maxSeqLen'), i64(1)], + // Output: logits over model vocabulary + [f32(1, 'vocabSize')] + ), + ...metadata, + }, + // vlm: { + // ...method('text_decoder', [f32(1, 'maxSeqLen', 'hid'), i64('seq')], [f32(1, 'vocabSize')]), + // ...method('vision_encoder', [f32(1, 3, 'H', 'W')], [f32(1, 'patches', 'hid')]), + // ...method('token_embedding', [i64(1, 'seq')], [f32(1, 'seq', 'hid')]), + // ...metadata, + // }, + // gemma: { + // ...method( + // 'token_embedding', + // [i64(1, 'seq')], + // [f32(1, 'seq', 1536), f32(1, 'seq', 35, 256)] + // ), + // ...method( + // 'audio_encoder', + // [f32(1, 'numSamples')], + // [f32(1, 'audioTokens', 1536), f32(1, 'audioTokens')] + // ), + // ...method( + // 'text_decoder', + // [f32(1, 'seq', 1536), f32(1, 'seq', 35, 256), i64('seq')], + // [f32(1, 1, 262144)] + // ), + // ...method('vision_encoder', [f32(1, 3, 448, 640)], [f32(1, 280, 1536)]), + // ...metadata, + // }, + }); + + if (variant === 'llm' && modalities && modalities.length > 0) { + throw RnExecuTorchError('INVALID_ARGUMENT', ''); + } + // if (variant === 'vlm' && !modalities?.includes('image')) { + // throw RnExecuTorchError('INVALID_ARGUMENT', ''); + // } + // if (variant === 'gemma' && !(modalities?.includes('audio') && modalities.includes('image'))) { + // throw RnExecuTorchError('INVALID_ARGUMENT', ''); + // } + + const [maxSeqLen, vocabSize] = dims.constant('maxSeqLen', 'vocabSize'); + const [maxContextLen] = model.execute('get_max_context_len', [], []) as [number]; + const [eosIds] = model.execute('get_eos_ids', [], []) as [number[]]; + + if (maxSeqLen !== model.execute('get_max_seq_len', [], [])[0]) { + throw RnExecuTorchError('SCHEMA_MISMATCH', ''); + } + if (vocabSize !== model.execute('get_vocab_size', [], [])[0]) { + throw RnExecuTorchError('SCHEMA_MISMATCH', ''); + } + if (vocabSize !== tokenizer.getVocabSize()) { + throw RnExecuTorchError('SCHEMA_MISMATCH', ''); + } + if (model.execute('use_kv_cache', [], [])[0] !== true) { + throw RnExecuTorchError('SCHEMA_MISMATCH', ''); + } + if (model.execute('enable_dynamic_shape', [], [])[0] !== true) { + throw RnExecuTorchError('SCHEMA_MISMATCH', ''); + } + if (maxSeqLen <= 0 || !Number.isInteger(maxSeqLen)) { + throw RnExecuTorchError('SCHEMA_MISMATCH', ''); + } + if (maxContextLen < maxSeqLen || !Number.isInteger(maxContextLen)) { + throw RnExecuTorchError('SCHEMA_MISMATCH', ''); + } + if (!Array.isArray(eosIds) || eosIds.length === 0) { + throw RnExecuTorchError('SCHEMA_MISMATCH', ''); + } + + return createTextRunner(model, tokenizer, { maxSeqLen, maxContextLen, vocabSize, eosIds }); + } catch (e) { + scope.dispose(); + throw e; + } } diff --git a/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts b/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts index 137e5d3d3e..42dbcc587d 100644 --- a/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts +++ b/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts @@ -248,12 +248,10 @@ export async function createLLMChatSession( } try { - let prefillStartMs = Date.now(); - // Prefill newly committed messages up to current user message without generation prompt const toCommit = history.length - committed; const userPrompt = chatPreprocessor.process(history, toCommit, { addGenPrompt: false }); - await prefill(userPrompt); + const prefillStats = await prefill(userPrompt); chatPreprocessor.clear(); // Record exact position at the end of the user message (before assistant generation header) @@ -266,11 +264,10 @@ export async function createLLMChatSession( const uncommitted = history.length - committed; const prompt = chatPreprocessor.process(history, uncommitted, { addGenPrompt: true }); - const prefillDurationMs = Date.now() - prefillStartMs; - const { response, stats } = await generateChatTurn(runner, prompt, generationOpts); + const cumPrefillStats = currentTurn === 0 ? prefillStats : stats.prefill; chatPreprocessor.clear(); - generationStatsList.push({ ...stats, prefillDurationMs }); + generationStatsList.push({ ...stats, prefill: cumPrefillStats }); // Always rewind KV cache back to posAtEndOfUser so next turn prefills // cleanly formatted message with tool outputs @@ -313,8 +310,6 @@ export async function createLLMChatSession( content: toolContent, }); } - - prefillStartMs = Date.now(); } // Prefill all uncommitted assistant & tool messages so KV cache contains From 87f443951693b9b6ecedbeed20e8ce6f1b00cdb1 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Fri, 2 Oct 2026 16:54:10 +0200 Subject: [PATCH 06/83] feat(llm): add runners/ directory with textRunner, multimodalRunner, gemmaRunner --- .../src/extensions/llm/runners/gemmaRunner.ts | 9 + .../src/extensions/llm/runners/index.ts | 3 + .../llm/runners/multimodalRunner.ts | 9 + .../src/extensions/llm/runners/textRunner.ts | 186 ++++++++++++++++++ 4 files changed, 207 insertions(+) create mode 100644 packages/react-native-executorch/src/extensions/llm/runners/gemmaRunner.ts create mode 100644 packages/react-native-executorch/src/extensions/llm/runners/index.ts create mode 100644 packages/react-native-executorch/src/extensions/llm/runners/multimodalRunner.ts create mode 100644 packages/react-native-executorch/src/extensions/llm/runners/textRunner.ts diff --git a/packages/react-native-executorch/src/extensions/llm/runners/gemmaRunner.ts b/packages/react-native-executorch/src/extensions/llm/runners/gemmaRunner.ts new file mode 100644 index 0000000000..8d3fa78e4b --- /dev/null +++ b/packages/react-native-executorch/src/extensions/llm/runners/gemmaRunner.ts @@ -0,0 +1,9 @@ +import type { Model } from '../../../core/model'; +import type { Tokenizer } from '../../nlp'; +import type { LLMRunner } from '../llmRunner'; + +export function createGemmaRunner(model: Model, tokenizer: Tokenizer): LLMRunner { + void model; + void tokenizer; + throw new Error('Not implemented'); +} diff --git a/packages/react-native-executorch/src/extensions/llm/runners/index.ts b/packages/react-native-executorch/src/extensions/llm/runners/index.ts new file mode 100644 index 0000000000..55b37910d1 --- /dev/null +++ b/packages/react-native-executorch/src/extensions/llm/runners/index.ts @@ -0,0 +1,3 @@ +export * from './textRunner'; +export * from './multimodalRunner'; +export * from './gemmaRunner'; diff --git a/packages/react-native-executorch/src/extensions/llm/runners/multimodalRunner.ts b/packages/react-native-executorch/src/extensions/llm/runners/multimodalRunner.ts new file mode 100644 index 0000000000..b5156364f6 --- /dev/null +++ b/packages/react-native-executorch/src/extensions/llm/runners/multimodalRunner.ts @@ -0,0 +1,9 @@ +import type { Model } from '../../../core/model'; +import type { Tokenizer } from '../../nlp'; +import type { LLMRunner } from '../llmRunner'; + +export function createMultimodalRunner(model: Model, tokenizer: Tokenizer): LLMRunner { + void model; + void tokenizer; + throw new Error('Not implemented'); +} diff --git a/packages/react-native-executorch/src/extensions/llm/runners/textRunner.ts b/packages/react-native-executorch/src/extensions/llm/runners/textRunner.ts new file mode 100644 index 0000000000..a93058faf8 --- /dev/null +++ b/packages/react-native-executorch/src/extensions/llm/runners/textRunner.ts @@ -0,0 +1,186 @@ +import { createSynchronizable } from 'react-native-worklets'; + +import { tensor } from '../../../core/tensor'; +import type { Model } from '../../../core/model'; +import type { Tokenizer } from '../../nlp'; +import { RnExecuTorchError } from '../../../core/error'; + +import type { + LLMGenerationConfig, + LLMGenerationStats, + LLMKVCacheState, + LLMPrefillStats, + LLMRunner, + Prompt, +} from '../llmRunner'; + +export function createTextRunner( + model: Model, + tokenizer: Tokenizer, + meta: { + readonly maxSeqLen: number; + readonly maxContextLen: number; + readonly vocabSize: number; + readonly eosIds: readonly number[]; + } +): LLMRunner { + const { maxSeqLen, maxContextLen, vocabSize, eosIds } = meta; + const tLogits = tensor('float32', [1, vocabSize]); + + const dispose = () => { + model.dispose(); + tLogits.dispose(); + tokenizer.dispose(); + }; + + // ================================== + // KV Cache management + // ================================== + let pos = 0; + + const reset = (targetPos?: number): void => { + if (targetPos !== undefined && (targetPos < 0 || targetPos >= maxContextLen)) { + throw RnExecuTorchError('INVALID_ARGUMENT', ''); + } + pos = targetPos ?? 0; + }; + + const getKVCacheState = (): LLMKVCacheState => ({ + pos, + maxContextLen, + remainingTokens: maxContextLen - pos, + usageRatio: pos / maxContextLen, + }); + + // ================================== + // Generation methods + // ================================== + const isCancelled = createSynchronizable(false); + const stop = (): void => isCancelled.setBlocking(true); + + const prefill = (prompt: Prompt): LLMPrefillStats => { + 'worklet'; + isCancelled.setBlocking(false); + + if (typeof prompt !== 'string') { + throw RnExecuTorchError('INVALID_ARGUMENT', ''); + } + + const startMs = Date.now(); + const startPos = pos; + const tokens = tokenizer.encode(prompt); + + let offset = 0; + while (offset < tokens.length && !isCancelled.getBlocking()) { + const seqLen = Math.min(tokens.length - offset, maxSeqLen); + const chunk = tokens.subarray(offset, offset + seqLen); + + if (pos + seqLen > maxContextLen) { + throw RnExecuTorchError('EXECUTION_FAILED', ''); + } + + // prettier-ignore + const tTokens = tensor('int64', [1, seqLen], BigInt64Array.from(chunk, (v) => BigInt(v))); + const tCurPos = tensor('int64', [1], BigInt64Array.of(BigInt(pos))); + try { + model.execute('forward', [tTokens, tCurPos], [tLogits]); + } finally { + tTokens.dispose(); + tCurPos.dispose(); + } + + pos += seqLen; + offset += seqLen; + } + + const durationMs = Date.now() - startMs; + const numTokens = pos - startPos; + const tokensPerSecond = durationMs > 0 ? (numTokens / durationMs) * 1000 : 0; + + return { + numTokens, + durationMs, + tokensPerSecond, + }; + }; + + const generate = ( + prompt: Prompt, + config?: LLMGenerationConfig, + onToken?: (token: string) => void + ): LLMGenerationStats => { + 'worklet'; + isCancelled.setBlocking(false); + + if (typeof prompt !== 'string') { + throw RnExecuTorchError('INVALID_ARGUMENT', ''); + } + + const prefillStats = prefill(prompt); + + const generateStartMs = Date.now(); + const maxNewTokens = config?.maxNewTokens ?? Infinity; + const logits = tLogits.getData(new Float32Array(vocabSize)); + + let numTokens = 0; + + const tToken = tensor('int64', [1, 1]); + const tCurPos = tensor('int64', [1]); + + try { + while (numTokens < maxNewTokens && !isCancelled.getBlocking()) { + if (pos >= maxContextLen) break; + + // TODO: Implement sampling (e.g. temperature, top-k, top-p, penalties) + let nextToken = 0; + + if (!config?.ignoreEos && eosIds.includes(nextToken)) { + break; + } + + onToken?.(tokenizer.decode(Int32Array.of(nextToken))); + + tToken.setData(BigInt64Array.of(BigInt(nextToken))); + tCurPos.setData(BigInt64Array.of(BigInt(pos))); + + model.execute('forward', [tToken, tCurPos], [tLogits]); + + pos += 1; + numTokens += 1; + tLogits.getData(logits); + } + } finally { + tToken.dispose(); + tCurPos.dispose(); + } + + const durationMs = Date.now() - generateStartMs; + const tokensPerSecond = durationMs > 0 ? (numTokens / durationMs) * 1000 : 0; + + return { + numTokens, + durationMs, + tokensPerSecond, + prefill: prefillStats, + }; + }; + + return { + // Metadata + modelPath: model.path, + tokenizerPath: tokenizer.path, + modalities: [], + + // Lifecycle & Execution + dispose, + stop, + + // KV Cache + reset, + getKVCacheState, + + // Generation + prefill, + generate, + } as unknown as LLMRunner; +} From 731edfdfd131361a39a97ec216049702130d876a Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Fri, 2 Oct 2026 17:01:44 +0200 Subject: [PATCH 07/83] style --- .../react-native-executorch/src/extensions/llm/llmRunner.ts | 2 +- .../src/extensions/llm/runners/textRunner.ts | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/react-native-executorch/src/extensions/llm/llmRunner.ts b/packages/react-native-executorch/src/extensions/llm/llmRunner.ts index b58943fe21..6f3037b391 100644 --- a/packages/react-native-executorch/src/extensions/llm/llmRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/llmRunner.ts @@ -4,11 +4,11 @@ import type { Tensor } from '../../core/tensor'; import { wrapAsync } from '../../core/runtime'; import { loadModel } from '../../core/model'; +import { RnExecuTorchError } from '../../core/error'; import { createResourceScope } from '../../core/lifetime'; import { f32, i64, method, validateSpec } from '../../core/schema'; import { loadTokenizer } from '../nlp'; -import { RnExecuTorchError } from '../../core/error'; import { createTextRunner } from './runners/textRunner'; declare const llmRunnerBrand: unique symbol; diff --git a/packages/react-native-executorch/src/extensions/llm/runners/textRunner.ts b/packages/react-native-executorch/src/extensions/llm/runners/textRunner.ts index a93058faf8..a8f0e6cde2 100644 --- a/packages/react-native-executorch/src/extensions/llm/runners/textRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/runners/textRunner.ts @@ -2,9 +2,10 @@ import { createSynchronizable } from 'react-native-worklets'; import { tensor } from '../../../core/tensor'; import type { Model } from '../../../core/model'; -import type { Tokenizer } from '../../nlp'; import { RnExecuTorchError } from '../../../core/error'; +import type { Tokenizer } from '../../nlp'; + import type { LLMGenerationConfig, LLMGenerationStats, From 4ec801d907a42dbe6f40b4f857f57b280a6ddccd Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Fri, 2 Oct 2026 17:08:39 +0200 Subject: [PATCH 08/83] style --- .../src/extensions/llm/runners/textRunner.ts | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/packages/react-native-executorch/src/extensions/llm/runners/textRunner.ts b/packages/react-native-executorch/src/extensions/llm/runners/textRunner.ts index a8f0e6cde2..f9218fbff7 100644 --- a/packages/react-native-executorch/src/extensions/llm/runners/textRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/runners/textRunner.ts @@ -158,12 +158,7 @@ export function createTextRunner( const durationMs = Date.now() - generateStartMs; const tokensPerSecond = durationMs > 0 ? (numTokens / durationMs) * 1000 : 0; - return { - numTokens, - durationMs, - tokensPerSecond, - prefill: prefillStats, - }; + return { numTokens, durationMs, tokensPerSecond, prefill: prefillStats }; }; return { From 7961f062197b5d7c9f4b76fe45aff2d3bcb9ec93 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Fri, 2 Oct 2026 17:19:54 +0200 Subject: [PATCH 09/83] feat(llm): add sampling pipeline (wip) - sampler.ts: LogitProcessor chain + Sampler interface - Built-in processors: temperature, top-k, top-p, min-p, repetition penalty, logit bias - Built-in samplers: greedySampler (default), multinomialSampler - buildSampler() composes config into SamplingPipeline - runSampling() applies pipeline in the generation loop - Add 'multinomial' to cspell wordlist --- .cspell-wordlist.txt | 1 + .../src/extensions/llm/sampler.ts | 397 ++++++++++++++++++ 2 files changed, 398 insertions(+) create mode 100644 packages/react-native-executorch/src/extensions/llm/sampler.ts diff --git a/.cspell-wordlist.txt b/.cspell-wordlist.txt index 2e3e47e2a5..dabdb3c665 100644 --- a/.cspell-wordlist.txt +++ b/.cspell-wordlist.txt @@ -400,6 +400,7 @@ skiplist skiplisted startoftext maxsim +multinomial adb benchmarkable benchprobe diff --git a/packages/react-native-executorch/src/extensions/llm/sampler.ts b/packages/react-native-executorch/src/extensions/llm/sampler.ts new file mode 100644 index 0000000000..9b41fd8958 --- /dev/null +++ b/packages/react-native-executorch/src/extensions/llm/sampler.ts @@ -0,0 +1,397 @@ +/** + * LLM sampling pipeline — logit processors and samplers. + * + * The pipeline applies a sequence of {@link LogitProcessor}s to the raw logit + * array, then passes the result to a {@link Sampler} to pick the next token. + * + * Standard built-in order (matches llama.cpp / HuggingFace conventions): + * repetition penalty → logit bias → temperature → top-k → top-p / min-p → sample + * + * Custom processors can be injected at the end of the chain via + * {@link SamplingConfig.logitProcessors}, and the sampler can be replaced + * entirely via {@link SamplingConfig.sampler}. + */ + +// ── Types ─────────────────────────────────────────────────────────────────── + +/** + * Contextual information available to logit processors during generation. + * @category LLM / Types + */ +export type SamplingContext = { + /** All token ids emitted so far in the current generation call. */ + readonly generatedTokens: readonly number[]; +}; + +/** + * Modifies a logit array **in-place** before sampling. + * Multiple processors are applied sequentially in the order they are registered. + * @category LLM / Types + */ +export type LogitProcessor = (logits: Float32Array, ctx: SamplingContext) => void; + +/** + * Picks a single next-token id from a (processed) logit array. + * + * Note: greedy and multinomial sampling are distinct algorithms, not + * limit-cases of each other in practice. Greedy is a single-pass argmax — + * exact and deterministic. Multinomial requires softmax + a random draw, and + * even at very low temperatures, floating-point precision can cause it to + * diverge from the true argmax near logit ties. Keep both; {@link buildSampler} + * selects the right default based on {@link SamplingConfig.temperature}. + * @category LLM / Types + */ +export type Sampler = (logits: Float32Array) => number; + +/** + * Configuration for the sampling pipeline. + * + * Shorthand fields (`temperature`, `topK`, …) are expanded into built-in + * {@link LogitProcessor}s automatically by {@link buildSampler}. + * + * Set `temperature` to `0` (or leave it unset) for deterministic greedy + * decoding. Set it to any positive value to enable stochastic multinomial + * sampling. + * @category LLM / Types + */ +export type SamplingConfig = { + /** + * Softmax temperature. Values below 1 sharpen the distribution (more + * focused), values above 1 flatten it (more random). Set to 0 or omit for + * greedy decoding. + */ + readonly temperature?: number; + /** + * Keep only the `topK` highest-logit tokens; set the rest to `-Infinity`. + * Applied after temperature scaling. + */ + readonly topK?: number; + /** + * Nucleus sampling: keep the smallest set of tokens whose cumulative + * probability (after softmax) is at least `topP`. Applied after temperature + * and top-k. Values in `(0, 1]`. + */ + readonly topP?: number; + /** + * Min-p sampling: keep tokens whose probability is at least `minP × p_max`, + * where `p_max` is the highest token probability. Simpler and often more + * robust than top-p. Applied after temperature and top-k. Values in `(0, 1]`. + */ + readonly minP?: number; + /** + * Repetition penalty factor (> 1 penalizes repetition, 1 = no effect). + * Logits of already-generated tokens are divided (if positive) or multiplied + * (if negative) by this value. Applied before temperature. + */ + readonly repetitionPenalty?: number; + /** + * Fixed logit offsets keyed by token id. Positive values boost a token, + * negative values suppress it. Use `-Infinity` to completely ban a token. + * Applied before temperature. + */ + readonly logitBias?: ReadonlyMap; + /** + * Additional {@link LogitProcessor}s appended after all built-in ones. + * Use this for constrained decoding (grammar, JSON schema, regex, …). + */ + readonly logitProcessors?: readonly LogitProcessor[]; + /** + * Custom sampler. When omitted, defaults to {@link greedySampler} if + * `temperature` is 0 or unset, and {@link multinomialSampler} otherwise. + */ + readonly sampler?: Sampler; +}; + +/** + * Compiled sampling pipeline, ready to be used in a generation loop. + * @category LLM / Types + */ +export type SamplingPipeline = { + readonly processors: readonly LogitProcessor[]; + readonly sampler: Sampler; +}; + +// ── Internal utilities ─────────────────────────────────────────────────────── + +/** + * Computes softmax of `logits` into `out` in-place. + * Numerically stable via max subtraction. `-Infinity` logits contribute 0. + * @param logits Source logit array. + * @param out Destination probability array (same length as `logits`). + */ +function softmax(logits: Float32Array, out: Float32Array): void { + 'worklet'; + let max = -Infinity; + for (let i = 0; i < logits.length; i++) { + if (logits[i]! > max) max = logits[i]!; + } + let sum = 0; + for (let i = 0; i < logits.length; i++) { + out[i] = Math.exp(logits[i]! - max); + sum += out[i]!; + } + for (let i = 0; i < logits.length; i++) { + out[i]! /= sum; + } +} + +// ── Built-in logit processors ──────────────────────────────────────────────── + +/** + * Scales logits by `1 / temperature`. Steers the distribution toward the + * peak (temperature < 1) or away from it (temperature > 1). + * @param temperature Scaling factor (must be > 0). + * @returns A {@link LogitProcessor} that applies the temperature scale. + * @category LLM / Functions + */ +export function temperatureProcessor(temperature: number): LogitProcessor { + return (logits) => { + 'worklet'; + for (let i = 0; i < logits.length; i++) { + logits[i]! /= temperature; + } + }; +} + +/** + * Keeps only the `k` tokens with the highest logits; all others are set to + * `-Infinity`. Pass `Infinity` to disable. + * @param k Number of top tokens to keep. + * @returns A {@link LogitProcessor} that applies top-k masking. + * @category LLM / Functions + */ +export function topKProcessor(k: number): LogitProcessor { + return (logits) => { + 'worklet'; + if (k <= 0 || k >= logits.length) return; + // Sort a copy ascending; the k-th largest is at index (n - k). + const sorted = logits.slice().sort(); + const threshold = sorted[sorted.length - k]!; + for (let i = 0; i < logits.length; i++) { + if (logits[i]! < threshold) logits[i] = -Infinity; + } + }; +} + +/** + * Nucleus (top-p) sampling: keeps the smallest set of tokens whose cumulative + * softmax probability is at least `p`, then masks the rest to `-Infinity`. + * Must be applied **after** temperature scaling. + * @param p Cumulative probability threshold, in `(0, 1]`. + * @returns A {@link LogitProcessor} that applies nucleus masking. + * @category LLM / Functions + */ +export function topPProcessor(p: number): LogitProcessor { + return (logits) => { + 'worklet'; + const n = logits.length; + const probs = new Float32Array(n); + softmax(logits, probs); + + // Build index array sorted by probability descending. + const indices = Array.from({ length: n }, (_, i) => i); + indices.sort((a, b) => probs[b]! - probs[a]!); + + // Walk until cumulative probability >= p; mark everything else as masked. + const keep = new Uint8Array(n); + let cumProb = 0; + for (const idx of indices) { + keep[idx] = 1; + cumProb += probs[idx]!; + if (cumProb >= p) break; + } + + for (let i = 0; i < n; i++) { + if (!keep[i]) logits[i] = -Infinity; + } + }; +} + +/** + * Min-p sampling: keeps tokens whose probability is at least `minP × p_max`, + * where `p_max` is the maximum token probability after softmax. Simpler and + * often more robust than top-p. Must be applied **after** temperature scaling. + * @param minP Minimum probability ratio relative to the top token, in `(0, 1]`. + * @returns A {@link LogitProcessor} that applies min-p masking. + * @category LLM / Functions + */ +export function minPProcessor(minP: number): LogitProcessor { + return (logits) => { + 'worklet'; + const n = logits.length; + const probs = new Float32Array(n); + softmax(logits, probs); + + let maxProb = 0; + for (let i = 0; i < n; i++) { + if (probs[i]! > maxProb) maxProb = probs[i]!; + } + + const threshold = minP * maxProb; + for (let i = 0; i < n; i++) { + if (probs[i]! < threshold) logits[i] = -Infinity; + } + }; +} + +/** + * Applies a repetition penalty to already-generated tokens. Positive logits + * are divided by `penalty`; negative logits are multiplied, so the penalty + * always pushes logits toward 0 (less likely). `penalty` must be > 1 to have + * any effect. + * @param penalty Penalty factor (must be > 1 to penalize repetition). + * @returns A {@link LogitProcessor} that applies the repetition penalty. + * @category LLM / Functions + */ +export function repetitionPenaltyProcessor(penalty: number): LogitProcessor { + return (logits, ctx) => { + 'worklet'; + for (const token of ctx.generatedTokens) { + const logit = logits[token]; + if (logit === undefined) continue; + logits[token] = logit > 0 ? logit / penalty : logit * penalty; + } + }; +} + +/** + * Adds fixed offsets to specific token logits. Positive values boost a token, + * negative values suppress it. Pass `-Infinity` to ban a token entirely. + * @param bias Map from token id to logit offset. + * @returns A {@link LogitProcessor} that applies the logit bias. + * @category LLM / Functions + */ +export function logitBiasProcessor(bias: ReadonlyMap): LogitProcessor { + return (logits) => { + 'worklet'; + for (const [token, value] of bias) { + if (token >= 0 && token < logits.length) { + logits[token]! += value; + } + } + }; +} + +// ── Built-in samplers ──────────────────────────────────────────────────────── + +/** + * Returns the token with the highest logit (argmax). Deterministic and ~3× + * faster than {@link multinomialSampler} since it requires no softmax. + * @returns A {@link Sampler} that picks the argmax token. + * @category LLM / Functions + */ +export function greedySampler(): Sampler { + return (logits) => { + 'worklet'; + let best = 0; + let bestVal = -Infinity; + for (let i = 0; i < logits.length; i++) { + if (logits[i]! > bestVal) { + bestVal = logits[i]!; + best = i; + } + } + return best; + }; +} + +/** + * Samples a token from the softmax probability distribution of the (processed) + * logits. Non-deterministic; use with a positive `temperature`. + * + * Although greedy decoding is the `temperature → 0` limit of multinomial + * sampling in theory, floating-point precision near logit ties makes + * {@link greedySampler} the correct choice for deterministic inference. + * @returns A {@link Sampler} that draws from the softmax distribution. + * @category LLM / Functions + */ +export function multinomialSampler(): Sampler { + return (logits) => { + 'worklet'; + const n = logits.length; + const probs = new Float32Array(n); + softmax(logits, probs); + + const r = Math.random(); + let cumProb = 0; + for (let i = 0; i < n; i++) { + cumProb += probs[i]!; + if (r <= cumProb) return i; + } + return n - 1; + }; +} + +// ── Pipeline builder ───────────────────────────────────────────────────────── + +/** + * Builds a {@link SamplingPipeline} from a {@link SamplingConfig}. + * + * Built-in processors are registered in this order: + * 1. Repetition penalty + * 2. Logit bias + * 3. Temperature + * 4. Top-k + * 5. Top-p + * 6. Min-p + * 7. Custom `logitProcessors` (appended last — ideal for constrained decoding) + * + * The sampler defaults to {@link greedySampler} when `temperature` is 0 or + * unset, and {@link multinomialSampler} otherwise. + * @param config Sampling configuration. + * @returns A compiled {@link SamplingPipeline}. + * @category LLM / Functions + */ +export function buildSampler(config?: SamplingConfig): SamplingPipeline { + const processors: LogitProcessor[] = []; + + if (config?.repetitionPenalty !== undefined && config.repetitionPenalty !== 1) { + processors.push(repetitionPenaltyProcessor(config.repetitionPenalty)); + } + if (config?.logitBias !== undefined && config.logitBias.size > 0) { + processors.push(logitBiasProcessor(config.logitBias)); + } + + const temperature = config?.temperature ?? 0; + const stochastic = temperature > 0; + + if (stochastic) { + processors.push(temperatureProcessor(temperature)); + } + if (config?.topK !== undefined) { + processors.push(topKProcessor(config.topK)); + } + if (config?.topP !== undefined) { + processors.push(topPProcessor(config.topP)); + } + if (config?.minP !== undefined) { + processors.push(minPProcessor(config.minP)); + } + for (const p of config?.logitProcessors ?? []) { + processors.push(p); + } + + const sampler = config?.sampler ?? (stochastic ? multinomialSampler() : greedySampler()); + + return { processors, sampler }; +} + +/** + * Applies `pipeline.processors` to `logits` in-place, then calls + * `pipeline.sampler` to pick the next token. Convenience wrapper for the + * generation loop. + * @param logits Raw logit array from the model (modified in-place). + * @param pipeline Compiled {@link SamplingPipeline} from {@link buildSampler}. + * @param ctx Contextual information for processors that need generation history. + * @returns The sampled token id. + * @category LLM / Functions + */ +export function runSampling( + logits: Float32Array, + pipeline: SamplingPipeline, + ctx: SamplingContext +): number { + 'worklet'; + for (const p of pipeline.processors) p(logits, ctx); + return pipeline.sampler(logits); +} From d9eaa1eeb0906288c9f5c1ab8bfabb17998db69a Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Sat, 3 Oct 2026 02:29:07 +0200 Subject: [PATCH 10/83] feat(llm): add token sampling pipeline and wire into textRunner - math.ts: add multinomial logit sampling utility - sampler.ts: logit processors (temp, top-k, top-p, min-p, repetition penalty, bias) and createSampler factory returning callable Sampler - llmRunner.ts: compose LLMGenerationConfig with SamplingConfig - textRunner.ts: wire createSampler into decoding loop --- .../src/extensions/llm/index.ts | 1 + .../src/extensions/llm/llmRunner.ts | 6 +- .../src/extensions/llm/runners/textRunner.ts | 13 +- .../src/extensions/llm/sampler.ts | 372 +++++++----------- .../src/extensions/math.ts | 60 +++ 5 files changed, 224 insertions(+), 228 deletions(-) diff --git a/packages/react-native-executorch/src/extensions/llm/index.ts b/packages/react-native-executorch/src/extensions/llm/index.ts index bae6c85305..44557676fe 100644 --- a/packages/react-native-executorch/src/extensions/llm/index.ts +++ b/packages/react-native-executorch/src/extensions/llm/index.ts @@ -4,6 +4,7 @@ */ export * from './llmRunner'; +export * from './sampler'; export * from './utils/chatPreprocessor'; export * from './utils/tokenizerConfig'; export * from './utils/toolCalling'; diff --git a/packages/react-native-executorch/src/extensions/llm/llmRunner.ts b/packages/react-native-executorch/src/extensions/llm/llmRunner.ts index 6f3037b391..143efbfed2 100644 --- a/packages/react-native-executorch/src/extensions/llm/llmRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/llmRunner.ts @@ -9,7 +9,9 @@ import { createResourceScope } from '../../core/lifetime'; import { f32, i64, method, validateSpec } from '../../core/schema'; import { loadTokenizer } from '../nlp'; + import { createTextRunner } from './runners/textRunner'; +import type { SamplingConfig } from './sampler'; declare const llmRunnerBrand: unique symbol; @@ -18,13 +20,11 @@ declare const llmRunnerBrand: unique symbol; * @experimental This API is experimental and might change in future releases. * @category LLM / Types */ -export type LLMGenerationConfig = { +export type LLMGenerationConfig = SamplingConfig & { /** Whether to ignore EOS tokens during generation. */ readonly ignoreEos?: boolean; /** Maximum number of new tokens to generate. */ readonly maxNewTokens?: number; - /** Sampling temperature for token selection. */ - readonly temperature?: number; }; /** diff --git a/packages/react-native-executorch/src/extensions/llm/runners/textRunner.ts b/packages/react-native-executorch/src/extensions/llm/runners/textRunner.ts index f9218fbff7..fbc580409d 100644 --- a/packages/react-native-executorch/src/extensions/llm/runners/textRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/runners/textRunner.ts @@ -5,7 +5,7 @@ import type { Model } from '../../../core/model'; import { RnExecuTorchError } from '../../../core/error'; import type { Tokenizer } from '../../nlp'; - +import { createSampler } from '../sampler'; import type { LLMGenerationConfig, LLMGenerationStats, @@ -120,8 +120,11 @@ export function createTextRunner( const prefillStats = prefill(prompt); const generateStartMs = Date.now(); + const generatedTokens: number[] = []; const maxNewTokens = config?.maxNewTokens ?? Infinity; + const logits = tLogits.getData(new Float32Array(vocabSize)); + const sample = createSampler(config); let numTokens = 0; @@ -132,15 +135,15 @@ export function createTextRunner( while (numTokens < maxNewTokens && !isCancelled.getBlocking()) { if (pos >= maxContextLen) break; - // TODO: Implement sampling (e.g. temperature, top-k, top-p, penalties) - let nextToken = 0; + const nextToken = sample(logits, { generatedTokens }); + + onToken?.(tokenizer.decode(Int32Array.of(nextToken))); + generatedTokens.push(nextToken); if (!config?.ignoreEos && eosIds.includes(nextToken)) { break; } - onToken?.(tokenizer.decode(Int32Array.of(nextToken))); - tToken.setData(BigInt64Array.of(BigInt(nextToken))); tCurPos.setData(BigInt64Array.of(BigInt(pos))); diff --git a/packages/react-native-executorch/src/extensions/llm/sampler.ts b/packages/react-native-executorch/src/extensions/llm/sampler.ts index 9b41fd8958..6c8f672e7d 100644 --- a/packages/react-native-executorch/src/extensions/llm/sampler.ts +++ b/packages/react-native-executorch/src/extensions/llm/sampler.ts @@ -1,18 +1,8 @@ /** - * LLM sampling pipeline — logit processors and samplers. - * - * The pipeline applies a sequence of {@link LogitProcessor}s to the raw logit - * array, then passes the result to a {@link Sampler} to pick the next token. - * - * Standard built-in order (matches llama.cpp / HuggingFace conventions): - * repetition penalty → logit bias → temperature → top-k → top-p / min-p → sample - * - * Custom processors can be injected at the end of the chain via - * {@link SamplingConfig.logitProcessors}, and the sampler can be replaced - * entirely via {@link SamplingConfig.sampler}. + * LLM generation sampling pipeline and logit processors. */ -// ── Types ─────────────────────────────────────────────────────────────────── +import { multinomial } from '../math'; /** * Contextual information available to logit processors during generation. @@ -24,124 +14,69 @@ export type SamplingContext = { }; /** - * Modifies a logit array **in-place** before sampling. - * Multiple processors are applied sequentially in the order they are registered. + * Modifies a logit array in-place before token selection. * @category LLM / Types */ export type LogitProcessor = (logits: Float32Array, ctx: SamplingContext) => void; /** - * Picks a single next-token id from a (processed) logit array. - * - * Note: greedy and multinomial sampling are distinct algorithms, not - * limit-cases of each other in practice. Greedy is a single-pass argmax — - * exact and deterministic. Multinomial requires softmax + a random draw, and - * even at very low temperatures, floating-point precision can cause it to - * diverge from the true argmax near logit ties. Keep both; {@link buildSampler} - * selects the right default based on {@link SamplingConfig.temperature}. + * Selects a token id from a processed logit distribution. * @category LLM / Types */ -export type Sampler = (logits: Float32Array) => number; +export type TokenSelector = (logits: Float32Array) => number; /** - * Configuration for the sampling pipeline. - * - * Shorthand fields (`temperature`, `topK`, …) are expanded into built-in - * {@link LogitProcessor}s automatically by {@link buildSampler}. - * - * Set `temperature` to `0` (or leave it unset) for deterministic greedy - * decoding. Set it to any positive value to enable stochastic multinomial - * sampling. + * Samples the next token id given raw logits and generation context. + * @category LLM / Types + */ +export type Sampler = (logits: Float32Array, ctx: SamplingContext) => number; + +/** + * Configuration options for LLM token sampling. * @category LLM / Types */ export type SamplingConfig = { /** - * Softmax temperature. Values below 1 sharpen the distribution (more - * focused), values above 1 flatten it (more random). Set to 0 or omit for - * greedy decoding. + * Sampling temperature. Values below 1 sharpen the distribution, while values + * above 1 flatten it. Defaults to greedy decoding when unset or set to 0. */ readonly temperature?: number; - /** - * Keep only the `topK` highest-logit tokens; set the rest to `-Infinity`. - * Applied after temperature scaling. - */ + /** Number of highest-probability tokens to retain. */ readonly topK?: number; - /** - * Nucleus sampling: keep the smallest set of tokens whose cumulative - * probability (after softmax) is at least `topP`. Applied after temperature - * and top-k. Values in `(0, 1]`. - */ + /** Cumulative probability threshold for nucleus sampling, in `(0, 1]`. */ readonly topP?: number; - /** - * Min-p sampling: keep tokens whose probability is at least `minP × p_max`, - * where `p_max` is the highest token probability. Simpler and often more - * robust than top-p. Applied after temperature and top-k. Values in `(0, 1]`. - */ + /** Minimum token probability relative to the most likely token, in `(0, 1]`. */ readonly minP?: number; - /** - * Repetition penalty factor (> 1 penalizes repetition, 1 = no effect). - * Logits of already-generated tokens are divided (if positive) or multiplied - * (if negative) by this value. Applied before temperature. - */ + /** Penalty applied to previously generated tokens. Values greater than 1 discourage repetition. */ readonly repetitionPenalty?: number; - /** - * Fixed logit offsets keyed by token id. Positive values boost a token, - * negative values suppress it. Use `-Infinity` to completely ban a token. - * Applied before temperature. - */ + /** Additive logit offsets keyed by token id. */ readonly logitBias?: ReadonlyMap; - /** - * Additional {@link LogitProcessor}s appended after all built-in ones. - * Use this for constrained decoding (grammar, JSON schema, regex, …). - */ + /** Custom logit processors applied after standard processors. */ readonly logitProcessors?: readonly LogitProcessor[]; - /** - * Custom sampler. When omitted, defaults to {@link greedySampler} if - * `temperature` is 0 or unset, and {@link multinomialSampler} otherwise. - */ - readonly sampler?: Sampler; -}; - -/** - * Compiled sampling pipeline, ready to be used in a generation loop. - * @category LLM / Types - */ -export type SamplingPipeline = { - readonly processors: readonly LogitProcessor[]; - readonly sampler: Sampler; + /** Custom token selector overriding greedy or multinomial sampling. */ + readonly selector?: TokenSelector; }; -// ── Internal utilities ─────────────────────────────────────────────────────── - /** - * Computes softmax of `logits` into `out` in-place. - * Numerically stable via max subtraction. `-Infinity` logits contribute 0. - * @param logits Source logit array. - * @param out Destination probability array (same length as `logits`). + * Inserts a value into a sorted array maintaining ascending order. + * @param arr Target sorted array. + * @param val Value to insert. */ -function softmax(logits: Float32Array, out: Float32Array): void { +function insertSorted(arr: number[], val: number): void { 'worklet'; - let max = -Infinity; - for (let i = 0; i < logits.length; i++) { - if (logits[i]! > max) max = logits[i]!; - } - let sum = 0; - for (let i = 0; i < logits.length; i++) { - out[i] = Math.exp(logits[i]! - max); - sum += out[i]!; - } - for (let i = 0; i < logits.length; i++) { - out[i]! /= sum; + let [lo, hi] = [0, arr.length]; + while (lo < hi) { + const mid = Math.floor((lo + hi) / 2); + if (arr[mid]! < val) lo = mid + 1; + else hi = mid; } + arr.splice(lo, 0, val); } -// ── Built-in logit processors ──────────────────────────────────────────────── - /** - * Scales logits by `1 / temperature`. Steers the distribution toward the - * peak (temperature < 1) or away from it (temperature > 1). - * @param temperature Scaling factor (must be > 0). - * @returns A {@link LogitProcessor} that applies the temperature scale. + * Scales logits by the inverse temperature factor. + * @param temperature Sampling temperature. + * @returns A logit processor that applies temperature scaling. * @category LLM / Functions */ export function temperatureProcessor(temperature: number): LogitProcessor { @@ -154,111 +89,151 @@ export function temperatureProcessor(temperature: number): LogitProcessor { } /** - * Keeps only the `k` tokens with the highest logits; all others are set to - * `-Infinity`. Pass `Infinity` to disable. + * Retains only the top-k highest logits and masks the rest to `-Infinity`. * @param k Number of top tokens to keep. - * @returns A {@link LogitProcessor} that applies top-k masking. + * @returns A logit processor that applies top-k filtering. * @category LLM / Functions */ export function topKProcessor(k: number): LogitProcessor { return (logits) => { 'worklet'; if (k <= 0 || k >= logits.length) return; - // Sort a copy ascending; the k-th largest is at index (n - k). - const sorted = logits.slice().sort(); - const threshold = sorted[sorted.length - k]!; + + const top: number[] = []; + for (let i = 0; i < logits.length; i++) { + const val = logits[i]!; + if (top.length < k) { + insertSorted(top, val); + } else if (val > top[0]!) { + top.shift(); + insertSorted(top, val); + } + } + + const threshold = top[0]!; for (let i = 0; i < logits.length; i++) { - if (logits[i]! < threshold) logits[i] = -Infinity; + if (logits[i]! < threshold) { + logits[i] = -Infinity; + } } }; } /** - * Nucleus (top-p) sampling: keeps the smallest set of tokens whose cumulative - * softmax probability is at least `p`, then masks the rest to `-Infinity`. - * Must be applied **after** temperature scaling. - * @param p Cumulative probability threshold, in `(0, 1]`. - * @returns A {@link LogitProcessor} that applies nucleus masking. + * Applies nucleus (top-p) filtering, retaining tokens up to cumulative probability `p`. + * @param p Cumulative probability threshold. + * @returns A logit processor that applies top-p filtering. * @category LLM / Functions */ export function topPProcessor(p: number): LogitProcessor { return (logits) => { 'worklet'; - const n = logits.length; - const probs = new Float32Array(n); - softmax(logits, probs); + if (p <= 0 || p >= 1) return; - // Build index array sorted by probability descending. - const indices = Array.from({ length: n }, (_, i) => i); - indices.sort((a, b) => probs[b]! - probs[a]!); + let maxLogit = -Infinity; + for (let i = 0; i < logits.length; i++) { + if (logits[i]! > maxLogit) { + maxLogit = logits[i]!; + } + } + if (!Number.isFinite(maxLogit)) return; - // Walk until cumulative probability >= p; mark everything else as masked. - const keep = new Uint8Array(n); - let cumProb = 0; - for (const idx of indices) { - keep[idx] = 1; - cumProb += probs[idx]!; - if (cumProb >= p) break; + const minLogitThreshold = maxLogit - 16; + let totalWeight = 0; + const candidates: number[] = []; + + for (let i = 0; i < logits.length; i++) { + const val = logits[i]!; + if (val > minLogitThreshold) { + totalWeight += Math.exp(val - maxLogit); + candidates.push(i); + } else { + logits[i] = -Infinity; + } } - for (let i = 0; i < n; i++) { - if (!keep[i]) logits[i] = -Infinity; + if (candidates.length > 512) { + candidates.sort((a, b) => logits[b]! - logits[a]!); + for (let i = 512; i < candidates.length; i++) { + logits[candidates[i]!] = -Infinity; + } + candidates.length = 512; + } else { + candidates.sort((a, b) => logits[b]! - logits[a]!); + } + + const targetWeight = p * totalWeight; + let cumWeight = 0; + let cutoff = candidates.length - 1; + + for (let i = 0; i < candidates.length; i++) { + const idx = candidates[i]!; + cumWeight += Math.exp(logits[idx]! - maxLogit); + if (cumWeight >= targetWeight) { + cutoff = i; + break; + } + } + + for (let i = cutoff + 1; i < candidates.length; i++) { + logits[candidates[i]!] = -Infinity; } }; } /** - * Min-p sampling: keeps tokens whose probability is at least `minP × p_max`, - * where `p_max` is the maximum token probability after softmax. Simpler and - * often more robust than top-p. Must be applied **after** temperature scaling. - * @param minP Minimum probability ratio relative to the top token, in `(0, 1]`. - * @returns A {@link LogitProcessor} that applies min-p masking. + * Filters out tokens whose probability is below `minP` times the top token's probability. + * @param minP Minimum relative probability threshold. + * @returns A logit processor that applies min-p filtering. * @category LLM / Functions */ export function minPProcessor(minP: number): LogitProcessor { return (logits) => { 'worklet'; - const n = logits.length; - const probs = new Float32Array(n); - softmax(logits, probs); - - let maxProb = 0; - for (let i = 0; i < n; i++) { - if (probs[i]! > maxProb) maxProb = probs[i]!; + if (minP <= 0) return; + let maxLogit = -Infinity; + for (let i = 0; i < logits.length; i++) { + if (logits[i]! > maxLogit) { + maxLogit = logits[i]!; + } } + if (!Number.isFinite(maxLogit)) return; - const threshold = minP * maxProb; - for (let i = 0; i < n; i++) { - if (probs[i]! < threshold) logits[i] = -Infinity; + const threshold = maxLogit + Math.log(minP); + for (let i = 0; i < logits.length; i++) { + if (logits[i]! < threshold) { + logits[i] = -Infinity; + } } }; } /** - * Applies a repetition penalty to already-generated tokens. Positive logits - * are divided by `penalty`; negative logits are multiplied, so the penalty - * always pushes logits toward 0 (less likely). `penalty` must be > 1 to have - * any effect. - * @param penalty Penalty factor (must be > 1 to penalize repetition). - * @returns A {@link LogitProcessor} that applies the repetition penalty. + * Penalizes logits of previously emitted tokens to discourage repetition. + * @param penalty Multiplicative penalty factor. + * @returns A logit processor that penalizes repeated tokens. * @category LLM / Functions */ export function repetitionPenaltyProcessor(penalty: number): LogitProcessor { return (logits, ctx) => { 'worklet'; + if (penalty === 1 || ctx.generatedTokens.length === 0) return; + const seen = new Set(); for (const token of ctx.generatedTokens) { + if (seen.has(token)) continue; + seen.add(token); const logit = logits[token]; - if (logit === undefined) continue; - logits[token] = logit > 0 ? logit / penalty : logit * penalty; + if (logit !== undefined) { + logits[token] = logit > 0 ? logit / penalty : logit * penalty; + } } }; } /** - * Adds fixed offsets to specific token logits. Positive values boost a token, - * negative values suppress it. Pass `-Infinity` to ban a token entirely. - * @param bias Map from token id to logit offset. - * @returns A {@link LogitProcessor} that applies the logit bias. + * Applies additive biases to specific token logits. + * @param bias Map from token id to additive logit offset. + * @returns A logit processor that applies the offsets. * @category LLM / Functions */ export function logitBiasProcessor(bias: ReadonlyMap): LogitProcessor { @@ -272,15 +247,12 @@ export function logitBiasProcessor(bias: ReadonlyMap): LogitProc }; } -// ── Built-in samplers ──────────────────────────────────────────────────────── - /** - * Returns the token with the highest logit (argmax). Deterministic and ~3× - * faster than {@link multinomialSampler} since it requires no softmax. - * @returns A {@link Sampler} that picks the argmax token. + * Selects the token with the highest logit. + * @returns A deterministic greedy token selector. * @category LLM / Functions */ -export function greedySampler(): Sampler { +export function greedySelector(): TokenSelector { return (logits) => { 'worklet'; let best = 0; @@ -296,53 +268,25 @@ export function greedySampler(): Sampler { } /** - * Samples a token from the softmax probability distribution of the (processed) - * logits. Non-deterministic; use with a positive `temperature`. - * - * Although greedy decoding is the `temperature → 0` limit of multinomial - * sampling in theory, floating-point precision near logit ties makes - * {@link greedySampler} the correct choice for deterministic inference. - * @returns A {@link Sampler} that draws from the softmax distribution. + * Draws a token from the categorical probability distribution of logits. + * @param rng Optional custom uniform random number generator producing values in `[0, 1)`. + * @returns A stochastic multinomial token selector. * @category LLM / Functions */ -export function multinomialSampler(): Sampler { +export function multinomialSelector(rng?: () => number): TokenSelector { return (logits) => { 'worklet'; - const n = logits.length; - const probs = new Float32Array(n); - softmax(logits, probs); - - const r = Math.random(); - let cumProb = 0; - for (let i = 0; i < n; i++) { - cumProb += probs[i]!; - if (r <= cumProb) return i; - } - return n - 1; + return multinomial(logits, { rng }); }; } -// ── Pipeline builder ───────────────────────────────────────────────────────── - /** - * Builds a {@link SamplingPipeline} from a {@link SamplingConfig}. - * - * Built-in processors are registered in this order: - * 1. Repetition penalty - * 2. Logit bias - * 3. Temperature - * 4. Top-k - * 5. Top-p - * 6. Min-p - * 7. Custom `logitProcessors` (appended last — ideal for constrained decoding) - * - * The sampler defaults to {@link greedySampler} when `temperature` is 0 or - * unset, and {@link multinomialSampler} otherwise. + * Creates a sampler function configured from generation options. * @param config Sampling configuration. - * @returns A compiled {@link SamplingPipeline}. + * @returns A {@link Sampler} function. * @category LLM / Functions */ -export function buildSampler(config?: SamplingConfig): SamplingPipeline { +export function createSampler(config?: SamplingConfig): Sampler { const processors: LogitProcessor[] = []; if (config?.repetitionPenalty !== undefined && config.repetitionPenalty !== 1) { @@ -367,31 +311,19 @@ export function buildSampler(config?: SamplingConfig): SamplingPipeline { if (config?.minP !== undefined) { processors.push(minPProcessor(config.minP)); } - for (const p of config?.logitProcessors ?? []) { - processors.push(p); + if (config?.logitProcessors) { + for (const processor of config.logitProcessors) { + processors.push(processor); + } } - const sampler = config?.sampler ?? (stochastic ? multinomialSampler() : greedySampler()); - - return { processors, sampler }; -} + const selector = config?.selector ?? (stochastic ? multinomialSelector() : greedySelector()); -/** - * Applies `pipeline.processors` to `logits` in-place, then calls - * `pipeline.sampler` to pick the next token. Convenience wrapper for the - * generation loop. - * @param logits Raw logit array from the model (modified in-place). - * @param pipeline Compiled {@link SamplingPipeline} from {@link buildSampler}. - * @param ctx Contextual information for processors that need generation history. - * @returns The sampled token id. - * @category LLM / Functions - */ -export function runSampling( - logits: Float32Array, - pipeline: SamplingPipeline, - ctx: SamplingContext -): number { - 'worklet'; - for (const p of pipeline.processors) p(logits, ctx); - return pipeline.sampler(logits); + return (logits: Float32Array, ctx: SamplingContext): number => { + 'worklet'; + for (const processor of processors) { + processor(logits, ctx); + } + return selector(logits); + }; } diff --git a/packages/react-native-executorch/src/extensions/math.ts b/packages/react-native-executorch/src/extensions/math.ts index a3feccc823..558ef089f4 100644 --- a/packages/react-native-executorch/src/extensions/math.ts +++ b/packages/react-native-executorch/src/extensions/math.ts @@ -22,6 +22,18 @@ export type RandomNormalOptions = { readonly seed?: number; }; +/** + * Configuration options for categorical multinomial sampling from logits. + * @category Math / Types + */ +export type MultinomialOptions = { + /** + * Custom uniform random number generator producing values in `[0, 1)`. + * Defaults to `Math.random`. + */ + readonly rng?: () => number; +}; + /** * Computes the element-wise sigmoid activation on a float32 source tensor and * writes the result to a destination tensor. @@ -179,6 +191,54 @@ export function randomNormal(size: number, options?: RandomNormalOptions): Float return out; } +/** + * Draws a single index from a categorical distribution defined by unnormalized logits. + * @category Math / Functions + * @param logits Unnormalized log-probabilities to sample from. + * @param options Sampling options including an optional custom random number generator. + * @returns The sampled index. + * @throws {RnExecuTorchError} With code `INVALID_ARGUMENT` if `logits` is empty. + */ +export function multinomial(logits: ArrayLike, options?: MultinomialOptions): number { + 'worklet'; + const n = logits.length; + if (n === 0) { + throw RnExecuTorchError( + 'INVALID_ARGUMENT', + 'multinomial: cannot sample from an empty array of logits.' + ); + } + + const rng = options?.rng ?? Math.random; + + let maxLogit = -Infinity; + for (let i = 0; i < n; i++) { + const val = logits[i]!; + if (val > maxLogit) maxLogit = val; + } + if (!Number.isFinite(maxLogit)) return 0; + + let sum = 0; + for (let i = 0; i < n; i++) { + const val = logits[i]!; + if (val !== -Infinity) { + sum += Math.exp(val - maxLogit); + } + } + if (sum === 0) return 0; + + const target = rng() * sum; + let cum = 0; + for (let i = 0; i < n; i++) { + const val = logits[i]!; + if (val !== -Infinity) { + cum += Math.exp(val - maxLogit); + if (cum >= target) return i; + } + } + return n - 1; +} + /** * Repeats each element of `values` as many times as the matching entry of * `repeats`, concatenating the runs into a newly allocated array of the same From 6d4b0ed9368fb5b2374bd7b8e75fc62a81a5f86b Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Sat, 3 Oct 2026 03:40:39 +0200 Subject: [PATCH 11/83] wip --- apps/nlp/app/llm/index.tsx | 14 ++- .../api/__snapshots__/apiSurface.test.ts.snap | 10 +++ .../cpp/core/model.cpp | 3 +- .../src/extensions/llm/llmRunner.ts | 78 ++++++++++++----- .../src/extensions/llm/runners/textRunner.ts | 86 ++++++++++++++----- .../src/extensions/llm/sampler.ts | 9 ++ .../extensions/llm/tasks/llmChatSession.ts | 4 +- 7 files changed, 149 insertions(+), 55 deletions(-) diff --git a/apps/nlp/app/llm/index.tsx b/apps/nlp/app/llm/index.tsx index 658435e62a..6f579ee6e1 100644 --- a/apps/nlp/app/llm/index.tsx +++ b/apps/nlp/app/llm/index.tsx @@ -31,15 +31,11 @@ type Turn = { }; function formatStats(stats: llm.LLMGenerationStats): string { - const decodeMs = stats.inferenceEndMs - stats.firstTokenMs; - const tokensPerSec = decodeMs > 0 ? (stats.numGeneratedTokens / decodeMs) * 1000 : 0; - const decodeTtftMs = stats.firstTokenMs - stats.inferenceStartMs; - const totalTtftMs = decodeTtftMs + (stats.prefillDurationMs ?? 0); - const totalMs = stats.inferenceEndMs - stats.inferenceStartMs + (stats.prefillDurationMs ?? 0); + const totalMs = stats.durationMs + stats.prefill.durationMs; return ( - `${stats.numGeneratedTokens} tokens · ` + - `${tokensPerSec.toFixed(1)} tok/s · ` + - `${totalTtftMs.toFixed(0)}ms ttft · ` + + `${stats.numTokens} tokens · ` + + `${stats.tokensPerSecond.toFixed(1)} tok/s · ` + + `${stats.prefill.durationMs.toFixed(0)}ms prefill · ` + `${(totalMs / 1000).toFixed(2)}s` ); } @@ -302,7 +298,7 @@ function LLMContent() { Context Window - {kvCacheState.pos} / {kvCacheState.maxSeqLen} tokens ( + {kvCacheState.pos} / {kvCacheState.maxContextLen} tokens ( {(kvCacheState.usageRatio * 100).toFixed(1)}%) diff --git a/packages/react-native-executorch/__tests__/api/__snapshots__/apiSurface.test.ts.snap b/packages/react-native-executorch/__tests__/api/__snapshots__/apiSurface.test.ts.snap index a6c6fa9291..45224ab750 100644 --- a/packages/react-native-executorch/__tests__/api/__snapshots__/apiSurface.test.ts.snap +++ b/packages/react-native-executorch/__tests__/api/__snapshots__/apiSurface.test.ts.snap @@ -54,12 +54,22 @@ exports[`public API surface matches the recorded contents of each namespace 1`] "llm": [ "createChatPreprocessor", "createLLMRunner", + "createSampler", + "greedySelector", + "logitBiasProcessor", + "minPProcessor", + "multinomialSelector", "parseTokenizerConfig", + "repetitionPenaltyProcessor", + "temperatureProcessor", + "topKProcessor", + "topPProcessor", ], "math": [ "argmax", "gather", "mulberry32", + "multinomial", "randomNormal", "repeatInterleave", "sigmoid", diff --git a/packages/react-native-executorch/cpp/core/model.cpp b/packages/react-native-executorch/cpp/core/model.cpp index 745b2bd6f7..ba99b5db73 100644 --- a/packages/react-native-executorch/cpp/core/model.cpp +++ b/packages/react-native-executorch/cpp/core/model.cpp @@ -48,11 +48,12 @@ namespace rnexecutorch::core::model { namespace jsi = facebook::jsi; namespace conversions = rnexecutorch::core::conversions; +using executorch::extension::Module; using rnexecutorch::core::tensor::TensorHostObject; ModelHostObject::ModelHostObject(const std::string &modelPath, const bool eagerLoadMethods) : modelPath_(modelPath), - etModule_(std::make_unique(modelPath)) { + etModule_(std::make_unique(modelPath, Module::LoadMode::Mmap)) { auto loadError = etModule_->load(); if (!etModule_->is_loaded()) { diff --git a/packages/react-native-executorch/src/extensions/llm/llmRunner.ts b/packages/react-native-executorch/src/extensions/llm/llmRunner.ts index 143efbfed2..55d8ecb34e 100644 --- a/packages/react-native-executorch/src/extensions/llm/llmRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/llmRunner.ts @@ -1,6 +1,9 @@ /** * Low-level native ExecuTorch LLM runner types and factory. */ + +import { type WorkletRuntime } from 'react-native-worklets'; + import type { Tensor } from '../../core/tensor'; import { wrapAsync } from '../../core/runtime'; import { loadModel } from '../../core/model'; @@ -170,17 +173,19 @@ export type LLMRunner = { * @param tokenizerPath Path to the local tokenizer configuration file (e.g. `tokenizer.json`). * @param modalities List of supported input non-text modalities (e.g. * `['image']`). When omitted, defaults to text-only. + * @param runtime ... * @returns A native {@link LLMRunner} instance. */ export async function createLLMRunner( modelPath: string, tokenizerPath: string, - modalities?: readonly Modality[] + modalities?: readonly Modality[], + runtime?: WorkletRuntime ): Promise { const scope = createResourceScope(); try { - const model = scope.track(await wrapAsync(loadModel)(modelPath)); - const tokenizer = scope.track(await wrapAsync(loadTokenizer)(tokenizerPath)); + const model = scope.track(await wrapAsync(loadModel, runtime)(modelPath)); + const tokenizer = scope.track(await wrapAsync(loadTokenizer, runtime)(tokenizerPath)); // TODO(@bh): update validate spec after reexporting LLMs with schema const method const metadata = { @@ -189,14 +194,14 @@ export async function createLLMRunner( ...method('get_vocab_size', [], [{ kind: 'Int' }]), ...method('use_kv_cache', [], [{ kind: 'Bool' }]), ...method('enable_dynamic_shape', [], [{ kind: 'Bool' }]), - ...method('get_eos_ids', [], [{ kind: 'ListInt' }]), + ...method('get_eos_ids', [], [{ kind: 'Int' }]), }; const { variant, dims } = validateSpec(model.schema, { llm: { ...method( 'forward', - // Input: ... + // Input: tokens [1, maxSeqLen], curPos [1] [i64(1, 'maxSeqLen'), i64(1)], // Output: logits over model vocabulary [f32(1, 'vocabSize')] @@ -231,7 +236,10 @@ export async function createLLMRunner( }); if (variant === 'llm' && modalities && modalities.length > 0) { - throw RnExecuTorchError('INVALID_ARGUMENT', ''); + throw RnExecuTorchError( + 'INVALID_ARGUMENT', + `Text-only model does not support modalities: ${modalities.join(', ')}.` + ); } // if (variant === 'vlm' && !modalities?.includes('image')) { // throw RnExecuTorchError('INVALID_ARGUMENT', ''); @@ -242,32 +250,58 @@ export async function createLLMRunner( const [maxSeqLen, vocabSize] = dims.constant('maxSeqLen', 'vocabSize'); const [maxContextLen] = model.execute('get_max_context_len', [], []) as [number]; - const [eosIds] = model.execute('get_eos_ids', [], []) as [number[]]; + const eosIds = model.execute('get_eos_ids', [], []) as [number]; - if (maxSeqLen !== model.execute('get_max_seq_len', [], [])[0]) { - throw RnExecuTorchError('SCHEMA_MISMATCH', ''); - } - if (vocabSize !== model.execute('get_vocab_size', [], [])[0]) { - throw RnExecuTorchError('SCHEMA_MISMATCH', ''); - } - if (vocabSize !== tokenizer.getVocabSize()) { - throw RnExecuTorchError('SCHEMA_MISMATCH', ''); + // const maxSeqLenMethod = model.execute('get_max_seq_len', [], [])[0]; + // if (maxSeqLen !== maxSeqLenMethod) { + // throw RnExecuTorchError( + // 'SCHEMA_MISMATCH', + // `Schema maxSeqLen (${maxSeqLen}) does not match get_max_seq_len (${maxSeqLenMethod}).` + // ); + // } + const vocabSizeMethod = model.execute('get_vocab_size', [], [])[0]; + if (vocabSize !== vocabSizeMethod) { + throw RnExecuTorchError( + 'SCHEMA_MISMATCH', + `Schema vocabSize (${vocabSize}) does not match get_vocab_size (${vocabSizeMethod}).` + ); } + // if (vocabSize !== tokenizer.getVocabSize()) { + // throw RnExecuTorchError( + // 'SCHEMA_MISMATCH', + // `Model vocabSize (${vocabSize}) does not match tokenizer size (${tokenizer.getVocabSize()}).` + // ); + // } if (model.execute('use_kv_cache', [], [])[0] !== true) { - throw RnExecuTorchError('SCHEMA_MISMATCH', ''); + throw RnExecuTorchError( + 'SCHEMA_MISMATCH', // prettier-ignore + 'Model must enable use_kv_cache.' + ); } if (model.execute('enable_dynamic_shape', [], [])[0] !== true) { - throw RnExecuTorchError('SCHEMA_MISMATCH', ''); + throw RnExecuTorchError( + 'SCHEMA_MISMATCH', // prettier-ignore + 'Model must enable enable_dynamic_shape.' + ); } if (maxSeqLen <= 0 || !Number.isInteger(maxSeqLen)) { - throw RnExecuTorchError('SCHEMA_MISMATCH', ''); + throw RnExecuTorchError( + 'SCHEMA_MISMATCH', + `maxSeqLen must be a positive integer, got ${maxSeqLen}.` + ); } if (maxContextLen < maxSeqLen || !Number.isInteger(maxContextLen)) { - throw RnExecuTorchError('SCHEMA_MISMATCH', ''); - } - if (!Array.isArray(eosIds) || eosIds.length === 0) { - throw RnExecuTorchError('SCHEMA_MISMATCH', ''); + throw RnExecuTorchError( + 'SCHEMA_MISMATCH', + `maxContextLen (${maxContextLen}) must be an integer >= maxSeqLen (${maxSeqLen}).` + ); } + // if (!Array.isArray(eosIds) || eosIds.length === 0) { + // throw RnExecuTorchError( + // 'SCHEMA_MISMATCH', + // 'Model get_eos_ids returned an empty or invalid array.' + // ); + // } return createTextRunner(model, tokenizer, { maxSeqLen, maxContextLen, vocabSize, eosIds }); } catch (e) { diff --git a/packages/react-native-executorch/src/extensions/llm/runners/textRunner.ts b/packages/react-native-executorch/src/extensions/llm/runners/textRunner.ts index fbc580409d..678a3dc64e 100644 --- a/packages/react-native-executorch/src/extensions/llm/runners/textRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/runners/textRunner.ts @@ -37,24 +37,30 @@ export function createTextRunner( // ================================== // KV Cache management // ================================== - let pos = 0; + const pos = createSynchronizable(0); const reset = (targetPos?: number): void => { if (targetPos !== undefined && (targetPos < 0 || targetPos >= maxContextLen)) { - throw RnExecuTorchError('INVALID_ARGUMENT', ''); + throw RnExecuTorchError( + 'INVALID_ARGUMENT', + `targetPos (${targetPos}) must be in range [0, ${maxContextLen}).` + ); } - pos = targetPos ?? 0; + pos.setBlocking(targetPos ?? 0); }; - const getKVCacheState = (): LLMKVCacheState => ({ - pos, - maxContextLen, - remainingTokens: maxContextLen - pos, - usageRatio: pos / maxContextLen, - }); + const getKVCacheState = (): LLMKVCacheState => { + const curPos = pos.getBlocking(); + return { + pos: curPos, + maxContextLen, + remainingTokens: maxContextLen - curPos, + usageRatio: curPos / maxContextLen, + }; + }; // ================================== - // Generation methods + // Prefill // ================================== const isCancelled = createSynchronizable(false); const stop = (): void => isCancelled.setBlocking(true); @@ -64,11 +70,14 @@ export function createTextRunner( isCancelled.setBlocking(false); if (typeof prompt !== 'string') { - throw RnExecuTorchError('INVALID_ARGUMENT', ''); + throw RnExecuTorchError( + 'INVALID_ARGUMENT', + `prefill: Expected text prompt string, got ${typeof prompt}.` + ); } const startMs = Date.now(); - const startPos = pos; + const startPos = pos.getBlocking(); const tokens = tokenizer.encode(prompt); let offset = 0; @@ -76,13 +85,16 @@ export function createTextRunner( const seqLen = Math.min(tokens.length - offset, maxSeqLen); const chunk = tokens.subarray(offset, offset + seqLen); - if (pos + seqLen > maxContextLen) { - throw RnExecuTorchError('EXECUTION_FAILED', ''); + if (startPos + offset + seqLen > maxContextLen) { + throw RnExecuTorchError( + 'EXECUTION_FAILED', + `prefill: Context length exceeded (${startPos + offset + seqLen} > ${maxContextLen}).` + ); } // prettier-ignore const tTokens = tensor('int64', [1, seqLen], BigInt64Array.from(chunk, (v) => BigInt(v))); - const tCurPos = tensor('int64', [1], BigInt64Array.of(BigInt(pos))); + const tCurPos = tensor('int64', [1], BigInt64Array.of(BigInt(startPos + offset))); try { model.execute('forward', [tTokens, tCurPos], [tLogits]); } finally { @@ -90,12 +102,12 @@ export function createTextRunner( tCurPos.dispose(); } - pos += seqLen; + pos.setBlocking(startPos + offset + seqLen); offset += seqLen; } const durationMs = Date.now() - startMs; - const numTokens = pos - startPos; + const numTokens = offset; const tokensPerSecond = durationMs > 0 ? (numTokens / durationMs) * 1000 : 0; return { @@ -105,6 +117,9 @@ export function createTextRunner( }; }; + // ================================== + // Decode + // ================================== const generate = ( prompt: Prompt, config?: LLMGenerationConfig, @@ -114,7 +129,10 @@ export function createTextRunner( isCancelled.setBlocking(false); if (typeof prompt !== 'string') { - throw RnExecuTorchError('INVALID_ARGUMENT', ''); + throw RnExecuTorchError( + 'INVALID_ARGUMENT', + `generate: Expected text prompt string, got ${typeof prompt}.` + ); } const prefillStats = prefill(prompt); @@ -127,17 +145,30 @@ export function createTextRunner( const sample = createSampler(config); let numTokens = 0; + // TO-REMOVE: Timing breakdown for benchmarking + let totalModelExecuteMs = 0; + let totalSampleMs = 0; + let totalOnTokenMs = 0; const tToken = tensor('int64', [1, 1]); const tCurPos = tensor('int64', [1]); + let curPos = pos.getBlocking(); + try { while (numTokens < maxNewTokens && !isCancelled.getBlocking()) { - if (pos >= maxContextLen) break; + if (curPos >= maxContextLen) break; + // TO-REMOVE: Sample timing + const tSample0 = Date.now(); const nextToken = sample(logits, { generatedTokens }); + totalSampleMs += Date.now() - tSample0; + // TO-REMOVE: onToken timing + const tOnToken0 = Date.now(); onToken?.(tokenizer.decode(Int32Array.of(nextToken))); + totalOnTokenMs += Date.now() - tOnToken0; + generatedTokens.push(nextToken); if (!config?.ignoreEos && eosIds.includes(nextToken)) { @@ -145,11 +176,15 @@ export function createTextRunner( } tToken.setData(BigInt64Array.of(BigInt(nextToken))); - tCurPos.setData(BigInt64Array.of(BigInt(pos))); + tCurPos.setData(BigInt64Array.of(BigInt(curPos))); + // TO-REMOVE: model.execute timing + const tExec0 = Date.now(); model.execute('forward', [tToken, tCurPos], [tLogits]); + totalModelExecuteMs += Date.now() - tExec0; - pos += 1; + curPos += 1; + pos.setBlocking(curPos); numTokens += 1; tLogits.getData(logits); } @@ -161,6 +196,15 @@ export function createTextRunner( const durationMs = Date.now() - generateStartMs; const tokensPerSecond = durationMs > 0 ? (numTokens / durationMs) * 1000 : 0; + // TO-REMOVE: Console log breakdown + // eslint-disable-next-line no-console + console.log( + `[textRunner] Generated ${numTokens} tokens in ${durationMs}ms (${tokensPerSecond.toFixed(1)} tok/s) | ` + + `model.execute: ${totalModelExecuteMs}ms (${durationMs > 0 ? ((totalModelExecuteMs / durationMs) * 100).toFixed(1) : 0}%) | ` + + `sample: ${totalSampleMs}ms (${durationMs > 0 ? ((totalSampleMs / durationMs) * 100).toFixed(1) : 0}%) | ` + + `onToken: ${totalOnTokenMs}ms (${durationMs > 0 ? ((totalOnTokenMs / durationMs) * 100).toFixed(1) : 0}%)` + ); + return { numTokens, durationMs, tokensPerSecond, prefill: prefillStats }; }; diff --git a/packages/react-native-executorch/src/extensions/llm/sampler.ts b/packages/react-native-executorch/src/extensions/llm/sampler.ts index 6c8f672e7d..00609de58e 100644 --- a/packages/react-native-executorch/src/extensions/llm/sampler.ts +++ b/packages/react-native-executorch/src/extensions/llm/sampler.ts @@ -80,6 +80,7 @@ function insertSorted(arr: number[], val: number): void { * @category LLM / Functions */ export function temperatureProcessor(temperature: number): LogitProcessor { + 'worklet'; return (logits) => { 'worklet'; for (let i = 0; i < logits.length; i++) { @@ -95,6 +96,7 @@ export function temperatureProcessor(temperature: number): LogitProcessor { * @category LLM / Functions */ export function topKProcessor(k: number): LogitProcessor { + 'worklet'; return (logits) => { 'worklet'; if (k <= 0 || k >= logits.length) return; @@ -126,6 +128,7 @@ export function topKProcessor(k: number): LogitProcessor { * @category LLM / Functions */ export function topPProcessor(p: number): LogitProcessor { + 'worklet'; return (logits) => { 'worklet'; if (p <= 0 || p >= 1) return; @@ -188,6 +191,7 @@ export function topPProcessor(p: number): LogitProcessor { * @category LLM / Functions */ export function minPProcessor(minP: number): LogitProcessor { + 'worklet'; return (logits) => { 'worklet'; if (minP <= 0) return; @@ -215,6 +219,7 @@ export function minPProcessor(minP: number): LogitProcessor { * @category LLM / Functions */ export function repetitionPenaltyProcessor(penalty: number): LogitProcessor { + 'worklet'; return (logits, ctx) => { 'worklet'; if (penalty === 1 || ctx.generatedTokens.length === 0) return; @@ -237,6 +242,7 @@ export function repetitionPenaltyProcessor(penalty: number): LogitProcessor { * @category LLM / Functions */ export function logitBiasProcessor(bias: ReadonlyMap): LogitProcessor { + 'worklet'; return (logits) => { 'worklet'; for (const [token, value] of bias) { @@ -253,6 +259,7 @@ export function logitBiasProcessor(bias: ReadonlyMap): LogitProc * @category LLM / Functions */ export function greedySelector(): TokenSelector { + 'worklet'; return (logits) => { 'worklet'; let best = 0; @@ -274,6 +281,7 @@ export function greedySelector(): TokenSelector { * @category LLM / Functions */ export function multinomialSelector(rng?: () => number): TokenSelector { + 'worklet'; return (logits) => { 'worklet'; return multinomial(logits, { rng }); @@ -287,6 +295,7 @@ export function multinomialSelector(rng?: () => number): TokenSelector { * @category LLM / Functions */ export function createSampler(config?: SamplingConfig): Sampler { + 'worklet'; const processors: LogitProcessor[] = []; if (config?.repetitionPenalty !== undefined && config.repetitionPenalty !== 1) { diff --git a/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts b/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts index 42dbcc587d..6e00eac11b 100644 --- a/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts +++ b/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts @@ -203,8 +203,8 @@ export async function createLLMChatSession( const chatPreprocessor = scope.track(createChatPreprocessor(chatPreprocessorConfig)); // Prepare runner - const load = wrapAsync(createLLMRunner, runtime); - const runner = scope.track(await load(modelPath, tokenizerPath, modalities)); + // prettier-ignore + const runner = scope.track(await createLLMRunner(modelPath, tokenizerPath, modalities, runtime)); const prefill = wrapAsync(runner.prefill, runtime); const history: ChatMessage[] = []; From 3b877890e54dadda6b76f64a893f2d4cf4a4efe3 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Sat, 3 Oct 2026 04:02:32 +0200 Subject: [PATCH 12/83] revert(model): revert primitive list support from execute and bridge docstring --- .../cpp/core/conversions.h | 25 +++++++++---------- .../cpp/core/model.cpp | 9 ------- .../react-native-executorch/src/core/model.ts | 9 +------ .../src/native/bridge.ts | 2 +- 4 files changed, 14 insertions(+), 31 deletions(-) diff --git a/packages/react-native-executorch/cpp/core/conversions.h b/packages/react-native-executorch/cpp/core/conversions.h index cfaba990d1..d894e066c0 100644 --- a/packages/react-native-executorch/cpp/core/conversions.h +++ b/packages/react-native-executorch/cpp/core/conversions.h @@ -164,27 +164,26 @@ template inline constexpr bool kAlwaysFalse = false; /** - * Converts a container (e.g. std::vector, ArrayRef) of values to a new facebook::jsi::Array. + * Converts a std::vector of values to a new facebook::jsi::Array. * Handles strings, booleans, and numeric types appropriately. * - * @tparam Container The container type (must support .size() and operator[]). + * @tparam T The element type in the vector. * @param rt The JSI runtime instance. - * @param items The source container to convert. - * @return A new facebook::jsi::Array containing the elements from the container. + * @param vec The source vector to convert. + * @return A new facebook::jsi::Array containing the elements from the vector. */ -template -jsi::Array toJsiArray(jsi::Runtime &rt, const Container &items) { - jsi::Array arr(rt, items.size()); - for (size_t i = 0; i < items.size(); ++i) { - using T = std::decay_t; +template +jsi::Array toJsiArray(jsi::Runtime &rt, const std::vector &vec) { + jsi::Array arr(rt, vec.size()); + for (size_t i = 0; i < vec.size(); ++i) { if constexpr (std::is_same_v) { - arr.setValueAtIndex(rt, i, jsi::String::createFromUtf8(rt, items[i])); + arr.setValueAtIndex(rt, i, jsi::String::createFromUtf8(rt, vec[i])); } else if constexpr (std::is_same_v) { - arr.setValueAtIndex(rt, i, jsi::Value(static_cast(items[i]))); + arr.setValueAtIndex(rt, i, jsi::Value(static_cast(vec[i]))); } else if constexpr (std::is_arithmetic_v) { - arr.setValueAtIndex(rt, i, jsi::Value(static_cast(items[i]))); + arr.setValueAtIndex(rt, i, jsi::Value(static_cast(vec[i]))); } else { - static_assert(kAlwaysFalse, "Unsupported element type for toJsiArray"); + static_assert(kAlwaysFalse, "Unsupported vector element type for toJsiArray"); } } return arr; diff --git a/packages/react-native-executorch/cpp/core/model.cpp b/packages/react-native-executorch/cpp/core/model.cpp index ba99b5db73..49c8786831 100644 --- a/packages/react-native-executorch/cpp/core/model.cpp +++ b/packages/react-native-executorch/cpp/core/model.cpp @@ -288,15 +288,6 @@ jsi::Value ModelHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) { case executorch::runtime::Tag::String: jsOutputArray.setValueAtIndex(rt, outputIdx, jsi::String::createFromUtf8(rt, std::string(output.toString()))); break; - case executorch::runtime::Tag::ListInt: - jsOutputArray.setValueAtIndex(rt, outputIdx, conversions::toJsiArray(rt, output.toIntList())); - break; - case executorch::runtime::Tag::ListDouble: - jsOutputArray.setValueAtIndex(rt, outputIdx, conversions::toJsiArray(rt, output.toDoubleList())); - break; - case executorch::runtime::Tag::ListBool: - jsOutputArray.setValueAtIndex(rt, outputIdx, conversions::toJsiArray(rt, output.toBoolList())); - break; default: throw error::ExecutionFailed(std::format("execute: Unsupported return type: {}", executorch::runtime::tag_to_string(output.tag))); diff --git a/packages/react-native-executorch/src/core/model.ts b/packages/react-native-executorch/src/core/model.ts index 14219aa08b..da42f818eb 100644 --- a/packages/react-native-executorch/src/core/model.ts +++ b/packages/react-native-executorch/src/core/model.ts @@ -22,14 +22,7 @@ export type ModelInput = Tensor | number | boolean | null; * A value returned from a model's `execute` method. * @category Core / Types */ -export type ModelOutput = - | Tensor - | number - | boolean - | string - | null - | readonly number[] - | readonly boolean[]; +export type ModelOutput = Tensor | number | boolean | string | null; /** * A compiled, ready-to-run ExecuTorch model loaded into native memory. diff --git a/packages/react-native-executorch/src/native/bridge.ts b/packages/react-native-executorch/src/native/bridge.ts index 998e3c644e..3d17ab8fc0 100644 --- a/packages/react-native-executorch/src/native/bridge.ts +++ b/packages/react-native-executorch/src/native/bridge.ts @@ -13,7 +13,7 @@ if (!globalObj.__rnexecutorch_jsi__) { * Provides low-level, synchronous bindings for ExecuTorch core runtime * operations (model loading, tensor allocation and manipulation, registered * backends inspection, emulator detection) and native extension namespaces - * (`cv`, `math`, `nlp`, `speech`). + * (`cv`, `llm`, `math`, `nlp`, `speech`). * @internal */ export const rnexecutorchJsi = globalObj.__rnexecutorch_jsi__; From 3d27e96d3822e82cd909f20ff355e5399384b2a9 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Sat, 3 Oct 2026 04:27:59 +0200 Subject: [PATCH 13/83] style --- .../src/extensions/llm/runners/textRunner.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/react-native-executorch/src/extensions/llm/runners/textRunner.ts b/packages/react-native-executorch/src/extensions/llm/runners/textRunner.ts index 678a3dc64e..e71cd06d9f 100644 --- a/packages/react-native-executorch/src/extensions/llm/runners/textRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/runners/textRunner.ts @@ -60,7 +60,7 @@ export function createTextRunner( }; // ================================== - // Prefill + // Generation methods // ================================== const isCancelled = createSynchronizable(false); const stop = (): void => isCancelled.setBlocking(true); @@ -117,9 +117,6 @@ export function createTextRunner( }; }; - // ================================== - // Decode - // ================================== const generate = ( prompt: Prompt, config?: LLMGenerationConfig, From 16d3d168f75e3850f637844c09013913cabf0331 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Mon, 5 Oct 2026 16:34:51 +0200 Subject: [PATCH 14/83] feat(llm): add native C++ sampler and install map Add the LLM extension's native sampler and the JSI wiring that exposes it: - cpp/extensions/llm/sampler.cpp|h: sampling pipeline (constraints -> penalties -> dry -> top-k -> top-p -> min-p -> xtc -> temperature -> draw) operating in place on the logits tensor, matching the llama.cpp sampler chain order. - cpp/extensions/llm/install.cpp|h: install map for the llm module. - RnExecutorch.cpp: call the llm install. - CMakeLists: glob extensions/llm sources into the Android and test builds. - sampler.ts: SamplingConfig bridge that resolves llama.cpp-style defaults and calls the native sampler. --- .../android/CMakeLists.txt | 2 + .../cpp/RnExecutorch.cpp | 2 + .../cpp/extensions/llm/install.cpp | 14 + .../cpp/extensions/llm/install.h | 13 + .../cpp/extensions/llm/sampler.cpp | 656 ++++++++++++++++++ .../cpp/extensions/llm/sampler.h | 7 + .../cpp/tests/CMakeLists.txt | 3 +- .../src/extensions/llm/sampler.ts | 412 ++++------- 8 files changed, 814 insertions(+), 295 deletions(-) create mode 100644 packages/react-native-executorch/cpp/extensions/llm/install.cpp create mode 100644 packages/react-native-executorch/cpp/extensions/llm/install.h create mode 100644 packages/react-native-executorch/cpp/extensions/llm/sampler.cpp create mode 100644 packages/react-native-executorch/cpp/extensions/llm/sampler.h diff --git a/packages/react-native-executorch/android/CMakeLists.txt b/packages/react-native-executorch/android/CMakeLists.txt index 1dc7f2f6bb..d4df1c0887 100644 --- a/packages/react-native-executorch/android/CMakeLists.txt +++ b/packages/react-native-executorch/android/CMakeLists.txt @@ -38,6 +38,7 @@ find_package(fbjni REQUIRED CONFIG) # explicit, extensible list of opencv-dependent sources: when multimodal LLM # lands in nlp, its vision-encoder sources should be appended here too. file(GLOB CORE_SOURCES ${CPP_DIR}/core/*.cpp) +file(GLOB LLM_SOURCES ${CPP_DIR}/extensions/llm/*.cpp) file(GLOB MATH_SOURCES ${CPP_DIR}/extensions/math/*.cpp) file(GLOB NLP_SOURCES ${CPP_DIR}/extensions/nlp/*.cpp) file(GLOB SPEECH_SOURCES ${CPP_DIR}/extensions/speech/*.cpp) @@ -76,6 +77,7 @@ list(FILTER SPEECH_SOURCES EXCLUDE REGEX "/phonemizer\\.cpp$") set(RNE_SOURCES ${CPP_DIR}/RnExecutorch.cpp ${CORE_SOURCES} + ${LLM_SOURCES} ${MATH_SOURCES} ${NLP_SOURCES} ${SPEECH_SOURCES} diff --git a/packages/react-native-executorch/cpp/RnExecutorch.cpp b/packages/react-native-executorch/cpp/RnExecutorch.cpp index df1bcb9d4d..8eeab6e278 100644 --- a/packages/react-native-executorch/cpp/RnExecutorch.cpp +++ b/packages/react-native-executorch/cpp/RnExecutorch.cpp @@ -1,6 +1,7 @@ #include "RnExecutorch.h" #include "core/install.h" +#include "extensions/llm/install.h" #include "extensions/math/install.h" #include "extensions/nlp/install.h" #include "extensions/speech/install.h" @@ -21,6 +22,7 @@ void install(jsi::Runtime &jsiRuntime) { #endif rnexecutorch::extensions::math::install(jsiRuntime, module); rnexecutorch::extensions::nlp::install(jsiRuntime, module); + rnexecutorch::extensions::llm::install(jsiRuntime, module); rnexecutorch::extensions::speech::install(jsiRuntime, module); jsiRuntime.global().setProperty(jsiRuntime, "__rnexecutorch_jsi__", std::move(module)); diff --git a/packages/react-native-executorch/cpp/extensions/llm/install.cpp b/packages/react-native-executorch/cpp/extensions/llm/install.cpp new file mode 100644 index 0000000000..0545e70b83 --- /dev/null +++ b/packages/react-native-executorch/cpp/extensions/llm/install.cpp @@ -0,0 +1,14 @@ +#include "install.h" +#include "sampler.h" + +namespace rnexecutorch::extensions::llm { +namespace jsi = facebook::jsi; + +void install(jsi::Runtime &rt, jsi::Object &module) { + jsi::Object llmModule(rt); + + install_sample(rt, llmModule); + + module.setProperty(rt, "llm", llmModule); +} +} // namespace rnexecutorch::extensions::llm diff --git a/packages/react-native-executorch/cpp/extensions/llm/install.h b/packages/react-native-executorch/cpp/extensions/llm/install.h new file mode 100644 index 0000000000..f9b16f6fb8 --- /dev/null +++ b/packages/react-native-executorch/cpp/extensions/llm/install.h @@ -0,0 +1,13 @@ +#pragma once + +#include + +namespace rnexecutorch::extensions::llm { +/** + * Installs all LLM extension host functions and objects onto the target JSI module. + * + * @param rt The JSI runtime instance. + * @param module The root `rnexecutorchJsi` module object. + */ +void install(facebook::jsi::Runtime &rt, facebook::jsi::Object &module); +} // namespace rnexecutorch::extensions::llm diff --git a/packages/react-native-executorch/cpp/extensions/llm/sampler.cpp b/packages/react-native-executorch/cpp/extensions/llm/sampler.cpp new file mode 100644 index 0000000000..5e4980beab --- /dev/null +++ b/packages/react-native-executorch/cpp/extensions/llm/sampler.cpp @@ -0,0 +1,656 @@ +#include "sampler.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "core/conversions.h" +#include "core/error.h" +#include "core/tensor.h" +#include "core/tensor_helpers.h" + +namespace rnexecutorch::extensions::llm { +namespace jsi = facebook::jsi; +namespace error = rnexecutorch::core::error; +namespace tensor = rnexecutorch::core::tensor; + +using rnexecutorch::core::conversions::asType; +using rnexecutorch::core::conversions::fromJsiTypedArray; +using rnexecutorch::core::conversions::getOptionalProperty; +using rnexecutorch::core::conversions::getRequiredProperty; +using rnexecutorch::core::types::DType; + +namespace { + +/** + * A token candidate paired with its logit and normalized probability. + */ +struct Candidate { + int32_t token; + float logit; + float prob; +}; + +/** + * Applies repetition penalty to the unnormalized logits based on previously generated tokens. + * + * For each unique token previously generated: + * - Positive logits are scaled down by dividing by the penalty factor. + * - Non-positive logits are penalized by multiplying by the penalty factor. + * + * @param logits Logits span to modify in place. + * @param tokens History of previously generated token IDs. + * @param penalty Multiplicative repetition penalty factor (> 1.0 penalizes repeats). + */ +void applyRepetitionPenalty(std::span logits, const std::vector &tokens, float penalty) { + if (penalty == 1.0f || tokens.empty()) { + return; + } + + std::unordered_set seen; + for (int32_t tok : tokens) { + if (tok < 0 || !std::cmp_less(tok, logits.size()) || !seen.insert(tok).second) { + continue; + } + const auto idx = static_cast(tok); + const float val = logits[idx]; + logits[idx] = (val > 0.0f) ? (val / penalty) : (val * penalty); + } +} + +/** + * Applies Don't Repeat Yourself (DRY) penalty to unnormalized logits. + * + * Scans previous token history to find matching suffixes that would be extended + * if a previous continuation token is repeated. When a match length meets or exceeds + * `allowedLength`, an exponential penalty is subtracted from the continuation token's logit: + * penalty = multiplier * (base ^ (matchLength - allowedLength)) + * + * @param logits Logits span to modify in place. + * @param tokens History of previously generated token IDs. + * @param multiplier Penalty multiplier factor (> 0). + * @param base Exponential base (> 0). + * @param allowedLength Minimum match length before penalty triggers. + * @param penaltyLastN Maximum recent tokens window to scan (-1 or 0 for unlimited). + * @param breakers Set of token IDs that break sequence matches and are exempt from penalty. + */ +void applyDry(std::span logits, + const std::vector &tokens, + float multiplier, + float base, + int32_t allowedLength, + int32_t penaltyLastN, + const std::unordered_set &breakers) { + if (multiplier <= 0.0f || base <= 0.0f || tokens.size() <= 1 || allowedLength < 0) { + return; + } + + const size_t totalTokens = tokens.size(); + const size_t windowStart = (penaltyLastN > 0 && std::cmp_greater(totalTokens, penaltyLastN)) + ? (totalTokens - static_cast(penaltyLastN)) + : 0; + + const int32_t lastTok = tokens[totalTokens - 1]; + if (breakers.contains(lastTok)) { + return; + } + + std::vector> maxPenalties; + + // Scan backwards through token history ending before the last token. + for (size_t i = windowStart; i < totalTokens - 1; ++i) { + if (tokens[i] != lastTok) { + continue; + } + + // Measure how far backwards the match extends. + size_t matchLen = 1; + while (matchLen <= (i - windowStart)) { + const int32_t pastTok = tokens[i - matchLen]; + const int32_t currTok = tokens[totalTokens - 1 - matchLen]; + if (pastTok != currTok || breakers.contains(pastTok)) { + break; + } + ++matchLen; + } + + if (std::cmp_less(matchLen, allowedLength)) { + continue; + } + + const int32_t nextTok = tokens[i + 1]; + if (breakers.contains(nextTok) || nextTok < 0 || !std::cmp_less(nextTok, logits.size())) { + continue; + } + + const auto exponent = static_cast(matchLen - static_cast(allowedLength)); + float penalty = multiplier * std::pow(base, exponent); + if (!std::isfinite(penalty)) { + penalty = std::numeric_limits::infinity(); + } + + auto it = std::ranges::find_if(maxPenalties, [nextTok](const auto &p) { return p.first == nextTok; }); + if (it != maxPenalties.end()) { + it->second = std::max(it->second, penalty); + } else { + maxPenalties.emplace_back(nextTok, penalty); + } + } + + for (const auto &[tok, penalty] : maxPenalties) { + const auto idx = static_cast(tok); + if (std::isinf(penalty)) { + logits[idx] = -std::numeric_limits::infinity(); + } else { + logits[idx] -= penalty; + } + } +} + +/** + * Invokes the user-defined `constraints` callback from TypeScript to apply constrained decoding. + * + * Supports: + * - `allowedTokens`: Whitelists tokens via Int32Array, setting all other logits to -infinity. + * - `bannedTokens`: Blacklists tokens via Int32Array, setting their logits to -infinity. + * + * @param rt The JSI runtime instance. + * @param logits Logits span to modify in place. + * @param constraintsFn The TypeScript callback function. + * @param ctxVal The SamplingContext object passed to the callback. + */ +void applyConstraintsCallback(jsi::Runtime &rt, std::span logits, const jsi::Function &constraintsFn, const jsi::Value &ctxVal) { + auto resVal = constraintsFn.call(rt, ctxVal); + if (resVal.isUndefined() || resVal.isNull()) { + return; + } + const auto resObj = asType(rt, "constraints: return value", resVal); + const size_t vocabSize = logits.size(); + + // Whitelist: allowedTokens + const auto allowedVal = getOptionalProperty(rt, "constraints", resObj, "allowedTokens"); + if (allowedVal.has_value()) { + const auto allowed = fromJsiTypedArray(rt, "constraints: option 'allowedTokens'", *allowedVal); + std::vector> kept; + kept.reserve(allowed.size()); + for (int32_t tok : allowed) { + if (tok < 0 || !std::cmp_less(tok, vocabSize)) { + continue; + } + const auto idx = static_cast(tok); + kept.emplace_back(idx, logits[idx]); + } + std::ranges::fill(logits, -std::numeric_limits::infinity()); + for (const auto &[idx, logit] : kept) { + logits[idx] = logit; + } + } + + // Blacklist: bannedTokens + const auto bannedVal = getOptionalProperty(rt, "constraints", resObj, "bannedTokens"); + if (bannedVal.has_value()) { + const auto banned = fromJsiTypedArray(rt, "constraints: option 'bannedTokens'", *bannedVal); + for (int32_t tok : banned) { + if (tok < 0 || !std::cmp_less(tok, vocabSize)) { + continue; + } + logits[static_cast(tok)] = -std::numeric_limits::infinity(); + } + } +} + +/** + * Extracts the highest-scoring candidate tokens from unnormalized logits using a bounded min-heap. + * + * Performs a single pass over logits to maintain up to `maxCandidates` tokens, + * sorts them descending, prunes tail logits below `maxLogit - 16.0f`, and normalizes + * softmax probabilities. + * + * @param logits The shaped logit values. + * @param maxCandidates Maximum number of candidate tokens to retain (topK). + * @return A vector of candidates sorted by descending probability, normalized to sum to 1.0. + */ +std::vector applyTopK(std::span logits, size_t maxCandidates) { + if (maxCandidates == 0 || logits.empty()) { + return {}; + } + + std::vector candidates; + candidates.reserve(maxCandidates); + + auto minHeapCmp = [](const Candidate &a, const Candidate &b) { + return a.logit > b.logit; + }; + + for (size_t i = 0; i < logits.size(); ++i) { + const float val = logits[i]; + + if (candidates.size() < maxCandidates) { + if (!std::isfinite(val)) { + continue; + } + candidates.push_back({.token = static_cast(i), .logit = val, .prob = 0.0f}); + if (candidates.size() == maxCandidates) { + std::ranges::make_heap(candidates, minHeapCmp); + } + continue; + } + + if (!(val > candidates.front().logit)) { + continue; + } + + std::ranges::pop_heap(candidates, minHeapCmp); + candidates.back() = {.token = static_cast(i), .logit = val, .prob = 0.0f}; + std::ranges::push_heap(candidates, minHeapCmp); + } + + if (candidates.empty()) { + return {}; + } + + std::ranges::sort_heap(candidates, minHeapCmp); + + const float maxLogit = candidates.front().logit; + const float cutoff = maxLogit - 16.0f; + while (candidates.size() > 1 && candidates.back().logit < cutoff) { + candidates.pop_back(); + } + + float totalWeight = 0.0f; + for (auto &c : candidates) { + c.prob = std::exp(c.logit - maxLogit); + totalWeight += c.prob; + } + + if (totalWeight <= 0.0f) { + return {}; + } + + for (auto &c : candidates) { + c.prob /= totalWeight; + } + + return candidates; +} + +/** + * Applies Top-P (nucleus) filtering to the candidates. + * + * Truncates candidates to the smallest subset whose cumulative probability + * exceeds topP, then re-normalizes candidate probabilities to sum to 1.0. + * + * @param candidates Candidates sorted by descending probability, modified in place. + * @param topP Cumulative probability threshold in the range (0.0, 1.0). + */ +void applyTopP(std::vector &candidates, float topP) { + if (topP <= 0.0f || topP >= 1.0f || candidates.empty()) { + return; + } + + float cumWeight = 0.0f; + size_t cutoff = candidates.size() - 1; + for (size_t i = 0; i < candidates.size(); ++i) { + cumWeight += candidates[i].prob; + if (cumWeight >= topP) { + cutoff = i; + break; + } + } + candidates.resize(cutoff + 1); + + float total = 0.0f; + for (const auto &c : candidates) { + total += c.prob; + } + if (total <= 0.0f) { + return; + } + for (auto &c : candidates) { + c.prob /= total; + } +} + +/** + * Filters out candidates whose probability is below minP * max_probability. + * + * Candidates are assumed to be sorted by descending probability. + * + * @param candidates Candidates sorted by descending probability, modified in place. + * @param minP Minimum relative probability threshold in the range (0.0, 1.0). + */ +void applyMinP(std::vector &candidates, float minP) { + if (minP <= 0.0f || minP >= 1.0f || candidates.empty()) { + return; + } + + const float threshold = candidates[0].prob * minP; + size_t cutoff = candidates.size(); + for (size_t i = 1; i < candidates.size(); ++i) { + if (candidates[i].prob < threshold) { + cutoff = i; + break; + } + } + candidates.resize(cutoff); + + float total = 0.0f; + for (const auto &c : candidates) { + total += c.prob; + } + if (total <= 0.0f) { + return; + } + for (auto &c : candidates) { + c.prob /= total; + } +} + +/** + * Applies Exclude Top Choices (XTC) filtering to the candidate distribution. + * + * When two or more candidates have probabilities >= threshold, the top + * candidate is removed with probability `probability`, and remaining candidate + * probabilities are re-normalized. + * + * @param candidates Candidates sorted by descending probability, modified in place. + * @param threshold Probability threshold for considering tokens as "top choices". + * @param probability Likelihood (0.0 to 1.0) of excluding the top candidate. + * @param rng Pseudo-random number generator for coin flip. + */ +void applyXtc(std::vector &candidates, float threshold, float probability, std::mt19937 &rng) { + if (probability <= 0.0f || threshold <= 0.0f || candidates.size() < 2) { + return; + } + + size_t count = 0; + for (const auto &c : candidates) { + if (c.prob >= threshold) { + ++count; + } else { + break; + } + } + + if (count < 2) { + return; + } + + std::uniform_real_distribution dist(0.0f, 1.0f); + if (dist(rng) >= probability) { + return; + } + + candidates.erase(candidates.begin()); + float total = 0.0f; + for (const auto &c : candidates) { + total += c.prob; + } + if (total <= 0.0f) { + return; + } + for (auto &c : candidates) { + c.prob /= total; + } +} + +/** + * Rescales candidate probabilities by applying temperature to their original + * logits. + * + * @param candidates Candidates sorted by descending probability, modified in + * place. + * @param temperature Temperature scaling factor (> 0.0). If 1.0, no + * transformation is needed. + */ +void applyTemperature(std::vector &candidates, float temperature) { + if (temperature == 1.0f || temperature <= 0.0f || candidates.size() <= 1) { + return; + } + + const float invTemp = 1.0f / temperature; + float maxScaledLogit = -std::numeric_limits::infinity(); + for (const auto &c : candidates) { + maxScaledLogit = std::max(c.logit * invTemp, maxScaledLogit); + } + + float total = 0.0f; + for (auto &c : candidates) { + c.prob = std::exp(c.logit * invTemp - maxScaledLogit); + total += c.prob; + } + + if (total <= 0.0f) { + return; + } + for (auto &c : candidates) { + c.prob /= total; + } +} + +/** + * Selects the token index with the highest logit value (greedy decoding). + * + * @param logits The unnormalized or shaped logits. + * @return The 0-based token index with the maximum logit value. + * @throws error::ExecutionFailed If all logits are non-finite or masked. + */ +int32_t sampleArgmax(std::span logits) { + int32_t bestIdx = -1; + float bestVal = -std::numeric_limits::infinity(); + for (size_t i = 0; i < logits.size(); ++i) { + if (logits[i] > bestVal) { + bestVal = logits[i]; + bestIdx = static_cast(i); + } + } + if (bestIdx < 0) { + throw error::ExecutionFailed("sample: No valid candidates available (all logits are non-finite or masked)"); + } + return bestIdx; +} + +/** + * Samples a token from the candidate distribution using roulette-wheel (multinomial) sampling. + * + * @param candidates Candidates with normalized probabilities. + * @param rng Pseudo-random number generator. + * @return The selected token ID. + * @throws error::ExecutionFailed If candidates is empty. + */ +int32_t sampleMultinomial(const std::vector &candidates, std::mt19937 &rng) { + if (candidates.empty()) { + throw error::ExecutionFailed("sample: No valid candidates available (all logits are non-finite or masked)"); + } + if (candidates.size() == 1) { + return candidates[0].token; + } + + float total = 0.0f; + for (const auto &c : candidates) { + total += c.prob; + } + if (total <= 0.0f) { + return candidates[0].token; + } + + std::uniform_real_distribution dist(0.0f, total); + const float r = dist(rng); + + float cum = 0.0f; + for (const auto &c : candidates) { + cum += c.prob; + if (cum >= r) { + return c.token; + } + } + return candidates.back().token; +} + +} // namespace + +void install_sample(jsi::Runtime &rt, jsi::Object &module) { + const auto *name = "sample"; + auto fnBody = [](jsi::Runtime &rt, const jsi::Value & /*thisVal*/, const jsi::Value *args, size_t count) -> jsi::Value { + if (count != 3) { + throw error::InvalidArgument("sample: Usage: sample(logits, ctx, options)"); + } + + auto src = tensor::fromJs(rt, "sample: logits", args[0], DType::float32, {1, "vocab_size"}); + auto lock = tensor::tryLockUnique(rt, "sample: logits", src); + + if (src->numel_ == 0) { + throw error::InvalidArgument("sample: logits tensor is empty"); + } + + auto *data = reinterpret_cast(src->data_.get()); + std::span logits(data, src->numel_); + + const auto opts = asType(rt, "sample: options", args[2]); + + const auto topK = getRequiredProperty(rt, "sample", opts, "topK"); + const auto topP = getRequiredProperty(rt, "sample", opts, "topP"); + const auto minP = getRequiredProperty(rt, "sample", opts, "minP"); + + const auto temperature = getRequiredProperty(rt, "sample", opts, "temperature"); + const auto repetitionPenalty = getRequiredProperty(rt, "sample", opts, "repetitionPenalty"); + + const auto xtcOptions = getOptionalProperty(rt, "sample", opts, "xtc"); + const auto dryOptions = getOptionalProperty(rt, "sample", opts, "dry"); + + if (topK < 1) { + throw error::InvalidArgument("sample: option 'topK' must be at least 1"); + } + if (topP <= 0.0f || topP > 1.0f) { + throw error::InvalidArgument("sample: option 'topP' must be in (0, 1]"); + } + if (minP < 0.0f || minP >= 1.0f) { + throw error::InvalidArgument("sample: option 'minP' must be in [0, 1)"); + } + if (temperature < 0.0f) { + throw error::InvalidArgument("sample: option 'temperature' must be non-negative"); + } + if (repetitionPenalty <= 0.0f) { + throw error::InvalidArgument("sample: option 'repetitionPenalty' must be positive"); + } + + static thread_local std::mt19937 rng(std::random_device{}()); + + /** + * The sampling pipeline follows the llama.cpp execution sequence: + * constraints -> penalties -> dry -> top-k -> top-p -> min-p -> xtc -> temperature -> multinomial draw + * + * Constraints, penalties and DRY are applied first directly to raw logits + * so truncation filters operate on the penalized distribution. This matches + * llama.cpp, where `logit_bias` leads the chain ahead of `penalties` and + * `dry`. + * + * Top-K then hard-prunes the logit space prior to softmax candidate + * distribution construction. + * + * Top-P (nucleus) and Min-P subsequently prune tail candidates based on + * cumulative and relative probability thresholds. + * + * XTC conditionally removes the top choice when multiple competitive + * candidates exist. + * + * Finally, temperature rescales probabilities across surviving + * candidates before the multinomial draw. + * + * Greedy decoding (temperature non-positive or topK one) short-circuits to + * argmax, but only when XTC is disabled: XTC runs before token selection in + * llama.cpp and must be able to remove the argmax. + */ + + const auto ctxObj = asType(rt, "sample: ctx", args[1]); + const auto genToksVal = getOptionalProperty(rt, "sample: ctx", ctxObj, "generatedTokens"); + const auto genToks = genToksVal.has_value() + ? fromJsiTypedArray(rt, "sample: ctx: 'generatedTokens'", *genToksVal) + : std::vector{}; + + auto constraintsFn = getOptionalProperty(rt, "sample: constraints", opts, "constraints"); + if (constraintsFn.has_value()) { + applyConstraintsCallback(rt, logits, constraintsFn.value(), args[1]); + } + + applyRepetitionPenalty(logits, genToks, repetitionPenalty); + + if (dryOptions.has_value()) { + const auto dryMultiplier = getRequiredProperty(rt, "sample: dry", *dryOptions, "multiplier"); + const auto dryBase = getRequiredProperty(rt, "sample: dry", *dryOptions, "base"); + const auto dryPenaltyLastN = getRequiredProperty(rt, "sample: dry", *dryOptions, "penaltyLastN"); + const auto dryAllowedLength = getRequiredProperty(rt, "sample: dry", *dryOptions, "allowedLength"); + + const auto breakersVal = getRequiredProperty(rt, "sample: dry", *dryOptions, "sequenceBreakers"); + const auto breakersVec = fromJsiTypedArray(rt, "sample: dry: option 'sequenceBreakers'", breakersVal); + const std::unordered_set breakers(breakersVec.begin(), breakersVec.end()); + + if (dryMultiplier < 0.0f) { + throw error::InvalidArgument("sample: dry option 'multiplier' must be non-negative"); + } + if (dryBase <= 1.0f) { + throw error::InvalidArgument("sample: dry option 'base' must be greater than 1.0"); + } + if (dryAllowedLength < 0) { + throw error::InvalidArgument("sample: dry option 'allowedLength' must be non-negative"); + } + if (dryPenaltyLastN < -1) { + throw error::InvalidArgument("sample: dry option 'penaltyLastN' must be -1 or non-negative"); + } + + applyDry(logits, genToks, dryMultiplier, dryBase, dryAllowedLength, dryPenaltyLastN, breakers); + } + + const bool greedy = temperature <= 0.0f || topK == 1; + + if (greedy && !xtcOptions.has_value()) { + const int32_t chosen = sampleArgmax(logits); + return jsi::Value(static_cast(chosen)); + } + + const auto maxCandidates = std::min(static_cast(topK), logits.size()); + + auto candidates = applyTopK(logits, maxCandidates); + + if (candidates.empty()) { + throw error::ExecutionFailed("sample: No valid candidates available (all logits are non-finite or masked)"); + } + + applyTopP(candidates, topP); + applyMinP(candidates, minP); + + if (xtcOptions.has_value()) { + const auto xtcProbability = getRequiredProperty(rt, "sample: xtc", *xtcOptions, "probability"); + const auto xtcThreshold = getRequiredProperty(rt, "sample: xtc", *xtcOptions, "threshold"); + + if (xtcProbability < 0.0f || xtcProbability > 1.0f) { + throw error::InvalidArgument("sample: xtc option 'probability' must be in [0, 1]"); + } + if (xtcThreshold < 0.0f || xtcThreshold > 0.5f) { + throw error::InvalidArgument("sample: xtc option 'threshold' must be in [0, 0.5]"); + } + applyXtc(candidates, xtcThreshold, xtcProbability, rng); + } + + applyTemperature(candidates, temperature); + + const int32_t chosen = greedy ? candidates.front().token : sampleMultinomial(candidates, rng); + + return jsi::Value(static_cast(chosen)); + }; + + module.setProperty(rt, name, jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, name), 3, error::guarded(fnBody))); +} + +} // namespace rnexecutorch::extensions::llm diff --git a/packages/react-native-executorch/cpp/extensions/llm/sampler.h b/packages/react-native-executorch/cpp/extensions/llm/sampler.h new file mode 100644 index 0000000000..a3901401bb --- /dev/null +++ b/packages/react-native-executorch/cpp/extensions/llm/sampler.h @@ -0,0 +1,7 @@ +#pragma once + +#include + +namespace rnexecutorch::extensions::llm { +void install_sample(facebook::jsi::Runtime &rt, facebook::jsi::Object &module); +} // namespace rnexecutorch::extensions::llm diff --git a/packages/react-native-executorch/cpp/tests/CMakeLists.txt b/packages/react-native-executorch/cpp/tests/CMakeLists.txt index 28ba81983b..336cf6795d 100644 --- a/packages/react-native-executorch/cpp/tests/CMakeLists.txt +++ b/packages/react-native-executorch/cpp/tests/CMakeLists.txt @@ -144,6 +144,7 @@ set_property(TARGET et_tokenizers APPEND PROPERTY # OpenCV flag and phonemizer.cpp behind the phonemis flag. Built as one static # library so every test binary shares it. file(GLOB CORE_SOURCES "${CPP_DIR}/core/*.cpp") +file(GLOB LLM_SOURCES "${CPP_DIR}/extensions/llm/*.cpp") file(GLOB MATH_SOURCES "${CPP_DIR}/extensions/math/*.cpp") file(GLOB NLP_SOURCES "${CPP_DIR}/extensions/nlp/*.cpp") file(GLOB SPEECH_SOURCES "${CPP_DIR}/extensions/speech/*.cpp") @@ -156,7 +157,7 @@ list(FILTER SPEECH_SOURCES EXCLUDE REGEX "/phonemizer\\.cpp$") set(RNE_SOURCES "${CPP_DIR}/RnExecutorch.cpp" - ${CORE_SOURCES} ${MATH_SOURCES} ${NLP_SOURCES} ${SPEECH_SOURCES}) + ${CORE_SOURCES} ${LLM_SOURCES} ${MATH_SOURCES} ${NLP_SOURCES} ${SPEECH_SOURCES}) # The cv extension uses only core and imgproc. Prefer OpenCV's CMake package # when it is installed (Homebrew, or a full distro OpenCV), but fall back to diff --git a/packages/react-native-executorch/src/extensions/llm/sampler.ts b/packages/react-native-executorch/src/extensions/llm/sampler.ts index 00609de58e..4b0dd3158c 100644 --- a/packages/react-native-executorch/src/extensions/llm/sampler.ts +++ b/packages/react-native-executorch/src/extensions/llm/sampler.ts @@ -1,35 +1,92 @@ /** - * LLM generation sampling pipeline and logit processors. + * LLM generation sampling types and native sampler bridge function. */ -import { multinomial } from '../math'; +import { rnexecutorchJsi } from '../../native/bridge'; +import type { Tensor } from '../../core/tensor'; /** - * Contextual information available to logit processors during generation. + * Contextual information available to constraint callbacks and sampling + * strategies during generation. * @category LLM / Types */ export type SamplingContext = { /** All token ids emitted so far in the current generation call. */ - readonly generatedTokens: readonly number[]; + readonly generatedTokens: Int32Array; }; /** - * Modifies a logit array in-place before token selection. + * Constraints returned by a {@link SamplingConstraintsCallback} to constrain token + * selection. * @category LLM / Types */ -export type LogitProcessor = (logits: Float32Array, ctx: SamplingContext) => void; +export type SamplingConstraints = { + /** + * If specified, only these tokens are allowed. All other token logits are + * masked to `-Infinity`. + */ + readonly allowedTokens?: Int32Array; + /** + * If specified, these tokens are forbidden. Their logits are masked to + * `-Infinity`. + */ + readonly bannedTokens?: Int32Array; +}; /** - * Selects a token id from a processed logit distribution. + * Callback invoked before token selection to dynamically apply token constraints. + * @param ctx Generation context containing emitted tokens. + * @returns Constraints specifying allowed or banned tokens. * @category LLM / Types */ -export type TokenSelector = (logits: Float32Array) => number; +export type SamplingConstraintsCallback = (ctx: SamplingContext) => SamplingConstraints | undefined; /** - * Samples the next token id given raw logits and generation context. + * Configuration options for Don't Repeat Yourself (DRY) repetition penalty. + * + * Penalizes tokens that would extend an already repeated n-gram sequence. * @category LLM / Types */ -export type Sampler = (logits: Float32Array, ctx: SamplingContext) => number; +export type DryConfig = { + /** + * Multiplier penalty factor (e.g. 0.8). Set to 0 to disable. + */ + readonly multiplier: number; + /** + * Exponential penalty base. Defaults to 1.75. + */ + readonly base?: number; + /** + * Minimum match length before penalty is applied. Defaults to 2. + */ + readonly allowedLength?: number; + /** + * Maximum number of recent tokens to consider (window). Defaults to -1 + * (entire context). + */ + readonly penaltyLastN?: number; + /** + * Sequence-breaking token IDs (e.g. newlines) that reset n-gram matching. + */ + readonly sequenceBreakers?: Int32Array; +}; + +/** + * Configuration options for Exclude Top Choices (XTC) sampling. + * @category LLM / Types + */ +export type XTCConfig = { + /** + * Probability of excluding the top choice when multiple candidates meet the + * threshold, in `[0, 1]`. Required: it is the knob that turns XTC on, so it + * has no default — a `0` default would make an enabled `xtc` a silent no-op. + */ + readonly probability: number; + /** + * Probability threshold for plausible tokens, in `[0, 0.5]`. Defaults to 0.1. + */ + readonly threshold?: number; +}; /** * Configuration options for LLM token sampling. @@ -38,301 +95,68 @@ export type Sampler = (logits: Float32Array, ctx: SamplingContext) => number; export type SamplingConfig = { /** * Sampling temperature. Values below 1 sharpen the distribution, while values - * above 1 flatten it. Defaults to greedy decoding when unset or set to 0. + * above 1 flatten it. Set to 0 for greedy decoding. Defaults to 0.8. */ readonly temperature?: number; - /** Number of highest-probability tokens to retain. */ + /** Number of highest-probability tokens to retain, `>= 1`. Use `1` for greedy decoding. Defaults to 40. */ readonly topK?: number; - /** Cumulative probability threshold for nucleus sampling, in `(0, 1]`. */ + /** Cumulative probability threshold for nucleus sampling, in `(0, 1]`. Defaults to 0.95. */ readonly topP?: number; - /** Minimum token probability relative to the most likely token, in `(0, 1]`. */ + /** Minimum token probability relative to the most likely token, in `[0, 1)`. Defaults to 0.05. */ readonly minP?: number; - /** Penalty applied to previously generated tokens. Values greater than 1 discourage repetition. */ + /** + * Penalty applied to previously generated tokens. Values greater than 1 + * discourage repetition. Defaults to 1.0 (disabled). + */ readonly repetitionPenalty?: number; - /** Additive logit offsets keyed by token id. */ - readonly logitBias?: ReadonlyMap; - /** Custom logit processors applied after standard processors. */ - readonly logitProcessors?: readonly LogitProcessor[]; - /** Custom token selector overriding greedy or multinomial sampling. */ - readonly selector?: TokenSelector; + /** Configuration for Don't Repeat Yourself (DRY) repetition penalty. */ + readonly dry?: DryConfig; + /** Configuration for Exclude Top Choices (XTC) sampling. */ + readonly xtc?: XTCConfig; + /** + * Optional callback for dynamic token constraints and guided decoding (e.g. + * JSON schemas, grammars, choice whitelists, stop tokens). + */ + readonly constraints?: SamplingConstraintsCallback; }; /** - * Inserts a value into a sorted array maintaining ascending order. - * @param arr Target sorted array. - * @param val Value to insert. - */ -function insertSorted(arr: number[], val: number): void { - 'worklet'; - let [lo, hi] = [0, arr.length]; - while (lo < hi) { - const mid = Math.floor((lo + hi) / 2); - if (arr[mid]! < val) lo = mid + 1; - else hi = mid; - } - arr.splice(lo, 0, val); -} - -/** - * Scales logits by the inverse temperature factor. - * @param temperature Sampling temperature. - * @returns A logit processor that applies temperature scaling. - * @category LLM / Functions - */ -export function temperatureProcessor(temperature: number): LogitProcessor { - 'worklet'; - return (logits) => { - 'worklet'; - for (let i = 0; i < logits.length; i++) { - logits[i]! /= temperature; - } - }; -} - -/** - * Retains only the top-k highest logits and masks the rest to `-Infinity`. - * @param k Number of top tokens to keep. - * @returns A logit processor that applies top-k filtering. + * Samples a token directly from a native logits tensor given generation context and config. + * + * The logits buffer is written to in place (penalties, DRY and constraints are + * applied directly to it), so its contents are undefined after this call. Copy + * the tensor first if the original logits are still needed. + * @param logits 2D float32 logits Tensor of shape `[1, vocabSize]`. Clobbered by the call. + * @param ctx Generation context containing emitted token history. + * @param config Optional sampling configuration (defaults to `{}`). + * @returns The sampled token id. + * @throws {RnExecuTorchError} With code `RESOURCE_BUSY` if the logits tensor is + * locked by another thread, or `INVALID_ARGUMENT` if the options are out of range. * @category LLM / Functions */ -export function topKProcessor(k: number): LogitProcessor { +export function sample(logits: Tensor, ctx: SamplingContext, config: SamplingConfig = {}): number { 'worklet'; - return (logits) => { - 'worklet'; - if (k <= 0 || k >= logits.length) return; - - const top: number[] = []; - for (let i = 0; i < logits.length; i++) { - const val = logits[i]!; - if (top.length < k) { - insertSorted(top, val); - } else if (val > top[0]!) { - top.shift(); - insertSorted(top, val); - } - } - - const threshold = top[0]!; - for (let i = 0; i < logits.length; i++) { - if (logits[i]! < threshold) { - logits[i] = -Infinity; - } - } - }; -} - -/** - * Applies nucleus (top-p) filtering, retaining tokens up to cumulative probability `p`. - * @param p Cumulative probability threshold. - * @returns A logit processor that applies top-p filtering. - * @category LLM / Functions - */ -export function topPProcessor(p: number): LogitProcessor { - 'worklet'; - return (logits) => { - 'worklet'; - if (p <= 0 || p >= 1) return; - - let maxLogit = -Infinity; - for (let i = 0; i < logits.length; i++) { - if (logits[i]! > maxLogit) { - maxLogit = logits[i]!; - } - } - if (!Number.isFinite(maxLogit)) return; - - const minLogitThreshold = maxLogit - 16; - let totalWeight = 0; - const candidates: number[] = []; - - for (let i = 0; i < logits.length; i++) { - const val = logits[i]!; - if (val > minLogitThreshold) { - totalWeight += Math.exp(val - maxLogit); - candidates.push(i); - } else { - logits[i] = -Infinity; - } - } - - if (candidates.length > 512) { - candidates.sort((a, b) => logits[b]! - logits[a]!); - for (let i = 512; i < candidates.length; i++) { - logits[candidates[i]!] = -Infinity; - } - candidates.length = 512; - } else { - candidates.sort((a, b) => logits[b]! - logits[a]!); - } - - const targetWeight = p * totalWeight; - let cumWeight = 0; - let cutoff = candidates.length - 1; - - for (let i = 0; i < candidates.length; i++) { - const idx = candidates[i]!; - cumWeight += Math.exp(logits[idx]! - maxLogit); - if (cumWeight >= targetWeight) { - cutoff = i; - break; - } - } - - for (let i = cutoff + 1; i < candidates.length; i++) { - logits[candidates[i]!] = -Infinity; - } - }; -} - -/** - * Filters out tokens whose probability is below `minP` times the top token's probability. - * @param minP Minimum relative probability threshold. - * @returns A logit processor that applies min-p filtering. - * @category LLM / Functions - */ -export function minPProcessor(minP: number): LogitProcessor { - 'worklet'; - return (logits) => { - 'worklet'; - if (minP <= 0) return; - let maxLogit = -Infinity; - for (let i = 0; i < logits.length; i++) { - if (logits[i]! > maxLogit) { - maxLogit = logits[i]!; - } - } - if (!Number.isFinite(maxLogit)) return; - - const threshold = maxLogit + Math.log(minP); - for (let i = 0; i < logits.length; i++) { - if (logits[i]! < threshold) { - logits[i] = -Infinity; - } - } - }; -} - -/** - * Penalizes logits of previously emitted tokens to discourage repetition. - * @param penalty Multiplicative penalty factor. - * @returns A logit processor that penalizes repeated tokens. - * @category LLM / Functions - */ -export function repetitionPenaltyProcessor(penalty: number): LogitProcessor { - 'worklet'; - return (logits, ctx) => { - 'worklet'; - if (penalty === 1 || ctx.generatedTokens.length === 0) return; - const seen = new Set(); - for (const token of ctx.generatedTokens) { - if (seen.has(token)) continue; - seen.add(token); - const logit = logits[token]; - if (logit !== undefined) { - logits[token] = logit > 0 ? logit / penalty : logit * penalty; - } - } - }; -} - -/** - * Applies additive biases to specific token logits. - * @param bias Map from token id to additive logit offset. - * @returns A logit processor that applies the offsets. - * @category LLM / Functions - */ -export function logitBiasProcessor(bias: ReadonlyMap): LogitProcessor { - 'worklet'; - return (logits) => { - 'worklet'; - for (const [token, value] of bias) { - if (token >= 0 && token < logits.length) { - logits[token]! += value; - } - } - }; -} - -/** - * Selects the token with the highest logit. - * @returns A deterministic greedy token selector. - * @category LLM / Functions - */ -export function greedySelector(): TokenSelector { - 'worklet'; - return (logits) => { - 'worklet'; - let best = 0; - let bestVal = -Infinity; - for (let i = 0; i < logits.length; i++) { - if (logits[i]! > bestVal) { - bestVal = logits[i]!; - best = i; - } - } - return best; - }; -} - -/** - * Draws a token from the categorical probability distribution of logits. - * @param rng Optional custom uniform random number generator producing values in `[0, 1)`. - * @returns A stochastic multinomial token selector. - * @category LLM / Functions - */ -export function multinomialSelector(rng?: () => number): TokenSelector { - 'worklet'; - return (logits) => { - 'worklet'; - return multinomial(logits, { rng }); - }; -} - -/** - * Creates a sampler function configured from generation options. - * @param config Sampling configuration. - * @returns A {@link Sampler} function. - * @category LLM / Functions - */ -export function createSampler(config?: SamplingConfig): Sampler { - 'worklet'; - const processors: LogitProcessor[] = []; - - if (config?.repetitionPenalty !== undefined && config.repetitionPenalty !== 1) { - processors.push(repetitionPenaltyProcessor(config.repetitionPenalty)); - } - if (config?.logitBias !== undefined && config.logitBias.size > 0) { - processors.push(logitBiasProcessor(config.logitBias)); - } - - const temperature = config?.temperature ?? 0; - const stochastic = temperature > 0; - - if (stochastic) { - processors.push(temperatureProcessor(temperature)); - } - if (config?.topK !== undefined) { - processors.push(topKProcessor(config.topK)); - } - if (config?.topP !== undefined) { - processors.push(topPProcessor(config.topP)); - } - if (config?.minP !== undefined) { - processors.push(minPProcessor(config.minP)); - } - if (config?.logitProcessors) { - for (const processor of config.logitProcessors) { - processors.push(processor); - } - } - - const selector = config?.selector ?? (stochastic ? multinomialSelector() : greedySelector()); - - return (logits: Float32Array, ctx: SamplingContext): number => { - 'worklet'; - for (const processor of processors) { - processor(logits, ctx); - } - return selector(logits); + // Defaults mirror llama.cpp's `common_params_sampling` and are resolved here + // so the native sampler receives a fully-specified options object and treats + // every field as required. + const options = { + temperature: config.temperature ?? 0.8, + repetitionPenalty: config.repetitionPenalty ?? 1.0, + topK: config.topK ?? 40, + topP: config.topP ?? 0.95, + minP: config.minP ?? 0.05, + dry: config.dry && { + multiplier: config.dry.multiplier, + base: config.dry.base ?? 1.75, + penaltyLastN: config.dry.penaltyLastN ?? -1, + allowedLength: config.dry.allowedLength ?? 2, + sequenceBreakers: config.dry.sequenceBreakers ?? new Int32Array(0), + }, + xtc: config.xtc && { + threshold: config.xtc.threshold ?? 0.1, + probability: config.xtc.probability, + }, + constraints: config.constraints, }; + return rnexecutorchJsi.llm.sample(logits, ctx, options); } From fecf0fd157f373f4b68989ca2e058578087c5e56 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Mon, 5 Oct 2026 16:45:01 +0200 Subject: [PATCH 15/83] style(llm): drop churn from sampler wiring Restore the LLM install map and CMakeLists to match main where the sampler change did not require touching them: - install.h, RnExecutorch.cpp, android/CMakeLists.txt, cpp/tests/CMakeLists.txt back to main's contents (zero diff). - install.cpp keeps only the two lines the swap needs: llm_runner.h -> sampler.h and install_createLLMRunner -> install_sample. --- .../android/CMakeLists.txt | 4 ++-- .../cpp/RnExecutorch.cpp | 2 +- .../cpp/extensions/llm/install.cpp | 4 ++-- .../cpp/extensions/llm/install.h | 6 ------ .../cpp/tests/CMakeLists.txt | 18 ++++++++++++------ 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/packages/react-native-executorch/android/CMakeLists.txt b/packages/react-native-executorch/android/CMakeLists.txt index d4df1c0887..56793ecd17 100644 --- a/packages/react-native-executorch/android/CMakeLists.txt +++ b/packages/react-native-executorch/android/CMakeLists.txt @@ -38,10 +38,10 @@ find_package(fbjni REQUIRED CONFIG) # explicit, extensible list of opencv-dependent sources: when multimodal LLM # lands in nlp, its vision-encoder sources should be appended here too. file(GLOB CORE_SOURCES ${CPP_DIR}/core/*.cpp) -file(GLOB LLM_SOURCES ${CPP_DIR}/extensions/llm/*.cpp) file(GLOB MATH_SOURCES ${CPP_DIR}/extensions/math/*.cpp) file(GLOB NLP_SOURCES ${CPP_DIR}/extensions/nlp/*.cpp) file(GLOB SPEECH_SOURCES ${CPP_DIR}/extensions/speech/*.cpp) +file(GLOB LLM_SOURCES ${CPP_DIR}/extensions/llm/*.cpp) file(GLOB OPENCV_SOURCES ${CPP_DIR}/extensions/cv/*.cpp) # ============================================================================== @@ -77,10 +77,10 @@ list(FILTER SPEECH_SOURCES EXCLUDE REGEX "/phonemizer\\.cpp$") set(RNE_SOURCES ${CPP_DIR}/RnExecutorch.cpp ${CORE_SOURCES} - ${LLM_SOURCES} ${MATH_SOURCES} ${NLP_SOURCES} ${SPEECH_SOURCES} + ${LLM_SOURCES} cpp-adapter.cpp ) diff --git a/packages/react-native-executorch/cpp/RnExecutorch.cpp b/packages/react-native-executorch/cpp/RnExecutorch.cpp index 8eeab6e278..98fb8c97ff 100644 --- a/packages/react-native-executorch/cpp/RnExecutorch.cpp +++ b/packages/react-native-executorch/cpp/RnExecutorch.cpp @@ -22,8 +22,8 @@ void install(jsi::Runtime &jsiRuntime) { #endif rnexecutorch::extensions::math::install(jsiRuntime, module); rnexecutorch::extensions::nlp::install(jsiRuntime, module); - rnexecutorch::extensions::llm::install(jsiRuntime, module); rnexecutorch::extensions::speech::install(jsiRuntime, module); + rnexecutorch::extensions::llm::install(jsiRuntime, module); jsiRuntime.global().setProperty(jsiRuntime, "__rnexecutorch_jsi__", std::move(module)); } diff --git a/packages/react-native-executorch/cpp/extensions/llm/install.cpp b/packages/react-native-executorch/cpp/extensions/llm/install.cpp index 0545e70b83..45f406f0ec 100644 --- a/packages/react-native-executorch/cpp/extensions/llm/install.cpp +++ b/packages/react-native-executorch/cpp/extensions/llm/install.cpp @@ -4,8 +4,8 @@ namespace rnexecutorch::extensions::llm { namespace jsi = facebook::jsi; -void install(jsi::Runtime &rt, jsi::Object &module) { - jsi::Object llmModule(rt); +void install(facebook::jsi::Runtime &rt, facebook::jsi::Object &module) { + jsi::Object llmModule = jsi::Object(rt); install_sample(rt, llmModule); diff --git a/packages/react-native-executorch/cpp/extensions/llm/install.h b/packages/react-native-executorch/cpp/extensions/llm/install.h index f9b16f6fb8..89d3de5e6b 100644 --- a/packages/react-native-executorch/cpp/extensions/llm/install.h +++ b/packages/react-native-executorch/cpp/extensions/llm/install.h @@ -3,11 +3,5 @@ #include namespace rnexecutorch::extensions::llm { -/** - * Installs all LLM extension host functions and objects onto the target JSI module. - * - * @param rt The JSI runtime instance. - * @param module The root `rnexecutorchJsi` module object. - */ void install(facebook::jsi::Runtime &rt, facebook::jsi::Object &module); } // namespace rnexecutorch::extensions::llm diff --git a/packages/react-native-executorch/cpp/tests/CMakeLists.txt b/packages/react-native-executorch/cpp/tests/CMakeLists.txt index 336cf6795d..8de778f7d2 100644 --- a/packages/react-native-executorch/cpp/tests/CMakeLists.txt +++ b/packages/react-native-executorch/cpp/tests/CMakeLists.txt @@ -100,13 +100,18 @@ rne_import_et_lib(et_flat_tensor "extension/flat_tensor" extension_flat_tensor) rne_import_et_lib(et_named_data_map "extension/named_data_map" extension_named_data_map) rne_import_et_lib(et_tokenizers "extension/llm/tokenizers" tokenizers) -# The memory allocator and the portable-kernel utilities. +# The llm extension wraps ExecuTorch's LLM runner, which pulls in the sampler, +# the memory allocator and the portable-kernel utilities it shares with them. +rne_import_et_lib(et_llm_runner "extension/llm/runner" extension_llm_runner) +rne_import_et_lib(et_llm_sampler "extension/llm/sampler" extension_llm_sampler) rne_import_et_lib(et_memory_allocator "extension/memory_allocator" extension_memory_allocator) rne_import_et_lib(et_kernels_util "kernels/portable/cpu/util" kernels_util_all_deps) -# Listed consumer-first: GNU ld resolves static archives left to right. +# Listed consumer-first: GNU ld resolves static archives left to right, so the +# runner has to precede the sampler/allocator/kernel archives it pulls from, and +# the ExecuTorch core has to follow all of them. set(ET_LIBS - et_memory_allocator et_kernels_util + et_llm_runner et_llm_sampler et_memory_allocator et_kernels_util et_module et_tensor et_flat_tensor et_named_data_map et_data_loader et_full et_core et_tokenizers) @@ -144,10 +149,10 @@ set_property(TARGET et_tokenizers APPEND PROPERTY # OpenCV flag and phonemizer.cpp behind the phonemis flag. Built as one static # library so every test binary shares it. file(GLOB CORE_SOURCES "${CPP_DIR}/core/*.cpp") -file(GLOB LLM_SOURCES "${CPP_DIR}/extensions/llm/*.cpp") file(GLOB MATH_SOURCES "${CPP_DIR}/extensions/math/*.cpp") file(GLOB NLP_SOURCES "${CPP_DIR}/extensions/nlp/*.cpp") file(GLOB SPEECH_SOURCES "${CPP_DIR}/extensions/speech/*.cpp") +file(GLOB LLM_SOURCES "${CPP_DIR}/extensions/llm/*.cpp") file(GLOB OPENCV_SOURCES "${CPP_DIR}/extensions/cv/*.cpp") # phonemizer.cpp is the one speech source gated on phonemis, exactly as in @@ -157,7 +162,7 @@ list(FILTER SPEECH_SOURCES EXCLUDE REGEX "/phonemizer\\.cpp$") set(RNE_SOURCES "${CPP_DIR}/RnExecutorch.cpp" - ${CORE_SOURCES} ${LLM_SOURCES} ${MATH_SOURCES} ${NLP_SOURCES} ${SPEECH_SOURCES}) + ${CORE_SOURCES} ${MATH_SOURCES} ${NLP_SOURCES} ${SPEECH_SOURCES} ${LLM_SOURCES}) # The cv extension uses only core and imgproc. Prefer OpenCV's CMake package # when it is installed (Homebrew, or a full distro OpenCV), but fall back to @@ -351,7 +356,8 @@ set(TEST_SUITES core/SchemaTest.cpp core/UtilsTest.cpp extensions/MathOpsTest.cpp - extensions/SpeechOpsTest.cpp) + extensions/SpeechOpsTest.cpp + extensions/LlmRunnerTest.cpp) if(RNE_TESTS_ENABLE_OPENCV) list(APPEND TEST_SUITES extensions/CvOpsTest.cpp extensions/OcrOpsTest.cpp) From d2563c6ef1b178e5ff8221eb45ccd272e22cff67 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Mon, 5 Oct 2026 16:47:41 +0200 Subject: [PATCH 16/83] fix(tests): drop et_llm_runner and LlmRunnerTest from the test build Restoring cpp/tests/CMakeLists.txt to main brought back the ExecuTorch LLM runner/sampler imports and LlmRunnerTest.cpp, both of which went away with the native runner. Keep the extensions/llm glob, drop the dead libs and the deleted test, and leave the glob where main has it. --- .../cpp/tests/CMakeLists.txt | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/packages/react-native-executorch/cpp/tests/CMakeLists.txt b/packages/react-native-executorch/cpp/tests/CMakeLists.txt index 8de778f7d2..0c2ab3872d 100644 --- a/packages/react-native-executorch/cpp/tests/CMakeLists.txt +++ b/packages/react-native-executorch/cpp/tests/CMakeLists.txt @@ -100,18 +100,13 @@ rne_import_et_lib(et_flat_tensor "extension/flat_tensor" extension_flat_tensor) rne_import_et_lib(et_named_data_map "extension/named_data_map" extension_named_data_map) rne_import_et_lib(et_tokenizers "extension/llm/tokenizers" tokenizers) -# The llm extension wraps ExecuTorch's LLM runner, which pulls in the sampler, -# the memory allocator and the portable-kernel utilities it shares with them. -rne_import_et_lib(et_llm_runner "extension/llm/runner" extension_llm_runner) -rne_import_et_lib(et_llm_sampler "extension/llm/sampler" extension_llm_sampler) +# The memory allocator and the portable-kernel utilities. rne_import_et_lib(et_memory_allocator "extension/memory_allocator" extension_memory_allocator) rne_import_et_lib(et_kernels_util "kernels/portable/cpu/util" kernels_util_all_deps) -# Listed consumer-first: GNU ld resolves static archives left to right, so the -# runner has to precede the sampler/allocator/kernel archives it pulls from, and -# the ExecuTorch core has to follow all of them. +# Listed consumer-first: GNU ld resolves static archives left to right. set(ET_LIBS - et_llm_runner et_llm_sampler et_memory_allocator et_kernels_util + et_memory_allocator et_kernels_util et_module et_tensor et_flat_tensor et_named_data_map et_data_loader et_full et_core et_tokenizers) @@ -356,8 +351,7 @@ set(TEST_SUITES core/SchemaTest.cpp core/UtilsTest.cpp extensions/MathOpsTest.cpp - extensions/SpeechOpsTest.cpp - extensions/LlmRunnerTest.cpp) + extensions/SpeechOpsTest.cpp) if(RNE_TESTS_ENABLE_OPENCV) list(APPEND TEST_SUITES extensions/CvOpsTest.cpp extensions/OcrOpsTest.cpp) From afc7ded5aa4c2b16d56b56b37876da5b72c32c2a Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Mon, 5 Oct 2026 16:56:15 +0200 Subject: [PATCH 17/83] refactor(llm): prefix runner modules and drop math multinomial - Rename runners/textRunner.ts, multimodalRunner.ts, gemmaRunner.ts to llmTextRunner.ts, llmMultimodalRunner.ts, llmGemmaRunner.ts, and createRunner to createLLMRunner, matching the llmRunner.ts / createLLMRunner pairing. - Revert src/extensions/math.ts to main: the multinomial helper landed for the old JS sampler and nothing imports it now that sampling is native. --- .../src/extensions/llm/llmRunner.ts | 23 +------ .../src/extensions/llm/runners/index.ts | 6 +- .../{gemmaRunner.ts => llmGemmaRunner.ts} | 2 +- ...imodalRunner.ts => llmMultimodalRunner.ts} | 2 +- .../{textRunner.ts => llmTextRunner.ts} | 31 ++++++---- .../src/extensions/math.ts | 60 ------------------- 6 files changed, 25 insertions(+), 99 deletions(-) rename packages/react-native-executorch/src/extensions/llm/runners/{gemmaRunner.ts => llmGemmaRunner.ts} (71%) rename packages/react-native-executorch/src/extensions/llm/runners/{multimodalRunner.ts => llmMultimodalRunner.ts} (70%) rename packages/react-native-executorch/src/extensions/llm/runners/{textRunner.ts => llmTextRunner.ts} (88%) diff --git a/packages/react-native-executorch/src/extensions/llm/llmRunner.ts b/packages/react-native-executorch/src/extensions/llm/llmRunner.ts index 55d8ecb34e..883c857448 100644 --- a/packages/react-native-executorch/src/extensions/llm/llmRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/llmRunner.ts @@ -13,7 +13,7 @@ import { f32, i64, method, validateSpec } from '../../core/schema'; import { loadTokenizer } from '../nlp'; -import { createTextRunner } from './runners/textRunner'; +import { createLLMTextRunner } from './runners/llmTextRunner'; import type { SamplingConfig } from './sampler'; declare const llmRunnerBrand: unique symbol; @@ -252,13 +252,6 @@ export async function createLLMRunner( const [maxContextLen] = model.execute('get_max_context_len', [], []) as [number]; const eosIds = model.execute('get_eos_ids', [], []) as [number]; - // const maxSeqLenMethod = model.execute('get_max_seq_len', [], [])[0]; - // if (maxSeqLen !== maxSeqLenMethod) { - // throw RnExecuTorchError( - // 'SCHEMA_MISMATCH', - // `Schema maxSeqLen (${maxSeqLen}) does not match get_max_seq_len (${maxSeqLenMethod}).` - // ); - // } const vocabSizeMethod = model.execute('get_vocab_size', [], [])[0]; if (vocabSize !== vocabSizeMethod) { throw RnExecuTorchError( @@ -266,12 +259,6 @@ export async function createLLMRunner( `Schema vocabSize (${vocabSize}) does not match get_vocab_size (${vocabSizeMethod}).` ); } - // if (vocabSize !== tokenizer.getVocabSize()) { - // throw RnExecuTorchError( - // 'SCHEMA_MISMATCH', - // `Model vocabSize (${vocabSize}) does not match tokenizer size (${tokenizer.getVocabSize()}).` - // ); - // } if (model.execute('use_kv_cache', [], [])[0] !== true) { throw RnExecuTorchError( 'SCHEMA_MISMATCH', // prettier-ignore @@ -296,14 +283,8 @@ export async function createLLMRunner( `maxContextLen (${maxContextLen}) must be an integer >= maxSeqLen (${maxSeqLen}).` ); } - // if (!Array.isArray(eosIds) || eosIds.length === 0) { - // throw RnExecuTorchError( - // 'SCHEMA_MISMATCH', - // 'Model get_eos_ids returned an empty or invalid array.' - // ); - // } - return createTextRunner(model, tokenizer, { maxSeqLen, maxContextLen, vocabSize, eosIds }); + return createLLMTextRunner(model, tokenizer, { maxSeqLen, maxContextLen, vocabSize, eosIds }); } catch (e) { scope.dispose(); throw e; diff --git a/packages/react-native-executorch/src/extensions/llm/runners/index.ts b/packages/react-native-executorch/src/extensions/llm/runners/index.ts index 55b37910d1..99d6326023 100644 --- a/packages/react-native-executorch/src/extensions/llm/runners/index.ts +++ b/packages/react-native-executorch/src/extensions/llm/runners/index.ts @@ -1,3 +1,3 @@ -export * from './textRunner'; -export * from './multimodalRunner'; -export * from './gemmaRunner'; +export * from './llmTextRunner'; +export * from './llmMultimodalRunner'; +export * from './llmGemmaRunner'; diff --git a/packages/react-native-executorch/src/extensions/llm/runners/gemmaRunner.ts b/packages/react-native-executorch/src/extensions/llm/runners/llmGemmaRunner.ts similarity index 71% rename from packages/react-native-executorch/src/extensions/llm/runners/gemmaRunner.ts rename to packages/react-native-executorch/src/extensions/llm/runners/llmGemmaRunner.ts index 8d3fa78e4b..b2c3ecccaa 100644 --- a/packages/react-native-executorch/src/extensions/llm/runners/gemmaRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/runners/llmGemmaRunner.ts @@ -2,7 +2,7 @@ import type { Model } from '../../../core/model'; import type { Tokenizer } from '../../nlp'; import type { LLMRunner } from '../llmRunner'; -export function createGemmaRunner(model: Model, tokenizer: Tokenizer): LLMRunner { +export function createLLMGemmaRunner(model: Model, tokenizer: Tokenizer): LLMRunner { void model; void tokenizer; throw new Error('Not implemented'); diff --git a/packages/react-native-executorch/src/extensions/llm/runners/multimodalRunner.ts b/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts similarity index 70% rename from packages/react-native-executorch/src/extensions/llm/runners/multimodalRunner.ts rename to packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts index b5156364f6..8c04f62f5b 100644 --- a/packages/react-native-executorch/src/extensions/llm/runners/multimodalRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts @@ -2,7 +2,7 @@ import type { Model } from '../../../core/model'; import type { Tokenizer } from '../../nlp'; import type { LLMRunner } from '../llmRunner'; -export function createMultimodalRunner(model: Model, tokenizer: Tokenizer): LLMRunner { +export function createLLMMultimodalRunner(model: Model, tokenizer: Tokenizer): LLMRunner { void model; void tokenizer; throw new Error('Not implemented'); diff --git a/packages/react-native-executorch/src/extensions/llm/runners/textRunner.ts b/packages/react-native-executorch/src/extensions/llm/runners/llmTextRunner.ts similarity index 88% rename from packages/react-native-executorch/src/extensions/llm/runners/textRunner.ts rename to packages/react-native-executorch/src/extensions/llm/runners/llmTextRunner.ts index e71cd06d9f..c1b66aff56 100644 --- a/packages/react-native-executorch/src/extensions/llm/runners/textRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/runners/llmTextRunner.ts @@ -5,7 +5,7 @@ import type { Model } from '../../../core/model'; import { RnExecuTorchError } from '../../../core/error'; import type { Tokenizer } from '../../nlp'; -import { createSampler } from '../sampler'; +import { sample } from '../sampler'; import type { LLMGenerationConfig, LLMGenerationStats, @@ -15,7 +15,7 @@ import type { Prompt, } from '../llmRunner'; -export function createTextRunner( +export function createLLMTextRunner( model: Model, tokenizer: Tokenizer, meta: { @@ -26,6 +26,8 @@ export function createTextRunner( } ): LLMRunner { const { maxSeqLen, maxContextLen, vocabSize, eosIds } = meta; + // Reused as the `forward` output buffer and clobbered in place by `sample()`; + // always refilled by `model.execute` before the next `sample()` call. const tLogits = tensor('float32', [1, vocabSize]); const dispose = () => { @@ -34,9 +36,9 @@ export function createTextRunner( tokenizer.dispose(); }; - // ================================== + // ====================================================== // KV Cache management - // ================================== + // ====================================================== const pos = createSynchronizable(0); const reset = (targetPos?: number): void => { @@ -59,11 +61,14 @@ export function createTextRunner( }; }; - // ================================== + // ====================================================== // Generation methods - // ================================== + // ====================================================== const isCancelled = createSynchronizable(false); - const stop = (): void => isCancelled.setBlocking(true); + const stop = (): void => { + 'worklet'; + isCancelled.setBlocking(true); + }; const prefill = (prompt: Prompt): LLMPrefillStats => { 'worklet'; @@ -138,9 +143,6 @@ export function createTextRunner( const generatedTokens: number[] = []; const maxNewTokens = config?.maxNewTokens ?? Infinity; - const logits = tLogits.getData(new Float32Array(vocabSize)); - const sample = createSampler(config); - let numTokens = 0; // TO-REMOVE: Timing breakdown for benchmarking let totalModelExecuteMs = 0; @@ -158,7 +160,11 @@ export function createTextRunner( // TO-REMOVE: Sample timing const tSample0 = Date.now(); - const nextToken = sample(logits, { generatedTokens }); + const nextToken = sample( + tLogits, + { generatedTokens: Int32Array.from(generatedTokens) }, + config + ); totalSampleMs += Date.now() - tSample0; // TO-REMOVE: onToken timing @@ -183,7 +189,6 @@ export function createTextRunner( curPos += 1; pos.setBlocking(curPos); numTokens += 1; - tLogits.getData(logits); } } finally { tToken.dispose(); @@ -196,7 +201,7 @@ export function createTextRunner( // TO-REMOVE: Console log breakdown // eslint-disable-next-line no-console console.log( - `[textRunner] Generated ${numTokens} tokens in ${durationMs}ms (${tokensPerSecond.toFixed(1)} tok/s) | ` + + `[llmTextRunner] Generated ${numTokens} tokens in ${durationMs}ms (${tokensPerSecond.toFixed(1)} tok/s) | ` + `model.execute: ${totalModelExecuteMs}ms (${durationMs > 0 ? ((totalModelExecuteMs / durationMs) * 100).toFixed(1) : 0}%) | ` + `sample: ${totalSampleMs}ms (${durationMs > 0 ? ((totalSampleMs / durationMs) * 100).toFixed(1) : 0}%) | ` + `onToken: ${totalOnTokenMs}ms (${durationMs > 0 ? ((totalOnTokenMs / durationMs) * 100).toFixed(1) : 0}%)` diff --git a/packages/react-native-executorch/src/extensions/math.ts b/packages/react-native-executorch/src/extensions/math.ts index 558ef089f4..a3feccc823 100644 --- a/packages/react-native-executorch/src/extensions/math.ts +++ b/packages/react-native-executorch/src/extensions/math.ts @@ -22,18 +22,6 @@ export type RandomNormalOptions = { readonly seed?: number; }; -/** - * Configuration options for categorical multinomial sampling from logits. - * @category Math / Types - */ -export type MultinomialOptions = { - /** - * Custom uniform random number generator producing values in `[0, 1)`. - * Defaults to `Math.random`. - */ - readonly rng?: () => number; -}; - /** * Computes the element-wise sigmoid activation on a float32 source tensor and * writes the result to a destination tensor. @@ -191,54 +179,6 @@ export function randomNormal(size: number, options?: RandomNormalOptions): Float return out; } -/** - * Draws a single index from a categorical distribution defined by unnormalized logits. - * @category Math / Functions - * @param logits Unnormalized log-probabilities to sample from. - * @param options Sampling options including an optional custom random number generator. - * @returns The sampled index. - * @throws {RnExecuTorchError} With code `INVALID_ARGUMENT` if `logits` is empty. - */ -export function multinomial(logits: ArrayLike, options?: MultinomialOptions): number { - 'worklet'; - const n = logits.length; - if (n === 0) { - throw RnExecuTorchError( - 'INVALID_ARGUMENT', - 'multinomial: cannot sample from an empty array of logits.' - ); - } - - const rng = options?.rng ?? Math.random; - - let maxLogit = -Infinity; - for (let i = 0; i < n; i++) { - const val = logits[i]!; - if (val > maxLogit) maxLogit = val; - } - if (!Number.isFinite(maxLogit)) return 0; - - let sum = 0; - for (let i = 0; i < n; i++) { - const val = logits[i]!; - if (val !== -Infinity) { - sum += Math.exp(val - maxLogit); - } - } - if (sum === 0) return 0; - - const target = rng() * sum; - let cum = 0; - for (let i = 0; i < n; i++) { - const val = logits[i]!; - if (val !== -Infinity) { - cum += Math.exp(val - maxLogit); - if (cum >= target) return i; - } - } - return n - 1; -} - /** * Repeats each element of `values` as many times as the matching entry of * `repeats`, concatenating the runs into a newly allocated array of the same From a69ea589bbe23fcbdfa21d5d76b04a96f905e295 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Tue, 6 Oct 2026 02:42:18 +0200 Subject: [PATCH 18/83] refactor(llm): clean up default sampling options formatting --- .../src/extensions/llm/sampler.ts | 47 ++++++++++++------- 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/packages/react-native-executorch/src/extensions/llm/sampler.ts b/packages/react-native-executorch/src/extensions/llm/sampler.ts index 4b0dd3158c..5025dc026f 100644 --- a/packages/react-native-executorch/src/extensions/llm/sampler.ts +++ b/packages/react-native-executorch/src/extensions/llm/sampler.ts @@ -120,6 +120,31 @@ export type SamplingConfig = { readonly constraints?: SamplingConstraintsCallback; }; +// Defaults mirror llama.cpp's `common_params_sampling` and are resolved here so +// the native sampler receives a fully-specified options object and treats every +// field as required. + +const EMPTY_SEQUENCE_BREAKERS = new Int32Array(0); + +const DEFAULT_SAMPLING_OPTIONS = { + temperature: 0.8, + repetitionPenalty: 1.0, + topK: 40, + topP: 0.95, + minP: 0.05, +} as const; + +const DEFAULT_DRY_OPTIONS = { + base: 1.75, + allowedLength: 2, + penaltyLastN: -1, + sequenceBreakers: EMPTY_SEQUENCE_BREAKERS, +} as const; + +const DEFAULT_XTC_OPTIONS = { + threshold: 0.1, +} as const; + /** * Samples a token directly from a native logits tensor given generation context and config. * @@ -136,27 +161,17 @@ export type SamplingConfig = { */ export function sample(logits: Tensor, ctx: SamplingContext, config: SamplingConfig = {}): number { 'worklet'; - // Defaults mirror llama.cpp's `common_params_sampling` and are resolved here - // so the native sampler receives a fully-specified options object and treats - // every field as required. const options = { - temperature: config.temperature ?? 0.8, - repetitionPenalty: config.repetitionPenalty ?? 1.0, - topK: config.topK ?? 40, - topP: config.topP ?? 0.95, - minP: config.minP ?? 0.05, + ...DEFAULT_SAMPLING_OPTIONS, + ...config, dry: config.dry && { - multiplier: config.dry.multiplier, - base: config.dry.base ?? 1.75, - penaltyLastN: config.dry.penaltyLastN ?? -1, - allowedLength: config.dry.allowedLength ?? 2, - sequenceBreakers: config.dry.sequenceBreakers ?? new Int32Array(0), + ...DEFAULT_DRY_OPTIONS, + ...config.dry, }, xtc: config.xtc && { - threshold: config.xtc.threshold ?? 0.1, - probability: config.xtc.probability, + ...DEFAULT_XTC_OPTIONS, + ...config.xtc, }, - constraints: config.constraints, }; return rnexecutorchJsi.llm.sample(logits, ctx, options); } From 01820529a90a1da9e3953ffa3a4dff9ecafdd34c Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Tue, 6 Oct 2026 02:52:25 +0200 Subject: [PATCH 19/83] docs(llm): emphasize in-place logits modification in sample JSDoc --- .../src/extensions/llm/sampler.ts | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/packages/react-native-executorch/src/extensions/llm/sampler.ts b/packages/react-native-executorch/src/extensions/llm/sampler.ts index 5025dc026f..2b7a6e1af6 100644 --- a/packages/react-native-executorch/src/extensions/llm/sampler.ts +++ b/packages/react-native-executorch/src/extensions/llm/sampler.ts @@ -148,7 +148,7 @@ const DEFAULT_XTC_OPTIONS = { /** * Samples a token directly from a native logits tensor given generation context and config. * - * The logits buffer is written to in place (penalties, DRY and constraints are + * **Important:** The logits buffer is written to in place (penalties, DRY and constraints are * applied directly to it), so its contents are undefined after this call. Copy * the tensor first if the original logits are still needed. * @param logits 2D float32 logits Tensor of shape `[1, vocabSize]`. Clobbered by the call. @@ -164,14 +164,8 @@ export function sample(logits: Tensor, ctx: SamplingContext, config: SamplingCon const options = { ...DEFAULT_SAMPLING_OPTIONS, ...config, - dry: config.dry && { - ...DEFAULT_DRY_OPTIONS, - ...config.dry, - }, - xtc: config.xtc && { - ...DEFAULT_XTC_OPTIONS, - ...config.xtc, - }, + dry: config.dry && { ...DEFAULT_DRY_OPTIONS, ...config.dry }, + xtc: config.xtc && { ...DEFAULT_XTC_OPTIONS, ...config.xtc }, }; return rnexecutorchJsi.llm.sample(logits, ctx, options); } From e868dfa0a15823917a68eb76c103369cec14b434 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Tue, 6 Oct 2026 02:57:24 +0200 Subject: [PATCH 20/83] refactor(llm): remove profiling code from llmTextRunner --- .../extensions/llm/runners/llmTextRunner.ts | 42 ++++--------------- 1 file changed, 8 insertions(+), 34 deletions(-) diff --git a/packages/react-native-executorch/src/extensions/llm/runners/llmTextRunner.ts b/packages/react-native-executorch/src/extensions/llm/runners/llmTextRunner.ts index c1b66aff56..ea99ed96b1 100644 --- a/packages/react-native-executorch/src/extensions/llm/runners/llmTextRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/runners/llmTextRunner.ts @@ -65,6 +65,7 @@ export function createLLMTextRunner( // Generation methods // ====================================================== const isCancelled = createSynchronizable(false); + const stop = (): void => { 'worklet'; isCancelled.setBlocking(true); @@ -83,6 +84,7 @@ export function createLLMTextRunner( const startMs = Date.now(); const startPos = pos.getBlocking(); + const tokens = tokenizer.encode(prompt); let offset = 0; @@ -143,52 +145,33 @@ export function createLLMTextRunner( const generatedTokens: number[] = []; const maxNewTokens = config?.maxNewTokens ?? Infinity; - let numTokens = 0; - // TO-REMOVE: Timing breakdown for benchmarking - let totalModelExecuteMs = 0; - let totalSampleMs = 0; - let totalOnTokenMs = 0; - const tToken = tensor('int64', [1, 1]); const tCurPos = tensor('int64', [1]); + let numTokens = 0; let curPos = pos.getBlocking(); try { while (numTokens < maxNewTokens && !isCancelled.getBlocking()) { if (curPos >= maxContextLen) break; - // TO-REMOVE: Sample timing - const tSample0 = Date.now(); - const nextToken = sample( - tLogits, - { generatedTokens: Int32Array.from(generatedTokens) }, - config - ); - totalSampleMs += Date.now() - tSample0; + const genTokArray = Int32Array.from(generatedTokens); + const nextToken = sample(tLogits, { generatedTokens: genTokArray }, config); - // TO-REMOVE: onToken timing - const tOnToken0 = Date.now(); onToken?.(tokenizer.decode(Int32Array.of(nextToken))); - totalOnTokenMs += Date.now() - tOnToken0; - generatedTokens.push(nextToken); - if (!config?.ignoreEos && eosIds.includes(nextToken)) { - break; - } + if (!config?.ignoreEos && eosIds.includes(nextToken)) break; tToken.setData(BigInt64Array.of(BigInt(nextToken))); tCurPos.setData(BigInt64Array.of(BigInt(curPos))); - // TO-REMOVE: model.execute timing - const tExec0 = Date.now(); model.execute('forward', [tToken, tCurPos], [tLogits]); - totalModelExecuteMs += Date.now() - tExec0; + numTokens += 1; curPos += 1; + pos.setBlocking(curPos); - numTokens += 1; } } finally { tToken.dispose(); @@ -198,15 +181,6 @@ export function createLLMTextRunner( const durationMs = Date.now() - generateStartMs; const tokensPerSecond = durationMs > 0 ? (numTokens / durationMs) * 1000 : 0; - // TO-REMOVE: Console log breakdown - // eslint-disable-next-line no-console - console.log( - `[llmTextRunner] Generated ${numTokens} tokens in ${durationMs}ms (${tokensPerSecond.toFixed(1)} tok/s) | ` + - `model.execute: ${totalModelExecuteMs}ms (${durationMs > 0 ? ((totalModelExecuteMs / durationMs) * 100).toFixed(1) : 0}%) | ` + - `sample: ${totalSampleMs}ms (${durationMs > 0 ? ((totalSampleMs / durationMs) * 100).toFixed(1) : 0}%) | ` + - `onToken: ${totalOnTokenMs}ms (${durationMs > 0 ? ((totalOnTokenMs / durationMs) * 100).toFixed(1) : 0}%)` - ); - return { numTokens, durationMs, tokensPerSecond, prefill: prefillStats }; }; From 5a040bb338b56df72b7f1c70f0b7ff3db7b1ea2e Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Wed, 7 Oct 2026 18:03:56 +0200 Subject: [PATCH 21/83] style --- .../src/extensions/llm/runners/llmTextRunner.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/react-native-executorch/src/extensions/llm/runners/llmTextRunner.ts b/packages/react-native-executorch/src/extensions/llm/runners/llmTextRunner.ts index ea99ed96b1..be7499d287 100644 --- a/packages/react-native-executorch/src/extensions/llm/runners/llmTextRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/runners/llmTextRunner.ts @@ -26,8 +26,8 @@ export function createLLMTextRunner( } ): LLMRunner { const { maxSeqLen, maxContextLen, vocabSize, eosIds } = meta; - // Reused as the `forward` output buffer and clobbered in place by `sample()`; - // always refilled by `model.execute` before the next `sample()` call. + + // Output buffer for `forward` logits: shape [1, vocabSize] const tLogits = tensor('float32', [1, vocabSize]); const dispose = () => { From 5fdec2d32130833b776c0a4edff9e140b34f6d77 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Wed, 7 Oct 2026 20:01:43 +0200 Subject: [PATCH 22/83] feat(llm): implement multimodal runner and clean up text runner --- .../llm/runners/llmMultimodalRunner.ts | 313 +++++++++++++++++- .../extensions/llm/runners/llmTextRunner.ts | 7 +- 2 files changed, 309 insertions(+), 11 deletions(-) diff --git a/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts b/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts index 8c04f62f5b..4948241280 100644 --- a/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts @@ -1,9 +1,312 @@ +import { createSynchronizable } from 'react-native-worklets'; + +import { tensor } from '../../../core/tensor'; import type { Model } from '../../../core/model'; +import { RnExecuTorchError } from '../../../core/error'; + import type { Tokenizer } from '../../nlp'; -import type { LLMRunner } from '../llmRunner'; +import { sample } from '../sampler'; +import type { + LLMGenerationConfig, + LLMGenerationStats, + LLMKVCacheState, + LLMPrefillStats, + LLMRunner, + MediaInput, + Prompt, +} from '../llmRunner'; + +export type LLMMultimodalMeta = { + readonly maxSeqLen: number; + readonly maxContextLen: number; + readonly vocabSize: number; + readonly hiddenDim: number; + readonly eosIds: readonly number[]; + readonly imgShape: readonly [number, number, number, number]; + readonly numVisualTokens: number; +}; + +export function createLLMMultimodalRunner( + model: Model, + tokenizer: Tokenizer, + meta: LLMMultimodalMeta +): LLMRunner { + const { maxSeqLen, maxContextLen, vocabSize, hiddenDim, numVisualTokens, eosIds, imgShape } = + meta; + + // Output buffer for text_decoder logits: shape [1, vocabSize] + const tLogits = tensor('float32', [1, vocabSize]); + + const dispose = () => { + model.dispose(); + tLogits.dispose(); + tokenizer.dispose(); + }; + + // ====================================================== + // KV Cache management + // ====================================================== + const pos = createSynchronizable(0); + + const reset = (targetPos?: number): void => { + if (targetPos !== undefined && (targetPos < 0 || targetPos >= maxContextLen)) { + throw RnExecuTorchError( + 'INVALID_ARGUMENT', + `targetPos (${targetPos}) must be in range [0, ${maxContextLen}).` + ); + } + pos.setBlocking(targetPos ?? 0); + }; + + const getKVCacheState = (): LLMKVCacheState => { + const curPos = pos.getBlocking(); + return { + pos: curPos, + maxContextLen, + remainingTokens: maxContextLen - curPos, + usageRatio: curPos / maxContextLen, + }; + }; + + // ====================================================== + // Generation & Prefill methods + // ====================================================== + const isCancelled = createSynchronizable(false); + + const stop = (): void => { + 'worklet'; + isCancelled.setBlocking(true); + }; + + const prefillText = (prompt: string): LLMPrefillStats => { + 'worklet'; + isCancelled.setBlocking(false); + + const startMs = Date.now(); + const startPos = pos.getBlocking(); + + const tokens = tokenizer.encode(prompt); + + let offset = 0; + while (offset < tokens.length && !isCancelled.getBlocking()) { + const seqLen = Math.min(tokens.length - offset, maxSeqLen); + const chunk = tokens.subarray(offset, offset + seqLen); + + if (startPos + offset + seqLen > maxContextLen) { + throw RnExecuTorchError( + 'EXECUTION_FAILED', + `prefill: Context length exceeded (${startPos + offset + seqLen} > ${maxContextLen}).` + ); + } + + const positions = BigInt64Array.from( + { length: seqLen }, // prettier-ignore + (_, i) => BigInt(startPos + offset + i) + ); + const tensors = [ + // prettier-ignore + tensor('int64', [1, seqLen], BigInt64Array.from(chunk, (v) => BigInt(v))), + tensor('int64', [seqLen], positions), + tensor('float32', [1, seqLen, hiddenDim]), + ] as const; + + const [tTokens, tPositions, tTextEmbeds] = tensors; + + try { + model.execute('token_embedding', [tTokens], [tTextEmbeds]); + model.execute('text_decoder', [tTextEmbeds, tPositions], [tLogits]); + } finally { + tensors.forEach((t) => t.dispose()); + } + + pos.setBlocking(startPos + offset + seqLen); + offset += seqLen; + } + + const durationMs = Date.now() - startMs; + const numTokens = offset; + const tokensPerSecond = durationMs > 0 ? (numTokens / durationMs) * 1000 : 0; + + return { numTokens, durationMs, tokensPerSecond }; + }; + + const prefillImage = (prompt: Extract): LLMPrefillStats => { + 'worklet'; + isCancelled.setBlocking(false); + + if (prompt.image.dtype !== 'float32') { + throw RnExecuTorchError( + 'INVALID_ARGUMENT', + `Expected image tensor of dtype 'float32', received '${prompt.image.dtype}'.` + ); + } + if (imgShape.some((d, i) => d !== prompt.image.shape[i])) { + throw RnExecuTorchError( + 'INVALID_ARGUMENT', + `Image tensor shape [${prompt.image.shape.join(', ')}] != [${imgShape.join(', ')}].` + ); + } + + const startMs = Date.now(); + const startPos = pos.getBlocking(); + + const tVisionEmbed = tensor('float32', [1, numVisualTokens, hiddenDim]); + try { + model.execute('vision_encoder', [prompt.image], [tVisionEmbed]); + + const embedding = tVisionEmbed.getData(new Float32Array(tVisionEmbed.numel)); + let offset = 0; + while (offset < numVisualTokens && !isCancelled.getBlocking()) { + const seqLen = Math.min(numVisualTokens - offset, maxSeqLen); + const chunk = embedding.subarray(offset * hiddenDim, (offset + seqLen) * hiddenDim); + + if (startPos + offset + seqLen > maxContextLen) { + throw RnExecuTorchError( + 'EXECUTION_FAILED', + `prefill: Context length exceeded (${startPos + offset + seqLen} > ${maxContextLen}).` + ); + } + + const positions = BigInt64Array.from( + { length: seqLen }, // prettier-ignore + (_, i) => BigInt(startPos + offset + i) + ); + const tPositions = tensor('int64', [seqLen], positions); + const tChunk = tensor('float32', [1, seqLen, hiddenDim], chunk); + + try { + model.execute('text_decoder', [tChunk, tPositions], [tLogits]); + } finally { + tPositions.dispose(); + tChunk.dispose(); + } + + pos.setBlocking(startPos + offset + seqLen); + offset += seqLen; + } + } finally { + tVisionEmbed.dispose(); + } + + const durationMs = Date.now() - startMs; + const numTokens = numVisualTokens; + const tokensPerSecond = durationMs > 0 ? (numTokens / durationMs) * 1000 : 0; + + return { numTokens, durationMs, tokensPerSecond }; + }; + + const prefill = (prompt: Prompt): LLMPrefillStats => { + 'worklet'; + isCancelled.setBlocking(false); + + const segments = typeof prompt === 'string' ? [prompt] : prompt; + if (segments.length === 0) { + throw RnExecuTorchError('INVALID_ARGUMENT', 'prefill: Empty prompt provided.'); + } + + let totStats: LLMPrefillStats | undefined; + const combine = (st1: LLMPrefillStats, st2?: LLMPrefillStats): LLMPrefillStats => { + if (st2 === undefined) return st1; + const numTokens = st1.numTokens + st2.numTokens; + const durationMs = st1.durationMs + st2.durationMs; + const tokensPerSecond = durationMs > 0 ? (numTokens / durationMs) * 1000 : 0; + return { numTokens, durationMs, tokensPerSecond }; + }; + + for (const segment of segments) { + if (typeof segment === 'string') { + const stats = prefillText(segment); + totStats = combine(stats, totStats); + continue; + } + + switch (segment.kind) { + case 'image': { + const stats = prefillImage(segment); + totStats = combine(stats, totStats); + break; + } + default: + throw RnExecuTorchError('INVALID_ARGUMENT', ''); + } + } + + return totStats!; + }; + + const generate = ( + prompt: Prompt, + config?: LLMGenerationConfig, + onToken?: (token: string) => void + ): LLMGenerationStats => { + 'worklet'; + isCancelled.setBlocking(false); + + const prefillStats = prefill(prompt); + + const generateStartMs = Date.now(); + const generatedTokens: number[] = []; + const maxNewTokens = config?.maxNewTokens ?? Infinity; + + const tCurPos = tensor('int64', [1]); + const tTokenId = tensor('int64', [1, 1]); + const tTokenEmbed = tensor('float32', [1, 1, hiddenDim]); + + let numTokens = 0; + let curPos = pos.getBlocking(); + + try { + while (numTokens < maxNewTokens && !isCancelled.getBlocking()) { + if (curPos >= maxContextLen) break; + + const genTokArray = Int32Array.from(generatedTokens); + const nextToken = sample(tLogits, { generatedTokens: genTokArray }, config); + + onToken?.(tokenizer.decode(Int32Array.of(nextToken))); + generatedTokens.push(nextToken); + + if (!config?.ignoreEos && eosIds.includes(nextToken)) break; + + // Embed single token + tTokenId.setData(BigInt64Array.of(BigInt(nextToken))); + model.execute('token_embedding', [tTokenId], [tTokenEmbed]); + + // Decode single token embedding + tCurPos.setData(BigInt64Array.of(BigInt(curPos))); + model.execute('text_decoder', [tTokenEmbed, tCurPos], [tLogits]); + + numTokens += 1; + curPos += 1; + pos.setBlocking(curPos); + } + } finally { + tTokenId.dispose(); + tTokenEmbed.dispose(); + tCurPos.dispose(); + } + + const durationMs = Date.now() - generateStartMs; + const tokensPerSecond = durationMs > 0 ? (numTokens / durationMs) * 1000 : 0; + + return { numTokens, durationMs, tokensPerSecond, prefill: prefillStats }; + }; + + return { + // Metadata + modelPath: model.path, + tokenizerPath: tokenizer.path, + modalities: ['image'], + + // Lifecycle & Execution + dispose, + stop, + + // KV Cache + reset, + getKVCacheState, -export function createLLMMultimodalRunner(model: Model, tokenizer: Tokenizer): LLMRunner { - void model; - void tokenizer; - throw new Error('Not implemented'); + // Generation + prefill, + generate, + } as unknown as LLMRunner; } diff --git a/packages/react-native-executorch/src/extensions/llm/runners/llmTextRunner.ts b/packages/react-native-executorch/src/extensions/llm/runners/llmTextRunner.ts index be7499d287..f4bdb8bd80 100644 --- a/packages/react-native-executorch/src/extensions/llm/runners/llmTextRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/runners/llmTextRunner.ts @@ -117,11 +117,7 @@ export function createLLMTextRunner( const numTokens = offset; const tokensPerSecond = durationMs > 0 ? (numTokens / durationMs) * 1000 : 0; - return { - numTokens, - durationMs, - tokensPerSecond, - }; + return { numTokens, durationMs, tokensPerSecond }; }; const generate = ( @@ -170,7 +166,6 @@ export function createLLMTextRunner( numTokens += 1; curPos += 1; - pos.setBlocking(curPos); } } finally { From 07695a33f6608e89727aeb596d65153794647c97 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Wed, 7 Oct 2026 20:29:49 +0200 Subject: [PATCH 23/83] fix(llm): allow rank-4 targetShape in chatPreprocessor --- .../src/extensions/llm/utils/chatPreprocessor.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/react-native-executorch/src/extensions/llm/utils/chatPreprocessor.ts b/packages/react-native-executorch/src/extensions/llm/utils/chatPreprocessor.ts index 77b5b37ab0..5b74aafdd2 100644 --- a/packages/react-native-executorch/src/extensions/llm/utils/chatPreprocessor.ts +++ b/packages/react-native-executorch/src/extensions/llm/utils/chatPreprocessor.ts @@ -47,8 +47,8 @@ export type ChatMessage = Readonly< export type LLMImagePreprocessorConfig = { /** Sentinel token delimiters inserted into Jinja prompts. */ readonly visionToken: { readonly start: string; readonly end: string }; - /** Fixed target shape expected by native LLM `[C, H, W]`. */ - readonly targetShape: readonly [number, number, number]; + /** Fixed target shape expected by native LLM `[C, H, W]` or `[1, C, H, W]`. */ + readonly targetShape: readonly [number, number, number] | readonly [1, number, number, number]; /** Image preprocessing options (normalization, resize mode, interpolation). */ readonly preprocessorOpts: ImagePreprocessorOptions; }; @@ -306,7 +306,7 @@ export function createChatPreprocessor(config: ChatPreprocessorConfig): ChatPrep const template = new Template(chatTemplate); let imgPreprocessor: ReturnType | undefined; - let imgShape: [number, number, number] | undefined; + let imgShape: [number, number, number] | [1, number, number, number] | undefined; if (preprocessorConfig?.image !== undefined) { imgShape = [...preprocessorConfig.image.targetShape]; From 5368533ab52238452c453c228269e616253090bc Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Wed, 7 Oct 2026 20:46:32 +0200 Subject: [PATCH 24/83] refactor(llm): decouple specialized runner specs and polymorphically dispatch from llmRunner --- .../src/extensions/llm/llmRunner.ts | 112 +++--------------- .../llm/runners/llmMultimodalRunner.ts | 89 ++++++++++++-- .../extensions/llm/runners/llmTextRunner.ts | 52 ++++++-- 3 files changed, 139 insertions(+), 114 deletions(-) diff --git a/packages/react-native-executorch/src/extensions/llm/llmRunner.ts b/packages/react-native-executorch/src/extensions/llm/llmRunner.ts index 883c857448..ed926fbbff 100644 --- a/packages/react-native-executorch/src/extensions/llm/llmRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/llmRunner.ts @@ -9,12 +9,16 @@ import { wrapAsync } from '../../core/runtime'; import { loadModel } from '../../core/model'; import { RnExecuTorchError } from '../../core/error'; import { createResourceScope } from '../../core/lifetime'; -import { f32, i64, method, validateSpec } from '../../core/schema'; +import { validateSpec } from '../../core/schema'; import { loadTokenizer } from '../nlp'; -import { createLLMTextRunner } from './runners/llmTextRunner'; import type { SamplingConfig } from './sampler'; +import { createLLMTextRunner, LLM_TEXT_RUNNER_SPEC } from './runners/llmTextRunner'; +import { + createLLMMultimodalRunner, + LLM_MULTIMODAL_RUNNER_SPEC, +} from './runners/llmMultimodalRunner'; declare const llmRunnerBrand: unique symbol; @@ -187,104 +191,20 @@ export async function createLLMRunner( const model = scope.track(await wrapAsync(loadModel, runtime)(modelPath)); const tokenizer = scope.track(await wrapAsync(loadTokenizer, runtime)(tokenizerPath)); - // TODO(@bh): update validate spec after reexporting LLMs with schema const method - const metadata = { - ...method('get_max_seq_len', [], [{ kind: 'Int' }]), - ...method('get_max_context_len', [], [{ kind: 'Int' }]), - ...method('get_vocab_size', [], [{ kind: 'Int' }]), - ...method('use_kv_cache', [], [{ kind: 'Bool' }]), - ...method('enable_dynamic_shape', [], [{ kind: 'Bool' }]), - ...method('get_eos_ids', [], [{ kind: 'Int' }]), - }; - const { variant, dims } = validateSpec(model.schema, { - llm: { - ...method( - 'forward', - // Input: tokens [1, maxSeqLen], curPos [1] - [i64(1, 'maxSeqLen'), i64(1)], - // Output: logits over model vocabulary - [f32(1, 'vocabSize')] - ), - ...metadata, - }, - // vlm: { - // ...method('text_decoder', [f32(1, 'maxSeqLen', 'hid'), i64('seq')], [f32(1, 'vocabSize')]), - // ...method('vision_encoder', [f32(1, 3, 'H', 'W')], [f32(1, 'patches', 'hid')]), - // ...method('token_embedding', [i64(1, 'seq')], [f32(1, 'seq', 'hid')]), - // ...metadata, - // }, - // gemma: { - // ...method( - // 'token_embedding', - // [i64(1, 'seq')], - // [f32(1, 'seq', 1536), f32(1, 'seq', 35, 256)] - // ), - // ...method( - // 'audio_encoder', - // [f32(1, 'numSamples')], - // [f32(1, 'audioTokens', 1536), f32(1, 'audioTokens')] - // ), - // ...method( - // 'text_decoder', - // [f32(1, 'seq', 1536), f32(1, 'seq', 35, 256), i64('seq')], - // [f32(1, 1, 262144)] - // ), - // ...method('vision_encoder', [f32(1, 3, 448, 640)], [f32(1, 280, 1536)]), - // ...metadata, - // }, + ...LLM_TEXT_RUNNER_SPEC, + ...LLM_MULTIMODAL_RUNNER_SPEC, }); - if (variant === 'llm' && modalities && modalities.length > 0) { - throw RnExecuTorchError( - 'INVALID_ARGUMENT', - `Text-only model does not support modalities: ${modalities.join(', ')}.` - ); - } - // if (variant === 'vlm' && !modalities?.includes('image')) { - // throw RnExecuTorchError('INVALID_ARGUMENT', ''); - // } - // if (variant === 'gemma' && !(modalities?.includes('audio') && modalities.includes('image'))) { - // throw RnExecuTorchError('INVALID_ARGUMENT', ''); - // } - - const [maxSeqLen, vocabSize] = dims.constant('maxSeqLen', 'vocabSize'); - const [maxContextLen] = model.execute('get_max_context_len', [], []) as [number]; - const eosIds = model.execute('get_eos_ids', [], []) as [number]; - - const vocabSizeMethod = model.execute('get_vocab_size', [], [])[0]; - if (vocabSize !== vocabSizeMethod) { - throw RnExecuTorchError( - 'SCHEMA_MISMATCH', - `Schema vocabSize (${vocabSize}) does not match get_vocab_size (${vocabSizeMethod}).` - ); - } - if (model.execute('use_kv_cache', [], [])[0] !== true) { - throw RnExecuTorchError( - 'SCHEMA_MISMATCH', // prettier-ignore - 'Model must enable use_kv_cache.' - ); + switch (variant) { + case 'text': + return createLLMTextRunner(model, tokenizer, dims, modalities); + case 'vision': + case 'visionS': + return createLLMMultimodalRunner(model, tokenizer, dims, modalities); + default: + throw RnExecuTorchError('LOAD_FAILED', `llmRunner: unrecognized variant ${variant}`); } - if (model.execute('enable_dynamic_shape', [], [])[0] !== true) { - throw RnExecuTorchError( - 'SCHEMA_MISMATCH', // prettier-ignore - 'Model must enable enable_dynamic_shape.' - ); - } - if (maxSeqLen <= 0 || !Number.isInteger(maxSeqLen)) { - throw RnExecuTorchError( - 'SCHEMA_MISMATCH', - `maxSeqLen must be a positive integer, got ${maxSeqLen}.` - ); - } - if (maxContextLen < maxSeqLen || !Number.isInteger(maxContextLen)) { - throw RnExecuTorchError( - 'SCHEMA_MISMATCH', - `maxContextLen (${maxContextLen}) must be an integer >= maxSeqLen (${maxSeqLen}).` - ); - } - - return createLLMTextRunner(model, tokenizer, { maxSeqLen, maxContextLen, vocabSize, eosIds }); } catch (e) { scope.dispose(); throw e; diff --git a/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts b/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts index 4948241280..1be30ed8f0 100644 --- a/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts @@ -2,10 +2,13 @@ import { createSynchronizable } from 'react-native-worklets'; import { tensor } from '../../../core/tensor'; import type { Model } from '../../../core/model'; +import type { SpecMatch } from '../../../core/schema'; import { RnExecuTorchError } from '../../../core/error'; +import { f32, i64, method, DynamicDim as Dyn } from '../../../core/schema'; import type { Tokenizer } from '../../nlp'; import { sample } from '../sampler'; + import type { LLMGenerationConfig, LLMGenerationStats, @@ -13,26 +16,90 @@ import type { LLMPrefillStats, LLMRunner, MediaInput, + Modality, Prompt, } from '../llmRunner'; -export type LLMMultimodalMeta = { - readonly maxSeqLen: number; - readonly maxContextLen: number; - readonly vocabSize: number; - readonly hiddenDim: number; - readonly eosIds: readonly number[]; - readonly imgShape: readonly [number, number, number, number]; - readonly numVisualTokens: number; +const METADATA_SPEC = { + ...method('get_max_seq_len', [], [{ kind: 'Int' }]), + ...method('get_max_context_len', [], [{ kind: 'Int' }]), + ...method('get_vocab_size', [], [{ kind: 'Int' }]), + ...method('use_kv_cache', [], [{ kind: 'Bool' }]), + ...method('enable_dynamic_shape', [], [{ kind: 'Bool' }]), }; +export const LLM_MULTIMODAL_RUNNER_SPEC = { + vision: { + ...method( + 'text_decoder', + [f32(1, Dyn('seqLen'), 'hiddenDim'), i64(Dyn('seqLen'))], + [f32(1, 'vocabSize')] + ), + ...method( + 'vision_encoder', // + [f32(1, 3, 'imgH', 'imgW')], + [f32(1, 'visualTokens', 'hiddenDim')] + ), + ...method( + 'token_embedding', // + [i64(1, Dyn('seqLen'))], + [f32(1, Dyn('seqLen'), 'hiddenDim')] + ), + ...METADATA_SPEC, + }, + // TO-REMOVE + visionS: { + ...method( + 'text_decoder', + [f32(1, 'seqLen', 'hiddenDim'), i64('seqLen')], + [f32(1, 'vocabSize')] + ), + ...method( + 'vision_encoder', // + [f32(1, 3, 'imgH', 'imgW')], + [f32(1, 'visualTokens', 'hiddenDim')] + ), + ...method( + 'token_embedding', // + [i64(1, 'seqLen')], + [f32(1, 'seqLen', 'hiddenDim')] + ), + ...METADATA_SPEC, + }, +} as const; + export function createLLMMultimodalRunner( model: Model, tokenizer: Tokenizer, - meta: LLMMultimodalMeta + dims: SpecMatch['dims'], + modalities?: readonly Modality[] ): LLMRunner { - const { maxSeqLen, maxContextLen, vocabSize, hiddenDim, numVisualTokens, eosIds, imgShape } = - meta; + if (!modalities?.includes('image')) { + throw RnExecuTorchError( + 'INVALID_ARGUMENT', + `Multimodal vision-language model requires 'image' in modalities.` + ); + } + + const [H, W] = dims.constant('imgH', 'imgW'); + const [hiddenDim, numVisualTokens] = dims.constant('hiddenDim', 'visualTokens'); + const [vocabSize] = dims.constant('vocabSize'); + const [maxSeqLen] = model.execute('get_max_seq_len', [], []) as [number]; // TODO: change to dims + const [maxContextLen] = model.execute('get_max_context_len', [], []) as [number]; + + const imgShape = [1, 3, H, W] as const; + const eosIds = model.execute('get_eos_ids', [], []) as number[]; + + if (maxContextLen < maxSeqLen || !Number.isInteger(maxContextLen)) { + throw RnExecuTorchError( + 'SCHEMA_MISMATCH', + `maxContextLen (${maxContextLen}) must be an integer >= maxSeqLen (${maxSeqLen}).` + ); + } + + if (model.execute('use_kv_cache', [], [])[0] !== true) { + throw RnExecuTorchError('SCHEMA_MISMATCH', 'Model must enable use_kv_cache.'); + } // Output buffer for text_decoder logits: shape [1, vocabSize] const tLogits = tensor('float32', [1, vocabSize]); diff --git a/packages/react-native-executorch/src/extensions/llm/runners/llmTextRunner.ts b/packages/react-native-executorch/src/extensions/llm/runners/llmTextRunner.ts index f4bdb8bd80..8bc16e0c2a 100644 --- a/packages/react-native-executorch/src/extensions/llm/runners/llmTextRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/runners/llmTextRunner.ts @@ -2,30 +2,68 @@ import { createSynchronizable } from 'react-native-worklets'; import { tensor } from '../../../core/tensor'; import type { Model } from '../../../core/model'; +import type { SpecMatch } from '../../../core/schema'; import { RnExecuTorchError } from '../../../core/error'; +import { f32, i64, method, DynamicDim as Dyn } from '../../../core/schema'; import type { Tokenizer } from '../../nlp'; import { sample } from '../sampler'; + import type { LLMGenerationConfig, LLMGenerationStats, LLMKVCacheState, LLMPrefillStats, LLMRunner, + Modality, Prompt, } from '../llmRunner'; +export const LLM_TEXT_RUNNER_SPEC = { + text: { + ...method( + 'forward', // prettier-ignore + [i64(1, Dyn('seqLen')), i64(1)], + [f32(1, 'vocabSize')] + ), + ...method('get_max_seq_len', [], [{ kind: 'Int' }]), + ...method('get_max_context_len', [], [{ kind: 'Int' }]), + ...method('get_vocab_size', [], [{ kind: 'Int' }]), + ...method('use_kv_cache', [], [{ kind: 'Bool' }]), + ...method('enable_dynamic_shape', [], [{ kind: 'Bool' }]), + }, +} as const; + export function createLLMTextRunner( model: Model, tokenizer: Tokenizer, - meta: { - readonly maxSeqLen: number; - readonly maxContextLen: number; - readonly vocabSize: number; - readonly eosIds: readonly number[]; - } + dims: SpecMatch['dims'], + modalities?: readonly Modality[] ): LLMRunner { - const { maxSeqLen, maxContextLen, vocabSize, eosIds } = meta; + if (modalities && modalities.length > 0) { + throw RnExecuTorchError( + 'INVALID_ARGUMENT', + `Text-only model does not support modalities: ${modalities.join(', ')}.` + ); + } + + const [vocabSize] = dims.constant('vocabSize'); + const [seqLenRange] = dims.range('seqLen'); + const [maxContextLen] = model.execute('get_max_context_len', [], []) as [number]; + + const maxSeqLen = seqLenRange.max; + const eosIds = model.execute('get_eos_ids', [], []) as number[]; + + if (maxContextLen < maxSeqLen || !Number.isInteger(maxContextLen)) { + throw RnExecuTorchError( + 'SCHEMA_MISMATCH', + `maxContextLen (${maxContextLen}) must be an integer >= maxSeqLen (${maxSeqLen}).` + ); + } + + if (model.execute('use_kv_cache', [], [])[0] !== true) { + throw RnExecuTorchError('SCHEMA_MISMATCH', 'Model must enable use_kv_cache.'); + } // Output buffer for `forward` logits: shape [1, vocabSize] const tLogits = tensor('float32', [1, vocabSize]); From ab82eb1c82f1b8acad6948a86f203656308067dc Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Wed, 7 Oct 2026 20:51:26 +0200 Subject: [PATCH 25/83] fix(llm): add descriptive error message for unsupported media segment kind --- .../src/extensions/llm/runners/llmMultimodalRunner.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts b/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts index 1be30ed8f0..e97d57363a 100644 --- a/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts @@ -222,7 +222,9 @@ export function createLLMMultimodalRunner( model.execute('vision_encoder', [prompt.image], [tVisionEmbed]); const embedding = tVisionEmbed.getData(new Float32Array(tVisionEmbed.numel)); + let offset = 0; + while (offset < numVisualTokens && !isCancelled.getBlocking()) { const seqLen = Math.min(numVisualTokens - offset, maxSeqLen); const chunk = embedding.subarray(offset * hiddenDim, (offset + seqLen) * hiddenDim); @@ -293,8 +295,13 @@ export function createLLMMultimodalRunner( totStats = combine(stats, totStats); break; } - default: - throw RnExecuTorchError('INVALID_ARGUMENT', ''); + default: { + const kind = (segment as { readonly kind?: string }).kind; + throw RnExecuTorchError( + 'INVALID_ARGUMENT', + `prefill: Unsupported media input kind '${kind}'.` + ); + } } } From e7cdb6969b57c2e2081c6f43b7483b5dc5ab812f Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Wed, 7 Oct 2026 20:53:20 +0200 Subject: [PATCH 26/83] refactor(llm): simplify segment kind access in prefill error --- .../src/extensions/llm/runners/llmMultimodalRunner.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts b/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts index e97d57363a..066f14826d 100644 --- a/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts @@ -295,13 +295,11 @@ export function createLLMMultimodalRunner( totStats = combine(stats, totStats); break; } - default: { - const kind = (segment as { readonly kind?: string }).kind; + default: throw RnExecuTorchError( 'INVALID_ARGUMENT', - `prefill: Unsupported media input kind '${kind}'.` + `prefill: Unsupported media input kind '${segment.kind}'.` ); - } } } From 2c63dd04f96722d4c22c4dde63d36726f902a621 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Wed, 7 Oct 2026 20:57:56 +0200 Subject: [PATCH 27/83] refactor(llm): flatten specialized runner specs into model specs --- .../src/extensions/llm/llmRunner.ts | 5 +- .../llm/runners/llmMultimodalRunner.ts | 77 +++++++++---------- .../extensions/llm/runners/llmTextRunner.ts | 22 +++--- 3 files changed, 50 insertions(+), 54 deletions(-) diff --git a/packages/react-native-executorch/src/extensions/llm/llmRunner.ts b/packages/react-native-executorch/src/extensions/llm/llmRunner.ts index ed926fbbff..dd6a552c3b 100644 --- a/packages/react-native-executorch/src/extensions/llm/llmRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/llmRunner.ts @@ -192,15 +192,14 @@ export async function createLLMRunner( const tokenizer = scope.track(await wrapAsync(loadTokenizer, runtime)(tokenizerPath)); const { variant, dims } = validateSpec(model.schema, { - ...LLM_TEXT_RUNNER_SPEC, - ...LLM_MULTIMODAL_RUNNER_SPEC, + text: LLM_TEXT_RUNNER_SPEC, + vision: LLM_MULTIMODAL_RUNNER_SPEC, }); switch (variant) { case 'text': return createLLMTextRunner(model, tokenizer, dims, modalities); case 'vision': - case 'visionS': return createLLMMultimodalRunner(model, tokenizer, dims, modalities); default: throw RnExecuTorchError('LOAD_FAILED', `llmRunner: unrecognized variant ${variant}`); diff --git a/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts b/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts index 066f14826d..4e0a2b29d9 100644 --- a/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts @@ -4,7 +4,7 @@ import { tensor } from '../../../core/tensor'; import type { Model } from '../../../core/model'; import type { SpecMatch } from '../../../core/schema'; import { RnExecuTorchError } from '../../../core/error'; -import { f32, i64, method, DynamicDim as Dyn } from '../../../core/schema'; +import { f32, i64, method } from '../../../core/schema'; import type { Tokenizer } from '../../nlp'; import { sample } from '../sampler'; @@ -28,45 +28,44 @@ const METADATA_SPEC = { ...method('enable_dynamic_shape', [], [{ kind: 'Bool' }]), }; +// REPLACE WITH THIS AFTER REEXPORT +// { +// ...method( +// 'text_decoder', +// [f32(1, Dyn('seqLen'), 'hiddenDim'), i64(Dyn('seqLen'))], +// [f32(1, 'vocabSize')] +// ), +// ...method( +// 'vision_encoder', // +// [f32(1, 3, 'imgH', 'imgW')], +// [f32(1, 'visualTokens', 'hiddenDim')] +// ), +// ...method( +// 'token_embedding', // +// [i64(1, Dyn('seqLen'))], +// [f32(1, Dyn('seqLen'), 'hiddenDim')] +// ), +// ...METADATA_SPEC, +// } + export const LLM_MULTIMODAL_RUNNER_SPEC = { - vision: { - ...method( - 'text_decoder', - [f32(1, Dyn('seqLen'), 'hiddenDim'), i64(Dyn('seqLen'))], - [f32(1, 'vocabSize')] - ), - ...method( - 'vision_encoder', // - [f32(1, 3, 'imgH', 'imgW')], - [f32(1, 'visualTokens', 'hiddenDim')] - ), - ...method( - 'token_embedding', // - [i64(1, Dyn('seqLen'))], - [f32(1, Dyn('seqLen'), 'hiddenDim')] - ), - ...METADATA_SPEC, - }, - // TO-REMOVE - visionS: { - ...method( - 'text_decoder', - [f32(1, 'seqLen', 'hiddenDim'), i64('seqLen')], - [f32(1, 'vocabSize')] - ), - ...method( - 'vision_encoder', // - [f32(1, 3, 'imgH', 'imgW')], - [f32(1, 'visualTokens', 'hiddenDim')] - ), - ...method( - 'token_embedding', // - [i64(1, 'seqLen')], - [f32(1, 'seqLen', 'hiddenDim')] - ), - ...METADATA_SPEC, - }, -} as const; + ...method( + 'text_decoder', // + [f32(1, 'seqLen', 'hiddenDim'), i64('seqLen')], + [f32(1, 'vocabSize')] + ), + ...method( + 'vision_encoder', // + [f32(1, 3, 'imgH', 'imgW')], + [f32(1, 'visualTokens', 'hiddenDim')] + ), + ...method( + 'token_embedding', // + [i64(1, 'seqLen')], + [f32(1, 'seqLen', 'hiddenDim')] + ), + ...METADATA_SPEC, +}; export function createLLMMultimodalRunner( model: Model, diff --git a/packages/react-native-executorch/src/extensions/llm/runners/llmTextRunner.ts b/packages/react-native-executorch/src/extensions/llm/runners/llmTextRunner.ts index 8bc16e0c2a..e446081e14 100644 --- a/packages/react-native-executorch/src/extensions/llm/runners/llmTextRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/runners/llmTextRunner.ts @@ -20,18 +20,16 @@ import type { } from '../llmRunner'; export const LLM_TEXT_RUNNER_SPEC = { - text: { - ...method( - 'forward', // prettier-ignore - [i64(1, Dyn('seqLen')), i64(1)], - [f32(1, 'vocabSize')] - ), - ...method('get_max_seq_len', [], [{ kind: 'Int' }]), - ...method('get_max_context_len', [], [{ kind: 'Int' }]), - ...method('get_vocab_size', [], [{ kind: 'Int' }]), - ...method('use_kv_cache', [], [{ kind: 'Bool' }]), - ...method('enable_dynamic_shape', [], [{ kind: 'Bool' }]), - }, + ...method( + 'forward', // prettier-ignore + [i64(1, Dyn('seqLen')), i64(1)], + [f32(1, 'vocabSize')] + ), + ...method('get_max_seq_len', [], [{ kind: 'Int' }]), + ...method('get_max_context_len', [], [{ kind: 'Int' }]), + ...method('get_vocab_size', [], [{ kind: 'Int' }]), + ...method('use_kv_cache', [], [{ kind: 'Bool' }]), + ...method('enable_dynamic_shape', [], [{ kind: 'Bool' }]), } as const; export function createLLMTextRunner( From fc0ba83c36fe36eae47c027e0d85f544675d4d8e Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Wed, 7 Oct 2026 20:59:50 +0200 Subject: [PATCH 28/83] refactor(llm): self-validate runner specs inside specialized runner constructors --- .../react-native-executorch/src/extensions/llm/llmRunner.ts | 6 +++--- .../src/extensions/llm/runners/llmMultimodalRunner.ts | 6 +++--- .../src/extensions/llm/runners/llmTextRunner.ts | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/react-native-executorch/src/extensions/llm/llmRunner.ts b/packages/react-native-executorch/src/extensions/llm/llmRunner.ts index dd6a552c3b..1a00181e46 100644 --- a/packages/react-native-executorch/src/extensions/llm/llmRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/llmRunner.ts @@ -191,16 +191,16 @@ export async function createLLMRunner( const model = scope.track(await wrapAsync(loadModel, runtime)(modelPath)); const tokenizer = scope.track(await wrapAsync(loadTokenizer, runtime)(tokenizerPath)); - const { variant, dims } = validateSpec(model.schema, { + const { variant } = validateSpec(model.schema, { text: LLM_TEXT_RUNNER_SPEC, vision: LLM_MULTIMODAL_RUNNER_SPEC, }); switch (variant) { case 'text': - return createLLMTextRunner(model, tokenizer, dims, modalities); + return createLLMTextRunner(model, tokenizer, modalities); case 'vision': - return createLLMMultimodalRunner(model, tokenizer, dims, modalities); + return createLLMMultimodalRunner(model, tokenizer, modalities); default: throw RnExecuTorchError('LOAD_FAILED', `llmRunner: unrecognized variant ${variant}`); } diff --git a/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts b/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts index 4e0a2b29d9..3af31c0e5f 100644 --- a/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts @@ -2,9 +2,8 @@ import { createSynchronizable } from 'react-native-worklets'; import { tensor } from '../../../core/tensor'; import type { Model } from '../../../core/model'; -import type { SpecMatch } from '../../../core/schema'; import { RnExecuTorchError } from '../../../core/error'; -import { f32, i64, method } from '../../../core/schema'; +import { f32, i64, method, validateSpec } from '../../../core/schema'; import type { Tokenizer } from '../../nlp'; import { sample } from '../sampler'; @@ -70,7 +69,6 @@ export const LLM_MULTIMODAL_RUNNER_SPEC = { export function createLLMMultimodalRunner( model: Model, tokenizer: Tokenizer, - dims: SpecMatch['dims'], modalities?: readonly Modality[] ): LLMRunner { if (!modalities?.includes('image')) { @@ -80,6 +78,8 @@ export function createLLMMultimodalRunner( ); } + const { dims } = validateSpec(model.schema, { default: LLM_MULTIMODAL_RUNNER_SPEC }); + const [H, W] = dims.constant('imgH', 'imgW'); const [hiddenDim, numVisualTokens] = dims.constant('hiddenDim', 'visualTokens'); const [vocabSize] = dims.constant('vocabSize'); diff --git a/packages/react-native-executorch/src/extensions/llm/runners/llmTextRunner.ts b/packages/react-native-executorch/src/extensions/llm/runners/llmTextRunner.ts index e446081e14..831dd57f4c 100644 --- a/packages/react-native-executorch/src/extensions/llm/runners/llmTextRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/runners/llmTextRunner.ts @@ -2,9 +2,8 @@ import { createSynchronizable } from 'react-native-worklets'; import { tensor } from '../../../core/tensor'; import type { Model } from '../../../core/model'; -import type { SpecMatch } from '../../../core/schema'; import { RnExecuTorchError } from '../../../core/error'; -import { f32, i64, method, DynamicDim as Dyn } from '../../../core/schema'; +import { f32, i64, method, DynamicDim as Dyn, validateSpec } from '../../../core/schema'; import type { Tokenizer } from '../../nlp'; import { sample } from '../sampler'; @@ -35,7 +34,6 @@ export const LLM_TEXT_RUNNER_SPEC = { export function createLLMTextRunner( model: Model, tokenizer: Tokenizer, - dims: SpecMatch['dims'], modalities?: readonly Modality[] ): LLMRunner { if (modalities && modalities.length > 0) { @@ -45,6 +43,8 @@ export function createLLMTextRunner( ); } + const { dims } = validateSpec(model.schema, { default: LLM_TEXT_RUNNER_SPEC }); + const [vocabSize] = dims.constant('vocabSize'); const [seqLenRange] = dims.range('seqLen'); const [maxContextLen] = model.execute('get_max_context_len', [], []) as [number]; From 964eb57eca9c663a0eb0680c318ca74efc781164 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Wed, 7 Oct 2026 21:04:40 +0200 Subject: [PATCH 29/83] style --- .../src/extensions/llm/runners/llmMultimodalRunner.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts b/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts index 3af31c0e5f..94e252aea9 100644 --- a/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts @@ -49,17 +49,17 @@ const METADATA_SPEC = { export const LLM_MULTIMODAL_RUNNER_SPEC = { ...method( - 'text_decoder', // + 'text_decoder', // prettier-ignore [f32(1, 'seqLen', 'hiddenDim'), i64('seqLen')], [f32(1, 'vocabSize')] ), ...method( - 'vision_encoder', // + 'vision_encoder', // prettier-ignore [f32(1, 3, 'imgH', 'imgW')], [f32(1, 'visualTokens', 'hiddenDim')] ), ...method( - 'token_embedding', // + 'token_embedding', // prettier-ignore [i64(1, 'seqLen')], [f32(1, 'seqLen', 'hiddenDim')] ), From 2ce17cc8b3e94b5b4f68f6c50ed2cf35b5e98190 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Wed, 7 Oct 2026 21:18:14 +0200 Subject: [PATCH 30/83] style --- .../src/extensions/llm/runners/llmMultimodalRunner.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts b/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts index 94e252aea9..94c6d6309e 100644 --- a/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts @@ -80,13 +80,12 @@ export function createLLMMultimodalRunner( const { dims } = validateSpec(model.schema, { default: LLM_MULTIMODAL_RUNNER_SPEC }); - const [H, W] = dims.constant('imgH', 'imgW'); + const [imgH, imgW, vocabSize] = dims.constant('imgH', 'imgW', 'vocabSize'); const [hiddenDim, numVisualTokens] = dims.constant('hiddenDim', 'visualTokens'); - const [vocabSize] = dims.constant('vocabSize'); const [maxSeqLen] = model.execute('get_max_seq_len', [], []) as [number]; // TODO: change to dims const [maxContextLen] = model.execute('get_max_context_len', [], []) as [number]; - const imgShape = [1, 3, H, W] as const; + const imgShape = [1, 3, imgH, imgW] as const; const eosIds = model.execute('get_eos_ids', [], []) as number[]; if (maxContextLen < maxSeqLen || !Number.isInteger(maxContextLen)) { From f13dc536bd21f55eb19fc119ee21e1050273ab85 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Wed, 7 Oct 2026 22:15:19 +0200 Subject: [PATCH 31/83] style --- .../src/extensions/llm/llmRunner.ts | 11 ++++------- .../src/extensions/llm/runners/llmMultimodalRunner.ts | 4 ++-- .../src/extensions/llm/runners/llmTextRunner.ts | 4 ++-- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/packages/react-native-executorch/src/extensions/llm/llmRunner.ts b/packages/react-native-executorch/src/extensions/llm/llmRunner.ts index 1a00181e46..28c649a3c2 100644 --- a/packages/react-native-executorch/src/extensions/llm/llmRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/llmRunner.ts @@ -14,11 +14,8 @@ import { validateSpec } from '../../core/schema'; import { loadTokenizer } from '../nlp'; import type { SamplingConfig } from './sampler'; -import { createLLMTextRunner, LLM_TEXT_RUNNER_SPEC } from './runners/llmTextRunner'; -import { - createLLMMultimodalRunner, - LLM_MULTIMODAL_RUNNER_SPEC, -} from './runners/llmMultimodalRunner'; +import { createLLMTextRunner, LLM_TEXT_SPEC } from './runners/llmTextRunner'; +import { createLLMMultimodalRunner, LLM_MULTIMODAL_SPEC } from './runners/llmMultimodalRunner'; declare const llmRunnerBrand: unique symbol; @@ -192,8 +189,8 @@ export async function createLLMRunner( const tokenizer = scope.track(await wrapAsync(loadTokenizer, runtime)(tokenizerPath)); const { variant } = validateSpec(model.schema, { - text: LLM_TEXT_RUNNER_SPEC, - vision: LLM_MULTIMODAL_RUNNER_SPEC, + text: LLM_TEXT_SPEC, + vision: LLM_MULTIMODAL_SPEC, }); switch (variant) { diff --git a/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts b/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts index 94c6d6309e..c9fcff783f 100644 --- a/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts @@ -47,7 +47,7 @@ const METADATA_SPEC = { // ...METADATA_SPEC, // } -export const LLM_MULTIMODAL_RUNNER_SPEC = { +export const LLM_MULTIMODAL_SPEC = { ...method( 'text_decoder', // prettier-ignore [f32(1, 'seqLen', 'hiddenDim'), i64('seqLen')], @@ -78,7 +78,7 @@ export function createLLMMultimodalRunner( ); } - const { dims } = validateSpec(model.schema, { default: LLM_MULTIMODAL_RUNNER_SPEC }); + const { dims } = validateSpec(model.schema, { default: LLM_MULTIMODAL_SPEC }); const [imgH, imgW, vocabSize] = dims.constant('imgH', 'imgW', 'vocabSize'); const [hiddenDim, numVisualTokens] = dims.constant('hiddenDim', 'visualTokens'); diff --git a/packages/react-native-executorch/src/extensions/llm/runners/llmTextRunner.ts b/packages/react-native-executorch/src/extensions/llm/runners/llmTextRunner.ts index 831dd57f4c..311e6b1fd8 100644 --- a/packages/react-native-executorch/src/extensions/llm/runners/llmTextRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/runners/llmTextRunner.ts @@ -18,7 +18,7 @@ import type { Prompt, } from '../llmRunner'; -export const LLM_TEXT_RUNNER_SPEC = { +export const LLM_TEXT_SPEC = { ...method( 'forward', // prettier-ignore [i64(1, Dyn('seqLen')), i64(1)], @@ -43,7 +43,7 @@ export function createLLMTextRunner( ); } - const { dims } = validateSpec(model.schema, { default: LLM_TEXT_RUNNER_SPEC }); + const { dims } = validateSpec(model.schema, { default: LLM_TEXT_SPEC }); const [vocabSize] = dims.constant('vocabSize'); const [seqLenRange] = dims.range('seqLen'); From 7c74aa9c43ca44587f612cc92ff986db9e7b347f Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Fri, 9 Oct 2026 02:25:13 +0200 Subject: [PATCH 32/83] update --- .../cpp/core/model.cpp | 2 +- .../src/extensions/llm/llmRunner.ts | 10 ++++- .../llm/runners/llmMultimodalRunner.ts | 43 ++++++++++--------- 3 files changed, 33 insertions(+), 22 deletions(-) diff --git a/packages/react-native-executorch/cpp/core/model.cpp b/packages/react-native-executorch/cpp/core/model.cpp index 49c8786831..2066730250 100644 --- a/packages/react-native-executorch/cpp/core/model.cpp +++ b/packages/react-native-executorch/cpp/core/model.cpp @@ -168,7 +168,7 @@ jsi::Value ModelHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) { switch (tag) { case executorch::runtime::Tag::Tensor: { const auto &tSpec = methodSpec.inputs[i]; - auto tensorHostObject = tensor::fromJs(rt, ctx, val, tSpec.dtype, tSpec.shape); + auto tensorHostObject = tensor::fromJs(rt, ctx, val, tSpec.dtype, std::nullopt); // TODO: tSpec.shape); if (!lockedTensors.insert(tensorHostObject.get()).second) { throw error::InvalidArgument("execute: Tensor aliasing detected. " diff --git a/packages/react-native-executorch/src/extensions/llm/llmRunner.ts b/packages/react-native-executorch/src/extensions/llm/llmRunner.ts index 28c649a3c2..90e2636d20 100644 --- a/packages/react-native-executorch/src/extensions/llm/llmRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/llmRunner.ts @@ -15,7 +15,11 @@ import { loadTokenizer } from '../nlp'; import type { SamplingConfig } from './sampler'; import { createLLMTextRunner, LLM_TEXT_SPEC } from './runners/llmTextRunner'; -import { createLLMMultimodalRunner, LLM_MULTIMODAL_SPEC } from './runners/llmMultimodalRunner'; +import { + createLLMMultimodalRunner, + LLM_MULTIMODAL_DYN_SPEC, + LLM_MULTIMODAL_SPEC, +} from './runners/llmMultimodalRunner'; declare const llmRunnerBrand: unique symbol; @@ -191,12 +195,16 @@ export async function createLLMRunner( const { variant } = validateSpec(model.schema, { text: LLM_TEXT_SPEC, vision: LLM_MULTIMODAL_SPEC, + visionDyn: LLM_MULTIMODAL_DYN_SPEC, }); + console.log(variant); + switch (variant) { case 'text': return createLLMTextRunner(model, tokenizer, modalities); case 'vision': + case 'visionDyn': return createLLMMultimodalRunner(model, tokenizer, modalities); default: throw RnExecuTorchError('LOAD_FAILED', `llmRunner: unrecognized variant ${variant}`); diff --git a/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts b/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts index c9fcff783f..d67f7f2936 100644 --- a/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts @@ -3,7 +3,7 @@ import { createSynchronizable } from 'react-native-worklets'; import { tensor } from '../../../core/tensor'; import type { Model } from '../../../core/model'; import { RnExecuTorchError } from '../../../core/error'; -import { f32, i64, method, validateSpec } from '../../../core/schema'; +import { f32, i64, method, validateSpec, DynamicDim as Dyn } from '../../../core/schema'; import type { Tokenizer } from '../../nlp'; import { sample } from '../sampler'; @@ -28,24 +28,24 @@ const METADATA_SPEC = { }; // REPLACE WITH THIS AFTER REEXPORT -// { -// ...method( -// 'text_decoder', -// [f32(1, Dyn('seqLen'), 'hiddenDim'), i64(Dyn('seqLen'))], -// [f32(1, 'vocabSize')] -// ), -// ...method( -// 'vision_encoder', // -// [f32(1, 3, 'imgH', 'imgW')], -// [f32(1, 'visualTokens', 'hiddenDim')] -// ), -// ...method( -// 'token_embedding', // -// [i64(1, Dyn('seqLen'))], -// [f32(1, Dyn('seqLen'), 'hiddenDim')] -// ), -// ...METADATA_SPEC, -// } +export const LLM_MULTIMODAL_DYN_SPEC = { + ...method( + 'text_decoder', + [f32(1, Dyn('seqLen'), 'hiddenDim'), i64(Dyn('seqLen'))], + [f32(1, 'vocabSize')] + ), + ...method( + 'vision_encoder', // + [f32(1, 3, 'imgH', 'imgW')], + [f32(1, 'visualTokens', 'hiddenDim')] + ), + ...method( + 'token_embedding', // + [i64(1, Dyn('seqLen'))], + [f32(1, Dyn('seqLen'), 'hiddenDim')] + ), + ...METADATA_SPEC, +}; export const LLM_MULTIMODAL_SPEC = { ...method( @@ -78,7 +78,10 @@ export function createLLMMultimodalRunner( ); } - const { dims } = validateSpec(model.schema, { default: LLM_MULTIMODAL_SPEC }); + const { dims } = validateSpec(model.schema, { + static: LLM_MULTIMODAL_SPEC, + dynamic: LLM_MULTIMODAL_DYN_SPEC, + }); const [imgH, imgW, vocabSize] = dims.constant('imgH', 'imgW', 'vocabSize'); const [hiddenDim, numVisualTokens] = dims.constant('hiddenDim', 'visualTokens'); From 005a09ddaecec9823858267451d191707f2237eb Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Fri, 9 Oct 2026 04:20:12 +0200 Subject: [PATCH 33/83] update --- apps/nlp/constants/llm.ts | 279 +++++++++++++---- .../src/extensions/llm/llmRunner.ts | 2 - .../extensions/llm/runners/llmGemmaRunner.ts | 4 +- .../react-native-executorch/src/models.ts | 294 +++++++++--------- 4 files changed, 374 insertions(+), 205 deletions(-) diff --git a/apps/nlp/constants/llm.ts b/apps/nlp/constants/llm.ts index 161a663e9c..10ab22bc82 100644 --- a/apps/nlp/constants/llm.ts +++ b/apps/nlp/constants/llm.ts @@ -314,9 +314,12 @@ export const LLAMA_TOOL_OPTS: LLMToolOpts = { // --- Model Config Type --- +export type LLMBackend = 'xnnpack' | 'mlx' | 'vulkan'; + export interface LLMModelConfig { id: string; name: string; + backend: LLMBackend; model: LLMModel; generationConfig: llm.LLMGenerationConfig; systemPrompt?: string; @@ -331,7 +334,8 @@ export const LLM_MODELS: LLMModelConfig[] = [ // LFM 2.5 family { id: 'lfm-2.5-1.2b-xnnpack', - name: 'LFM 2.5 1.2B (8da4w)', + name: 'LFM 2.5 1.2B', + backend: 'xnnpack', model: models.llm.LFM2_5_1_2B.XNNPACK_8DA4W, generationConfig: DEFAULT_GENERATION_CONFIG, systemPrompt: DEFAULT_SYSTEM_PROMPT, @@ -340,7 +344,8 @@ export const LLM_MODELS: LLMModelConfig[] = [ }, { id: 'lfm-2.5-1.2b-mlx', - name: 'LFM 2.5 1.2B (MLX Int4)', + name: 'LFM 2.5 1.2B', + backend: 'mlx', model: models.llm.LFM2_5_1_2B.MLX_INT4, generationConfig: DEFAULT_GENERATION_CONFIG, systemPrompt: DEFAULT_SYSTEM_PROMPT, @@ -350,7 +355,8 @@ export const LLM_MODELS: LLMModelConfig[] = [ }, { id: 'lfm-2.5-350m-xnnpack', - name: 'LFM 2.5 350M (8da4w)', + name: 'LFM 2.5 350M', + backend: 'xnnpack', model: models.llm.LFM2_5_350M.XNNPACK_8DA4W, generationConfig: DEFAULT_GENERATION_CONFIG, systemPrompt: DEFAULT_SYSTEM_PROMPT, @@ -359,7 +365,8 @@ export const LLM_MODELS: LLMModelConfig[] = [ }, { id: 'lfm-2.5-350m-mlx', - name: 'LFM 2.5 350M (MLX Int4)', + name: 'LFM 2.5 350M', + backend: 'mlx', model: models.llm.LFM2_5_350M.MLX_INT4, generationConfig: DEFAULT_GENERATION_CONFIG, systemPrompt: DEFAULT_SYSTEM_PROMPT, @@ -369,7 +376,8 @@ export const LLM_MODELS: LLMModelConfig[] = [ }, { id: 'lfm-2.5-vl-450m-xnnpack', - name: 'LFM 2.5 VL 450M (8da4w)', + name: 'LFM 2.5 VL 450M', + backend: 'xnnpack', model: models.llm.LFM2_5_VL_450M.XNNPACK_8DA4W, generationConfig: DEFAULT_GENERATION_CONFIG, systemPrompt: DEFAULT_SYSTEM_PROMPT, @@ -378,7 +386,8 @@ export const LLM_MODELS: LLMModelConfig[] = [ }, { id: 'lfm-2.5-vl-450m-mlx', - name: 'LFM 2.5 VL 450M (MLX Int4)', + name: 'LFM 2.5 VL 450M', + backend: 'mlx', model: models.llm.LFM2_5_VL_450M.MLX_INT4, generationConfig: DEFAULT_GENERATION_CONFIG, systemPrompt: DEFAULT_SYSTEM_PROMPT, @@ -388,7 +397,8 @@ export const LLM_MODELS: LLMModelConfig[] = [ }, { id: 'lfm-2.5-vl-1.6b-xnnpack', - name: 'LFM 2.5 VL 1.6B (8da4w)', + name: 'LFM 2.5 VL 1.6B', + backend: 'xnnpack', model: models.llm.LFM2_5_VL_1_6B.XNNPACK_8DA4W, generationConfig: DEFAULT_GENERATION_CONFIG, systemPrompt: DEFAULT_SYSTEM_PROMPT, @@ -396,98 +406,174 @@ export const LLM_MODELS: LLMModelConfig[] = [ toolOpts: undefined, }, { - id: 'lfm-2.5-vl-450m-vulkan', - name: 'LFM 2.5 VL 450M (Vulkan 8da4w)', - model: models.llm.LFM2_5_VL_450M.VULKAN_8DA4W, - generationConfig: DEFAULT_GENERATION_CONFIG, - systemPrompt: DEFAULT_SYSTEM_PROMPT, - stopRegex: undefined, - toolOpts: undefined, - }, - { - id: 'lfm-2.5-vl-1.6b-vulkan', - name: 'LFM 2.5 VL 1.6B (Vulkan 8da4w)', - model: models.llm.LFM2_5_VL_1_6B.VULKAN_8DA4W, + id: 'lfm-2.5-vl-1.6b-mlx', + name: 'LFM 2.5 VL 1.6B', + backend: 'mlx', + model: models.llm.LFM2_5_VL_1_6B.MLX_INT4, generationConfig: DEFAULT_GENERATION_CONFIG, systemPrompt: DEFAULT_SYSTEM_PROMPT, stopRegex: undefined, toolOpts: undefined, + iosOnly: true, }, + // { + // id: 'lfm-2.5-vl-450m-vulkan', + // name: 'LFM 2.5 VL 450M', + // backend: 'vulkan', + // model: models.llm.LFM2_5_VL_450M.VULKAN_8DA4W, + // generationConfig: DEFAULT_GENERATION_CONFIG, + // systemPrompt: DEFAULT_SYSTEM_PROMPT, + // stopRegex: undefined, + // toolOpts: undefined, + // }, + // { + // id: 'lfm-2.5-vl-1.6b-vulkan', + // name: 'LFM 2.5 VL 1.6B', + // backend: 'vulkan', + // model: models.llm.LFM2_5_VL_1_6B.VULKAN_8DA4W, + // generationConfig: DEFAULT_GENERATION_CONFIG, + // systemPrompt: DEFAULT_SYSTEM_PROMPT, + // stopRegex: undefined, + // toolOpts: undefined, + // }, // Gemma 4 family + // { + // id: 'gemma-4-e2b-xnnpack', + // name: 'Gemma 4 E2B', + // backend: 'xnnpack', + // model: models.llm.GEMMA4_E2B.XNNPACK_8DA4W, + // generationConfig: DEFAULT_GENERATION_CONFIG, + // systemPrompt: DEFAULT_SYSTEM_PROMPT, + // stopRegex: GEMMA_TOOL_STOP_REGEX, + // toolOpts: GEMMA_TOOL_OPTS, + // }, + // { + // id: 'gemma-4-e2b-mlx', + // name: 'Gemma 4 E2B', + // backend: 'mlx', + // model: models.llm.GEMMA4_E2B.MLX_INT4, + // generationConfig: DEFAULT_GENERATION_CONFIG, + // systemPrompt: DEFAULT_SYSTEM_PROMPT, + // stopRegex: GEMMA_TOOL_STOP_REGEX, + // toolOpts: GEMMA_TOOL_OPTS, + // iosOnly: true, + // }, + + // Hammer 2.1 family { - id: 'gemma-4-e2b-xnnpack', - name: 'Gemma 4 E2B (8da4w)', - model: models.llm.GEMMA4_E2B.XNNPACK_8DA4W, + id: 'hammer-2.1-0.5b-xnnpack', + name: 'Hammer 2.1 0.5B', + backend: 'xnnpack', + model: models.llm.HAMMER2_1_0_5B.XNNPACK_8DA4W, generationConfig: DEFAULT_GENERATION_CONFIG, - systemPrompt: DEFAULT_SYSTEM_PROMPT, - stopRegex: GEMMA_TOOL_STOP_REGEX, - toolOpts: GEMMA_TOOL_OPTS, + systemPrompt: undefined, + stopRegex: HAMMER_TOOL_STOP_REGEX, + toolOpts: HAMMER_TOOL_OPTS, }, { - id: 'gemma-4-e2b-mlx', - name: 'Gemma 4 E2B (MLX Int4)', - model: models.llm.GEMMA4_E2B.MLX_INT4, + id: 'hammer-2.1-0.5b-mlx', + name: 'Hammer 2.1 0.5B', + backend: 'mlx', + model: models.llm.HAMMER2_1_0_5B.MLX_INT4, generationConfig: DEFAULT_GENERATION_CONFIG, - systemPrompt: DEFAULT_SYSTEM_PROMPT, - stopRegex: GEMMA_TOOL_STOP_REGEX, - toolOpts: GEMMA_TOOL_OPTS, + systemPrompt: undefined, + stopRegex: HAMMER_TOOL_STOP_REGEX, + toolOpts: HAMMER_TOOL_OPTS, iosOnly: true, }, - - // Hammer 2.1 family { - id: 'hammer-2.1-0.5b-xnnpack', - name: 'Hammer 2.1 0.5B (8da4w)', - model: models.llm.HAMMER2_1_0_5B.XNNPACK_8DA4W, + id: 'hammer-2.1-1.5b-xnnpack', + name: 'Hammer 2.1 1.5B', + backend: 'xnnpack', + model: models.llm.HAMMER2_1_1_5B.XNNPACK_8DA4W, generationConfig: DEFAULT_GENERATION_CONFIG, systemPrompt: undefined, stopRegex: HAMMER_TOOL_STOP_REGEX, toolOpts: HAMMER_TOOL_OPTS, }, { - id: 'hammer-2.1-1.5b-xnnpack', - name: 'Hammer 2.1 1.5B (8da4w)', - model: models.llm.HAMMER2_1_1_5B.XNNPACK_8DA4W, + id: 'hammer-2.1-1.5b-mlx', + name: 'Hammer 2.1 1.5B', + backend: 'mlx', + model: models.llm.HAMMER2_1_1_5B.MLX_INT4, generationConfig: DEFAULT_GENERATION_CONFIG, systemPrompt: undefined, stopRegex: HAMMER_TOOL_STOP_REGEX, toolOpts: HAMMER_TOOL_OPTS, + iosOnly: true, }, { id: 'hammer-2.1-3b-xnnpack', - name: 'Hammer 2.1 3B (8da4w)', + name: 'Hammer 2.1 3B', + backend: 'xnnpack', model: models.llm.HAMMER2_1_3B.XNNPACK_8DA4W, generationConfig: DEFAULT_GENERATION_CONFIG, systemPrompt: undefined, stopRegex: HAMMER_TOOL_STOP_REGEX, toolOpts: HAMMER_TOOL_OPTS, }, + { + id: 'hammer-2.1-3b-mlx', + name: 'Hammer 2.1 3B', + backend: 'mlx', + model: models.llm.HAMMER2_1_3B.MLX_INT4, + generationConfig: DEFAULT_GENERATION_CONFIG, + systemPrompt: undefined, + stopRegex: HAMMER_TOOL_STOP_REGEX, + toolOpts: HAMMER_TOOL_OPTS, + iosOnly: true, + }, // Llama 3.2 family { id: 'llama-3.2-1b-spinquant', - name: 'Llama 3.2 1B (SpinQuant)', + name: 'Llama 3.2 1B', + backend: 'xnnpack', model: models.llm.LLAMA3_2_1B.XNNPACK_SPINQUANT, generationConfig: DEFAULT_GENERATION_CONFIG, systemPrompt: DEFAULT_SYSTEM_PROMPT, stopRegex: LLAMA_TOOL_STOP_REGEX, toolOpts: LLAMA_TOOL_OPTS, }, + { + id: 'llama-3.2-1b-mlx', + name: 'Llama 3.2 1B', + backend: 'mlx', + model: models.llm.LLAMA3_2_1B.MLX_INT4, + generationConfig: DEFAULT_GENERATION_CONFIG, + systemPrompt: DEFAULT_SYSTEM_PROMPT, + stopRegex: LLAMA_TOOL_STOP_REGEX, + toolOpts: LLAMA_TOOL_OPTS, + iosOnly: true, + }, { id: 'llama-3.2-3b-spinquant', - name: 'Llama 3.2 3B (SpinQuant)', + name: 'Llama 3.2 3B', + backend: 'xnnpack', model: models.llm.LLAMA3_2_3B.XNNPACK_SPINQUANT, generationConfig: DEFAULT_GENERATION_CONFIG, systemPrompt: DEFAULT_SYSTEM_PROMPT, stopRegex: LLAMA_TOOL_STOP_REGEX, toolOpts: LLAMA_TOOL_OPTS, }, + { + id: 'llama-3.2-3b-mlx', + name: 'Llama 3.2 3B', + backend: 'mlx', + model: models.llm.LLAMA3_2_3B.MLX_INT4, + generationConfig: DEFAULT_GENERATION_CONFIG, + systemPrompt: DEFAULT_SYSTEM_PROMPT, + stopRegex: LLAMA_TOOL_STOP_REGEX, + toolOpts: LLAMA_TOOL_OPTS, + iosOnly: true, + }, // SmolLM2 family { id: 'smollm2-135m-xnnpack', - name: 'SmolLM2 135M (8da8w)', + name: 'SmolLM2 135M', + backend: 'xnnpack', model: models.llm.SMOLLM2_135M.XNNPACK_8DA8W, generationConfig: DEFAULT_GENERATION_CONFIG, systemPrompt: DEFAULT_SYSTEM_PROMPT, @@ -496,7 +582,8 @@ export const LLM_MODELS: LLMModelConfig[] = [ }, { id: 'smollm2-360m-xnnpack', - name: 'SmolLM2 360M (8da8w)', + name: 'SmolLM2 360M', + backend: 'xnnpack', model: models.llm.SMOLLM2_360M.XNNPACK_8DA8W, generationConfig: DEFAULT_GENERATION_CONFIG, systemPrompt: DEFAULT_SYSTEM_PROMPT, @@ -505,7 +592,8 @@ export const LLM_MODELS: LLMModelConfig[] = [ }, { id: 'smollm2-1.7b-xnnpack', - name: 'SmolLM2 1.7B (8da8w)', + name: 'SmolLM2 1.7B', + backend: 'xnnpack', model: models.llm.SMOLLM2_1_7B.XNNPACK_8DA8W, generationConfig: DEFAULT_GENERATION_CONFIG, systemPrompt: DEFAULT_SYSTEM_PROMPT, @@ -516,76 +604,161 @@ export const LLM_MODELS: LLMModelConfig[] = [ // Qwen 2.5 family { id: 'qwen-2.5-0.5b-xnnpack', - name: 'Qwen 2.5 0.5B (8da4w)', + name: 'Qwen 2.5 0.5B', + backend: 'xnnpack', model: models.llm.QWEN2_5_0_5B.XNNPACK_8DA4W, generationConfig: DEFAULT_GENERATION_CONFIG, systemPrompt: DEFAULT_SYSTEM_PROMPT, stopRegex: QWEN_TOOL_STOP_REGEX, toolOpts: QWEN_TOOL_OPTS, }, + { + id: 'qwen-2.5-0.5b-mlx', + name: 'Qwen 2.5 0.5B', + backend: 'mlx', + model: models.llm.QWEN2_5_0_5B.MLX_INT4, + generationConfig: DEFAULT_GENERATION_CONFIG, + systemPrompt: DEFAULT_SYSTEM_PROMPT, + stopRegex: QWEN_TOOL_STOP_REGEX, + toolOpts: QWEN_TOOL_OPTS, + iosOnly: true, + }, { id: 'qwen-2.5-1.5b-xnnpack', - name: 'Qwen 2.5 1.5B (8da4w)', + name: 'Qwen 2.5 1.5B', + backend: 'xnnpack', model: models.llm.QWEN2_5_1_5B.XNNPACK_8DA4W, generationConfig: DEFAULT_GENERATION_CONFIG, systemPrompt: DEFAULT_SYSTEM_PROMPT, stopRegex: QWEN_TOOL_STOP_REGEX, toolOpts: QWEN_TOOL_OPTS, }, + { + id: 'qwen-2.5-1.5b-mlx', + name: 'Qwen 2.5 1.5B', + backend: 'mlx', + model: models.llm.QWEN2_5_1_5B.MLX_INT4, + generationConfig: DEFAULT_GENERATION_CONFIG, + systemPrompt: DEFAULT_SYSTEM_PROMPT, + stopRegex: QWEN_TOOL_STOP_REGEX, + toolOpts: QWEN_TOOL_OPTS, + iosOnly: true, + }, { id: 'qwen-2.5-3b-xnnpack', - name: 'Qwen 2.5 3B (8da4w)', + name: 'Qwen 2.5 3B', + backend: 'xnnpack', model: models.llm.QWEN2_5_3B.XNNPACK_8DA4W, generationConfig: DEFAULT_GENERATION_CONFIG, systemPrompt: DEFAULT_SYSTEM_PROMPT, stopRegex: QWEN_TOOL_STOP_REGEX, toolOpts: QWEN_TOOL_OPTS, }, + { + id: 'qwen-2.5-3b-mlx', + name: 'Qwen 2.5 3B', + backend: 'mlx', + model: models.llm.QWEN2_5_3B.MLX_INT4, + generationConfig: DEFAULT_GENERATION_CONFIG, + systemPrompt: DEFAULT_SYSTEM_PROMPT, + stopRegex: QWEN_TOOL_STOP_REGEX, + toolOpts: QWEN_TOOL_OPTS, + iosOnly: true, + }, // Qwen 3 family { id: 'qwen-3-0.6b-xnnpack', - name: 'Qwen 3 0.6B (8da4w)', + name: 'Qwen 3 0.6B', + backend: 'xnnpack', model: models.llm.QWEN3_0_6B.XNNPACK_8DA4W, generationConfig: DEFAULT_GENERATION_CONFIG, systemPrompt: DEFAULT_SYSTEM_PROMPT, stopRegex: QWEN_TOOL_STOP_REGEX, toolOpts: QWEN_TOOL_OPTS, }, + { + id: 'qwen-3-0.6b-mlx', + name: 'Qwen 3 0.6B', + backend: 'mlx', + model: models.llm.QWEN3_0_6B.MLX_INT4, + generationConfig: DEFAULT_GENERATION_CONFIG, + systemPrompt: DEFAULT_SYSTEM_PROMPT, + stopRegex: QWEN_TOOL_STOP_REGEX, + toolOpts: QWEN_TOOL_OPTS, + iosOnly: true, + }, { id: 'qwen-3-1.7b-xnnpack', - name: 'Qwen 3 1.7B (8da4w)', + name: 'Qwen 3 1.7B', + backend: 'xnnpack', model: models.llm.QWEN3_1_7B.XNNPACK_8DA4W, generationConfig: DEFAULT_GENERATION_CONFIG, systemPrompt: DEFAULT_SYSTEM_PROMPT, stopRegex: QWEN_TOOL_STOP_REGEX, toolOpts: QWEN_TOOL_OPTS, }, + { + id: 'qwen-3-1.7b-mlx', + name: 'Qwen 3 1.7B', + backend: 'mlx', + model: models.llm.QWEN3_1_7B.MLX_INT4, + generationConfig: DEFAULT_GENERATION_CONFIG, + systemPrompt: DEFAULT_SYSTEM_PROMPT, + stopRegex: QWEN_TOOL_STOP_REGEX, + toolOpts: QWEN_TOOL_OPTS, + iosOnly: true, + }, { id: 'qwen-3-4b-xnnpack', - name: 'Qwen 3 4B (8da4w)', + name: 'Qwen 3 4B', + backend: 'xnnpack', model: models.llm.QWEN3_4B.XNNPACK_8DA4W, generationConfig: DEFAULT_GENERATION_CONFIG, systemPrompt: DEFAULT_SYSTEM_PROMPT, stopRegex: QWEN_TOOL_STOP_REGEX, toolOpts: QWEN_TOOL_OPTS, }, + { + id: 'qwen-3-4b-mlx', + name: 'Qwen 3 4B', + backend: 'mlx', + model: models.llm.QWEN3_4B.MLX_INT4, + generationConfig: DEFAULT_GENERATION_CONFIG, + systemPrompt: DEFAULT_SYSTEM_PROMPT, + stopRegex: QWEN_TOOL_STOP_REGEX, + toolOpts: QWEN_TOOL_OPTS, + iosOnly: true, + }, // Phi-4 Mini { id: 'phi-4-mini-xnnpack', - name: 'Phi-4 Mini 3.8B (8da4w)', + name: 'Phi-4 Mini 3.8B', + backend: 'xnnpack', model: models.llm.PHI4_MINI.XNNPACK_8DA4W, generationConfig: DEFAULT_GENERATION_CONFIG, systemPrompt: DEFAULT_SYSTEM_PROMPT, stopRegex: undefined, toolOpts: undefined, }, + { + id: 'phi-4-mini-mlx', + name: 'Phi-4 Mini 3.8B', + backend: 'mlx', + model: models.llm.PHI4_MINI.MLX_INT4, + generationConfig: DEFAULT_GENERATION_CONFIG, + systemPrompt: DEFAULT_SYSTEM_PROMPT, + stopRegex: undefined, + toolOpts: undefined, + iosOnly: true, + }, // Bielik v3 { id: 'bielik-v3-1.5b-xnnpack', - name: 'Bielik v3 1.5B (8da4w)', + name: 'Bielik v3 1.5B', + backend: 'xnnpack', model: models.llm.BIELIK_V3_1_5B.XNNPACK_8DA4W, generationConfig: DEFAULT_GENERATION_CONFIG, // eslint-disable-next-line @cspell/spellchecker diff --git a/packages/react-native-executorch/src/extensions/llm/llmRunner.ts b/packages/react-native-executorch/src/extensions/llm/llmRunner.ts index 90e2636d20..00821c6187 100644 --- a/packages/react-native-executorch/src/extensions/llm/llmRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/llmRunner.ts @@ -198,8 +198,6 @@ export async function createLLMRunner( visionDyn: LLM_MULTIMODAL_DYN_SPEC, }); - console.log(variant); - switch (variant) { case 'text': return createLLMTextRunner(model, tokenizer, modalities); diff --git a/packages/react-native-executorch/src/extensions/llm/runners/llmGemmaRunner.ts b/packages/react-native-executorch/src/extensions/llm/runners/llmGemmaRunner.ts index b2c3ecccaa..01f2930600 100644 --- a/packages/react-native-executorch/src/extensions/llm/runners/llmGemmaRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/runners/llmGemmaRunner.ts @@ -3,7 +3,5 @@ import type { Tokenizer } from '../../nlp'; import type { LLMRunner } from '../llmRunner'; export function createLLMGemmaRunner(model: Model, tokenizer: Tokenizer): LLMRunner { - void model; - void tokenizer; - throw new Error('Not implemented'); + throw new Error(`Not implemented ${model.path}, ${tokenizer.path}`); } diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index f889717b01..733dfde2e0 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -2069,24 +2069,25 @@ const PPOCRV6_SMALL_VULKAN_FP16: PaddleOcrModel = { // LLMs // ============================================================================= const LFM2_5_BASE_URL = `${BASE_URL}-lfm-2.5/${VERSION_TAG}`; +const SCRATCH_LLMS_BASE_URL = 'https://huggingface.co/bhanc/scratch/resolve/main/llms'; const LFM2_5_1_2B_XNNPACK_8DA4W: LLMModel = { - modelPath: `${LFM2_5_BASE_URL}/1_2b/xnnpack/lfm_2_5_1_2b_xnnpack_8da4w.pte`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/lfm2_5_1_2b_xnnpack_8da4w_16k.pte`, tokenizerPath: `${LFM2_5_BASE_URL}/1_2b/tokenizer.json`, tokenizerConfigPath: `${LFM2_5_BASE_URL}/1_2b/tokenizer_config.json`, }; const LFM2_5_1_2B_MLX_INT4: LLMModel = { - modelPath: `${LFM2_5_BASE_URL}/1_2b/mlx/lfm_2_5_1_2b_mlx_int4.pte`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/lfm_2_5_1_2b_mlx_int4_16k.pte`, tokenizerPath: `${LFM2_5_BASE_URL}/1_2b/tokenizer.json`, tokenizerConfigPath: `${LFM2_5_BASE_URL}/1_2b/tokenizer_config.json`, }; const LFM2_5_350M_XNNPACK_8DA4W: LLMModel = { - modelPath: `${LFM2_5_BASE_URL}/350m/xnnpack/lfm_2_5_350m_xnnpack_8da4w.pte`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/lfm_2_5_350m_xnnpack_8da4w_16k.pte`, tokenizerPath: `${LFM2_5_BASE_URL}/350m/tokenizer.json`, tokenizerConfigPath: `${LFM2_5_BASE_URL}/350m/tokenizer_config.json`, }; const LFM2_5_350M_MLX_INT4: LLMModel = { - modelPath: `${LFM2_5_BASE_URL}/350m/mlx/lfm_2_5_350m_mlx_int4.pte`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/lfm_2_5_350m_mlx_int4_16k.pte`, tokenizerPath: `${LFM2_5_BASE_URL}/350m/tokenizer.json`, tokenizerConfigPath: `${LFM2_5_BASE_URL}/350m/tokenizer_config.json`, }; @@ -2094,7 +2095,7 @@ const LFM2_5_350M_MLX_INT4: LLMModel = { const LFM2_5_VL_PREPROCESSOR_CONFIG = { image: { visionToken: { start: '<|image_start|>', end: '<|image_end|>' }, - targetShape: [3, 512, 512] as const, + targetShape: [1, 3, 512, 512] as const, preprocessorOpts: { resizeMode: 'letterbox' as const, interpolation: 'linear' as const, @@ -2103,59 +2104,59 @@ const LFM2_5_VL_PREPROCESSOR_CONFIG = { }, }; const LFM2_5_VL_450M_XNNPACK_8DA4W: LLMModel = { - modelPath: `${LFM2_5_BASE_URL}/vl_450m/xnnpack/lfm_2_5_vl_450m_xnnpack_8da4w.pte`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/lfm_2_5_vl_450m_xnnpack_8da4w.pte`, tokenizerPath: `${LFM2_5_BASE_URL}/vl_450m/tokenizer.json`, tokenizerConfigPath: `${LFM2_5_BASE_URL}/vl_450m/tokenizer_config.json`, modalities: ['image'], preprocessorConfig: LFM2_5_VL_PREPROCESSOR_CONFIG, }; const LFM2_5_VL_450M_MLX_INT4: LLMModel = { - modelPath: `${LFM2_5_BASE_URL}/vl_450m/mlx/lfm_2_5_vl_450m_mlx_int4.pte`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/lfm_2_5_vl_450m_mlx_int4.pte`, tokenizerPath: `${LFM2_5_BASE_URL}/vl_450m/tokenizer.json`, tokenizerConfigPath: `${LFM2_5_BASE_URL}/vl_450m/tokenizer_config.json`, modalities: ['image'], preprocessorConfig: LFM2_5_VL_PREPROCESSOR_CONFIG, }; const LFM2_5_VL_1_6B_XNNPACK_8DA4W: LLMModel = { - modelPath: `${LFM2_5_BASE_URL}/vl_1_6b/xnnpack/lfm_2_5_vl_1_6b_xnnpack_8da4w.pte`, - tokenizerPath: `${LFM2_5_BASE_URL}/vl_1_6b/tokenizer.json`, - tokenizerConfigPath: `${LFM2_5_BASE_URL}/vl_1_6b/tokenizer_config.json`, - modalities: ['image'], - preprocessorConfig: LFM2_5_VL_PREPROCESSOR_CONFIG, -}; -const LFM2_5_VL_450M_VULKAN_8DA4W: LLMModel = { - modelPath: `${LFM2_5_BASE_URL}/vl_450m/vulkan/lfm_2_5_vl_450m_vulkan_8da4w.pte`, - tokenizerPath: `${LFM2_5_BASE_URL}/vl_450m/tokenizer.json`, - tokenizerConfigPath: `${LFM2_5_BASE_URL}/vl_450m/tokenizer_config.json`, - modalities: ['image'], - preprocessorConfig: LFM2_5_VL_PREPROCESSOR_CONFIG, -}; -const LFM2_5_VL_1_6B_VULKAN_8DA4W: LLMModel = { - modelPath: `${LFM2_5_BASE_URL}/vl_1_6b/vulkan/lfm_2_5_vl_1_6b_vulkan_8da4w.pte`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/lfm_2_5_vl_1_6b_xnnpack_8da4w.pte`, tokenizerPath: `${LFM2_5_BASE_URL}/vl_1_6b/tokenizer.json`, tokenizerConfigPath: `${LFM2_5_BASE_URL}/vl_1_6b/tokenizer_config.json`, modalities: ['image'], preprocessorConfig: LFM2_5_VL_PREPROCESSOR_CONFIG, }; +// const LFM2_5_VL_450M_VULKAN_8DA4W: LLMModel = { +// modelPath: `${LFM2_5_BASE_URL}/vl_450m/vulkan/lfm_2_5_vl_450m_vulkan_8da4w.pte`, +// tokenizerPath: `${LFM2_5_BASE_URL}/vl_450m/tokenizer.json`, +// tokenizerConfigPath: `${LFM2_5_BASE_URL}/vl_450m/tokenizer_config.json`, +// modalities: ['image'], +// preprocessorConfig: LFM2_5_VL_PREPROCESSOR_CONFIG, +// }; +// const LFM2_5_VL_1_6B_VULKAN_8DA4W: LLMModel = { +// modelPath: `${LFM2_5_BASE_URL}/vl_1_6b/vulkan/lfm_2_5_vl_1_6b_vulkan_8da4w.pte`, +// tokenizerPath: `${LFM2_5_BASE_URL}/vl_1_6b/tokenizer.json`, +// tokenizerConfigPath: `${LFM2_5_BASE_URL}/vl_1_6b/tokenizer_config.json`, +// modalities: ['image'], +// preprocessorConfig: LFM2_5_VL_PREPROCESSOR_CONFIG, +// }; const LFM2_5_VL_1_6B_MLX_INT4: LLMModel = { - modelPath: `${LFM2_5_BASE_URL}/vl_1_6b/mlx/lfm_2_5_vl_1_6b_mlx_int4.pte`, - tokenizerPath: `${LFM2_5_BASE_URL}/vl_1_6b/tokenizer.json`, - tokenizerConfigPath: `${LFM2_5_BASE_URL}/vl_1_6b/tokenizer_config.json`, - modalities: ['image'], - preprocessorConfig: LFM2_5_VL_PREPROCESSOR_CONFIG, -}; -const LFM2_5_VL_1_6B_MLX_INT8: LLMModel = { - modelPath: `${LFM2_5_BASE_URL}/vl_1_6b/mlx/lfm_2_5_vl_1_6b_mlx_int8.pte`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/lfm_2_5_vl_1_6b_mlx_int4.pte`, tokenizerPath: `${LFM2_5_BASE_URL}/vl_1_6b/tokenizer.json`, tokenizerConfigPath: `${LFM2_5_BASE_URL}/vl_1_6b/tokenizer_config.json`, modalities: ['image'], preprocessorConfig: LFM2_5_VL_PREPROCESSOR_CONFIG, }; +// const LFM2_5_VL_1_6B_MLX_INT8: LLMModel = { +// modelPath: `${LFM2_5_BASE_URL}/vl_1_6b/mlx/lfm_2_5_vl_1_6b_mlx_int8.pte`, +// tokenizerPath: `${LFM2_5_BASE_URL}/vl_1_6b/tokenizer.json`, +// tokenizerConfigPath: `${LFM2_5_BASE_URL}/vl_1_6b/tokenizer_config.json`, +// modalities: ['image'], +// preprocessorConfig: LFM2_5_VL_PREPROCESSOR_CONFIG, +// }; const BIELIK_V3_1_5B_BASE_URL = `${BASE_URL}-bielik-v3.0/${VERSION_TAG}`; const BIELIK_V3_1_5B_XNNPACK_8DA4W: LLMModel = { - modelPath: `${BIELIK_V3_1_5B_BASE_URL}/xnnpack/bielik_v3_0_1_5b_xnnpack_8da4w.pte`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/bielik_v3_0_1_5b_xnnpack_8da4w_8k.pte`, tokenizerPath: `${BIELIK_V3_1_5B_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${BIELIK_V3_1_5B_BASE_URL}/tokenizer_config.json`, }; @@ -2163,22 +2164,22 @@ const BIELIK_V3_1_5B_XNNPACK_8DA4W: LLMModel = { const LLAMA3_2_BASE_URL = `${BASE_URL}-llama-3.2/${VERSION_TAG}`; const LLAMA3_2_3B_SPINQUANT: LLMModel = { - modelPath: `${LLAMA3_2_BASE_URL}/3b/xnnpack/llama_3_2_3b_xnnpack_spinquant.pte`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/llama_3_2_3b_xnnpack_spinquant_4k.pte`, tokenizerPath: `${LLAMA3_2_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${LLAMA3_2_BASE_URL}/tokenizer_config.json`, }; const LLAMA3_2_1B_SPINQUANT: LLMModel = { - modelPath: `${LLAMA3_2_BASE_URL}/1b/xnnpack/llama_3_2_1b_xnnpack_spinquant.pte`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/llama_3_2_1b_xnnpack_spinquant_16k.pte`, tokenizerPath: `${LLAMA3_2_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${LLAMA3_2_BASE_URL}/tokenizer_config.json`, }; const LLAMA3_2_1B_MLX_INT4: LLMModel = { - modelPath: `${LLAMA3_2_BASE_URL}/1b/mlx/llama_3_2_1b_mlx_int4.pte`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/llama_3_2_1b_mlx_4w_16k.pte`, tokenizerPath: `${LLAMA3_2_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${LLAMA3_2_BASE_URL}/tokenizer_config.json`, }; const LLAMA3_2_3B_MLX_INT4: LLMModel = { - modelPath: `${LLAMA3_2_BASE_URL}/3b/mlx/llama_3_2_3b_mlx_int4.pte`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/llama_3_2_3b_mlx_4w_4k.pte`, tokenizerPath: `${LLAMA3_2_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${LLAMA3_2_BASE_URL}/tokenizer_config.json`, }; @@ -2186,78 +2187,78 @@ const LLAMA3_2_3B_MLX_INT4: LLMModel = { const SMOLLM2_BASE_URL = `${BASE_URL}-smolLm-2/${VERSION_TAG}`; const SMOLLM2_135M_8DA8W: LLMModel = { - modelPath: `${SMOLLM2_BASE_URL}/135m/xnnpack/smollm2_135m_xnnpack_8da8w.pte`, - tokenizerPath: `${SMOLLM2_BASE_URL}/tokenizer.json`, - tokenizerConfigPath: `${SMOLLM2_BASE_URL}/tokenizer_config.json`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/smollm2_135m_xnnpack_8da8w_16k.pte`, + tokenizerPath: `${SMOLLM2_BASE_URL}/135m/tokenizer.json`, + tokenizerConfigPath: `${SMOLLM2_BASE_URL}/135m/tokenizer_config.json`, }; const SMOLLM2_360M_8DA8W: LLMModel = { - modelPath: `${SMOLLM2_BASE_URL}/360m/xnnpack/smollm2_360m_xnnpack_8da8w.pte`, - tokenizerPath: `${SMOLLM2_BASE_URL}/tokenizer.json`, - tokenizerConfigPath: `${SMOLLM2_BASE_URL}/tokenizer_config.json`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/smollm2_360m_xnnpack_8da8w_16k.pte`, + tokenizerPath: `${SMOLLM2_BASE_URL}/360m/tokenizer.json`, + tokenizerConfigPath: `${SMOLLM2_BASE_URL}/360m/tokenizer_config.json`, }; const SMOLLM2_1_7B_8DA8W: LLMModel = { - modelPath: `${SMOLLM2_BASE_URL}/1_7b/xnnpack/smollm2_1_7b_xnnpack_8da8w.pte`, - tokenizerPath: `${SMOLLM2_BASE_URL}/tokenizer.json`, - tokenizerConfigPath: `${SMOLLM2_BASE_URL}/tokenizer_config.json`, -}; -const SMOLLM2_135M_MLX_INT8: LLMModel = { - modelPath: `${SMOLLM2_BASE_URL}/135m/mlx/smollm2_135m_mlx_int8.pte`, - tokenizerPath: `${SMOLLM2_BASE_URL}/tokenizer.json`, - tokenizerConfigPath: `${SMOLLM2_BASE_URL}/tokenizer_config.json`, -}; -const SMOLLM2_360M_MLX_INT8: LLMModel = { - modelPath: `${SMOLLM2_BASE_URL}/360m/mlx/smollm2_360m_mlx_int8.pte`, - tokenizerPath: `${SMOLLM2_BASE_URL}/tokenizer.json`, - tokenizerConfigPath: `${SMOLLM2_BASE_URL}/tokenizer_config.json`, -}; -const SMOLLM2_1_7B_MLX_INT8: LLMModel = { - modelPath: `${SMOLLM2_BASE_URL}/1_7b/mlx/smollm2_1_7b_mlx_int8.pte`, - tokenizerPath: `${SMOLLM2_BASE_URL}/tokenizer.json`, - tokenizerConfigPath: `${SMOLLM2_BASE_URL}/tokenizer_config.json`, -}; + modelPath: `${SCRATCH_LLMS_BASE_URL}/smollm2_1_7b_xnnpack_8da8w_2k.pte`, + tokenizerPath: `${SMOLLM2_BASE_URL}/1_7b/tokenizer.json`, + tokenizerConfigPath: `${SMOLLM2_BASE_URL}/1_7b/tokenizer_config.json`, +}; +// const SMOLLM2_135M_MLX_INT8: LLMModel = { +// modelPath: `${SMOLLM2_BASE_URL}/135m/mlx/smollm2_135m_mlx_int8.pte`, +// tokenizerPath: `${SMOLLM2_BASE_URL}/135m/tokenizer.json`, +// tokenizerConfigPath: `${SMOLLM2_BASE_URL}/135m/tokenizer_config.json`, +// }; +// const SMOLLM2_360M_MLX_INT8: LLMModel = { +// modelPath: `${SMOLLM2_BASE_URL}/360m/mlx/smollm2_360m_mlx_int8.pte`, +// tokenizerPath: `${SMOLLM2_BASE_URL}/360m/tokenizer.json`, +// tokenizerConfigPath: `${SMOLLM2_BASE_URL}/360m/tokenizer_config.json`, +// }; +// const SMOLLM2_1_7B_MLX_INT8: LLMModel = { +// modelPath: `${SMOLLM2_BASE_URL}/1_7b/mlx/smollm2_1_7b_mlx_int8.pte`, +// tokenizerPath: `${SMOLLM2_BASE_URL}/1_7b/tokenizer.json`, +// tokenizerConfigPath: `${SMOLLM2_BASE_URL}/1_7b/tokenizer_config.json`, +// }; const HAMMER2_1_BASE_URL = `${BASE_URL}-hammer-2.1/${VERSION_TAG}`; const HAMMER2_1_0_5B_XNNPACK_8DA4W: LLMModel = { - modelPath: `${HAMMER2_1_BASE_URL}/0_5b/xnnpack/hammer_2_1_0_5b_xnnpack_8da4w.pte`, - tokenizerPath: `${HAMMER2_1_BASE_URL}/tokenizer.json`, - tokenizerConfigPath: `${HAMMER2_1_BASE_URL}/tokenizer_config.json`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/hammer_2_1_0_5b_xnnpack_8da4w_16k.pte`, + tokenizerPath: `${HAMMER2_1_BASE_URL}/0_5b/tokenizer.json`, + tokenizerConfigPath: `${HAMMER2_1_BASE_URL}/0_5b/tokenizer_config.json`, }; const HAMMER2_1_1_5B_XNNPACK_8DA4W: LLMModel = { - modelPath: `${HAMMER2_1_BASE_URL}/1_5b/xnnpack/hammer_2_1_1_5b_xnnpack_8da4w.pte`, - tokenizerPath: `${HAMMER2_1_BASE_URL}/tokenizer.json`, - tokenizerConfigPath: `${HAMMER2_1_BASE_URL}/tokenizer_config.json`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/hammer_2_1_1_5b_xnnpack_8da4w_16k.pte`, + tokenizerPath: `${HAMMER2_1_BASE_URL}/1_5b/tokenizer.json`, + tokenizerConfigPath: `${HAMMER2_1_BASE_URL}/1_5b/tokenizer_config.json`, }; const HAMMER2_1_3B_XNNPACK_8DA4W: LLMModel = { - modelPath: `${HAMMER2_1_BASE_URL}/3b/xnnpack/hammer_2_1_3b_xnnpack_8da4w.pte`, - tokenizerPath: `${HAMMER2_1_BASE_URL}/tokenizer.json`, - tokenizerConfigPath: `${HAMMER2_1_BASE_URL}/tokenizer_config.json`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/hammer_2_1_3b_xnnpack_8da4w_8k.pte`, + tokenizerPath: `${HAMMER2_1_BASE_URL}/3b/tokenizer.json`, + tokenizerConfigPath: `${HAMMER2_1_BASE_URL}/3b/tokenizer_config.json`, }; const HAMMER2_1_0_5B_MLX_INT4: LLMModel = { - modelPath: `${HAMMER2_1_BASE_URL}/0_5b/mlx/hammer_2_1_0_5b_mlx_int4.pte`, - tokenizerPath: `${HAMMER2_1_BASE_URL}/tokenizer.json`, - tokenizerConfigPath: `${HAMMER2_1_BASE_URL}/tokenizer_config.json`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/hammer_2_1_0_5b_mlx_4w_16k.pte`, + tokenizerPath: `${HAMMER2_1_BASE_URL}/0_5b/tokenizer.json`, + tokenizerConfigPath: `${HAMMER2_1_BASE_URL}/0_5b/tokenizer_config.json`, }; const HAMMER2_1_1_5B_MLX_INT4: LLMModel = { - modelPath: `${HAMMER2_1_BASE_URL}/1_5b/mlx/hammer_2_1_1_5b_mlx_int4.pte`, - tokenizerPath: `${HAMMER2_1_BASE_URL}/tokenizer.json`, - tokenizerConfigPath: `${HAMMER2_1_BASE_URL}/tokenizer_config.json`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/hammer_2_1_1_5b_mlx_4w_16k.pte`, + tokenizerPath: `${HAMMER2_1_BASE_URL}/1_5b/tokenizer.json`, + tokenizerConfigPath: `${HAMMER2_1_BASE_URL}/1_5b/tokenizer_config.json`, }; const HAMMER2_1_3B_MLX_INT4: LLMModel = { - modelPath: `${HAMMER2_1_BASE_URL}/3b/mlx/hammer_2_1_3b_mlx_int4.pte`, - tokenizerPath: `${HAMMER2_1_BASE_URL}/tokenizer.json`, - tokenizerConfigPath: `${HAMMER2_1_BASE_URL}/tokenizer_config.json`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/hammer_2_1_3b_mlx_4w_8k.pte`, + tokenizerPath: `${HAMMER2_1_BASE_URL}/3b/tokenizer.json`, + tokenizerConfigPath: `${HAMMER2_1_BASE_URL}/3b/tokenizer_config.json`, }; const PHI4_MINI_BASE_URL = `${BASE_URL}-phi-4-mini/${VERSION_TAG}`; const PHI4_MINI_XNNPACK_8DA4W: LLMModel = { - modelPath: `${PHI4_MINI_BASE_URL}/xnnpack/phi_4_mini_xnnpack_8da4w.pte`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/phi_4_mini_3_8b_xnnpack_8da4w_4k.pte`, tokenizerPath: `${PHI4_MINI_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${PHI4_MINI_BASE_URL}/tokenizer_config.json`, }; const PHI4_MINI_MLX_INT4: LLMModel = { - modelPath: `${PHI4_MINI_BASE_URL}/mlx/phi_4_mini_mlx_int4.pte`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/phi_4_mini_3_8b_mlx_4w_4k.pte`, tokenizerPath: `${PHI4_MINI_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${PHI4_MINI_BASE_URL}/tokenizer_config.json`, }; @@ -2265,85 +2266,85 @@ const PHI4_MINI_MLX_INT4: LLMModel = { const QWEN2_5_BASE_URL = `${BASE_URL}-qwen-2.5/${VERSION_TAG}`; const QWEN2_5_0_5B_XNNPACK_8DA4W: LLMModel = { - modelPath: `${QWEN2_5_BASE_URL}/0_5b/xnnpack/qwen_2_5_0_5b_xnnpack_8da4w.pte`, - tokenizerPath: `${QWEN2_5_BASE_URL}/tokenizer.json`, - tokenizerConfigPath: `${QWEN2_5_BASE_URL}/tokenizer_config.json`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/qwen_2_5_0_5b_xnnpack_8da4w_16k.pte`, + tokenizerPath: `${QWEN2_5_BASE_URL}/0_5b/tokenizer.json`, + tokenizerConfigPath: `${QWEN2_5_BASE_URL}/0_5b/tokenizer_config.json`, }; const QWEN2_5_1_5B_XNNPACK_8DA4W: LLMModel = { - modelPath: `${QWEN2_5_BASE_URL}/1_5b/xnnpack/qwen_2_5_1_5b_xnnpack_8da4w.pte`, - tokenizerPath: `${QWEN2_5_BASE_URL}/tokenizer.json`, - tokenizerConfigPath: `${QWEN2_5_BASE_URL}/tokenizer_config.json`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/qwen_2_5_1_5b_xnnpack_8da4w_16k.pte`, + tokenizerPath: `${QWEN2_5_BASE_URL}/1_5b/tokenizer.json`, + tokenizerConfigPath: `${QWEN2_5_BASE_URL}/1_5b/tokenizer_config.json`, }; const QWEN2_5_3B_XNNPACK_8DA4W: LLMModel = { - modelPath: `${QWEN2_5_BASE_URL}/3b/xnnpack/qwen_2_5_3b_xnnpack_8da4w.pte`, - tokenizerPath: `${QWEN2_5_BASE_URL}/tokenizer.json`, - tokenizerConfigPath: `${QWEN2_5_BASE_URL}/tokenizer_config.json`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/qwen_2_5_3b_xnnpack_8da4w_8k.pte`, + tokenizerPath: `${QWEN2_5_BASE_URL}/3b/tokenizer.json`, + tokenizerConfigPath: `${QWEN2_5_BASE_URL}/3b/tokenizer_config.json`, }; const QWEN2_5_0_5B_MLX_INT4: LLMModel = { - modelPath: `${QWEN2_5_BASE_URL}/0_5b/mlx/qwen_2_5_0_5b_mlx_int4.pte`, - tokenizerPath: `${QWEN2_5_BASE_URL}/tokenizer.json`, - tokenizerConfigPath: `${QWEN2_5_BASE_URL}/tokenizer_config.json`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/qwen_2_5_0_5b_mlx_4w_16k.pte`, + tokenizerPath: `${QWEN2_5_BASE_URL}/0_5b/tokenizer.json`, + tokenizerConfigPath: `${QWEN2_5_BASE_URL}/0_5b/tokenizer_config.json`, }; const QWEN2_5_1_5B_MLX_INT4: LLMModel = { - modelPath: `${QWEN2_5_BASE_URL}/1_5b/mlx/qwen_2_5_1_5b_mlx_int4.pte`, - tokenizerPath: `${QWEN2_5_BASE_URL}/tokenizer.json`, - tokenizerConfigPath: `${QWEN2_5_BASE_URL}/tokenizer_config.json`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/qwen_2_5_1_5b_mlx_4w_16k.pte`, + tokenizerPath: `${QWEN2_5_BASE_URL}/1_5b/tokenizer.json`, + tokenizerConfigPath: `${QWEN2_5_BASE_URL}/1_5b/tokenizer_config.json`, }; const QWEN2_5_3B_MLX_INT4: LLMModel = { - modelPath: `${QWEN2_5_BASE_URL}/3b/mlx/qwen_2_5_3b_mlx_int4.pte`, - tokenizerPath: `${QWEN2_5_BASE_URL}/tokenizer.json`, - tokenizerConfigPath: `${QWEN2_5_BASE_URL}/tokenizer_config.json`, -}; - -const GEMMA4_BASE_URL = `${BASE_URL}-gemma-4/${VERSION_TAG}`; - -const GEMMA4_E2B_XNNPACK_8DA4W: LLMModel = { - modelPath: `${GEMMA4_BASE_URL}/e2b/xnnpack/gemma_4_e2b_xnnpack_8da4w.pte`, - tokenizerPath: `${GEMMA4_BASE_URL}/e2b/tokenizer.json`, - tokenizerConfigPath: `${GEMMA4_BASE_URL}/e2b/tokenizer_config.json`, -}; -const GEMMA4_E2B_VULKAN_8DA4W: LLMModel = { - modelPath: `${GEMMA4_BASE_URL}/e2b/vulkan/gemma_4_e2b_vulkan_8da4w.pte`, - tokenizerPath: `${GEMMA4_BASE_URL}/e2b/tokenizer.json`, - tokenizerConfigPath: `${GEMMA4_BASE_URL}/e2b/tokenizer_config.json`, -}; -const GEMMA4_E2B_MLX_INT4: LLMModel = { - modelPath: `${GEMMA4_BASE_URL}/e2b/mlx/gemma4_e2b_mlx_int4.pte`, - tokenizerPath: `${GEMMA4_BASE_URL}/e2b/tokenizer.json`, - tokenizerConfigPath: `${GEMMA4_BASE_URL}/e2b/tokenizer_config.json`, -}; + modelPath: `${SCRATCH_LLMS_BASE_URL}/qwen_2_5_3b_mlx_4w_8k.pte`, + tokenizerPath: `${QWEN2_5_BASE_URL}/3b/tokenizer.json`, + tokenizerConfigPath: `${QWEN2_5_BASE_URL}/3b/tokenizer_config.json`, +}; + +// const GEMMA4_BASE_URL = `${BASE_URL}-gemma-4/${VERSION_TAG}`; + +// const GEMMA4_E2B_XNNPACK_8DA4W: LLMModel = { +// modelPath: `${GEMMA4_BASE_URL}/e2b/xnnpack/gemma_4_e2b_xnnpack_8da4w.pte`, +// tokenizerPath: `${GEMMA4_BASE_URL}/e2b/tokenizer.json`, +// tokenizerConfigPath: `${GEMMA4_BASE_URL}/e2b/tokenizer_config.json`, +// }; +// const GEMMA4_E2B_VULKAN_8DA4W: LLMModel = { +// modelPath: `${GEMMA4_BASE_URL}/e2b/vulkan/gemma_4_e2b_vulkan_8da4w.pte`, +// tokenizerPath: `${GEMMA4_BASE_URL}/e2b/tokenizer.json`, +// tokenizerConfigPath: `${GEMMA4_BASE_URL}/e2b/tokenizer_config.json`, +// }; +// const GEMMA4_E2B_MLX_INT4: LLMModel = { +// modelPath: `${GEMMA4_BASE_URL}/e2b/mlx/gemma4_e2b_mlx_int4.pte`, +// tokenizerPath: `${GEMMA4_BASE_URL}/e2b/tokenizer.json`, +// tokenizerConfigPath: `${GEMMA4_BASE_URL}/e2b/tokenizer_config.json`, +// }; const QWEN3_BASE_URL = `${BASE_URL}-qwen-3/${VERSION_TAG}`; const QWEN3_0_6B_XNNPACK_8DA4W: LLMModel = { - modelPath: `${QWEN3_BASE_URL}/0_6b/xnnpack/qwen_3_0_6b_xnnpack_8da4w.pte`, - tokenizerPath: `${QWEN3_BASE_URL}/tokenizer.json`, - tokenizerConfigPath: `${QWEN3_BASE_URL}/tokenizer_config.json`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/qwen3_0_6b_xnnpack_8da4w_8k.pte`, + tokenizerPath: `${QWEN3_BASE_URL}/0_6b/tokenizer.json`, + tokenizerConfigPath: `${QWEN3_BASE_URL}/0_6b/tokenizer_config.json`, }; const QWEN3_1_7B_XNNPACK_8DA4W: LLMModel = { - modelPath: `${QWEN3_BASE_URL}/1_7b/xnnpack/qwen_3_1_7b_xnnpack_8da4w.pte`, - tokenizerPath: `${QWEN3_BASE_URL}/tokenizer.json`, - tokenizerConfigPath: `${QWEN3_BASE_URL}/tokenizer_config.json`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/qwen3_1_7b_xnnpack_8da4w_4k.pte`, + tokenizerPath: `${QWEN3_BASE_URL}/1_7b/tokenizer.json`, + tokenizerConfigPath: `${QWEN3_BASE_URL}/1_7b/tokenizer_config.json`, }; const QWEN3_4B_XNNPACK_8DA4W: LLMModel = { - modelPath: `${QWEN3_BASE_URL}/4b/xnnpack/qwen_3_4b_xnnpack_8da4w.pte`, - tokenizerPath: `${QWEN3_BASE_URL}/tokenizer.json`, - tokenizerConfigPath: `${QWEN3_BASE_URL}/tokenizer_config.json`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/qwen3_4b_xnnpack_8da4w_4k.pte`, + tokenizerPath: `${QWEN3_BASE_URL}/4b/tokenizer.json`, + tokenizerConfigPath: `${QWEN3_BASE_URL}/4b/tokenizer_config.json`, }; const QWEN3_0_6B_MLX_INT4: LLMModel = { - modelPath: `${QWEN3_BASE_URL}/0_6b/mlx/qwen_3_0_6b_mlx_int4.pte`, - tokenizerPath: `${QWEN3_BASE_URL}/tokenizer.json`, - tokenizerConfigPath: `${QWEN3_BASE_URL}/tokenizer_config.json`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/qwen_3_0_6b_mlx_4w_8k.pte`, + tokenizerPath: `${QWEN3_BASE_URL}/0_6b/tokenizer.json`, + tokenizerConfigPath: `${QWEN3_BASE_URL}/0_6b/tokenizer_config.json`, }; const QWEN3_1_7B_MLX_INT4: LLMModel = { - modelPath: `${QWEN3_BASE_URL}/1_7b/mlx/qwen_3_1_7b_mlx_int4.pte`, - tokenizerPath: `${QWEN3_BASE_URL}/tokenizer.json`, - tokenizerConfigPath: `${QWEN3_BASE_URL}/tokenizer_config.json`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/qwen_3_1_7b_mlx_4w_4k.pte`, + tokenizerPath: `${QWEN3_BASE_URL}/1_7b/tokenizer.json`, + tokenizerConfigPath: `${QWEN3_BASE_URL}/1_7b/tokenizer_config.json`, }; const QWEN3_4B_MLX_INT4: LLMModel = { - modelPath: `${QWEN3_BASE_URL}/4b/mlx/qwen_3_4b_mlx_int4.pte`, - tokenizerPath: `${QWEN3_BASE_URL}/tokenizer.json`, - tokenizerConfigPath: `${QWEN3_BASE_URL}/tokenizer_config.json`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/qwen_3_4b_mlx_4w_4k.pte`, + tokenizerPath: `${QWEN3_BASE_URL}/4b/tokenizer.json`, + tokenizerConfigPath: `${QWEN3_BASE_URL}/4b/tokenizer_config.json`, }; /** @@ -3177,7 +3178,7 @@ export const models = { LFM2_5_VL_450M: variants({ XNNPACK_8DA4W: LFM2_5_VL_450M_XNNPACK_8DA4W, MLX_INT4: LFM2_5_VL_450M_MLX_INT4, - VULKAN_8DA4W: LFM2_5_VL_450M_VULKAN_8DA4W, + // VULKAN_8DA4W: LFM2_5_VL_450M_VULKAN_8DA4W, }), /** * Liquid AI LFM 2.5 VL 1.6B high-capacity vision-language model. Provides @@ -3187,9 +3188,9 @@ export const models = { */ LFM2_5_VL_1_6B: variants({ XNNPACK_8DA4W: LFM2_5_VL_1_6B_XNNPACK_8DA4W, - VULKAN_8DA4W: LFM2_5_VL_1_6B_VULKAN_8DA4W, + // VULKAN_8DA4W: LFM2_5_VL_1_6B_VULKAN_8DA4W, MLX_INT4: LFM2_5_VL_1_6B_MLX_INT4, - MLX_INT8: LFM2_5_VL_1_6B_MLX_INT8, + // MLX_INT8: LFM2_5_VL_1_6B_MLX_INT8, }), /** * Bielik v3 1.5B bilingual Polish & English language model, developed by @@ -3228,7 +3229,7 @@ export const models = { */ SMOLLM2_135M: variants({ XNNPACK_8DA8W: SMOLLM2_135M_8DA8W, - MLX_INT8: SMOLLM2_135M_MLX_INT8, + // MLX_INT8: SMOLLM2_135M_MLX_INT8, }), /** * Hugging Face SmolLM2 360M compact instruction-tuned model. Provides a @@ -3238,7 +3239,7 @@ export const models = { */ SMOLLM2_360M: variants({ XNNPACK_8DA8W: SMOLLM2_360M_8DA8W, - MLX_INT8: SMOLLM2_360M_MLX_INT8, + // MLX_INT8: SMOLLM2_360M_MLX_INT8, }), /** * Hugging Face SmolLM2 1.7B language model trained on curated educational, @@ -3248,7 +3249,7 @@ export const models = { */ SMOLLM2_1_7B: variants({ XNNPACK_8DA8W: SMOLLM2_1_7B_8DA8W, - MLX_INT8: SMOLLM2_1_7B_MLX_INT8, + // MLX_INT8: SMOLLM2_1_7B_MLX_INT8, }), /** * Hammer 2.1 0.5B specialized function-calling model. Fine-tuned @@ -3353,13 +3354,12 @@ export const models = { * following, creative text generation, and reasoning efficiency optimized * for mobile deployment. */ - GEMMA4_E2B: variants({ - XNNPACK_8DA4W: GEMMA4_E2B_XNNPACK_8DA4W, - VULKAN_8DA4W: GEMMA4_E2B_VULKAN_8DA4W, - MLX_INT4: GEMMA4_E2B_MLX_INT4, - }), + // GEMMA4_E2B: variants({ + // XNNPACK_8DA4W: GEMMA4_E2B_XNNPACK_8DA4W, + // VULKAN_8DA4W: GEMMA4_E2B_VULKAN_8DA4W, + // MLX_INT4: GEMMA4_E2B_MLX_INT4, + // }), }, - /** * Text embedding models mapping sentences and documents into dense vector * representations for semantic search and RAG. From 24ce0016ad65170fb3886ad3d05cdabb4a4e3a64 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Fri, 9 Oct 2026 04:26:40 +0200 Subject: [PATCH 34/83] test(llm): drive the chat session through the TypeScript LLM runner The LLM runner now lives in TypeScript on top of loadModel/loadTokenizer, so it no longer belongs in the fake JSI. Drop the scripted runner from fakeJsi and fake the layer below it instead: a scripted LLM .pte (support/fakeLLM.ts), an open-vocabulary fake tokenizer, and a greedy llm.sample op. Update the chat session suite and the API surface snapshot accordingly. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../__tests__/README.md | 11 +- .../api/__snapshots__/apiSurface.test.ts.snap | 11 +- .../__tests__/support/fakeJsi.ts | 211 ++---------------- .../__tests__/support/fakeLLM.ts | 164 ++++++++++++++ .../__tests__/support/fakeOps.ts | 40 +++- .../__tests__/support/setup.ts | 3 - .../tasks/constructionFailure.test.ts | 5 +- .../__tests__/tasks/llmChatSession.test.ts | 125 ++++++----- 8 files changed, 296 insertions(+), 274 deletions(-) create mode 100644 packages/react-native-executorch/__tests__/support/fakeLLM.ts diff --git a/packages/react-native-executorch/__tests__/README.md b/packages/react-native-executorch/__tests__/README.md index 1eda99b2eb..d79f5ab163 100644 --- a/packages/react-native-executorch/__tests__/README.md +++ b/packages/react-native-executorch/__tests__/README.md @@ -29,8 +29,9 @@ So `support/fakeJsi.ts` implements the native contract in JavaScript instead: | Piece | What it does | | --- | --- | | `support/fakeTensor.ts` | Typed-array-backed tensors with the real `setData`/`getData` byte semantics, `copyTo` windows, and use-after-dispose errors | -| `support/fakeOps.ts` | JS implementations of the `math`, `cv` and `speech` operators, plus the phonemizer host object | -| `support/fakeJsi.ts` | `createTensor`, `loadModel`, `loadTokenizer`, `createLLMRunner`, and the resource trackers | +| `support/fakeOps.ts` | JS implementations of the `math`, `cv` and `speech` operators, the phonemizer host object, and a greedy `llm.sample` | +| `support/fakeJsi.ts` | `createTensor`, `loadModel`, `loadTokenizer`, and the resource trackers | +| `support/fakeLLM.ts` | A scripted LLM `.pte` and its tokenizer, for the TypeScript LLM runner to drive | | `support/blobUtilMock.ts` | In-memory filesystem plus a programmable server (status, body, `Range` support, `stateChange`, and a gate to hold a download open) | | `support/workletsMock.ts` | Runs worklets inline — a worklet is an ordinary function marked for a second runtime | @@ -98,9 +99,9 @@ The line is drawn per pipeline rather than per suite, because it falls in a different place for each. Kokoro's waveform is weights, but its chunking, argument validation and streaming are not; the privacy filter's logits are weights, but the BIOES decode and the sliding window over them are pure -TypeScript and are driven end to end; the LLM's generation belongs to the -native runner, but the chat session's history, KV cache bookkeeping and -tool-calling loop are covered against a scripted one; PaddleOCR's probability +TypeScript and are driven end to end; the LLM's next-token logits are +weights, but the runner's prefill/decode loop and the chat session's history, +KV cache bookkeeping and tool-calling loop run over a scripted model; PaddleOCR's probability map is weights, but the quad decode, CTC collapse and reading order run over a map the test paints. diff --git a/packages/react-native-executorch/__tests__/api/__snapshots__/apiSurface.test.ts.snap b/packages/react-native-executorch/__tests__/api/__snapshots__/apiSurface.test.ts.snap index 45224ab750..7b39ced3d9 100644 --- a/packages/react-native-executorch/__tests__/api/__snapshots__/apiSurface.test.ts.snap +++ b/packages/react-native-executorch/__tests__/api/__snapshots__/apiSurface.test.ts.snap @@ -54,22 +54,13 @@ exports[`public API surface matches the recorded contents of each namespace 1`] "llm": [ "createChatPreprocessor", "createLLMRunner", - "createSampler", - "greedySelector", - "logitBiasProcessor", - "minPProcessor", - "multinomialSelector", "parseTokenizerConfig", - "repetitionPenaltyProcessor", - "temperatureProcessor", - "topKProcessor", - "topPProcessor", + "sample", ], "math": [ "argmax", "gather", "mulberry32", - "multinomial", "randomNormal", "repeatInterleave", "sigmoid", diff --git a/packages/react-native-executorch/__tests__/support/fakeJsi.ts b/packages/react-native-executorch/__tests__/support/fakeJsi.ts index 1b145c554e..1e68fe6d25 100644 --- a/packages/react-native-executorch/__tests__/support/fakeJsi.ts +++ b/packages/react-native-executorch/__tests__/support/fakeJsi.ts @@ -15,18 +15,22 @@ import type { ConcreteDim, ModelSpec } from '../../src/core/schema'; import type { DType } from '../../src/core/tensor'; import { FakeTensor, tensorTracker } from './fakeTensor'; -import { cv, fakePhonemizer, math, speech } from './fakeOps'; +import { cv, fakePhonemizer, llm, math, speech } from './fakeOps'; // ============================================================================ // Models // ============================================================================ -/** What `execute` does: reads the inputs and fills the pre-allocated outputs. */ +/** + * What `execute` does: reads the inputs and fills the pre-allocated outputs. + * A method with non-tensor outputs (an `Int`, a `Bool`) returns them instead, + * as the native `execute` does. + */ export type FakeExecute = ( methodName: string, inputs: readonly unknown[], outputs: readonly FakeTensor[] -) => void; +) => readonly unknown[] | void; /** A program `loadModel` can return. */ export type FakeProgram = { @@ -65,8 +69,8 @@ class FakeModel { throw new Error(`execute: method '${methodName}' is not exported by '${this.path}'`); } executions.push({ path: this.path, methodName }); - this.program.execute?.(methodName, inputs, outputTensors); - return outputTensors; + const values = this.program.execute?.(methodName, inputs, outputTensors); + return values ? [...values] : outputTensors; } dispose(): void { @@ -90,6 +94,12 @@ export type FakeVocabulary = { suffix?: readonly number[]; /** Ids skipped by `decode` when `skipSpecialTokens` is set. */ specialIds?: readonly number[]; + /** + * Appends unknown words to `tokens` on `encode` instead of mapping them to + * `0`, so any text round-trips through `encode`/`decode`. The `tokens` + * array is mutated in place, which lets a test resolve the ids it was given. + */ + open?: boolean; }; const vocabularies = new Map(); @@ -125,7 +135,11 @@ function createFakeTokenizer(path: string, vocabulary: FakeVocabulary) { encode: (text: string): Int32Array => { assertLive('encode'); const words = text.split(/\s+/).filter(Boolean); - const ids = words.map((word) => Math.max(0, vocabulary.tokens.indexOf(word))); + const ids = words.map((word) => { + const id = vocabulary.tokens.indexOf(word); + if (id !== -1 || !vocabulary.open) return Math.max(0, id); + return (vocabulary.tokens as string[]).push(word) - 1; + }); return Int32Array.from([...(vocabulary.prefix ?? []), ...ids, ...(vocabulary.suffix ?? [])]); }, @@ -166,159 +180,6 @@ function createFakeTokenizer(path: string, vocabulary: FakeVocabulary) { }; } -// ============================================================================ -// LLM runners -// ============================================================================ - -/** How a fake runner answers one `generate` call. */ -export type FakeGeneration = { - /** The text the runner produces, streamed token by token (words). */ - response: string; -}; - -/** A runner `createLLMRunner` can return. */ -export type FakeRunnerProgram = { - /** Context window the runner reports. Defaults to `128`. */ - maxSeqLen?: number; - /** - * Responses handed out in order, one per `generate` call. A runner that runs - * past the end of the list repeats the last entry, so a test only has to - * script the turns it cares about. - */ - generations?: readonly FakeGeneration[]; -}; - -/** One call made on a fake runner, in order, for a test to assert against. */ -export type RunnerCall = - | { kind: 'prefill'; text: string; pos: number } - | { kind: 'generate'; text: string; pos: number } - | { kind: 'reset'; targetPos: number } - | { kind: 'stop' }; - -const runnerPrograms = new Map(); -const liveRunners = new Set(); -const runnerCalls: RunnerCall[] = []; - -/** The text of a prompt, ignoring any media inputs a multimodal prompt carries. */ -const promptText = (prompt: unknown): string => - typeof prompt === 'string' - ? prompt - : Array.isArray(prompt) - ? prompt.filter((part) => typeof part === 'string').join('') - : ''; - -/** - * Builds a fake LLM runner. - * - * Its KV cache is modelled as a token count that prefill and generate both - * advance and `reset` rewinds, which is the only part of the native runner's - * state the chat session actually reasons about: it diffs the position across a - * turn to decide what still has to be prefilled, and rewinds to a recorded - * position between tool turns. - * @param modelPath The model path. - * @param tokenizerPath The tokenizer path. - * @param modalities The non-text modalities the runner accepts. - * @param program What the runner generates. - * @returns The fake runner. - */ -function createFakeRunner( - modelPath: string, - tokenizerPath: string, - modalities: readonly string[], - program: FakeRunnerProgram -) { - const maxSeqLen = program.maxSeqLen ?? 128; - let disposed = false; - let pos = 0; - let generationIndex = 0; - let stopped = false; - liveRunners.add(modelPath); - - const assertLive = (op: string): void => { - if (disposed) throw new Error(`${op}: runner '${modelPath}' has been disposed`); - }; - - // One token per whitespace-separated word, matching the fake tokenizer. - const countTokens = (text: string): number => text.split(/\s+/).filter(Boolean).length; - - return { - modelPath, - tokenizerPath, - modalities, - - prefill: (prompt: unknown): void => { - assertLive('prefill'); - const text = promptText(prompt); - pos = Math.min(maxSeqLen, pos + countTokens(text)); - runnerCalls.push({ kind: 'prefill', text, pos }); - }, - - generate: (prompt: unknown, _config?: unknown, onToken?: (token: string) => void) => { - assertLive('generate'); - stopped = false; - const text = promptText(prompt); - pos = Math.min(maxSeqLen, pos + countTokens(text)); - - const scripted = program.generations ?? []; - const generation = scripted[Math.min(generationIndex, scripted.length - 1)]; - generationIndex++; - const response = generation?.response ?? ''; - - let emitted = 0; - for (const token of response.split(/(?<=\s)/)) { - if (stopped) break; - onToken?.(token); - emitted++; - pos = Math.min(maxSeqLen, pos + 1); - } - runnerCalls.push({ kind: 'generate', text, pos }); - - return { - numPromptTokens: countTokens(text), - numGeneratedTokens: emitted, - firstTokenMs: 0, - inferenceStartMs: 0, - inferenceEndMs: 0, - modelLoadStartMs: 0, - modelLoadEndMs: 0, - }; - }, - - reset: (targetPos?: number): void => { - assertLive('reset'); - // The native runner only rewinds: a target past the current position is - // rejected rather than clamped. Modelling that here is what makes a - // rollback to a stale position observable in a test instead of silently - // succeeding. - if (targetPos !== undefined && (targetPos < 0 || targetPos > pos)) { - throw new Error( - `LLMRunner.reset: targetPos must be in range [0, ${pos}], but got ${targetPos}` - ); - } - pos = targetPos ?? 0; - runnerCalls.push({ kind: 'reset', targetPos: pos }); - }, - - stop: (): void => { - stopped = true; - runnerCalls.push({ kind: 'stop' }); - }, - - getKVCacheState: () => ({ - pos, - maxSeqLen, - remainingTokens: maxSeqLen - pos, - usageRatio: pos / maxSeqLen, - }), - - dispose: (): void => { - if (disposed) return; - disposed = true; - liveRunners.delete(modelPath); - }, - }; -} - // ============================================================================ // The global // ============================================================================ @@ -343,15 +204,7 @@ const jsi = { math, cv, speech, - llm: { - createLLMRunner: (modelPath: string, tokenizerPath: string, modalities: string[]) => { - const program = runnerPrograms.get(modelPath); - if (!program) { - throw new Error(`createLLMRunner: no runner registered at '${modelPath}'`); - } - return createFakeRunner(modelPath, tokenizerPath, modalities, program); - }, - }, + llm, nlp: { loadTokenizer: (path: string) => { const vocabulary = vocabularies.get(path); @@ -394,25 +247,6 @@ export const fakeJsi = { vocabularies.set(path, vocabulary); }, - /** - * Makes `createLLMRunner(path, ...)` succeed and return a scripted runner. - * @param path The model path a chat session will be pointed at. - * @param program The context window and the responses to generate. - */ - registerLLMRunner(path: string, program: FakeRunnerProgram = {}): void { - runnerPrograms.set(path, program); - }, - - /** @returns Every call made on a fake LLM runner so far, in order. */ - runnerCalls(): readonly RunnerCall[] { - return runnerCalls; - }, - - /** @returns Model paths of LLM runners that were created and not disposed. */ - liveRunners(): string[] { - return [...liveRunners].sort(); - }, - /** @returns Languages of phonemizers that were created and not disposed. */ livePhonemizers(): string[] { return fakePhonemizer.live(); @@ -463,12 +297,9 @@ export const fakeJsi = { reset(): void { programs.clear(); vocabularies.clear(); - runnerPrograms.clear(); liveModels.clear(); liveTokenizers.clear(); - liveRunners.clear(); executions.length = 0; - runnerCalls.length = 0; registeredBackends = ['XnnpackBackend', 'CoreMLBackend']; jsi.isEmulator = false; fakePhonemizer.reset(); diff --git a/packages/react-native-executorch/__tests__/support/fakeLLM.ts b/packages/react-native-executorch/__tests__/support/fakeLLM.ts new file mode 100644 index 0000000000..0ae2577ebf --- /dev/null +++ b/packages/react-native-executorch/__tests__/support/fakeLLM.ts @@ -0,0 +1,164 @@ +/** + * A scripted LLM `.pte` program for the fake runtime. + * + * The LLM runner is assembled in TypeScript on top of `loadModel`, + * `loadTokenizer` and the native sampler, so the fake sits below it: this + * registers a model exporting the text-runner methods and a tokenizer over an + * open vocabulary, and the real runner drives them. Nothing about the runner + * itself is faked. + * + * The model answers with scripted responses. Its `forward` emits one-hot + * logits for the next token of the current response, and then for the eos + * token once the response runs out. A `forward` that feeds back the token it + * just asked for continues the response; any other input is a new prompt, and + * a new prompt after a response has been (at least partly) emitted moves on to + * the next scripted one — so prefilling the user message and the generation + * prompt of the same turn does not skip a response. + */ +import type { ConcreteDim, ModelSpec } from '../../src/core/schema'; +import { fakeJsi } from './fakeJsi'; +import type { FakeTensor } from './fakeTensor'; + +/** How the model answers one generation. */ +export type FakeGeneration = { + /** The text produced, one token per whitespace-terminated word. */ + response: string; +}; + +export type FakeLLMProgram = { + /** Largest chunk a single `forward` accepts. Defaults to `128`. */ + maxSeqLen?: number; + /** KV cache capacity. Defaults to `1024`. */ + maxContextLen?: number; + /** The token the model ends every response with. */ + eosToken: string; + /** + * Responses handed out in order, one per generation. A model that runs past + * the end of the list repeats the last entry, so a test only has to script + * the turns it cares about. + */ + generations?: readonly FakeGeneration[]; +}; + +/** A prompt the model read: the text of one prefill, and where it started. */ +export type FakePromptRead = { text: string; startPos: number }; + +const VOCAB_SIZE = 4096; + +/** + * Registers a scripted LLM at `modelPath` and its tokenizer at `tokenizerPath`. + * @param modelPath The `.pte` path the runner will load. + * @param tokenizerPath The tokenizer path the runner will load. + * @param program The context window and the responses to generate. + * @returns Accessors for what the model was asked to read. + */ +export function registerFakeLLM(modelPath: string, tokenizerPath: string, program: FakeLLMProgram) { + const { maxSeqLen = 128, maxContextLen = 1024, eosToken, generations = [] } = program; + + const tokens: string[] = []; + const idOf = (token: string): number => { + const id = tokens.indexOf(token); + return id !== -1 ? id : tokens.push(token) - 1; + }; + const eosId = idOf(eosToken); + // `split` on a lookbehind keeps the whitespace on the token it follows, so + // the decoded tokens concatenate back to the response verbatim. + const responses = generations.map(({ response }) => [ + ...response + .split(/(?<=\s)/) + .filter(Boolean) + .map(idOf), + eosId, + ]); + + let generation = 0; + let step = 0; + let emitted = false; + const prompts: FakePromptRead[] = []; + + const current = (): readonly number[] => + responses[Math.min(generation, responses.length - 1)] ?? [eosId]; + + const emit = (logits: FakeTensor, id: number): void => { + for (let i = 0; i < logits.numel; i++) logits.setElement(i, i === id ? 1 : 0); + }; + + // The method names are the exported `.pte` method names. + /* eslint-disable camelcase */ + const schema: ModelSpec = { + forward: { + inputs: [ + { + kind: 'Tensor', + dtype: 'int64', + shape: [ + { kind: 'constant', value: 1 }, + { kind: 'range', range: { min: 1, max: maxSeqLen, step: 1 } }, + ], + }, + { kind: 'Tensor', dtype: 'int64', shape: [{ kind: 'constant', value: 1 }] }, + ], + outputs: [ + { + kind: 'Tensor', + dtype: 'float32', + shape: [ + { kind: 'constant', value: 1 }, + { kind: 'constant', value: VOCAB_SIZE }, + ], + }, + ], + runtimeConstraints: [], + }, + get_max_seq_len: { inputs: [], outputs: [{ kind: 'Int' }], runtimeConstraints: [] }, + get_max_context_len: { inputs: [], outputs: [{ kind: 'Int' }], runtimeConstraints: [] }, + get_vocab_size: { inputs: [], outputs: [{ kind: 'Int' }], runtimeConstraints: [] }, + use_kv_cache: { inputs: [], outputs: [{ kind: 'Bool' }], runtimeConstraints: [] }, + enable_dynamic_shape: { inputs: [], outputs: [{ kind: 'Bool' }], runtimeConstraints: [] }, + get_eos_ids: { inputs: [], outputs: [{ kind: 'Int' }], runtimeConstraints: [] }, + } as ModelSpec; + /* eslint-enable camelcase */ + + fakeJsi.registerModel(modelPath, { + schema, + execute: (methodName, inputs, outputs) => { + switch (methodName) { + case 'get_max_seq_len': + return [maxSeqLen]; + case 'get_max_context_len': + return [maxContextLen]; + case 'get_vocab_size': + return [VOCAB_SIZE]; + case 'use_kv_cache': + case 'enable_dynamic_shape': + return [true]; + case 'get_eos_ids': + return [eosId]; + } + + const [tTokens, tPos] = inputs as [FakeTensor, FakeTensor]; + const ids = Array.from({ length: tTokens.numel }, (_, i) => tTokens.getElement(i)); + const response = current(); + + if (ids.length === 1 && ids[0] === response[step]) { + emitted = true; + step++; + } else { + if (emitted) generation++; + emitted = false; + step = 0; + prompts.push({ text: ids.map((id) => tokens[id]).join(' '), startPos: tPos.getElement(0) }); + } + + emit(outputs[0]!, current()[step] ?? eosId); + return undefined; + }, + }); + + fakeJsi.registerTokenizer(tokenizerPath, { tokens, open: true }); + + return { + /** @returns Every prompt the model read, prefill by prefill, in order. */ + prompts: (): readonly FakePromptRead[] => prompts, + }; +} diff --git a/packages/react-native-executorch/__tests__/support/fakeOps.ts b/packages/react-native-executorch/__tests__/support/fakeOps.ts index 896cc58b34..85ac5f8481 100644 --- a/packages/react-native-executorch/__tests__/support/fakeOps.ts +++ b/packages/react-native-executorch/__tests__/support/fakeOps.ts @@ -1,6 +1,6 @@ /** * JavaScript implementations of the native operations exposed under - * `__rnexecutorch_jsi__.{math,cv,speech}`. + * `__rnexecutorch_jsi__.{math,cv,speech,llm}`. * * These follow the contracts documented on the TypeScript wrappers in * `src/extensions/`, so a pipeline that composes them produces the values a @@ -700,3 +700,41 @@ export const speech = { return dst; }, }; + +// ============================================================================ +// llm +// ============================================================================ + +type FakeSamplingConstraints = { allowedTokens?: Int32Array; bannedTokens?: Int32Array }; + +export const llm = { + /** + * Greedy stand-in for the native sampler: applies the `constraints` callback + * the way the native chain does (first, on the raw logits) and then takes + * the argmax. Every other option shapes a distribution the fake has no use + * for, since a scripted model emits one-hot logits anyway. + */ + sample( + logits: FakeTensor, + ctx: { generatedTokens?: Int32Array }, + options: { constraints?: (ctx: unknown) => FakeSamplingConstraints | undefined } + ): number { + expectShape(logits, [1, logits.numel], 'sample: logits'); + const { allowedTokens, bannedTokens } = options.constraints?.(ctx) ?? {}; + const allowed = allowedTokens && new Set(allowedTokens); + const banned = new Set(bannedTokens ?? []); + + let best = -1; + let bestValue = -Infinity; + for (let i = 0; i < logits.numel; i++) { + if ((allowed && !allowed.has(i)) || banned.has(i)) continue; + const value = logits.getElement(i); + if (best === -1 || value > bestValue) { + best = i; + bestValue = value; + } + } + if (best === -1) throw new Error('sample: constraints left no token to choose'); + return best; + }, +}; diff --git a/packages/react-native-executorch/__tests__/support/setup.ts b/packages/react-native-executorch/__tests__/support/setup.ts index 7782be5208..fcbfb97356 100644 --- a/packages/react-native-executorch/__tests__/support/setup.ts +++ b/packages/react-native-executorch/__tests__/support/setup.ts @@ -47,13 +47,11 @@ afterEach(async () => { const tensors = fakeJsi.liveTensorDescriptions(); const models = fakeJsi.liveModels(); const tokenizers = fakeJsi.liveTokenizers(); - const runners = fakeJsi.liveRunners(); const phonemizers = fakeJsi.livePhonemizers(); if ( tensors.length === 0 && models.length === 0 && tokenizers.length === 0 && - runners.length === 0 && phonemizers.length === 0 ) { return; @@ -65,7 +63,6 @@ afterEach(async () => { tensors.length > 0 ? ` tensors: ${tensors.join(', ')}` : '', models.length > 0 ? ` models: ${models.join(', ')}` : '', tokenizers.length > 0 ? ` tokenizers: ${tokenizers.join(', ')}` : '', - runners.length > 0 ? ` LLM runners: ${runners.join(', ')}` : '', phonemizers.length > 0 ? ` phonemizers: ${phonemizers.join(', ')}` : '', 'Dispose the pipeline, or call allowNativeLeaks() if the leak is the point of the test.', ] diff --git a/packages/react-native-executorch/__tests__/tasks/constructionFailure.test.ts b/packages/react-native-executorch/__tests__/tasks/constructionFailure.test.ts index f4520e350a..f5a7f2dbfb 100644 --- a/packages/react-native-executorch/__tests__/tasks/constructionFailure.test.ts +++ b/packages/react-native-executorch/__tests__/tasks/constructionFailure.test.ts @@ -2,7 +2,7 @@ * What a `create` factory leaves behind when it throws. * * Every factory follows the same shape: load a model (and, depending on the - * task, a tokenizer, a phonemizer, an LLM runner or a whole nested pipeline), + * task, a tokenizer, a phonemizer or a whole nested pipeline), * validate the schema, pre-allocate the execution tensors, and only then hand * back a `dispose`. A failure anywhere after the first allocation means the * caller never receives that `dispose`, so whatever was already allocated is @@ -146,12 +146,11 @@ const factories: Record Promise> = { const stillAllocated = () => ({ models: fakeJsi.liveModels(), tokenizers: fakeJsi.liveTokenizers(), - runners: fakeJsi.liveRunners(), phonemizers: fakeJsi.livePhonemizers(), tensors: fakeJsi.liveTensorDescriptions(), }); -const NOTHING = { models: [], tokenizers: [], runners: [], phonemizers: [], tensors: [] }; +const NOTHING = { models: [], tokenizers: [], phonemizers: [], tensors: [] }; describe('create — schema validation failure', () => { beforeEach(() => { diff --git a/packages/react-native-executorch/__tests__/tasks/llmChatSession.test.ts b/packages/react-native-executorch/__tests__/tasks/llmChatSession.test.ts index 34b7b74cb2..4cad2a0c75 100644 --- a/packages/react-native-executorch/__tests__/tasks/llmChatSession.test.ts +++ b/packages/react-native-executorch/__tests__/tasks/llmChatSession.test.ts @@ -1,22 +1,22 @@ /** * The LLM chat session. * - * The generation itself belongs to the native runner, so what this suite owns - * is everything the session does around it: the history it keeps, the prompt it - * renders through the model's own chat template, the KV cache bookkeeping that - * lets a turn prefill only what is new, the tool-calling loop, and the rollback - * that has to leave the session usable after a failed turn. + * What this suite owns is everything the session does around generation: the + * history it keeps, the prompt it renders through the model's own chat + * template, the KV cache bookkeeping that lets a turn prefill only what is new, + * the tool-calling loop, and the rollback that has to leave the session usable + * after a failed turn. * - * The fake runner models its KV cache as a token position that prefill and - * generate advance and `reset` rewinds — which is the only part of the native - * state the session reasons about — and hands out scripted responses, so a test - * can drive a tool loop without any weights. + * The session runs on the real TypeScript LLM runner; only the `.pte` below it + * is scripted (see `support/fakeLLM.ts`), so a test can drive a tool loop + * without any weights and observe exactly what the model was asked to read. */ import { createLLMChatSession } from '../../src/extensions/llm/tasks/llmChatSession'; import type { ToolCall, ToolParserResult } from '../../src/extensions/llm/utils/toolCalling'; import { fakeJsi } from '../support/fakeJsi'; import { fakeFs } from '../support/blobUtilMock'; import { tracked } from '../support/lifetime'; +import { registerFakeLLM, type FakeGeneration } from '../support/fakeLLM'; const MODEL_PATH = '/models/llm.pte'; const TOKENIZER_PATH = '/models/tokenizer.json'; @@ -63,16 +63,19 @@ const NAMED_TEMPLATES = { }; /* eslint-enable camelcase */ -/** Everything the session sent to the runner this test, prefill and generate. */ -const promptsSent = (): string[] => - fakeJsi - .runnerCalls() - .filter((call) => call.kind === 'prefill' || call.kind === 'generate') - .map((call) => (call as { text: string }).text); +let llm: ReturnType; + +/** Registers the model, answering with `generations` in order. */ +const script = (generations: readonly FakeGeneration[]) => { + llm = registerFakeLLM(MODEL_PATH, TOKENIZER_PATH, { eosToken: EOS, generations }); +}; + +/** Everything the model was asked to read this test, prefill by prefill. */ +const promptsSent = (): string[] => llm.prompts().map((prompt) => prompt.text); beforeEach(() => { writeTokenizerConfig(); - fakeJsi.registerLLMRunner(MODEL_PATH, { generations: [{ response: 'hello there ' }] }); + script([{ response: 'hello there ' }]); }); describe('createLLMChatSession — construction', () => { @@ -112,16 +115,17 @@ describe('createLLMChatSession — construction', () => { ); expect(session.getHistory()).toEqual([{ role: 'system', content: 'be brief' }]); - expect(fakeJsi.runnerCalls().map((call) => call.kind)).toEqual(['prefill']); + expect(promptsSent()).toEqual(['<|system|>be brief<|end|>']); expect(session.getKVCacheState().pos).toBeGreaterThan(0); }); - it('releases the runner on dispose', async () => { + it('releases the model and the tokenizer on dispose', async () => { const session = await createLLMChatSession(config); session.dispose(); - expect(fakeJsi.liveRunners()).toEqual([]); + expect(fakeJsi.liveModels()).toEqual([]); + expect(fakeJsi.liveTokenizers()).toEqual([]); }); }); @@ -152,7 +156,7 @@ describe('createLLMChatSession — a turn', () => { }); it('streams every token to the callback', async () => { - fakeJsi.registerLLMRunner(MODEL_PATH, { generations: [{ response: 'one two three' }] }); + script([{ response: 'one two three' }]); const session = tracked(await createLLMChatSession(config)); const tokens: string[] = []; @@ -164,7 +168,7 @@ describe('createLLMChatSession — a turn', () => { }); it('keeps the eos token out of the response and out of the stream', async () => { - fakeJsi.registerLLMRunner(MODEL_PATH, { generations: [{ response: `done ${EOS}` }] }); + script([{ response: `done ${EOS}` }]); const session = tracked(await createLLMChatSession(config)); const tokens: string[] = []; @@ -179,7 +183,7 @@ describe('createLLMChatSession — a turn', () => { // its config only names as `pad_token`, so `eos_token` alone does not cover it. it('keeps a terminal token named only as pad_token out of the response', async () => { writeTokenizerConfig({ pad_token: PAD }); // eslint-disable-line camelcase - fakeJsi.registerLLMRunner(MODEL_PATH, { generations: [{ response: `done ${PAD}` }] }); + script([{ response: `done ${PAD}` }]); const session = tracked(await createLLMChatSession(config)); const tokens: string[] = []; @@ -192,7 +196,7 @@ describe('createLLMChatSession — a turn', () => { it('keeps a terminal token named only as eot_token out of the response', async () => { writeTokenizerConfig({ eot_token: EOT }); // eslint-disable-line camelcase - fakeJsi.registerLLMRunner(MODEL_PATH, { generations: [{ response: `done ${EOT}` }] }); + script([{ response: `done ${EOT}` }]); const session = tracked(await createLLMChatSession(config)); const result = await session.sendMessage('hi'); @@ -201,7 +205,7 @@ describe('createLLMChatSession — a turn', () => { }); it('stops generating as soon as the stop pattern matches', async () => { - fakeJsi.registerLLMRunner(MODEL_PATH, { generations: [{ response: 'keep going STOP more' }] }); + script([{ response: 'keep going STOP more' }]); const session = tracked(await createLLMChatSession(config, { stopRegex: /STOP/ })); const result = await session.sendMessage('hi'); @@ -217,9 +221,14 @@ describe('createLLMChatSession — a turn', () => { expect(rest).toEqual([]); expect(stats).toMatchObject({ - numGeneratedTokens: expect.any(Number), - numPromptTokens: expect.any(Number), - prefillDurationMs: expect.any(Number), + numTokens: 2, + durationMs: expect.any(Number), + tokensPerSecond: expect.any(Number), + prefill: { + numTokens: expect.any(Number), + durationMs: expect.any(Number), + tokensPerSecond: expect.any(Number), + }, }); }); @@ -227,14 +236,10 @@ describe('createLLMChatSession — a turn', () => { const session = tracked(await createLLMChatSession(config)); await session.sendMessage('first question'); - const before = fakeJsi.runnerCalls().length; + const before = promptsSent().length; await session.sendMessage('second question'); - const secondTurn = fakeJsi - .runnerCalls() - .slice(before) - .filter((call) => call.kind === 'prefill') - .map((call) => (call as { text: string }).text); + const secondTurn = promptsSent().slice(before); // The first turn is already in the KV cache, so it must not be re-sent. expect(secondTurn.join('')).toContain('second question'); @@ -245,25 +250,28 @@ describe('createLLMChatSession — a turn', () => { const session = tracked(await createLLMChatSession(config, { resetOnTurn: true })); await session.sendMessage('first question'); - const before = fakeJsi.runnerCalls().length; + const before = llm.prompts().length; await session.sendMessage('second question'); - const secondTurn = fakeJsi.runnerCalls().slice(before); - expect(secondTurn[0]).toEqual({ kind: 'reset', targetPos: 0 }); - expect( - secondTurn - .filter((call) => call.kind === 'prefill') - .map((call) => (call as { text: string }).text) - .join('') - ).toContain('first question'); + const secondTurn = llm.prompts().slice(before); + expect(secondTurn[0]!.startPos).toBe(0); + expect(secondTurn.map((prompt) => prompt.text).join('')).toContain('first question'); }); - it('forwards stop() to the runner', async () => { + it('stops the generation in flight on stop()', async () => { + script([{ response: 'one two three' }]); const session = tracked(await createLLMChatSession(config)); - session.stop(); + // The constraints callback runs on every sampling step, which makes it a + // hook into the middle of a generation. + const result = await session.sendMessage('hi', undefined, { + constraints: () => { + session.stop(); + return undefined; + }, + }); - expect(fakeJsi.runnerCalls()).toContainEqual({ kind: 'stop' }); + expect(result.messages.at(-1)!.content).toBe('one '); }); }); @@ -290,9 +298,7 @@ describe('createLLMChatSession — tool calling', () => { beforeEach(() => weather.execute.mockClear()); it('runs the tool and feeds its result back for a second generation', async () => { - fakeJsi.registerLLMRunner(MODEL_PATH, { - generations: [{ response: 'TOOL weather' }, { response: 'it is sunny' }], - }); + script([{ response: 'TOOL weather' }, { response: 'it is sunny' }]); const session = tracked( await createLLMChatSession(config, { toolOpts: { tools: [weather], parseToolCalls } }) ); @@ -313,9 +319,7 @@ describe('createLLMChatSession — tool calling', () => { }); it('records the tool result against the call that asked for it', async () => { - fakeJsi.registerLLMRunner(MODEL_PATH, { - generations: [{ response: 'TOOL weather' }, { response: 'it is sunny' }], - }); + script([{ response: 'TOOL weather' }, { response: 'it is sunny' }]); const session = tracked( await createLLMChatSession(config, { toolOpts: { tools: [weather], parseToolCalls } }) ); @@ -331,9 +335,7 @@ describe('createLLMChatSession — tool calling', () => { }); it('reports an unknown tool back to the model rather than throwing', async () => { - fakeJsi.registerLLMRunner(MODEL_PATH, { - generations: [{ response: 'TOOL missing' }, { response: 'sorry' }], - }); + script([{ response: 'TOOL missing' }, { response: 'sorry' }]); const session = tracked( await createLLMChatSession(config, { toolOpts: { tools: [weather], parseToolCalls } }) ); @@ -344,9 +346,7 @@ describe('createLLMChatSession — tool calling', () => { }); it('reports a throwing tool back to the model rather than failing the turn', async () => { - fakeJsi.registerLLMRunner(MODEL_PATH, { - generations: [{ response: 'TOOL weather' }, { response: 'sorry' }], - }); + script([{ response: 'TOOL weather' }, { response: 'sorry' }]); weather.execute.mockRejectedValueOnce(new Error('the service is down')); const session = tracked( await createLLMChatSession(config, { toolOpts: { tools: [weather], parseToolCalls } }) @@ -360,7 +360,7 @@ describe('createLLMChatSession — tool calling', () => { it('gives up after maxToolTurns rather than looping forever', async () => { // A model that only ever asks for the tool again. - fakeJsi.registerLLMRunner(MODEL_PATH, { generations: [{ response: 'TOOL weather' }] }); + script([{ response: 'TOOL weather' }]); const session = tracked( await createLLMChatSession(config, { toolOpts: { tools: [weather], parseToolCalls, maxToolTurns: 3 }, @@ -462,12 +462,12 @@ describe('createLLMChatSession — failure', () => { expect(result.finishReason).toBe('stop'); }); - it('surfaces a missing runner rather than resolving with a broken session', async () => { + it('surfaces a missing model rather than resolving with a broken session', async () => { fakeJsi.reset(); writeTokenizerConfig(); - await expect(createLLMChatSession(config)).rejects.toThrow(/no runner registered/); - expect(fakeJsi.liveRunners()).toEqual([]); + await expect(createLLMChatSession(config)).rejects.toThrow(/no program registered/); + expect(fakeJsi.liveModels()).toEqual([]); }); it('releases the runner when the initial prefill fails', async () => { @@ -481,6 +481,7 @@ describe('createLLMChatSession — failure', () => { createLLMChatSession(config, { initialMessages: [{ role: 'user', content: 'hi' }] }) ).rejects.toThrow(); - expect(fakeJsi.liveRunners()).toEqual([]); + expect(fakeJsi.liveModels()).toEqual([]); + expect(fakeJsi.liveTokenizers()).toEqual([]); }); }); From 65a31a4d42af1984202c99ffd6646c44688ce968 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Fri, 9 Oct 2026 04:27:33 +0200 Subject: [PATCH 35/83] feat(nlp-app): add a backend picker to the LLM screen Filter the model list by the selected backend (XNNPACK everywhere, MLX on iOS) and reset the chat when the backend changes. Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/nlp/app/llm/index.tsx | 62 ++++++++++++++++++++++++++++++++++---- 1 file changed, 56 insertions(+), 6 deletions(-) diff --git a/apps/nlp/app/llm/index.tsx b/apps/nlp/app/llm/index.tsx index 6f579ee6e1..b03dd29f84 100644 --- a/apps/nlp/app/llm/index.tsx +++ b/apps/nlp/app/llm/index.tsx @@ -20,7 +20,7 @@ import ScreenWrapper from '../../components/ScreenWrapper'; import { ModelPicker, type ModelOption } from '../../components/ModelPicker'; import { Button } from '../../components/Button'; import { getImage, skImageToBuffer } from '../../utils'; -import { LLM_MODELS, type LLMModelConfig } from '../../constants/llm'; +import { LLM_MODELS, type LLMModelConfig, type LLMBackend } from '../../constants/llm'; type Turn = { role: 'user' | 'assistant' | 'tool'; @@ -42,21 +42,47 @@ function formatStats(stats: llm.LLMGenerationStats): string { function LLMContent() { const insets = useSafeAreaInsets(); - const [selectedModelId, setSelectedModelId] = useState(LLM_MODELS[0]!.id); + const [selectedBackend, setSelectedBackend] = useState('xnnpack'); + + const availableBackends: ModelOption[] = useMemo(() => { + const list: { label: string; value: LLMBackend }[] = [{ label: 'XNNPACK', value: 'xnnpack' }]; + if (Platform.OS === 'ios') { + list.push({ label: 'MLX (Apple GPU)', value: 'mlx' }); + } + // if (Platform.OS === 'android') { + // list.push({ label: 'Vulkan', value: 'vulkan' }); + // } + return list; + }, []); + + const modelsForBackend = useMemo( + () => + LLM_MODELS.filter( + (m) => m.backend === selectedBackend && (!m.iosOnly || Platform.OS === 'ios') + ), + [selectedBackend] + ); + + const [selectedModelId, setSelectedModelId] = useState( + () => modelsForBackend[0]?.id ?? LLM_MODELS[0]!.id + ); const [isDownloadStarted, setIsDownloadStarted] = useState(false); const activeModel: LLMModelConfig = useMemo( - () => LLM_MODELS.find((m) => m.id === selectedModelId) ?? LLM_MODELS[0]!, - [selectedModelId] + () => + modelsForBackend.find((m) => m.id === selectedModelId) ?? + modelsForBackend[0] ?? + LLM_MODELS[0]!, + [modelsForBackend, selectedModelId] ); const modelOptions: ModelOption[] = useMemo( () => - LLM_MODELS.filter((m) => !m.iosOnly || Platform.OS === 'ios').map((m) => ({ + modelsForBackend.map((m) => ({ label: m.name, value: m.id, })), - [] + [modelsForBackend] ); const initialMessages: llm.ChatMessage[] = useMemo( @@ -97,6 +123,22 @@ function LLMContent() { const supportsImages = Boolean(activeModel.model.modalities?.includes('image')); + const handleBackendChange = (newBackend: LLMBackend) => { + if (newBackend === selectedBackend) return; + setSelectedBackend(newBackend); + const firstForBackend = LLM_MODELS.find( + (m) => m.backend === newBackend && (!m.iosOnly || Platform.OS === 'ios') + ); + if (firstForBackend) { + setSelectedModelId(firstForBackend.id); + } + setIsDownloadStarted(false); + setTurns([]); + setStreamingResponse(null); + setAttachedImage(null); + setInput(''); + }; + const handleModelChange = (newModelId: string) => { if (newModelId === selectedModelId) return; setSelectedModelId(newModelId); @@ -232,6 +274,14 @@ function LLMContent() { keyboardVerticalOffset={Platform.OS === 'ios' ? 90 : 0} > + {availableBackends.length > 1 && ( + + )} Date: Fri, 9 Oct 2026 04:27:44 +0200 Subject: [PATCH 36/83] chore(core): attribute the input shape-check TODO in model.cpp Co-Authored-By: Claude Opus 5.5 (1M context) --- packages/react-native-executorch/cpp/core/model.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/react-native-executorch/cpp/core/model.cpp b/packages/react-native-executorch/cpp/core/model.cpp index 2066730250..4e113c9d7f 100644 --- a/packages/react-native-executorch/cpp/core/model.cpp +++ b/packages/react-native-executorch/cpp/core/model.cpp @@ -168,7 +168,7 @@ jsi::Value ModelHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) { switch (tag) { case executorch::runtime::Tag::Tensor: { const auto &tSpec = methodSpec.inputs[i]; - auto tensorHostObject = tensor::fromJs(rt, ctx, val, tSpec.dtype, std::nullopt); // TODO: tSpec.shape); + auto tensorHostObject = tensor::fromJs(rt, ctx, val, tSpec.dtype, std::nullopt); // TODO(bh): tSpec.shape); if (!lockedTensors.insert(tensorHostObject.get()).second) { throw error::InvalidArgument("execute: Tensor aliasing detected. " From a41def8367894859f69049a2c709ad97a08fc155 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Fri, 9 Oct 2026 04:36:51 +0200 Subject: [PATCH 37/83] update --- .../src/extensions/llm/llmRunner.ts | 8 +---- .../llm/runners/llmMultimodalRunner.ts | 32 +++---------------- 2 files changed, 6 insertions(+), 34 deletions(-) diff --git a/packages/react-native-executorch/src/extensions/llm/llmRunner.ts b/packages/react-native-executorch/src/extensions/llm/llmRunner.ts index 00821c6187..28c649a3c2 100644 --- a/packages/react-native-executorch/src/extensions/llm/llmRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/llmRunner.ts @@ -15,11 +15,7 @@ import { loadTokenizer } from '../nlp'; import type { SamplingConfig } from './sampler'; import { createLLMTextRunner, LLM_TEXT_SPEC } from './runners/llmTextRunner'; -import { - createLLMMultimodalRunner, - LLM_MULTIMODAL_DYN_SPEC, - LLM_MULTIMODAL_SPEC, -} from './runners/llmMultimodalRunner'; +import { createLLMMultimodalRunner, LLM_MULTIMODAL_SPEC } from './runners/llmMultimodalRunner'; declare const llmRunnerBrand: unique symbol; @@ -195,14 +191,12 @@ export async function createLLMRunner( const { variant } = validateSpec(model.schema, { text: LLM_TEXT_SPEC, vision: LLM_MULTIMODAL_SPEC, - visionDyn: LLM_MULTIMODAL_DYN_SPEC, }); switch (variant) { case 'text': return createLLMTextRunner(model, tokenizer, modalities); case 'vision': - case 'visionDyn': return createLLMMultimodalRunner(model, tokenizer, modalities); default: throw RnExecuTorchError('LOAD_FAILED', `llmRunner: unrecognized variant ${variant}`); diff --git a/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts b/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts index d67f7f2936..1631daa675 100644 --- a/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts @@ -28,7 +28,7 @@ const METADATA_SPEC = { }; // REPLACE WITH THIS AFTER REEXPORT -export const LLM_MULTIMODAL_DYN_SPEC = { +export const LLM_MULTIMODAL_SPEC = { ...method( 'text_decoder', [f32(1, Dyn('seqLen'), 'hiddenDim'), i64(Dyn('seqLen'))], @@ -47,25 +47,6 @@ export const LLM_MULTIMODAL_DYN_SPEC = { ...METADATA_SPEC, }; -export const LLM_MULTIMODAL_SPEC = { - ...method( - 'text_decoder', // prettier-ignore - [f32(1, 'seqLen', 'hiddenDim'), i64('seqLen')], - [f32(1, 'vocabSize')] - ), - ...method( - 'vision_encoder', // prettier-ignore - [f32(1, 3, 'imgH', 'imgW')], - [f32(1, 'visualTokens', 'hiddenDim')] - ), - ...method( - 'token_embedding', // prettier-ignore - [i64(1, 'seqLen')], - [f32(1, 'seqLen', 'hiddenDim')] - ), - ...METADATA_SPEC, -}; - export function createLLMMultimodalRunner( model: Model, tokenizer: Tokenizer, @@ -78,17 +59,14 @@ export function createLLMMultimodalRunner( ); } - const { dims } = validateSpec(model.schema, { - static: LLM_MULTIMODAL_SPEC, - dynamic: LLM_MULTIMODAL_DYN_SPEC, - }); + const { dims, dim } = validateSpec(model.schema, { default: LLM_MULTIMODAL_SPEC }); const [imgH, imgW, vocabSize] = dims.constant('imgH', 'imgW', 'vocabSize'); const [hiddenDim, numVisualTokens] = dims.constant('hiddenDim', 'visualTokens'); - const [maxSeqLen] = model.execute('get_max_seq_len', [], []) as [number]; // TODO: change to dims - const [maxContextLen] = model.execute('get_max_context_len', [], []) as [number]; - + const maxSeqLen = dim('seqLen', 'range').max; const imgShape = [1, 3, imgH, imgW] as const; + + const [maxContextLen] = model.execute('get_max_context_len', [], []) as [number]; const eosIds = model.execute('get_eos_ids', [], []) as number[]; if (maxContextLen < maxSeqLen || !Number.isInteger(maxContextLen)) { From 73bb49d1fff0f59b8d181ee83d4b9c7945949047 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Fri, 9 Oct 2026 04:43:10 +0200 Subject: [PATCH 38/83] test(cpp): cover the native LLM sampler Add an LlmOpsTest suite driving `llm.sample` through Hermes: argument and option validation, greedy decoding, top-k/top-p/min-p/temperature/XTC, repetition penalty, DRY, constraints, and the in-place logits rewrite. Restore `llm` in UtilsTest's installed-namespace check, since the module is installed again for the sampler. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../cpp/tests/CMakeLists.txt | 3 +- .../cpp/tests/core/UtilsTest.cpp | 2 +- .../cpp/tests/extensions/LlmOpsTest.cpp | 408 ++++++++++++++++++ 3 files changed, 411 insertions(+), 2 deletions(-) create mode 100644 packages/react-native-executorch/cpp/tests/extensions/LlmOpsTest.cpp diff --git a/packages/react-native-executorch/cpp/tests/CMakeLists.txt b/packages/react-native-executorch/cpp/tests/CMakeLists.txt index 0c2ab3872d..f25fa07ec2 100644 --- a/packages/react-native-executorch/cpp/tests/CMakeLists.txt +++ b/packages/react-native-executorch/cpp/tests/CMakeLists.txt @@ -351,7 +351,8 @@ set(TEST_SUITES core/SchemaTest.cpp core/UtilsTest.cpp extensions/MathOpsTest.cpp - extensions/SpeechOpsTest.cpp) + extensions/SpeechOpsTest.cpp + extensions/LlmOpsTest.cpp) if(RNE_TESTS_ENABLE_OPENCV) list(APPEND TEST_SUITES extensions/CvOpsTest.cpp extensions/OcrOpsTest.cpp) diff --git a/packages/react-native-executorch/cpp/tests/core/UtilsTest.cpp b/packages/react-native-executorch/cpp/tests/core/UtilsTest.cpp index 6093b34904..d198a82fb2 100644 --- a/packages/react-native-executorch/cpp/tests/core/UtilsTest.cpp +++ b/packages/react-native-executorch/cpp/tests/core/UtilsTest.cpp @@ -53,7 +53,7 @@ TEST_F(UtilsTest, InstallsTheModuleUnderItsProductionName) { EXPECT_TRUE(evalBool("return __rnexecutorch_jsi__.cv === undefined;")); #endif - for (const auto *name : {"math", "nlp", "speech"}) { + for (const auto *name : {"math", "nlp", "speech", "llm"}) { EXPECT_TRUE(evalBool(std::string("return typeof __rnexecutorch_jsi__.") + name + " === 'object';")) << "missing extension namespace: " << name; } diff --git a/packages/react-native-executorch/cpp/tests/extensions/LlmOpsTest.cpp b/packages/react-native-executorch/cpp/tests/extensions/LlmOpsTest.cpp new file mode 100644 index 0000000000..092d37e594 --- /dev/null +++ b/packages/react-native-executorch/cpp/tests/extensions/LlmOpsTest.cpp @@ -0,0 +1,408 @@ +#include +#include +#include +#include +#include +#include + +#include "support/JsiTestEnv.h" + +namespace rnexecutorch::tests { +namespace { + +using LlmOpsTest = JsiTestEnv; +using Case = std::pair; + +// `llm.sample` picks the next token from a `[1, vocab]` logits tensor. The +// TypeScript layer resolves every default before calling it, so the native side +// treats the core options as required and validates each one. +// +// The draw is random unless the options make it greedy, so the stochastic paths +// are checked over many draws against the set of tokens they are allowed to +// produce, never against a particular token. Note that `sample` rewrites the +// logits in place (constraints, penalties, DRY), so every draw refills them. + +constexpr const char *kNs = R"( + const sample = __rnexecutorch_jsi__.llm.sample; + const createTensor = __rnexecutorch_jsi__.createTensor; + const logits = (values) => { + const t = createTensor([1, values.length], 'float32'); + t.setData(new Float32Array(values)); + return t; + }; + const read = (t) => { const o = new Float32Array(t.numel); t.getData(o); return Array.from(o); }; + const ctx = (generated = []) => ({ generatedTokens: new Int32Array(generated) }); + // Options that leave the distribution untouched, so a test only states what it exercises. + const neutral = (overrides = {}) => ({ + temperature: 1, topK: 1000, topP: 1, minP: 0, repetitionPenalty: 1, ...overrides, + }); + const greedy = (overrides = {}) => neutral({ temperature: 0, ...overrides }); + // The distinct tokens `n` draws produce, sorted. + const drawSet = (values, options, n = 300, generated = []) => { + const seen = new Set(); + const t = createTensor([1, values.length], 'float32'); + for (let i = 0; i < n; i++) { + t.setData(new Float32Array(values)); + seen.add(sample(t, ctx(generated), options)); + } + return Array.from(seen).sort((a, b) => a - b); + }; +)"; + +std::string js(std::string_view body) { return std::format("{}\n{}", kNs, body); } + +// --- Installation & arguments ------------------------------------------------ + +TEST_F(LlmOpsTest, IsInstalledUnderTheLlmNamespace) { + EXPECT_EQ(evalString("return typeof __rnexecutorch_jsi__.llm.sample;"), "function"); +} + +TEST_F(LlmOpsTest, RejectsAWrongArgumentCount) { + EXPECT_TRUE(isCodedError(evalThrowing(js("sample(logits([1, 2]), ctx());")), + "INVALID_ARGUMENT", + "Usage: sample(logits, ctx, options)")); +} + +TEST_F(LlmOpsTest, RejectsLogitsOfTheWrongRank) { + EXPECT_TRUE(isCodedError(evalThrowing(js(R"( + const t = createTensor([4], 'float32'); + sample(t, ctx(), greedy()); + )")), + "INVALID_ARGUMENT", + "sample: logits")); +} + +TEST_F(LlmOpsTest, RejectsABatchOfLogits) { + // The runner decodes one sequence; a [2, vocab] tensor is a caller bug. + EXPECT_TRUE(isCodedError(evalThrowing(js(R"( + const t = createTensor([2, 4], 'float32'); + sample(t, ctx(), greedy()); + )")), + "INVALID_ARGUMENT", + "sample: logits")); +} + +TEST_F(LlmOpsTest, RejectsNonFloatLogits) { + EXPECT_TRUE(isCodedError(evalThrowing(js(R"( + const t = createTensor([1, 4], 'int32'); + sample(t, ctx(), greedy()); + )")), + "INVALID_ARGUMENT", + "sample: logits")); +} + +TEST_F(LlmOpsTest, RequiresEveryCoreOption) { + for (const auto *option : {"temperature", "topK", "topP", "minP", "repetitionPenalty"}) { + EXPECT_TRUE(isCodedError(evalThrowing(js(std::format(R"( + const options = neutral(); + delete options.{}; + sample(logits([1, 2]), ctx(), options); + )", + option))), + "INVALID_ARGUMENT", + option)) + << option; + } +} + +TEST_F(LlmOpsTest, ValidatesTheRangeOfEachCoreOption) { + const std::array cases{ + Case{"topK: 0", "'topK' must be at least 1"}, + Case{"topP: 0", "'topP' must be in (0, 1]"}, + Case{"topP: 1.5", "'topP' must be in (0, 1]"}, + Case{"minP: -0.1", "'minP' must be in [0, 1)"}, + Case{"minP: 1", "'minP' must be in [0, 1)"}, + Case{"temperature: -1", "'temperature' must be non-negative"}, + Case{"repetitionPenalty: 0", "'repetitionPenalty' must be positive"}, + }; + for (const auto &[override, message] : cases) { + EXPECT_TRUE(isCodedError( + evalThrowing(js(std::format("sample(logits([1, 2]), ctx(), neutral({{ {} }}));", override))), + "INVALID_ARGUMENT", + message)) + << override; + } +} + +TEST_F(LlmOpsTest, ValidatesTheDryOptions) { + const std::array cases{ + Case{"multiplier: -1", "'multiplier' must be non-negative"}, + Case{"base: 1", "'base' must be greater than 1.0"}, + Case{"allowedLength: -1", "'allowedLength' must be non-negative"}, + Case{"penaltyLastN: -2", "'penaltyLastN' must be -1 or non-negative"}, + }; + for (const auto &[override, message] : cases) { + EXPECT_TRUE(isCodedError(evalThrowing(js(std::format(R"( + const dry = {{ + multiplier: 1, base: 1.75, allowedLength: 2, penaltyLastN: -1, + sequenceBreakers: new Int32Array(0), {} + }}; + sample(logits([1, 2]), ctx([0, 1]), greedy({{ dry }})); + )", + override))), + "INVALID_ARGUMENT", + message)) + << override; + } +} + +TEST_F(LlmOpsTest, ValidatesTheXtcOptions) { + const std::array cases{ + Case{"probability: 1.5, threshold: 0.1", "'probability' must be in [0, 1]"}, + Case{"probability: 0.5, threshold: 0.6", "'threshold' must be in [0, 0.5]"}, + }; + for (const auto &[xtc, message] : cases) { + EXPECT_TRUE(isCodedError( + evalThrowing(js(std::format("sample(logits([1, 2]), ctx(), neutral({{ xtc: {{ {} }} }}));", xtc))), + "INVALID_ARGUMENT", + message)) + << xtc; + } +} + +// --- Greedy decoding --------------------------------------------------------- + +TEST_F(LlmOpsTest, ZeroTemperatureReturnsTheArgmax) { + EXPECT_EQ(evalNumber(js("return sample(logits([0.1, 3, -2, 2.9]), ctx(), greedy());")), 1); +} + +TEST_F(LlmOpsTest, TopKOfOneReturnsTheArgmax) { + EXPECT_EQ(evalNumber(js("return sample(logits([0.1, 3, -2, 2.9]), ctx(), neutral({ topK: 1 }));")), 1); +} + +TEST_F(LlmOpsTest, GreedyDecodingIsDeterministic) { + EXPECT_EQ(evalNumberArray(js("return drawSet([1, 1.01, 0.99], greedy(), 50);")), std::vector{1}); +} + +TEST_F(LlmOpsTest, FailsWhenEveryLogitIsMasked) { + // Greedy and sampled decoding take different paths to the same failure. + for (const auto *options : {"greedy()", "neutral()"}) { + EXPECT_TRUE(isCodedError( + evalThrowing(js(std::format("sample(logits([-Infinity, -Infinity, NaN]), ctx(), {});", options))), + "EXECUTION_FAILED", + "No valid candidates")) + << options; + } +} + +// --- Stochastic decoding ----------------------------------------------------- + +TEST_F(LlmOpsTest, SamplesAcrossComparableTokens) { + // Two near-equal tokens: a greedy shortcut taken by mistake would only ever + // return one of them. + EXPECT_EQ(evalNumberArray(js("return drawSet([1, 1, -50], neutral());")), (std::vector{0, 1})); +} + +TEST_F(LlmOpsTest, TopKRestrictsTheDrawToTheKBestTokens) { + EXPECT_EQ(evalNumberArray(js("return drawSet([1, 3, 2.9, 0.5, 2.8], neutral({ topK: 2 }));")), + (std::vector{1, 2})); +} + +TEST_F(LlmOpsTest, DropsCandidatesFarBelowTheBest) { + // Anything more than 16 logits under the best is pruned before softmax, + // however wide top-k is. + EXPECT_EQ(evalNumberArray(js("return drawSet([20, 3.9, 4.1], neutral({ temperature: 100 }));")), + (std::vector{0, 2})); +} + +TEST_F(LlmOpsTest, TopPKeepsTheSmallestNucleusReachingP) { + // Probabilities ~0.5, 0.3, 0.2: reaching 0.6 takes the first two tokens. + EXPECT_EQ(evalNumberArray( + js("return drawSet([Math.log(0.5), Math.log(0.3), Math.log(0.2)], neutral({ topP: 0.6 }));")), + (std::vector{0, 1})); +} + +TEST_F(LlmOpsTest, MinPDropsTokensFarLessLikelyThanTheBest) { + // Probabilities ~0.5, 0.3, 0.2 with minP 0.5: only tokens at least half as + // likely as the best survive. + EXPECT_EQ(evalNumberArray( + js("return drawSet([Math.log(0.5), Math.log(0.3), Math.log(0.2)], neutral({ minP: 0.5 }));")), + (std::vector{0, 1})); +} + +TEST_F(LlmOpsTest, LowTemperatureConcentratesOnTheBest) { + EXPECT_EQ(evalNumberArray(js("return drawSet([1, 0.5, 0], neutral({ temperature: 0.01 }));")), + std::vector{0}); +} + +TEST_F(LlmOpsTest, XtcRemovesTheTopChoiceEvenWhenGreedy) { + // Both tokens clear the threshold, so XTC drops the best one. That has to + // happen before greedy selection, or XTC would be a no-op at temperature 0. + EXPECT_EQ(evalNumberArray(js(R"( + return drawSet([1, 0.9, -50], greedy({ xtc: { probability: 1, threshold: 0.1 } }), 50); + )")), + std::vector{1}); + EXPECT_EQ(evalNumberArray(js(R"( + return drawSet([1, 0.9, -50], greedy({ xtc: { probability: 0, threshold: 0.1 } }), 50); + )")), + std::vector{0}); +} + +TEST_F(LlmOpsTest, XtcKeepsTheTopChoiceWhenItIsTheOnlyLikelyOne) { + EXPECT_EQ(evalNumberArray(js(R"( + return drawSet([10, 0, 0], greedy({ xtc: { probability: 1, threshold: 0.1 } }), 50); + )")), + std::vector{0}); +} + +// --- Repetition penalty ------------------------------------------------------ + +TEST_F(LlmOpsTest, RepetitionPenaltyShrinksAPositiveRepeatedLogit) { + // 2 / 1.5 = 1.33 falls below 1.9. + EXPECT_EQ(evalNumber(js("return sample(logits([2, 1.9]), ctx([0]), greedy({ repetitionPenalty: 1.5 }));")), + 1); +} + +TEST_F(LlmOpsTest, RepetitionPenaltyPushesANegativeRepeatedLogitFurtherDown) { + // -1 * 1.5 = -1.5 falls below -1.2; dividing would have raised it instead. + EXPECT_EQ( + evalNumber(js("return sample(logits([-1, -1.2]), ctx([0]), greedy({ repetitionPenalty: 1.5 }));")), + 1); +} + +TEST_F(LlmOpsTest, RepetitionPenaltyAppliesOncePerDistinctToken) { + // Penalised once, 2 / 1.2 = 1.67 still beats 1.5; penalised for each of the + // three occurrences it would not. + EXPECT_EQ( + evalNumber(js("return sample(logits([2, 1.5]), ctx([0, 0, 0]), greedy({ repetitionPenalty: 1.2 }));")), + 0); +} + +TEST_F(LlmOpsTest, RepetitionPenaltyIgnoresOutOfRangeHistory) { + EXPECT_EQ( + evalNumber(js("return sample(logits([2, 1.9]), ctx([-1, 7]), greedy({ repetitionPenalty: 1.5 }));")), + 0); +} + +// --- DRY --------------------------------------------------------------------- + +// History 1 2 3 1 2: the suffix "1 2" already occurred, followed by 3, so +// emitting 3 now would repeat the sequence. +constexpr const char *kDry = R"( + const dry = (overrides = {}) => ({ + multiplier: 2, base: 1.75, allowedLength: 2, penaltyLastN: -1, + sequenceBreakers: new Int32Array(0), ...overrides, + }); + const history = [1, 2, 3, 1, 2]; + const values = [0, 0, 0, 1]; +)"; + +TEST_F(LlmOpsTest, DryPenalisesTheTokenThatWouldRepeatASequence) { + EXPECT_EQ(evalNumber(js(std::format("{} return sample(logits(values), ctx(history), greedy());", kDry))), 3); + EXPECT_EQ(evalNumber(js(std::format("{} return sample(logits(values), ctx(history), greedy({{ dry: dry() }}));", + kDry))), + 0); +} + +TEST_F(LlmOpsTest, DryToleratesRepeatsShorterThanTheAllowedLength) { + EXPECT_EQ(evalNumber(js(std::format( + "{} return sample(logits(values), ctx(history), greedy({{ dry: dry({{ allowedLength: 3 }}) }}));", + kDry))), + 3); +} + +TEST_F(LlmOpsTest, DryExemptsSequenceBreakers) { + EXPECT_EQ(evalNumber(js(std::format(R"( + {} + const options = greedy({{ dry: dry({{ sequenceBreakers: new Int32Array([3]) }}) }}); + return sample(logits(values), ctx(history), options); + )", + kDry))), + 3); +} + +TEST_F(LlmOpsTest, DryOnlyLooksAtTheLastNTokens) { + // The earlier "1 2 3" falls outside a window of the last three tokens. + EXPECT_EQ(evalNumber(js(std::format( + "{} return sample(logits(values), ctx(history), greedy({{ dry: dry({{ penaltyLastN: 3 }}) }}));", + kDry))), + 3); +} + +// --- Constraints ------------------------------------------------------------- + +TEST_F(LlmOpsTest, AllowedTokensRestrictTheChoice) { + EXPECT_EQ(evalNumber(js(R"( + const constraints = () => ({ allowedTokens: new Int32Array([0, 2]) }); + return sample(logits([1, 5, 2]), ctx(), greedy({ constraints })); + )")), + 2); +} + +TEST_F(LlmOpsTest, BannedTokensAreNeverChosen) { + EXPECT_EQ(evalNumberArray(js(R"( + const constraints = () => ({ bannedTokens: new Int32Array([1]) }); + return drawSet([1, 5, 1], neutral({ constraints })); + )")), + (std::vector{0, 2})); +} + +TEST_F(LlmOpsTest, ConstraintsIgnoreOutOfRangeIds) { + EXPECT_EQ(evalNumber(js(R"( + const constraints = () => ({ + allowedTokens: new Int32Array([-1, 2, 99]), + bannedTokens: new Int32Array([-5, 42]), + }); + return sample(logits([1, 5, 2]), ctx(), greedy({ constraints })); + )")), + 2); +} + +TEST_F(LlmOpsTest, ConstraintsReturningNothingLeaveTheLogitsAlone) { + for (const auto *ret : {"undefined", "null"}) { + EXPECT_EQ(evalNumber(js(std::format(R"( + const constraints = () => {{ return {}; }}; + return sample(logits([1, 5, 2]), ctx(), greedy({{ constraints }})); + )", + ret))), + 1) + << ret; + } +} + +TEST_F(LlmOpsTest, ConstraintsReceiveTheSamplingContext) { + EXPECT_EQ(evalNumberArray(js(R"( + let seen; + const constraints = (c) => { seen = Array.from(c.generatedTokens); }; + sample(logits([1, 5, 2]), ctx([4, 2]), greedy({ constraints })); + return seen; + )")), + (std::vector{4, 2})); +} + +TEST_F(LlmOpsTest, AnEmptyAllowListMasksEverything) { + EXPECT_TRUE(isCodedError(evalThrowing(js(R"( + const constraints = () => ({ allowedTokens: new Int32Array(0) }); + sample(logits([1, 5, 2]), ctx(), greedy({ constraints })); + )")), + "EXECUTION_FAILED", + "No valid candidates")); +} + +TEST_F(LlmOpsTest, PropagatesAThrowingConstraintsCallback) { + EXPECT_THAT(evalThrowingMessage(js(R"( + const constraints = () => { throw new Error('grammar exploded'); }; + sample(logits([1, 5, 2]), ctx(), greedy({ constraints })); + )")), + ::testing::HasSubstr("grammar exploded")); +} + +// --- In-place effects -------------------------------------------------------- + +TEST_F(LlmOpsTest, RewritesTheLogitsInPlace) { + // Documented on the TypeScript `sample`: the masked and penalised logits are + // written back into the caller's tensor rather than a copy. + auto result = evalNumberArray(js(R"( + const t = logits([1, 2, 3]); + const constraints = () => ({ bannedTokens: new Int32Array([2]) }); + sample(t, ctx([1]), greedy({ constraints, repetitionPenalty: 2 })); + return read(t); + )")); + ASSERT_EQ(result.size(), 3u); + EXPECT_DOUBLE_EQ(result[0], 1); + EXPECT_DOUBLE_EQ(result[1], 1); + EXPECT_TRUE(std::isinf(result[2]) && result[2] < 0); +} + +} // namespace +} // namespace rnexecutorch::tests From f99ffe509c46f4594bd9f61266a152b7b0532c27 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Fri, 9 Oct 2026 04:48:58 +0200 Subject: [PATCH 39/83] style --- .../src/extensions/llm/runners/llmMultimodalRunner.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts b/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts index 1631daa675..7356c712c1 100644 --- a/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts @@ -27,7 +27,6 @@ const METADATA_SPEC = { ...method('enable_dynamic_shape', [], [{ kind: 'Bool' }]), }; -// REPLACE WITH THIS AFTER REEXPORT export const LLM_MULTIMODAL_SPEC = { ...method( 'text_decoder', From e4669fe799a4c631dce0c8e789e8fbcc7b2c7a4f Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Fri, 9 Oct 2026 04:57:21 +0200 Subject: [PATCH 40/83] docs(llm): document the TypeScript LLM runner and the native sampler Drop the @experimental tags from the LLM runner JSDoc and describe it as the TypeScript prefill/decode loop it now is. Update the docs to the new generation stats and KV cache fields, await createLLMRunner, and cover llm.sample. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../02-llm-chat-and-generation.md | 12 +++--- .../04-operations-and-utilities.md | 17 +++++++-- .../src/extensions/llm/llmRunner.ts | 37 +++++++++---------- 3 files changed, 37 insertions(+), 29 deletions(-) diff --git a/docs/docs/02-extensions/natural-language/02-llm-chat-and-generation.md b/docs/docs/02-extensions/natural-language/02-llm-chat-and-generation.md index 3b4f209cf9..b9b40ac14d 100644 --- a/docs/docs/02-extensions/natural-language/02-llm-chat-and-generation.md +++ b/docs/docs/02-extensions/natural-language/02-llm-chat-and-generation.md @@ -145,7 +145,7 @@ type LLMChatTurnResult = { }; ``` -[`stats`](../../06-api-reference/type-aliases/LLMChatTurnResult.md#stats) is an array of [`LLMGenerationStats`](../../06-api-reference/react-native-executorch/namespaces/llm/type-aliases/LLMGenerationStats.md) because tool calling can trigger multiple consecutive generation steps in a single turn (e.g. `stats[0]` for the model generating the tool call, and `stats[1]` for generating the final answer after tool execution). Each entry provides [`numPromptTokens`](../../06-api-reference/react-native-executorch/namespaces/llm/type-aliases/LLMGenerationStats.md#numprompttokens), [`numGeneratedTokens`](../../06-api-reference/react-native-executorch/namespaces/llm/type-aliases/LLMGenerationStats.md#numgeneratedtokens), [`prefillDurationMs`](../../06-api-reference/react-native-executorch/namespaces/llm/type-aliases/LLMGenerationStats.md#prefilldurationms), and start/end timestamps ([`inferenceStartMs`](../../06-api-reference/react-native-executorch/namespaces/llm/type-aliases/LLMGenerationStats.md#inferencestartms) / [`inferenceEndMs`](../../06-api-reference/react-native-executorch/namespaces/llm/type-aliases/LLMGenerationStats.md#inferenceendms)) to compute tokens per second (`tok/s`). +[`stats`](../../06-api-reference/type-aliases/LLMChatTurnResult.md#stats) is an array of [`LLMGenerationStats`](../../06-api-reference/react-native-executorch/namespaces/llm/type-aliases/LLMGenerationStats.md) because tool calling can trigger multiple consecutive generation steps in a single turn (e.g. `stats[0]` for the model generating the tool call, and `stats[1]` for generating the final answer after tool execution). Each entry provides the number of generated tokens ([`numTokens`](../../06-api-reference/react-native-executorch/namespaces/llm/type-aliases/LLMGenerationStats.md#numtokens)), the decode time ([`durationMs`](../../06-api-reference/react-native-executorch/namespaces/llm/type-aliases/LLMGenerationStats.md#durationms)) and throughput ([`tokensPerSecond`](../../06-api-reference/react-native-executorch/namespaces/llm/type-aliases/LLMGenerationStats.md#tokenspersecond)), plus the same three metrics for the prompt in [`prefill`](../../06-api-reference/react-native-executorch/namespaces/llm/type-aliases/LLMGenerationStats.md#prefill). ## Chat Templates & Incremental KV Cache Diffing @@ -294,14 +294,14 @@ try { ## Low-Level Runner -While [`useLLMChatSession`](../../06-api-reference/functions/useLLMChatSession.md) and [`createLLMChatSession`](../../06-api-reference/functions/createLLMChatSession.md) handle chat formatting, message histories, and automated tool calling loops, you can drop down directly to the native [`LLMRunner`](../../06-api-reference/react-native-executorch/namespaces/llm/type-aliases/LLMRunner.md) via [`llm.createLLMRunner()`](../../06-api-reference/react-native-executorch/namespaces/llm/functions/createLLMRunner.md). +While [`useLLMChatSession`](../../06-api-reference/functions/useLLMChatSession.md) and [`createLLMChatSession`](../../06-api-reference/functions/createLLMChatSession.md) handle chat formatting, message histories, and automated tool calling loops, you can drop down directly to the [`LLMRunner`](../../06-api-reference/react-native-executorch/namespaces/llm/type-aliases/LLMRunner.md) via [`llm.createLLMRunner()`](../../06-api-reference/react-native-executorch/namespaces/llm/functions/createLLMRunner.md). -[`LLMRunner`](../../06-api-reference/react-native-executorch/namespaces/llm/type-aliases/LLMRunner.md) operates synchronously on a worklet runtime thread and provides low-level control: +[`LLMRunner`](../../06-api-reference/react-native-executorch/namespaces/llm/type-aliases/LLMRunner.md) runs the prefill and decode loop in TypeScript over the loaded model, picking each token with the native [`llm.sample()`](../../06-api-reference/react-native-executorch/namespaces/llm/functions/sample.md) kernel. It operates synchronously on a worklet runtime thread and provides low-level control: - **Raw Prompt Ingestion**: Pass raw prompt strings or preprocessed image tensors directly to the runner without role formatting or Jinja chat template rendering. - **Manual Prefill**: Execute [`runner.prefill(prompt)`](../../06-api-reference/react-native-executorch/namespaces/llm/type-aliases/LLMRunner.md#prefill) to populate the Key-Value (KV) cache with large background contexts, system prompts, or document chunks before starting interactive generation. - **Direct Synchronous Generation**: Call [`runner.generate(prompt, config, onToken)`](../../06-api-reference/react-native-executorch/namespaces/llm/type-aliases/LLMRunner.md#generate) to generate text continuations with zero Promise scheduling overhead, executing the `onToken` callback directly on each generated token. -- **KV Cache Inspection & Slicing**: Query [`runner.getKVCacheState()`](../../06-api-reference/react-native-executorch/namespaces/llm/type-aliases/LLMRunner.md#getkvcachestate) to check occupied tokens ([`pos`](../../06-api-reference/react-native-executorch/namespaces/llm/type-aliases/LLMKVCacheState.md#pos)), max context length ([`maxSeqLen`](../../06-api-reference/react-native-executorch/namespaces/llm/type-aliases/LLMKVCacheState.md#maxseqlen)), and context capacity ratio ([`usageRatio`](../../06-api-reference/react-native-executorch/namespaces/llm/type-aliases/LLMKVCacheState.md#usageratio)). +- **KV Cache Inspection & Slicing**: Query [`runner.getKVCacheState()`](../../06-api-reference/react-native-executorch/namespaces/llm/type-aliases/LLMRunner.md#getkvcachestate) to check occupied tokens ([`pos`](../../06-api-reference/react-native-executorch/namespaces/llm/type-aliases/LLMKVCacheState.md#pos)), max context length ([`maxContextLen`](../../06-api-reference/react-native-executorch/namespaces/llm/type-aliases/LLMKVCacheState.md#maxcontextlen)), and context capacity ratio ([`usageRatio`](../../06-api-reference/react-native-executorch/namespaces/llm/type-aliases/LLMKVCacheState.md#usageratio)). - **KV Cache Rewind & Branching**: Call [`runner.reset(targetPos)`](../../06-api-reference/react-native-executorch/namespaces/llm/type-aliases/LLMRunner.md#reset) to rewind the KV cache back to an exact token position. This enables speculative branching, sampling multiple divergent continuations from a shared prompt prefix without re-encoding, or manual conversation tree management. - **Cancellation**: Call [`runner.stop()`](../../06-api-reference/react-native-executorch/namespaces/llm/type-aliases/LLMRunner.md#stop) from any thread to abort active autoregressive generation immediately. @@ -360,7 +360,6 @@ The library provides ready-to-use models from the [Software Mansion HuggingFace | **Liquid LFM 2.5** | [`350M`](../../06-api-reference/variables/models.md#llmlfm2_5_350m), [`1.2B`](../../06-api-reference/variables/models.md#llmlfm2_5_1_2b) | 265 MB – 2.43 GB | XNNPACK (CPU), MLX (Apple) | Fast hybrid RNN/Transformer for low-latency chat. | | **Liquid LFM 2.5 VL** | [`VL 450M`](../../06-api-reference/variables/models.md#llmlfm2_5_vl_450m), [`VL 1.6B`](../../06-api-reference/variables/models.md#llmlfm2_5_vl_1_6b) | 376 MB – 2.36 GB | XNNPACK (CPU), MLX (Apple), Vulkan (Android) | Vision-language variants for image understanding and visual reasoning. | | **Meta Llama 3.2** | [`1B`](../../06-api-reference/variables/models.md#llmllama3_2_1b), [`3B`](../../06-api-reference/variables/models.md#llmllama3_2_3b) | 1.06 GB – 5.99 GB | XNNPACK (CPU), MLX (Apple) | High-quality reasoning, summarization, and instruction following. | -| **Google Gemma 4** | [`E2B`](../../06-api-reference/variables/models.md#llmgemma4_e2b) | 2.45 GB – 2.70 GB | XNNPACK (CPU), MLX (Apple), Vulkan (Android) | High-fidelity instruction following from Google DeepMind research. | | **Alibaba Qwen 3** | [`0.6B`](../../06-api-reference/variables/models.md#llmqwen3_0_6b), [`1.7B`](../../06-api-reference/variables/models.md#llmqwen3_1_7b), [`4B`](../../06-api-reference/variables/models.md#llmqwen3_4b) | 482 MB – 7.49 GB | XNNPACK (CPU), MLX (Apple) | Next-gen compact multilingual models supporting 29+ languages. | | **Alibaba Qwen 2.5** | [`0.5B`](../../06-api-reference/variables/models.md#llmqwen2_5_0_5b), [`1.5B`](../../06-api-reference/variables/models.md#llmqwen2_5_1_5b), [`3B`](../../06-api-reference/variables/models.md#llmqwen2_5_3b) | 417 MB – 5.75 GB | XNNPACK (CPU), MLX (Apple) | Proven multilingual instruction models across code, math, and chat. | | **Hammer 2.1** | [`0.5B`](../../06-api-reference/variables/models.md#llmhammer2_1_0_5b), [`1.5B`](../../06-api-reference/variables/models.md#llmhammer2_1_1_5b), [`3B`](../../06-api-reference/variables/models.md#llmhammer2_1_3b) | 398 MB – 5.75 GB | XNNPACK (CPU), MLX (Apple) | Fine-tuned function calling for automated tool execution & structured JSON. | @@ -389,6 +388,7 @@ The pipeline automatically verifies that the model's exported methods and KV cac - [`useLLMChatSession()`](../../06-api-reference/functions/useLLMChatSession.md) — React hook for managing LLM model downloading, KV cache, and conversational sessions. - [`createLLMChatSession()`](../../06-api-reference/functions/createLLMChatSession.md) — Imperative factory for multi-turn LLM chat sessions. - [`llm.createLLMRunner()`](../../06-api-reference/react-native-executorch/namespaces/llm/functions/createLLMRunner.md) — Low-level factory for direct prompt execution and KV cache manipulation. +- [`llm.sample()`](../../06-api-reference/react-native-executorch/namespaces/llm/functions/sample.md) — Native token sampler (temperature, top-k/top-p/min-p, repetition penalty, DRY, XTC, constraints) for custom decoding loops. - [`llm.createChatPreprocessor()`](../../06-api-reference/react-native-executorch/namespaces/llm/functions/createChatPreprocessor.md) — Jinja2 template renderer, media processor, and prompt diffing engine. ### Types & Options @@ -399,7 +399,7 @@ The pipeline automatically verifies that the model's exported methods and KV cac - [`ToolDefinition`](../../06-api-reference/react-native-executorch/namespaces/llm/type-aliases/ToolDefinition.md) — Tool declaration with JSON Schema parameters and `execute` callback. - [`ToolParser`](../../06-api-reference/react-native-executorch/namespaces/llm/type-aliases/ToolParser.md) — Parser function type for extracting tool calls from model output. - [`LLMChatTurnResult`](../../06-api-reference/type-aliases/LLMChatTurnResult.md) — Result of a chat turn with updated messages, finish reason, and performance statistics. -- [`LLMKVCacheState`](../../06-api-reference/react-native-executorch/namespaces/llm/type-aliases/LLMKVCacheState.md) — KV cache metrics (`pos`, `maxSeqLen`, `usageRatio`). +- [`LLMKVCacheState`](../../06-api-reference/react-native-executorch/namespaces/llm/type-aliases/LLMKVCacheState.md) — KV cache metrics (`pos`, `maxContextLen`, `usageRatio`). - [`LLMChatSessionOptions`](../../06-api-reference/type-aliases/LLMChatSessionOptions.md) — Session configuration options (`generationConfig`, `initialMessages`, `toolOpts`). - [`LLMModel`](../../06-api-reference/type-aliases/LLMModel.md) — Model configuration spec with model, tokenizer, and tokenizer config paths. - [`LLMGenerationConfig`](../../06-api-reference/react-native-executorch/namespaces/llm/type-aliases/LLMGenerationConfig.md) — Sampling and decoding parameters (`temperature`, `topP`, `maxNewTokens`). diff --git a/docs/docs/03-core-and-advanced/04-operations-and-utilities.md b/docs/docs/03-core-and-advanced/04-operations-and-utilities.md index 52ee4f2b77..0a3042d4cf 100644 --- a/docs/docs/03-core-and-advanced/04-operations-and-utilities.md +++ b/docs/docs/03-core-and-advanced/04-operations-and-utilities.md @@ -62,8 +62,9 @@ The `math`, `cv`, and `speech` operations below are all of this kind. **TypeScript utilities** are ordinary functions — seeded random number generation, bounding-box geometry, tokenizers, the LLM runner. They work on plain numbers, typed arrays, and strings rather than tensor destinations, and they allocate and -return their own result. The `nlp` and `llm` namespaces are entirely utilities, and -`math` and `cv` include a few alongside their native operations. +return their own result. The `nlp` namespace is entirely utilities, `llm` is too +apart from its native `sample` kernel, and `math` and `cv` include a few alongside +their native operations. ## Math operations @@ -208,9 +209,17 @@ multimodal inputs when the model supports them. ```typescript import { llm } from 'react-native-executorch'; -const runner = llm.createLLMRunner('/path/to/model.pte', '/path/to/tokenizer.json'); +const runner = await llm.createLLMRunner('/path/to/model.pte', '/path/to/tokenizer.json'); ``` +The runner picks each token with +[`sample`](../06-api-reference/react-native-executorch/namespaces/llm/functions/sample.md), a native kernel you +can also call directly to write your own decoding loop. It takes a `[1, vocabSize]` +logits tensor and returns the chosen token id, applying temperature, top-k, top-p, +min-p, repetition penalty, DRY, XTC and token constraints along the way. Unlike the +other kernels it has no destination: it rewrites the logits in place, so copy them +first if you still need the originals. + For chat models, [`createChatPreprocessor`](../06-api-reference/react-native-executorch/namespaces/llm/functions/createChatPreprocessor.md) applies the model's chat template to a list of messages and prepares any image or @@ -254,5 +263,5 @@ const tLatents = tensor('float32', [1, 4, 64, 64], latents); - CV: [`resize`](../06-api-reference/react-native-executorch/namespaces/cv/functions/resize.md) · [`cvtColor`](../06-api-reference/react-native-executorch/namespaces/cv/functions/cvtColor.md) · [`toChannelsFirst`](../06-api-reference/react-native-executorch/namespaces/cv/functions/toChannelsFirst.md) · [`toChannelsLast`](../06-api-reference/react-native-executorch/namespaces/cv/functions/toChannelsLast.md) · [`normalize`](../06-api-reference/react-native-executorch/namespaces/cv/functions/normalize.md) · [`applyColormap`](../06-api-reference/react-native-executorch/namespaces/cv/functions/applyColormap.md) - Speech: [`extractFrames`](../06-api-reference/react-native-executorch/namespaces/speech/functions/extractFrames.md) - NLP: [`loadTokenizer`](../06-api-reference/react-native-executorch/namespaces/nlp/functions/loadTokenizer.md) · [`piiSegments`](../06-api-reference/react-native-executorch/namespaces/nlp/functions/piiSegments.md) -- LLM: [`createLLMRunner`](../06-api-reference/react-native-executorch/namespaces/llm/functions/createLLMRunner.md) · [`createChatPreprocessor`](../06-api-reference/react-native-executorch/namespaces/llm/functions/createChatPreprocessor.md) +- LLM: [`createLLMRunner`](../06-api-reference/react-native-executorch/namespaces/llm/functions/createLLMRunner.md) · [`sample`](../06-api-reference/react-native-executorch/namespaces/llm/functions/sample.md) · [`createChatPreprocessor`](../06-api-reference/react-native-executorch/namespaces/llm/functions/createChatPreprocessor.md) - Numeric helpers: [`mulberry32`](../06-api-reference/react-native-executorch/namespaces/math/functions/mulberry32.md) · [`randomNormal`](../06-api-reference/react-native-executorch/namespaces/math/functions/randomNormal.md) · [`repeatInterleave`](../06-api-reference/react-native-executorch/namespaces/math/functions/repeatInterleave.md) diff --git a/packages/react-native-executorch/src/extensions/llm/llmRunner.ts b/packages/react-native-executorch/src/extensions/llm/llmRunner.ts index 28c649a3c2..ff63fa7bd5 100644 --- a/packages/react-native-executorch/src/extensions/llm/llmRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/llmRunner.ts @@ -1,5 +1,9 @@ /** - * Low-level native ExecuTorch LLM runner types and factory. + * Low-level ExecuTorch LLM runner types and factory. + * + * The runner drives a compiled `.pte` model through its prefill and decode loop + * in TypeScript, on top of `loadModel`, `loadTokenizer` and the native `sample` + * kernel. */ import { type WorkletRuntime } from 'react-native-worklets'; @@ -21,7 +25,6 @@ declare const llmRunnerBrand: unique symbol; /** * Configuration options for LLM text generation. - * @experimental This API is experimental and might change in future releases. * @category LLM / Types */ export type LLMGenerationConfig = SamplingConfig & { @@ -33,7 +36,6 @@ export type LLMGenerationConfig = SamplingConfig & { /** * Execution and performance statistics for a prefill phase. - * @experimental This API is experimental and might change in future releases. * @category LLM / Types */ export type LLMPrefillStats = { @@ -47,7 +49,6 @@ export type LLMPrefillStats = { /** * Execution and performance statistics for a generation call. - * @experimental This API is experimental and might change in future releases. * @category LLM / Types */ export type LLMGenerationStats = { @@ -63,7 +64,6 @@ export type LLMGenerationStats = { /** * Low-level non-text media input tensor payloads. - * @experimental This API is experimental and might change in future releases. * @category LLM / Types */ export type MediaInput = @@ -72,21 +72,18 @@ export type MediaInput = /** * Supported non-text input modality keys (e.g. `'image'`, `'audio'`). - * @experimental This API is experimental and might change in future releases. * @category LLM / Types */ export type Modality = MediaInput['kind']; /** * Text or interleaved multimodal prompt input for a low-level LLM runner. - * @experimental This API is experimental and might change in future releases. * @category LLM / Types */ export type Prompt = string | readonly (string | MediaInput)[]; /** * Current KV cache state and capacity metrics for an LLM runner. - * @experimental This API is experimental and might change in future releases. * @category LLM / Types */ export type LLMKVCacheState = { @@ -101,10 +98,11 @@ export type LLMKVCacheState = { }; /** - * Handle to a native ExecuTorch LLM runner. - * @experimental This API is experimental and might change in future releases. It - * relies on experimental ExecuTorch runtime extensions and injected member-pointer - * accessors to manage KV cache state that may evolve across releases. + * Handle to an ExecuTorch LLM runner. + * + * A runner owns a loaded model and its tokenizer, and tracks the KV cache + * position across calls, so successive `prefill` and `generate` calls continue + * the same sequence until it is `reset`. Obtain one via {@link createLLMRunner}. * @category LLM / Types */ export type LLMRunner = { @@ -165,17 +163,18 @@ export type LLMRunner = { }; /** - * Creates a native ExecuTorch LLM runner instance. - * @experimental This API is experimental and might change in future releases. It - * relies on experimental ExecuTorch runtime extensions and injected member-pointer - * accessors to manage KV cache state that may evolve across releases. + * Creates an ExecuTorch LLM runner instance. + * + * Loads the model and its tokenizer, validates the model's exported methods + * against the supported runner specs, and returns the matching runner. * @category LLM / Functions * @param modelPath Path to the local `.pte` model file. - * @param tokenizerPath Path to the local tokenizer configuration file (e.g. `tokenizer.json`). + * @param tokenizerPath Path to the local tokenizer file (e.g. `tokenizer.json`). * @param modalities List of supported input non-text modalities (e.g. * `['image']`). When omitted, defaults to text-only. - * @param runtime ... - * @returns A native {@link LLMRunner} instance. + * @param runtime Optional worklet runtime thread on which to load the model and + * tokenizer. + * @returns A promise resolving to the {@link LLMRunner} instance. */ export async function createLLMRunner( modelPath: string, From 8d7feb6cb6b622a790b606f6ca6839453ed5545b Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Fri, 9 Oct 2026 04:57:21 +0200 Subject: [PATCH 41/83] feat(models): restore the LLM variants pending a re-export Bring back the Gemma 4 E2B, LFM 2.5 VL Vulkan, LFM 2.5 VL 1.6B MLX int8 and SmolLM2 MLX int8 variants instead of commenting them out. Each is marked with a TODO: the published builds predate the TypeScript LLM runner and fail to load until they are re-exported. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../react-native-executorch/src/models.ts | 137 ++++++++++-------- 1 file changed, 73 insertions(+), 64 deletions(-) diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index 733dfde2e0..52fe8831f7 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -2124,20 +2124,22 @@ const LFM2_5_VL_1_6B_XNNPACK_8DA4W: LLMModel = { modalities: ['image'], preprocessorConfig: LFM2_5_VL_PREPROCESSOR_CONFIG, }; -// const LFM2_5_VL_450M_VULKAN_8DA4W: LLMModel = { -// modelPath: `${LFM2_5_BASE_URL}/vl_450m/vulkan/lfm_2_5_vl_450m_vulkan_8da4w.pte`, -// tokenizerPath: `${LFM2_5_BASE_URL}/vl_450m/tokenizer.json`, -// tokenizerConfigPath: `${LFM2_5_BASE_URL}/vl_450m/tokenizer_config.json`, -// modalities: ['image'], -// preprocessorConfig: LFM2_5_VL_PREPROCESSOR_CONFIG, -// }; -// const LFM2_5_VL_1_6B_VULKAN_8DA4W: LLMModel = { -// modelPath: `${LFM2_5_BASE_URL}/vl_1_6b/vulkan/lfm_2_5_vl_1_6b_vulkan_8da4w.pte`, -// tokenizerPath: `${LFM2_5_BASE_URL}/vl_1_6b/tokenizer.json`, -// tokenizerConfigPath: `${LFM2_5_BASE_URL}/vl_1_6b/tokenizer_config.json`, -// modalities: ['image'], -// preprocessorConfig: LFM2_5_VL_PREPROCESSOR_CONFIG, -// }; +// TODO: re-export for the TypeScript LLM runner; this build fails to load until then. +const LFM2_5_VL_450M_VULKAN_8DA4W: LLMModel = { + modelPath: `${LFM2_5_BASE_URL}/vl_450m/vulkan/lfm_2_5_vl_450m_vulkan_8da4w.pte`, + tokenizerPath: `${LFM2_5_BASE_URL}/vl_450m/tokenizer.json`, + tokenizerConfigPath: `${LFM2_5_BASE_URL}/vl_450m/tokenizer_config.json`, + modalities: ['image'], + preprocessorConfig: LFM2_5_VL_PREPROCESSOR_CONFIG, +}; +// TODO: re-export for the TypeScript LLM runner; this build fails to load until then. +const LFM2_5_VL_1_6B_VULKAN_8DA4W: LLMModel = { + modelPath: `${LFM2_5_BASE_URL}/vl_1_6b/vulkan/lfm_2_5_vl_1_6b_vulkan_8da4w.pte`, + tokenizerPath: `${LFM2_5_BASE_URL}/vl_1_6b/tokenizer.json`, + tokenizerConfigPath: `${LFM2_5_BASE_URL}/vl_1_6b/tokenizer_config.json`, + modalities: ['image'], + preprocessorConfig: LFM2_5_VL_PREPROCESSOR_CONFIG, +}; const LFM2_5_VL_1_6B_MLX_INT4: LLMModel = { modelPath: `${SCRATCH_LLMS_BASE_URL}/lfm_2_5_vl_1_6b_mlx_int4.pte`, tokenizerPath: `${LFM2_5_BASE_URL}/vl_1_6b/tokenizer.json`, @@ -2145,13 +2147,14 @@ const LFM2_5_VL_1_6B_MLX_INT4: LLMModel = { modalities: ['image'], preprocessorConfig: LFM2_5_VL_PREPROCESSOR_CONFIG, }; -// const LFM2_5_VL_1_6B_MLX_INT8: LLMModel = { -// modelPath: `${LFM2_5_BASE_URL}/vl_1_6b/mlx/lfm_2_5_vl_1_6b_mlx_int8.pte`, -// tokenizerPath: `${LFM2_5_BASE_URL}/vl_1_6b/tokenizer.json`, -// tokenizerConfigPath: `${LFM2_5_BASE_URL}/vl_1_6b/tokenizer_config.json`, -// modalities: ['image'], -// preprocessorConfig: LFM2_5_VL_PREPROCESSOR_CONFIG, -// }; +// TODO: re-export for the TypeScript LLM runner; this build fails to load until then. +const LFM2_5_VL_1_6B_MLX_INT8: LLMModel = { + modelPath: `${LFM2_5_BASE_URL}/vl_1_6b/mlx/lfm_2_5_vl_1_6b_mlx_int8.pte`, + tokenizerPath: `${LFM2_5_BASE_URL}/vl_1_6b/tokenizer.json`, + tokenizerConfigPath: `${LFM2_5_BASE_URL}/vl_1_6b/tokenizer_config.json`, + modalities: ['image'], + preprocessorConfig: LFM2_5_VL_PREPROCESSOR_CONFIG, +}; const BIELIK_V3_1_5B_BASE_URL = `${BASE_URL}-bielik-v3.0/${VERSION_TAG}`; @@ -2201,21 +2204,24 @@ const SMOLLM2_1_7B_8DA8W: LLMModel = { tokenizerPath: `${SMOLLM2_BASE_URL}/1_7b/tokenizer.json`, tokenizerConfigPath: `${SMOLLM2_BASE_URL}/1_7b/tokenizer_config.json`, }; -// const SMOLLM2_135M_MLX_INT8: LLMModel = { -// modelPath: `${SMOLLM2_BASE_URL}/135m/mlx/smollm2_135m_mlx_int8.pte`, -// tokenizerPath: `${SMOLLM2_BASE_URL}/135m/tokenizer.json`, -// tokenizerConfigPath: `${SMOLLM2_BASE_URL}/135m/tokenizer_config.json`, -// }; -// const SMOLLM2_360M_MLX_INT8: LLMModel = { -// modelPath: `${SMOLLM2_BASE_URL}/360m/mlx/smollm2_360m_mlx_int8.pte`, -// tokenizerPath: `${SMOLLM2_BASE_URL}/360m/tokenizer.json`, -// tokenizerConfigPath: `${SMOLLM2_BASE_URL}/360m/tokenizer_config.json`, -// }; -// const SMOLLM2_1_7B_MLX_INT8: LLMModel = { -// modelPath: `${SMOLLM2_BASE_URL}/1_7b/mlx/smollm2_1_7b_mlx_int8.pte`, -// tokenizerPath: `${SMOLLM2_BASE_URL}/1_7b/tokenizer.json`, -// tokenizerConfigPath: `${SMOLLM2_BASE_URL}/1_7b/tokenizer_config.json`, -// }; +// TODO: re-export for the TypeScript LLM runner; this build fails to load until then. +const SMOLLM2_135M_MLX_INT8: LLMModel = { + modelPath: `${SMOLLM2_BASE_URL}/135m/mlx/smollm2_135m_mlx_int8.pte`, + tokenizerPath: `${SMOLLM2_BASE_URL}/135m/tokenizer.json`, + tokenizerConfigPath: `${SMOLLM2_BASE_URL}/135m/tokenizer_config.json`, +}; +// TODO: re-export for the TypeScript LLM runner; this build fails to load until then. +const SMOLLM2_360M_MLX_INT8: LLMModel = { + modelPath: `${SMOLLM2_BASE_URL}/360m/mlx/smollm2_360m_mlx_int8.pte`, + tokenizerPath: `${SMOLLM2_BASE_URL}/360m/tokenizer.json`, + tokenizerConfigPath: `${SMOLLM2_BASE_URL}/360m/tokenizer_config.json`, +}; +// TODO: re-export for the TypeScript LLM runner; this build fails to load until then. +const SMOLLM2_1_7B_MLX_INT8: LLMModel = { + modelPath: `${SMOLLM2_BASE_URL}/1_7b/mlx/smollm2_1_7b_mlx_int8.pte`, + tokenizerPath: `${SMOLLM2_BASE_URL}/1_7b/tokenizer.json`, + tokenizerConfigPath: `${SMOLLM2_BASE_URL}/1_7b/tokenizer_config.json`, +}; const HAMMER2_1_BASE_URL = `${BASE_URL}-hammer-2.1/${VERSION_TAG}`; @@ -2296,23 +2302,26 @@ const QWEN2_5_3B_MLX_INT4: LLMModel = { tokenizerConfigPath: `${QWEN2_5_BASE_URL}/3b/tokenizer_config.json`, }; -// const GEMMA4_BASE_URL = `${BASE_URL}-gemma-4/${VERSION_TAG}`; - -// const GEMMA4_E2B_XNNPACK_8DA4W: LLMModel = { -// modelPath: `${GEMMA4_BASE_URL}/e2b/xnnpack/gemma_4_e2b_xnnpack_8da4w.pte`, -// tokenizerPath: `${GEMMA4_BASE_URL}/e2b/tokenizer.json`, -// tokenizerConfigPath: `${GEMMA4_BASE_URL}/e2b/tokenizer_config.json`, -// }; -// const GEMMA4_E2B_VULKAN_8DA4W: LLMModel = { -// modelPath: `${GEMMA4_BASE_URL}/e2b/vulkan/gemma_4_e2b_vulkan_8da4w.pte`, -// tokenizerPath: `${GEMMA4_BASE_URL}/e2b/tokenizer.json`, -// tokenizerConfigPath: `${GEMMA4_BASE_URL}/e2b/tokenizer_config.json`, -// }; -// const GEMMA4_E2B_MLX_INT4: LLMModel = { -// modelPath: `${GEMMA4_BASE_URL}/e2b/mlx/gemma4_e2b_mlx_int4.pte`, -// tokenizerPath: `${GEMMA4_BASE_URL}/e2b/tokenizer.json`, -// tokenizerConfigPath: `${GEMMA4_BASE_URL}/e2b/tokenizer_config.json`, -// }; +const GEMMA4_BASE_URL = `${BASE_URL}-gemma-4/${VERSION_TAG}`; + +// TODO: re-export for the TypeScript LLM runner; this build fails to load until then. +const GEMMA4_E2B_XNNPACK_8DA4W: LLMModel = { + modelPath: `${GEMMA4_BASE_URL}/e2b/xnnpack/gemma_4_e2b_xnnpack_8da4w.pte`, + tokenizerPath: `${GEMMA4_BASE_URL}/e2b/tokenizer.json`, + tokenizerConfigPath: `${GEMMA4_BASE_URL}/e2b/tokenizer_config.json`, +}; +// TODO: re-export for the TypeScript LLM runner; this build fails to load until then. +const GEMMA4_E2B_VULKAN_8DA4W: LLMModel = { + modelPath: `${GEMMA4_BASE_URL}/e2b/vulkan/gemma_4_e2b_vulkan_8da4w.pte`, + tokenizerPath: `${GEMMA4_BASE_URL}/e2b/tokenizer.json`, + tokenizerConfigPath: `${GEMMA4_BASE_URL}/e2b/tokenizer_config.json`, +}; +// TODO: re-export for the TypeScript LLM runner; this build fails to load until then. +const GEMMA4_E2B_MLX_INT4: LLMModel = { + modelPath: `${GEMMA4_BASE_URL}/e2b/mlx/gemma4_e2b_mlx_int4.pte`, + tokenizerPath: `${GEMMA4_BASE_URL}/e2b/tokenizer.json`, + tokenizerConfigPath: `${GEMMA4_BASE_URL}/e2b/tokenizer_config.json`, +}; const QWEN3_BASE_URL = `${BASE_URL}-qwen-3/${VERSION_TAG}`; @@ -3178,7 +3187,7 @@ export const models = { LFM2_5_VL_450M: variants({ XNNPACK_8DA4W: LFM2_5_VL_450M_XNNPACK_8DA4W, MLX_INT4: LFM2_5_VL_450M_MLX_INT4, - // VULKAN_8DA4W: LFM2_5_VL_450M_VULKAN_8DA4W, + VULKAN_8DA4W: LFM2_5_VL_450M_VULKAN_8DA4W, }), /** * Liquid AI LFM 2.5 VL 1.6B high-capacity vision-language model. Provides @@ -3188,9 +3197,9 @@ export const models = { */ LFM2_5_VL_1_6B: variants({ XNNPACK_8DA4W: LFM2_5_VL_1_6B_XNNPACK_8DA4W, - // VULKAN_8DA4W: LFM2_5_VL_1_6B_VULKAN_8DA4W, + VULKAN_8DA4W: LFM2_5_VL_1_6B_VULKAN_8DA4W, MLX_INT4: LFM2_5_VL_1_6B_MLX_INT4, - // MLX_INT8: LFM2_5_VL_1_6B_MLX_INT8, + MLX_INT8: LFM2_5_VL_1_6B_MLX_INT8, }), /** * Bielik v3 1.5B bilingual Polish & English language model, developed by @@ -3229,7 +3238,7 @@ export const models = { */ SMOLLM2_135M: variants({ XNNPACK_8DA8W: SMOLLM2_135M_8DA8W, - // MLX_INT8: SMOLLM2_135M_MLX_INT8, + MLX_INT8: SMOLLM2_135M_MLX_INT8, }), /** * Hugging Face SmolLM2 360M compact instruction-tuned model. Provides a @@ -3239,7 +3248,7 @@ export const models = { */ SMOLLM2_360M: variants({ XNNPACK_8DA8W: SMOLLM2_360M_8DA8W, - // MLX_INT8: SMOLLM2_360M_MLX_INT8, + MLX_INT8: SMOLLM2_360M_MLX_INT8, }), /** * Hugging Face SmolLM2 1.7B language model trained on curated educational, @@ -3249,7 +3258,7 @@ export const models = { */ SMOLLM2_1_7B: variants({ XNNPACK_8DA8W: SMOLLM2_1_7B_8DA8W, - // MLX_INT8: SMOLLM2_1_7B_MLX_INT8, + MLX_INT8: SMOLLM2_1_7B_MLX_INT8, }), /** * Hammer 2.1 0.5B specialized function-calling model. Fine-tuned @@ -3354,11 +3363,11 @@ export const models = { * following, creative text generation, and reasoning efficiency optimized * for mobile deployment. */ - // GEMMA4_E2B: variants({ - // XNNPACK_8DA4W: GEMMA4_E2B_XNNPACK_8DA4W, - // VULKAN_8DA4W: GEMMA4_E2B_VULKAN_8DA4W, - // MLX_INT4: GEMMA4_E2B_MLX_INT4, - // }), + GEMMA4_E2B: variants({ + XNNPACK_8DA4W: GEMMA4_E2B_XNNPACK_8DA4W, + VULKAN_8DA4W: GEMMA4_E2B_VULKAN_8DA4W, + MLX_INT4: GEMMA4_E2B_MLX_INT4, + }), }, /** * Text embedding models mapping sentences and documents into dense vector From f3c231b6202467e0dceb0fd4e77c3e4663366a23 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Fri, 9 Oct 2026 15:55:11 +0200 Subject: [PATCH 42/83] fix(models): point LLM tokenizers back at the shared repo-root files The Hammer 2.1, Qwen 2.5, Qwen 3 and SmolLM2 tokenizer URLs had been moved into per-size folders, which do not exist on the published repos (404). Each of those repos ships a single tokenizer at its root. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../react-native-executorch/src/models.ts | 96 +++++++++---------- 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index 52fe8831f7..07b3be7da3 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -2191,69 +2191,69 @@ const SMOLLM2_BASE_URL = `${BASE_URL}-smolLm-2/${VERSION_TAG}`; const SMOLLM2_135M_8DA8W: LLMModel = { modelPath: `${SCRATCH_LLMS_BASE_URL}/smollm2_135m_xnnpack_8da8w_16k.pte`, - tokenizerPath: `${SMOLLM2_BASE_URL}/135m/tokenizer.json`, - tokenizerConfigPath: `${SMOLLM2_BASE_URL}/135m/tokenizer_config.json`, + tokenizerPath: `${SMOLLM2_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${SMOLLM2_BASE_URL}/tokenizer_config.json`, }; const SMOLLM2_360M_8DA8W: LLMModel = { modelPath: `${SCRATCH_LLMS_BASE_URL}/smollm2_360m_xnnpack_8da8w_16k.pte`, - tokenizerPath: `${SMOLLM2_BASE_URL}/360m/tokenizer.json`, - tokenizerConfigPath: `${SMOLLM2_BASE_URL}/360m/tokenizer_config.json`, + tokenizerPath: `${SMOLLM2_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${SMOLLM2_BASE_URL}/tokenizer_config.json`, }; const SMOLLM2_1_7B_8DA8W: LLMModel = { modelPath: `${SCRATCH_LLMS_BASE_URL}/smollm2_1_7b_xnnpack_8da8w_2k.pte`, - tokenizerPath: `${SMOLLM2_BASE_URL}/1_7b/tokenizer.json`, - tokenizerConfigPath: `${SMOLLM2_BASE_URL}/1_7b/tokenizer_config.json`, + tokenizerPath: `${SMOLLM2_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${SMOLLM2_BASE_URL}/tokenizer_config.json`, }; // TODO: re-export for the TypeScript LLM runner; this build fails to load until then. const SMOLLM2_135M_MLX_INT8: LLMModel = { modelPath: `${SMOLLM2_BASE_URL}/135m/mlx/smollm2_135m_mlx_int8.pte`, - tokenizerPath: `${SMOLLM2_BASE_URL}/135m/tokenizer.json`, - tokenizerConfigPath: `${SMOLLM2_BASE_URL}/135m/tokenizer_config.json`, + tokenizerPath: `${SMOLLM2_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${SMOLLM2_BASE_URL}/tokenizer_config.json`, }; // TODO: re-export for the TypeScript LLM runner; this build fails to load until then. const SMOLLM2_360M_MLX_INT8: LLMModel = { modelPath: `${SMOLLM2_BASE_URL}/360m/mlx/smollm2_360m_mlx_int8.pte`, - tokenizerPath: `${SMOLLM2_BASE_URL}/360m/tokenizer.json`, - tokenizerConfigPath: `${SMOLLM2_BASE_URL}/360m/tokenizer_config.json`, + tokenizerPath: `${SMOLLM2_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${SMOLLM2_BASE_URL}/tokenizer_config.json`, }; // TODO: re-export for the TypeScript LLM runner; this build fails to load until then. const SMOLLM2_1_7B_MLX_INT8: LLMModel = { modelPath: `${SMOLLM2_BASE_URL}/1_7b/mlx/smollm2_1_7b_mlx_int8.pte`, - tokenizerPath: `${SMOLLM2_BASE_URL}/1_7b/tokenizer.json`, - tokenizerConfigPath: `${SMOLLM2_BASE_URL}/1_7b/tokenizer_config.json`, + tokenizerPath: `${SMOLLM2_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${SMOLLM2_BASE_URL}/tokenizer_config.json`, }; const HAMMER2_1_BASE_URL = `${BASE_URL}-hammer-2.1/${VERSION_TAG}`; const HAMMER2_1_0_5B_XNNPACK_8DA4W: LLMModel = { modelPath: `${SCRATCH_LLMS_BASE_URL}/hammer_2_1_0_5b_xnnpack_8da4w_16k.pte`, - tokenizerPath: `${HAMMER2_1_BASE_URL}/0_5b/tokenizer.json`, - tokenizerConfigPath: `${HAMMER2_1_BASE_URL}/0_5b/tokenizer_config.json`, + tokenizerPath: `${HAMMER2_1_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${HAMMER2_1_BASE_URL}/tokenizer_config.json`, }; const HAMMER2_1_1_5B_XNNPACK_8DA4W: LLMModel = { modelPath: `${SCRATCH_LLMS_BASE_URL}/hammer_2_1_1_5b_xnnpack_8da4w_16k.pte`, - tokenizerPath: `${HAMMER2_1_BASE_URL}/1_5b/tokenizer.json`, - tokenizerConfigPath: `${HAMMER2_1_BASE_URL}/1_5b/tokenizer_config.json`, + tokenizerPath: `${HAMMER2_1_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${HAMMER2_1_BASE_URL}/tokenizer_config.json`, }; const HAMMER2_1_3B_XNNPACK_8DA4W: LLMModel = { modelPath: `${SCRATCH_LLMS_BASE_URL}/hammer_2_1_3b_xnnpack_8da4w_8k.pte`, - tokenizerPath: `${HAMMER2_1_BASE_URL}/3b/tokenizer.json`, - tokenizerConfigPath: `${HAMMER2_1_BASE_URL}/3b/tokenizer_config.json`, + tokenizerPath: `${HAMMER2_1_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${HAMMER2_1_BASE_URL}/tokenizer_config.json`, }; const HAMMER2_1_0_5B_MLX_INT4: LLMModel = { modelPath: `${SCRATCH_LLMS_BASE_URL}/hammer_2_1_0_5b_mlx_4w_16k.pte`, - tokenizerPath: `${HAMMER2_1_BASE_URL}/0_5b/tokenizer.json`, - tokenizerConfigPath: `${HAMMER2_1_BASE_URL}/0_5b/tokenizer_config.json`, + tokenizerPath: `${HAMMER2_1_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${HAMMER2_1_BASE_URL}/tokenizer_config.json`, }; const HAMMER2_1_1_5B_MLX_INT4: LLMModel = { modelPath: `${SCRATCH_LLMS_BASE_URL}/hammer_2_1_1_5b_mlx_4w_16k.pte`, - tokenizerPath: `${HAMMER2_1_BASE_URL}/1_5b/tokenizer.json`, - tokenizerConfigPath: `${HAMMER2_1_BASE_URL}/1_5b/tokenizer_config.json`, + tokenizerPath: `${HAMMER2_1_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${HAMMER2_1_BASE_URL}/tokenizer_config.json`, }; const HAMMER2_1_3B_MLX_INT4: LLMModel = { modelPath: `${SCRATCH_LLMS_BASE_URL}/hammer_2_1_3b_mlx_4w_8k.pte`, - tokenizerPath: `${HAMMER2_1_BASE_URL}/3b/tokenizer.json`, - tokenizerConfigPath: `${HAMMER2_1_BASE_URL}/3b/tokenizer_config.json`, + tokenizerPath: `${HAMMER2_1_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${HAMMER2_1_BASE_URL}/tokenizer_config.json`, }; const PHI4_MINI_BASE_URL = `${BASE_URL}-phi-4-mini/${VERSION_TAG}`; @@ -2273,33 +2273,33 @@ const QWEN2_5_BASE_URL = `${BASE_URL}-qwen-2.5/${VERSION_TAG}`; const QWEN2_5_0_5B_XNNPACK_8DA4W: LLMModel = { modelPath: `${SCRATCH_LLMS_BASE_URL}/qwen_2_5_0_5b_xnnpack_8da4w_16k.pte`, - tokenizerPath: `${QWEN2_5_BASE_URL}/0_5b/tokenizer.json`, - tokenizerConfigPath: `${QWEN2_5_BASE_URL}/0_5b/tokenizer_config.json`, + tokenizerPath: `${QWEN2_5_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${QWEN2_5_BASE_URL}/tokenizer_config.json`, }; const QWEN2_5_1_5B_XNNPACK_8DA4W: LLMModel = { modelPath: `${SCRATCH_LLMS_BASE_URL}/qwen_2_5_1_5b_xnnpack_8da4w_16k.pte`, - tokenizerPath: `${QWEN2_5_BASE_URL}/1_5b/tokenizer.json`, - tokenizerConfigPath: `${QWEN2_5_BASE_URL}/1_5b/tokenizer_config.json`, + tokenizerPath: `${QWEN2_5_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${QWEN2_5_BASE_URL}/tokenizer_config.json`, }; const QWEN2_5_3B_XNNPACK_8DA4W: LLMModel = { modelPath: `${SCRATCH_LLMS_BASE_URL}/qwen_2_5_3b_xnnpack_8da4w_8k.pte`, - tokenizerPath: `${QWEN2_5_BASE_URL}/3b/tokenizer.json`, - tokenizerConfigPath: `${QWEN2_5_BASE_URL}/3b/tokenizer_config.json`, + tokenizerPath: `${QWEN2_5_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${QWEN2_5_BASE_URL}/tokenizer_config.json`, }; const QWEN2_5_0_5B_MLX_INT4: LLMModel = { modelPath: `${SCRATCH_LLMS_BASE_URL}/qwen_2_5_0_5b_mlx_4w_16k.pte`, - tokenizerPath: `${QWEN2_5_BASE_URL}/0_5b/tokenizer.json`, - tokenizerConfigPath: `${QWEN2_5_BASE_URL}/0_5b/tokenizer_config.json`, + tokenizerPath: `${QWEN2_5_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${QWEN2_5_BASE_URL}/tokenizer_config.json`, }; const QWEN2_5_1_5B_MLX_INT4: LLMModel = { modelPath: `${SCRATCH_LLMS_BASE_URL}/qwen_2_5_1_5b_mlx_4w_16k.pte`, - tokenizerPath: `${QWEN2_5_BASE_URL}/1_5b/tokenizer.json`, - tokenizerConfigPath: `${QWEN2_5_BASE_URL}/1_5b/tokenizer_config.json`, + tokenizerPath: `${QWEN2_5_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${QWEN2_5_BASE_URL}/tokenizer_config.json`, }; const QWEN2_5_3B_MLX_INT4: LLMModel = { modelPath: `${SCRATCH_LLMS_BASE_URL}/qwen_2_5_3b_mlx_4w_8k.pte`, - tokenizerPath: `${QWEN2_5_BASE_URL}/3b/tokenizer.json`, - tokenizerConfigPath: `${QWEN2_5_BASE_URL}/3b/tokenizer_config.json`, + tokenizerPath: `${QWEN2_5_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${QWEN2_5_BASE_URL}/tokenizer_config.json`, }; const GEMMA4_BASE_URL = `${BASE_URL}-gemma-4/${VERSION_TAG}`; @@ -2327,33 +2327,33 @@ const QWEN3_BASE_URL = `${BASE_URL}-qwen-3/${VERSION_TAG}`; const QWEN3_0_6B_XNNPACK_8DA4W: LLMModel = { modelPath: `${SCRATCH_LLMS_BASE_URL}/qwen3_0_6b_xnnpack_8da4w_8k.pte`, - tokenizerPath: `${QWEN3_BASE_URL}/0_6b/tokenizer.json`, - tokenizerConfigPath: `${QWEN3_BASE_URL}/0_6b/tokenizer_config.json`, + tokenizerPath: `${QWEN3_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${QWEN3_BASE_URL}/tokenizer_config.json`, }; const QWEN3_1_7B_XNNPACK_8DA4W: LLMModel = { modelPath: `${SCRATCH_LLMS_BASE_URL}/qwen3_1_7b_xnnpack_8da4w_4k.pte`, - tokenizerPath: `${QWEN3_BASE_URL}/1_7b/tokenizer.json`, - tokenizerConfigPath: `${QWEN3_BASE_URL}/1_7b/tokenizer_config.json`, + tokenizerPath: `${QWEN3_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${QWEN3_BASE_URL}/tokenizer_config.json`, }; const QWEN3_4B_XNNPACK_8DA4W: LLMModel = { modelPath: `${SCRATCH_LLMS_BASE_URL}/qwen3_4b_xnnpack_8da4w_4k.pte`, - tokenizerPath: `${QWEN3_BASE_URL}/4b/tokenizer.json`, - tokenizerConfigPath: `${QWEN3_BASE_URL}/4b/tokenizer_config.json`, + tokenizerPath: `${QWEN3_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${QWEN3_BASE_URL}/tokenizer_config.json`, }; const QWEN3_0_6B_MLX_INT4: LLMModel = { modelPath: `${SCRATCH_LLMS_BASE_URL}/qwen_3_0_6b_mlx_4w_8k.pte`, - tokenizerPath: `${QWEN3_BASE_URL}/0_6b/tokenizer.json`, - tokenizerConfigPath: `${QWEN3_BASE_URL}/0_6b/tokenizer_config.json`, + tokenizerPath: `${QWEN3_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${QWEN3_BASE_URL}/tokenizer_config.json`, }; const QWEN3_1_7B_MLX_INT4: LLMModel = { modelPath: `${SCRATCH_LLMS_BASE_URL}/qwen_3_1_7b_mlx_4w_4k.pte`, - tokenizerPath: `${QWEN3_BASE_URL}/1_7b/tokenizer.json`, - tokenizerConfigPath: `${QWEN3_BASE_URL}/1_7b/tokenizer_config.json`, + tokenizerPath: `${QWEN3_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${QWEN3_BASE_URL}/tokenizer_config.json`, }; const QWEN3_4B_MLX_INT4: LLMModel = { modelPath: `${SCRATCH_LLMS_BASE_URL}/qwen_3_4b_mlx_4w_4k.pte`, - tokenizerPath: `${QWEN3_BASE_URL}/4b/tokenizer.json`, - tokenizerConfigPath: `${QWEN3_BASE_URL}/4b/tokenizer_config.json`, + tokenizerPath: `${QWEN3_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${QWEN3_BASE_URL}/tokenizer_config.json`, }; /** From a34884f9037247c744128ef08d1eec4427fb7336 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Fri, 9 Oct 2026 20:24:00 +0200 Subject: [PATCH 43/83] style --- .../src/extensions/llm/runners/llmMultimodalRunner.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts b/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts index 7356c712c1..a09886585a 100644 --- a/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts @@ -34,12 +34,12 @@ export const LLM_MULTIMODAL_SPEC = { [f32(1, 'vocabSize')] ), ...method( - 'vision_encoder', // + 'vision_encoder', // prettier-ignore [f32(1, 3, 'imgH', 'imgW')], [f32(1, 'visualTokens', 'hiddenDim')] ), ...method( - 'token_embedding', // + 'token_embedding', // prettier-ignore [i64(1, Dyn('seqLen'))], [f32(1, Dyn('seqLen'), 'hiddenDim')] ), From d6f31193622ef37136c58f6e4682ff01a4b82e53 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Fri, 9 Oct 2026 23:50:28 +0200 Subject: [PATCH 44/83] feat(models): point the LLMs at the re-exported scratch builds Every LLM variant now resolves to a build that fits the TypeScript runner (get_model_schema, fp32 logits): the scratch files use the canonical react-native-executorch names, the MLX builds are re-exported with the SWM ExecuTorch fork (the PyPI wheel's MLX op numbering fails with InvalidProgram), and the Gemma 4 E2B, LFM 2.5 VL Vulkan/MLX int8 and SmolLM2 MLX variants that were waiting on a re-export are restored. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01XWvPGkm8P54hiXLs92s8nu --- .../react-native-executorch/src/models.ts | 91 +++++++++---------- 1 file changed, 41 insertions(+), 50 deletions(-) diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index 07b3be7da3..ba54cf27af 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -2072,22 +2072,22 @@ const LFM2_5_BASE_URL = `${BASE_URL}-lfm-2.5/${VERSION_TAG}`; const SCRATCH_LLMS_BASE_URL = 'https://huggingface.co/bhanc/scratch/resolve/main/llms'; const LFM2_5_1_2B_XNNPACK_8DA4W: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/lfm2_5_1_2b_xnnpack_8da4w_16k.pte`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/lfm_2_5_1_2b_xnnpack_8da4w.pte`, tokenizerPath: `${LFM2_5_BASE_URL}/1_2b/tokenizer.json`, tokenizerConfigPath: `${LFM2_5_BASE_URL}/1_2b/tokenizer_config.json`, }; const LFM2_5_1_2B_MLX_INT4: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/lfm_2_5_1_2b_mlx_int4_16k.pte`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/lfm_2_5_1_2b_mlx_int4.pte`, tokenizerPath: `${LFM2_5_BASE_URL}/1_2b/tokenizer.json`, tokenizerConfigPath: `${LFM2_5_BASE_URL}/1_2b/tokenizer_config.json`, }; const LFM2_5_350M_XNNPACK_8DA4W: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/lfm_2_5_350m_xnnpack_8da4w_16k.pte`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/lfm_2_5_350m_xnnpack_8da4w.pte`, tokenizerPath: `${LFM2_5_BASE_URL}/350m/tokenizer.json`, tokenizerConfigPath: `${LFM2_5_BASE_URL}/350m/tokenizer_config.json`, }; const LFM2_5_350M_MLX_INT4: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/lfm_2_5_350m_mlx_int4_16k.pte`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/lfm_2_5_350m_mlx_int4.pte`, tokenizerPath: `${LFM2_5_BASE_URL}/350m/tokenizer.json`, tokenizerConfigPath: `${LFM2_5_BASE_URL}/350m/tokenizer_config.json`, }; @@ -2124,17 +2124,15 @@ const LFM2_5_VL_1_6B_XNNPACK_8DA4W: LLMModel = { modalities: ['image'], preprocessorConfig: LFM2_5_VL_PREPROCESSOR_CONFIG, }; -// TODO: re-export for the TypeScript LLM runner; this build fails to load until then. const LFM2_5_VL_450M_VULKAN_8DA4W: LLMModel = { - modelPath: `${LFM2_5_BASE_URL}/vl_450m/vulkan/lfm_2_5_vl_450m_vulkan_8da4w.pte`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/lfm_2_5_vl_450m_vulkan_8da4w.pte`, tokenizerPath: `${LFM2_5_BASE_URL}/vl_450m/tokenizer.json`, tokenizerConfigPath: `${LFM2_5_BASE_URL}/vl_450m/tokenizer_config.json`, modalities: ['image'], preprocessorConfig: LFM2_5_VL_PREPROCESSOR_CONFIG, }; -// TODO: re-export for the TypeScript LLM runner; this build fails to load until then. const LFM2_5_VL_1_6B_VULKAN_8DA4W: LLMModel = { - modelPath: `${LFM2_5_BASE_URL}/vl_1_6b/vulkan/lfm_2_5_vl_1_6b_vulkan_8da4w.pte`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/lfm_2_5_vl_1_6b_vulkan_8da4w.pte`, tokenizerPath: `${LFM2_5_BASE_URL}/vl_1_6b/tokenizer.json`, tokenizerConfigPath: `${LFM2_5_BASE_URL}/vl_1_6b/tokenizer_config.json`, modalities: ['image'], @@ -2147,9 +2145,8 @@ const LFM2_5_VL_1_6B_MLX_INT4: LLMModel = { modalities: ['image'], preprocessorConfig: LFM2_5_VL_PREPROCESSOR_CONFIG, }; -// TODO: re-export for the TypeScript LLM runner; this build fails to load until then. const LFM2_5_VL_1_6B_MLX_INT8: LLMModel = { - modelPath: `${LFM2_5_BASE_URL}/vl_1_6b/mlx/lfm_2_5_vl_1_6b_mlx_int8.pte`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/lfm_2_5_vl_1_6b_mlx_int8.pte`, tokenizerPath: `${LFM2_5_BASE_URL}/vl_1_6b/tokenizer.json`, tokenizerConfigPath: `${LFM2_5_BASE_URL}/vl_1_6b/tokenizer_config.json`, modalities: ['image'], @@ -2159,7 +2156,7 @@ const LFM2_5_VL_1_6B_MLX_INT8: LLMModel = { const BIELIK_V3_1_5B_BASE_URL = `${BASE_URL}-bielik-v3.0/${VERSION_TAG}`; const BIELIK_V3_1_5B_XNNPACK_8DA4W: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/bielik_v3_0_1_5b_xnnpack_8da4w_8k.pte`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/bielik_v3_0_1_5b_xnnpack_8da4w.pte`, tokenizerPath: `${BIELIK_V3_1_5B_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${BIELIK_V3_1_5B_BASE_URL}/tokenizer_config.json`, }; @@ -2167,22 +2164,22 @@ const BIELIK_V3_1_5B_XNNPACK_8DA4W: LLMModel = { const LLAMA3_2_BASE_URL = `${BASE_URL}-llama-3.2/${VERSION_TAG}`; const LLAMA3_2_3B_SPINQUANT: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/llama_3_2_3b_xnnpack_spinquant_4k.pte`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/llama_3_2_3b_xnnpack_spinquant.pte`, tokenizerPath: `${LLAMA3_2_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${LLAMA3_2_BASE_URL}/tokenizer_config.json`, }; const LLAMA3_2_1B_SPINQUANT: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/llama_3_2_1b_xnnpack_spinquant_16k.pte`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/llama_3_2_1b_xnnpack_spinquant.pte`, tokenizerPath: `${LLAMA3_2_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${LLAMA3_2_BASE_URL}/tokenizer_config.json`, }; const LLAMA3_2_1B_MLX_INT4: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/llama_3_2_1b_mlx_4w_16k.pte`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/llama_3_2_1b_mlx_int4.pte`, tokenizerPath: `${LLAMA3_2_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${LLAMA3_2_BASE_URL}/tokenizer_config.json`, }; const LLAMA3_2_3B_MLX_INT4: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/llama_3_2_3b_mlx_4w_4k.pte`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/llama_3_2_3b_mlx_int4.pte`, tokenizerPath: `${LLAMA3_2_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${LLAMA3_2_BASE_URL}/tokenizer_config.json`, }; @@ -2190,35 +2187,32 @@ const LLAMA3_2_3B_MLX_INT4: LLMModel = { const SMOLLM2_BASE_URL = `${BASE_URL}-smolLm-2/${VERSION_TAG}`; const SMOLLM2_135M_8DA8W: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/smollm2_135m_xnnpack_8da8w_16k.pte`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/smollm2_135m_xnnpack_8da8w.pte`, tokenizerPath: `${SMOLLM2_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${SMOLLM2_BASE_URL}/tokenizer_config.json`, }; const SMOLLM2_360M_8DA8W: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/smollm2_360m_xnnpack_8da8w_16k.pte`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/smollm2_360m_xnnpack_8da8w.pte`, tokenizerPath: `${SMOLLM2_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${SMOLLM2_BASE_URL}/tokenizer_config.json`, }; const SMOLLM2_1_7B_8DA8W: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/smollm2_1_7b_xnnpack_8da8w_2k.pte`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/smollm2_1_7b_xnnpack_8da8w.pte`, tokenizerPath: `${SMOLLM2_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${SMOLLM2_BASE_URL}/tokenizer_config.json`, }; -// TODO: re-export for the TypeScript LLM runner; this build fails to load until then. const SMOLLM2_135M_MLX_INT8: LLMModel = { - modelPath: `${SMOLLM2_BASE_URL}/135m/mlx/smollm2_135m_mlx_int8.pte`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/smollm2_135m_mlx_int8.pte`, tokenizerPath: `${SMOLLM2_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${SMOLLM2_BASE_URL}/tokenizer_config.json`, }; -// TODO: re-export for the TypeScript LLM runner; this build fails to load until then. const SMOLLM2_360M_MLX_INT8: LLMModel = { - modelPath: `${SMOLLM2_BASE_URL}/360m/mlx/smollm2_360m_mlx_int8.pte`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/smollm2_360m_mlx_int8.pte`, tokenizerPath: `${SMOLLM2_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${SMOLLM2_BASE_URL}/tokenizer_config.json`, }; -// TODO: re-export for the TypeScript LLM runner; this build fails to load until then. const SMOLLM2_1_7B_MLX_INT8: LLMModel = { - modelPath: `${SMOLLM2_BASE_URL}/1_7b/mlx/smollm2_1_7b_mlx_int8.pte`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/smollm2_1_7b_mlx_int8.pte`, tokenizerPath: `${SMOLLM2_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${SMOLLM2_BASE_URL}/tokenizer_config.json`, }; @@ -2226,32 +2220,32 @@ const SMOLLM2_1_7B_MLX_INT8: LLMModel = { const HAMMER2_1_BASE_URL = `${BASE_URL}-hammer-2.1/${VERSION_TAG}`; const HAMMER2_1_0_5B_XNNPACK_8DA4W: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/hammer_2_1_0_5b_xnnpack_8da4w_16k.pte`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/hammer_2_1_0_5b_xnnpack_8da4w.pte`, tokenizerPath: `${HAMMER2_1_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${HAMMER2_1_BASE_URL}/tokenizer_config.json`, }; const HAMMER2_1_1_5B_XNNPACK_8DA4W: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/hammer_2_1_1_5b_xnnpack_8da4w_16k.pte`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/hammer_2_1_1_5b_xnnpack_8da4w.pte`, tokenizerPath: `${HAMMER2_1_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${HAMMER2_1_BASE_URL}/tokenizer_config.json`, }; const HAMMER2_1_3B_XNNPACK_8DA4W: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/hammer_2_1_3b_xnnpack_8da4w_8k.pte`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/hammer_2_1_3b_xnnpack_8da4w.pte`, tokenizerPath: `${HAMMER2_1_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${HAMMER2_1_BASE_URL}/tokenizer_config.json`, }; const HAMMER2_1_0_5B_MLX_INT4: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/hammer_2_1_0_5b_mlx_4w_16k.pte`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/hammer_2_1_0_5b_mlx_int4.pte`, tokenizerPath: `${HAMMER2_1_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${HAMMER2_1_BASE_URL}/tokenizer_config.json`, }; const HAMMER2_1_1_5B_MLX_INT4: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/hammer_2_1_1_5b_mlx_4w_16k.pte`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/hammer_2_1_1_5b_mlx_int4.pte`, tokenizerPath: `${HAMMER2_1_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${HAMMER2_1_BASE_URL}/tokenizer_config.json`, }; const HAMMER2_1_3B_MLX_INT4: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/hammer_2_1_3b_mlx_4w_8k.pte`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/hammer_2_1_3b_mlx_int4.pte`, tokenizerPath: `${HAMMER2_1_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${HAMMER2_1_BASE_URL}/tokenizer_config.json`, }; @@ -2259,12 +2253,12 @@ const HAMMER2_1_3B_MLX_INT4: LLMModel = { const PHI4_MINI_BASE_URL = `${BASE_URL}-phi-4-mini/${VERSION_TAG}`; const PHI4_MINI_XNNPACK_8DA4W: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/phi_4_mini_3_8b_xnnpack_8da4w_4k.pte`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/phi_4_mini_xnnpack_8da4w.pte`, tokenizerPath: `${PHI4_MINI_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${PHI4_MINI_BASE_URL}/tokenizer_config.json`, }; const PHI4_MINI_MLX_INT4: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/phi_4_mini_3_8b_mlx_4w_4k.pte`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/phi_4_mini_mlx_int4.pte`, tokenizerPath: `${PHI4_MINI_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${PHI4_MINI_BASE_URL}/tokenizer_config.json`, }; @@ -2272,53 +2266,50 @@ const PHI4_MINI_MLX_INT4: LLMModel = { const QWEN2_5_BASE_URL = `${BASE_URL}-qwen-2.5/${VERSION_TAG}`; const QWEN2_5_0_5B_XNNPACK_8DA4W: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/qwen_2_5_0_5b_xnnpack_8da4w_16k.pte`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/qwen_2_5_0_5b_xnnpack_8da4w.pte`, tokenizerPath: `${QWEN2_5_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${QWEN2_5_BASE_URL}/tokenizer_config.json`, }; const QWEN2_5_1_5B_XNNPACK_8DA4W: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/qwen_2_5_1_5b_xnnpack_8da4w_16k.pte`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/qwen_2_5_1_5b_xnnpack_8da4w.pte`, tokenizerPath: `${QWEN2_5_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${QWEN2_5_BASE_URL}/tokenizer_config.json`, }; const QWEN2_5_3B_XNNPACK_8DA4W: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/qwen_2_5_3b_xnnpack_8da4w_8k.pte`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/qwen_2_5_3b_xnnpack_8da4w.pte`, tokenizerPath: `${QWEN2_5_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${QWEN2_5_BASE_URL}/tokenizer_config.json`, }; const QWEN2_5_0_5B_MLX_INT4: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/qwen_2_5_0_5b_mlx_4w_16k.pte`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/qwen_2_5_0_5b_mlx_int4.pte`, tokenizerPath: `${QWEN2_5_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${QWEN2_5_BASE_URL}/tokenizer_config.json`, }; const QWEN2_5_1_5B_MLX_INT4: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/qwen_2_5_1_5b_mlx_4w_16k.pte`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/qwen_2_5_1_5b_mlx_int4.pte`, tokenizerPath: `${QWEN2_5_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${QWEN2_5_BASE_URL}/tokenizer_config.json`, }; const QWEN2_5_3B_MLX_INT4: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/qwen_2_5_3b_mlx_4w_8k.pte`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/qwen_2_5_3b_mlx_int4.pte`, tokenizerPath: `${QWEN2_5_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${QWEN2_5_BASE_URL}/tokenizer_config.json`, }; const GEMMA4_BASE_URL = `${BASE_URL}-gemma-4/${VERSION_TAG}`; -// TODO: re-export for the TypeScript LLM runner; this build fails to load until then. const GEMMA4_E2B_XNNPACK_8DA4W: LLMModel = { - modelPath: `${GEMMA4_BASE_URL}/e2b/xnnpack/gemma_4_e2b_xnnpack_8da4w.pte`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/gemma_4_e2b_xnnpack_8da4w.pte`, tokenizerPath: `${GEMMA4_BASE_URL}/e2b/tokenizer.json`, tokenizerConfigPath: `${GEMMA4_BASE_URL}/e2b/tokenizer_config.json`, }; -// TODO: re-export for the TypeScript LLM runner; this build fails to load until then. const GEMMA4_E2B_VULKAN_8DA4W: LLMModel = { - modelPath: `${GEMMA4_BASE_URL}/e2b/vulkan/gemma_4_e2b_vulkan_8da4w.pte`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/gemma_4_e2b_vulkan_8da4w.pte`, tokenizerPath: `${GEMMA4_BASE_URL}/e2b/tokenizer.json`, tokenizerConfigPath: `${GEMMA4_BASE_URL}/e2b/tokenizer_config.json`, }; -// TODO: re-export for the TypeScript LLM runner; this build fails to load until then. const GEMMA4_E2B_MLX_INT4: LLMModel = { - modelPath: `${GEMMA4_BASE_URL}/e2b/mlx/gemma4_e2b_mlx_int4.pte`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/gemma4_e2b_mlx_int4.pte`, tokenizerPath: `${GEMMA4_BASE_URL}/e2b/tokenizer.json`, tokenizerConfigPath: `${GEMMA4_BASE_URL}/e2b/tokenizer_config.json`, }; @@ -2326,32 +2317,32 @@ const GEMMA4_E2B_MLX_INT4: LLMModel = { const QWEN3_BASE_URL = `${BASE_URL}-qwen-3/${VERSION_TAG}`; const QWEN3_0_6B_XNNPACK_8DA4W: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/qwen3_0_6b_xnnpack_8da4w_8k.pte`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/qwen_3_0_6b_xnnpack_8da4w.pte`, tokenizerPath: `${QWEN3_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${QWEN3_BASE_URL}/tokenizer_config.json`, }; const QWEN3_1_7B_XNNPACK_8DA4W: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/qwen3_1_7b_xnnpack_8da4w_4k.pte`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/qwen_3_1_7b_xnnpack_8da4w.pte`, tokenizerPath: `${QWEN3_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${QWEN3_BASE_URL}/tokenizer_config.json`, }; const QWEN3_4B_XNNPACK_8DA4W: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/qwen3_4b_xnnpack_8da4w_4k.pte`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/qwen_3_4b_xnnpack_8da4w.pte`, tokenizerPath: `${QWEN3_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${QWEN3_BASE_URL}/tokenizer_config.json`, }; const QWEN3_0_6B_MLX_INT4: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/qwen_3_0_6b_mlx_4w_8k.pte`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/qwen_3_0_6b_mlx_int4.pte`, tokenizerPath: `${QWEN3_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${QWEN3_BASE_URL}/tokenizer_config.json`, }; const QWEN3_1_7B_MLX_INT4: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/qwen_3_1_7b_mlx_4w_4k.pte`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/qwen_3_1_7b_mlx_int4.pte`, tokenizerPath: `${QWEN3_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${QWEN3_BASE_URL}/tokenizer_config.json`, }; const QWEN3_4B_MLX_INT4: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/qwen_3_4b_mlx_4w_4k.pte`, + modelPath: `${SCRATCH_LLMS_BASE_URL}/qwen_3_4b_mlx_int4.pte`, tokenizerPath: `${QWEN3_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${QWEN3_BASE_URL}/tokenizer_config.json`, }; From 7b2c7c81e7127ebdc9f1ec16c71827a164adf66b Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Fri, 9 Oct 2026 23:50:28 +0200 Subject: [PATCH 45/83] feat(nlp-app): list every LLM variant on the LLM screen Add the SmolLM2 MLX, LFM 2.5 VL 1.6B MLX int8, LFM 2.5 VL Vulkan and Gemma 4 E2B (XNNPACK, MLX, Vulkan) entries, and offer the Vulkan backend on Android. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01XWvPGkm8P54hiXLs92s8nu --- apps/nlp/app/llm/index.tsx | 6 +- apps/nlp/constants/llm.ts | 136 ++++++++++++++++++++++++++----------- 2 files changed, 98 insertions(+), 44 deletions(-) diff --git a/apps/nlp/app/llm/index.tsx b/apps/nlp/app/llm/index.tsx index b03dd29f84..cab66b496e 100644 --- a/apps/nlp/app/llm/index.tsx +++ b/apps/nlp/app/llm/index.tsx @@ -49,9 +49,9 @@ function LLMContent() { if (Platform.OS === 'ios') { list.push({ label: 'MLX (Apple GPU)', value: 'mlx' }); } - // if (Platform.OS === 'android') { - // list.push({ label: 'Vulkan', value: 'vulkan' }); - // } + if (Platform.OS === 'android') { + list.push({ label: 'Vulkan (Android GPU)', value: 'vulkan' }); + } return list; }, []); diff --git a/apps/nlp/constants/llm.ts b/apps/nlp/constants/llm.ts index 10ab22bc82..e07ee0d63c 100644 --- a/apps/nlp/constants/llm.ts +++ b/apps/nlp/constants/llm.ts @@ -416,49 +416,70 @@ export const LLM_MODELS: LLMModelConfig[] = [ toolOpts: undefined, iosOnly: true, }, - // { - // id: 'lfm-2.5-vl-450m-vulkan', - // name: 'LFM 2.5 VL 450M', - // backend: 'vulkan', - // model: models.llm.LFM2_5_VL_450M.VULKAN_8DA4W, - // generationConfig: DEFAULT_GENERATION_CONFIG, - // systemPrompt: DEFAULT_SYSTEM_PROMPT, - // stopRegex: undefined, - // toolOpts: undefined, - // }, - // { - // id: 'lfm-2.5-vl-1.6b-vulkan', - // name: 'LFM 2.5 VL 1.6B', - // backend: 'vulkan', - // model: models.llm.LFM2_5_VL_1_6B.VULKAN_8DA4W, - // generationConfig: DEFAULT_GENERATION_CONFIG, - // systemPrompt: DEFAULT_SYSTEM_PROMPT, - // stopRegex: undefined, - // toolOpts: undefined, - // }, + { + id: 'lfm-2.5-vl-1.6b-mlx-int8', + name: 'LFM 2.5 VL 1.6B (int8)', + backend: 'mlx', + model: models.llm.LFM2_5_VL_1_6B.MLX_INT8, + generationConfig: DEFAULT_GENERATION_CONFIG, + systemPrompt: DEFAULT_SYSTEM_PROMPT, + stopRegex: undefined, + toolOpts: undefined, + iosOnly: true, + }, + { + id: 'lfm-2.5-vl-450m-vulkan', + name: 'LFM 2.5 VL 450M', + backend: 'vulkan', + model: models.llm.LFM2_5_VL_450M.VULKAN_8DA4W, + generationConfig: DEFAULT_GENERATION_CONFIG, + systemPrompt: DEFAULT_SYSTEM_PROMPT, + stopRegex: undefined, + toolOpts: undefined, + }, + { + id: 'lfm-2.5-vl-1.6b-vulkan', + name: 'LFM 2.5 VL 1.6B', + backend: 'vulkan', + model: models.llm.LFM2_5_VL_1_6B.VULKAN_8DA4W, + generationConfig: DEFAULT_GENERATION_CONFIG, + systemPrompt: DEFAULT_SYSTEM_PROMPT, + stopRegex: undefined, + toolOpts: undefined, + }, // Gemma 4 family - // { - // id: 'gemma-4-e2b-xnnpack', - // name: 'Gemma 4 E2B', - // backend: 'xnnpack', - // model: models.llm.GEMMA4_E2B.XNNPACK_8DA4W, - // generationConfig: DEFAULT_GENERATION_CONFIG, - // systemPrompt: DEFAULT_SYSTEM_PROMPT, - // stopRegex: GEMMA_TOOL_STOP_REGEX, - // toolOpts: GEMMA_TOOL_OPTS, - // }, - // { - // id: 'gemma-4-e2b-mlx', - // name: 'Gemma 4 E2B', - // backend: 'mlx', - // model: models.llm.GEMMA4_E2B.MLX_INT4, - // generationConfig: DEFAULT_GENERATION_CONFIG, - // systemPrompt: DEFAULT_SYSTEM_PROMPT, - // stopRegex: GEMMA_TOOL_STOP_REGEX, - // toolOpts: GEMMA_TOOL_OPTS, - // iosOnly: true, - // }, + { + id: 'gemma-4-e2b-xnnpack', + name: 'Gemma 4 E2B', + backend: 'xnnpack', + model: models.llm.GEMMA4_E2B.XNNPACK_8DA4W, + generationConfig: DEFAULT_GENERATION_CONFIG, + systemPrompt: DEFAULT_SYSTEM_PROMPT, + stopRegex: GEMMA_TOOL_STOP_REGEX, + toolOpts: GEMMA_TOOL_OPTS, + }, + { + id: 'gemma-4-e2b-mlx', + name: 'Gemma 4 E2B', + backend: 'mlx', + model: models.llm.GEMMA4_E2B.MLX_INT4, + generationConfig: DEFAULT_GENERATION_CONFIG, + systemPrompt: DEFAULT_SYSTEM_PROMPT, + stopRegex: GEMMA_TOOL_STOP_REGEX, + toolOpts: GEMMA_TOOL_OPTS, + iosOnly: true, + }, + { + id: 'gemma-4-e2b-vulkan', + name: 'Gemma 4 E2B', + backend: 'vulkan', + model: models.llm.GEMMA4_E2B.VULKAN_8DA4W, + generationConfig: DEFAULT_GENERATION_CONFIG, + systemPrompt: DEFAULT_SYSTEM_PROMPT, + stopRegex: GEMMA_TOOL_STOP_REGEX, + toolOpts: GEMMA_TOOL_OPTS, + }, // Hammer 2.1 family { @@ -580,6 +601,17 @@ export const LLM_MODELS: LLMModelConfig[] = [ stopRegex: undefined, toolOpts: undefined, }, + { + id: 'smollm2-135m-mlx', + name: 'SmolLM2 135M', + backend: 'mlx', + model: models.llm.SMOLLM2_135M.MLX_INT8, + generationConfig: DEFAULT_GENERATION_CONFIG, + systemPrompt: DEFAULT_SYSTEM_PROMPT, + stopRegex: undefined, + toolOpts: undefined, + iosOnly: true, + }, { id: 'smollm2-360m-xnnpack', name: 'SmolLM2 360M', @@ -590,6 +622,17 @@ export const LLM_MODELS: LLMModelConfig[] = [ stopRegex: undefined, toolOpts: undefined, }, + { + id: 'smollm2-360m-mlx', + name: 'SmolLM2 360M', + backend: 'mlx', + model: models.llm.SMOLLM2_360M.MLX_INT8, + generationConfig: DEFAULT_GENERATION_CONFIG, + systemPrompt: DEFAULT_SYSTEM_PROMPT, + stopRegex: undefined, + toolOpts: undefined, + iosOnly: true, + }, { id: 'smollm2-1.7b-xnnpack', name: 'SmolLM2 1.7B', @@ -600,6 +643,17 @@ export const LLM_MODELS: LLMModelConfig[] = [ stopRegex: undefined, toolOpts: undefined, }, + { + id: 'smollm2-1.7b-mlx', + name: 'SmolLM2 1.7B', + backend: 'mlx', + model: models.llm.SMOLLM2_1_7B.MLX_INT8, + generationConfig: DEFAULT_GENERATION_CONFIG, + systemPrompt: DEFAULT_SYSTEM_PROMPT, + stopRegex: undefined, + toolOpts: undefined, + iosOnly: true, + }, // Qwen 2.5 family { From bfc91c1b3a633be6d6a2b9f8222275be467f053b Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Sat, 10 Oct 2026 00:11:28 +0200 Subject: [PATCH 46/83] fix --- .../src/extensions/llm/runners/llmTextRunner.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/react-native-executorch/src/extensions/llm/runners/llmTextRunner.ts b/packages/react-native-executorch/src/extensions/llm/runners/llmTextRunner.ts index 311e6b1fd8..edd79ad0c4 100644 --- a/packages/react-native-executorch/src/extensions/llm/runners/llmTextRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/runners/llmTextRunner.ts @@ -190,7 +190,7 @@ export function createLLMTextRunner( const genTokArray = Int32Array.from(generatedTokens); const nextToken = sample(tLogits, { generatedTokens: genTokArray }, config); - onToken?.(tokenizer.decode(Int32Array.of(nextToken))); + onToken?.(tokenizer.decode(Int32Array.of(nextToken), /* skipSpecialTokens= */ false)); generatedTokens.push(nextToken); if (!config?.ignoreEos && eosIds.includes(nextToken)) break; From 3f5196c05a01fdc0925c01298dd36c668dbcc7a0 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Sat, 10 Oct 2026 00:24:33 +0200 Subject: [PATCH 47/83] fix --- packages/react-native-executorch/cpp/core/model.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/react-native-executorch/cpp/core/model.cpp b/packages/react-native-executorch/cpp/core/model.cpp index 4e113c9d7f..49c8786831 100644 --- a/packages/react-native-executorch/cpp/core/model.cpp +++ b/packages/react-native-executorch/cpp/core/model.cpp @@ -168,7 +168,7 @@ jsi::Value ModelHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) { switch (tag) { case executorch::runtime::Tag::Tensor: { const auto &tSpec = methodSpec.inputs[i]; - auto tensorHostObject = tensor::fromJs(rt, ctx, val, tSpec.dtype, std::nullopt); // TODO(bh): tSpec.shape); + auto tensorHostObject = tensor::fromJs(rt, ctx, val, tSpec.dtype, tSpec.shape); if (!lockedTensors.insert(tensorHostObject.get()).second) { throw error::InvalidArgument("execute: Tensor aliasing detected. " From ea5b543aaf0bf7c30cedcf7344209ff3c8118c18 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Sat, 10 Oct 2026 02:32:24 +0200 Subject: [PATCH 48/83] update --- .../src/extensions/llm/llmRunner.ts | 4 + .../extensions/llm/runners/llmGemmaRunner.ts | 433 +++++++++++++++++- .../extensions/llm/utils/chatPreprocessor.ts | 6 +- 3 files changed, 438 insertions(+), 5 deletions(-) diff --git a/packages/react-native-executorch/src/extensions/llm/llmRunner.ts b/packages/react-native-executorch/src/extensions/llm/llmRunner.ts index ff63fa7bd5..7eb8a51fba 100644 --- a/packages/react-native-executorch/src/extensions/llm/llmRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/llmRunner.ts @@ -20,6 +20,7 @@ import { loadTokenizer } from '../nlp'; import type { SamplingConfig } from './sampler'; import { createLLMTextRunner, LLM_TEXT_SPEC } from './runners/llmTextRunner'; import { createLLMMultimodalRunner, LLM_MULTIMODAL_SPEC } from './runners/llmMultimodalRunner'; +import { createLLMGemmaRunner, LLM_GEMMA_SPEC } from './runners/llmGemmaRunner'; declare const llmRunnerBrand: unique symbol; @@ -190,6 +191,7 @@ export async function createLLMRunner( const { variant } = validateSpec(model.schema, { text: LLM_TEXT_SPEC, vision: LLM_MULTIMODAL_SPEC, + gemma: LLM_GEMMA_SPEC, }); switch (variant) { @@ -197,6 +199,8 @@ export async function createLLMRunner( return createLLMTextRunner(model, tokenizer, modalities); case 'vision': return createLLMMultimodalRunner(model, tokenizer, modalities); + case 'gemma': + return createLLMGemmaRunner(model, tokenizer, modalities); default: throw RnExecuTorchError('LOAD_FAILED', `llmRunner: unrecognized variant ${variant}`); } diff --git a/packages/react-native-executorch/src/extensions/llm/runners/llmGemmaRunner.ts b/packages/react-native-executorch/src/extensions/llm/runners/llmGemmaRunner.ts index 01f2930600..3a13b46e6b 100644 --- a/packages/react-native-executorch/src/extensions/llm/runners/llmGemmaRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/runners/llmGemmaRunner.ts @@ -1,7 +1,434 @@ +import { createSynchronizable } from 'react-native-worklets'; + +import { tensor } from '../../../core/tensor'; import type { Model } from '../../../core/model'; +import { RnExecuTorchError } from '../../../core/error'; +import { f32, i64, method, DynamicDim as Dyn, validateSpec } from '../../../core/schema'; + import type { Tokenizer } from '../../nlp'; -import type { LLMRunner } from '../llmRunner'; +import { sample } from '../sampler'; + +import type { + LLMGenerationConfig, + LLMGenerationStats, + LLMKVCacheState, + LLMPrefillStats, + LLMRunner, + MediaInput, + Modality, + Prompt, +} from '../llmRunner'; + +const METADATA_SPEC = { + ...method('get_max_seq_len', [], [{ kind: 'Int' }]), + ...method('get_max_context_len', [], [{ kind: 'Int' }]), + ...method('get_vocab_size', [], [{ kind: 'Int' }]), + ...method('use_kv_cache', [], [{ kind: 'Bool' }]), + ...method('enable_dynamic_shape', [], [{ kind: 'Bool' }]), + // TODO: add `has_ple` metadata +}; + +export const LLM_GEMMA_SPEC = { + ...method( + 'token_embedding', + [i64(1, Dyn('seqLen'))], + [f32(1, Dyn('seqLen'), 'hiddenDim'), f32(1, Dyn('seqLen'), 'numLayers', 'pleDim')] + ), + ...method( + 'text_decoder', + [ + f32(1, Dyn('seqLen'), 'hiddenDim'), + f32(1, Dyn('seqLen'), 'numLayers', 'pleDim'), + i64(Dyn('seqLen')), + ], + [f32(1, 1, 'vocabSize')] + ), + ...method( + 'vision_encoder', // prettier-ignore + [f32(1, 3, 'imgH', 'imgW')], + [f32(1, 'visualTokens', 'hiddenDim')] + ), + ...method( + 'audio_encoder', + [f32(1, Dyn('audioSamples')), i64()], + [f32(1, Dyn('audioTokens'), 'hiddenDim'), f32(1, Dyn('audioTokens'))] + ), + ...METADATA_SPEC, +}; + +const AUDIO_SAMPLES_PER_BLOCK = 7680; + +export function createLLMGemmaRunner( + model: Model, + tokenizer: Tokenizer, + modalities?: readonly Modality[] +): LLMRunner { + const { dims, dim } = validateSpec(model.schema, { default: LLM_GEMMA_SPEC }); + + const [imgH, imgW, vocabSize] = dims.constant('imgH', 'imgW', 'vocabSize'); + const [hiddenDim, visualTokens] = dims.constant('hiddenDim', 'visualTokens'); + const [numLayers, pleDim] = dims.constant('numLayers', 'pleDim'); + const maxSeqLen = dim('seqLen', 'range').max; + const imgShape = [1, 3, imgH, imgW] as const; + + const [maxContextLen] = model.execute('get_max_context_len', [], []) as [number]; + const eosIds = model.execute('get_eos_ids', [], []) as number[]; + + // TODO(bh): validate modalities + + if (maxContextLen < maxSeqLen || !Number.isInteger(maxContextLen)) { + throw RnExecuTorchError( + 'SCHEMA_MISMATCH', + `maxContextLen (${maxContextLen}) must be an integer >= maxSeqLen (${maxSeqLen}).` + ); + } + + if (model.execute('use_kv_cache', [], [])[0] !== true) { + throw RnExecuTorchError('SCHEMA_MISMATCH', 'Model must enable use_kv_cache.'); + } + + // Output buffer for text_decoder logits: shape [1, 1, vocabSize] + const tLogits = tensor('float32', [1, 1, vocabSize]); + + const dispose = () => { + model.dispose(); + tLogits.dispose(); + tokenizer.dispose(); + }; + + // ====================================================== + // KV Cache management + // ====================================================== + const pos = createSynchronizable(0); + + const reset = (targetPos?: number): void => { + // TODO(bh): should be curPos not max ctx len + if (targetPos !== undefined && (targetPos < 0 || targetPos >= maxContextLen)) { + throw RnExecuTorchError( + 'INVALID_ARGUMENT', + `targetPos (${targetPos}) must be in range [0, ${maxContextLen}).` + ); + } + pos.setBlocking(targetPos ?? 0); + }; + + const getKVCacheState = (): LLMKVCacheState => { + const curPos = pos.getBlocking(); + return { + pos: curPos, + maxContextLen, + remainingTokens: maxContextLen - curPos, + usageRatio: curPos / maxContextLen, + }; + }; + + // ====================================================== + // Generation & Prefill methods + // ====================================================== + const isCancelled = createSynchronizable(false); + + const stop = (): void => { + 'worklet'; + isCancelled.setBlocking(true); + }; + + const prefillText = (prompt: string): LLMPrefillStats => { + 'worklet'; + isCancelled.setBlocking(false); + + const startMs = Date.now(); + const startPos = pos.getBlocking(); + + const tokens = tokenizer.encode(prompt); + + let offset = 0; + while (offset < tokens.length && !isCancelled.getBlocking()) { + const seqLen = Math.min(tokens.length - offset, maxSeqLen); + const chunk = tokens.subarray(offset, offset + seqLen); + + if (startPos + offset + seqLen > maxContextLen) { + throw RnExecuTorchError( + 'EXECUTION_FAILED', + `prefill: Context length exceeded (${startPos + offset + seqLen} > ${maxContextLen}).` + ); + } + + const positions = BigInt64Array.from( + { length: seqLen }, // prettier-ignore + (_, i) => BigInt(startPos + offset + i) + ); + + // TODO(bh): this pattern always when >= 3 tensors + const tensors = [ + // prettier-ignore + tensor('int64', [1, seqLen], BigInt64Array.from(chunk, (v) => BigInt(v))), + tensor('int64', [seqLen], positions), + tensor('float32', [1, seqLen, hiddenDim]), + tensor('float32', [1, seqLen, numLayers, pleDim]), + ] as const; + + const [tTokens, tPositions, tTextEmbeds, tPleTok] = tensors; + + try { + model.execute('token_embedding', [tTokens], [tTextEmbeds, tPleTok]); + model.execute('text_decoder', [tTextEmbeds, tPleTok, tPositions], [tLogits]); + } finally { + tensors.forEach((t) => t.dispose()); + } + + pos.setBlocking(startPos + offset + seqLen); + offset += seqLen; + } + + const durationMs = Date.now() - startMs; + const numTokens = offset; + const tokensPerSecond = durationMs > 0 ? (numTokens / durationMs) * 1000 : 0; + + return { numTokens, durationMs, tokensPerSecond }; + }; + + const prefillEmbeddings = (embeddings: Float32Array, numEmbedTokens: number): LLMPrefillStats => { + 'worklet'; + const startMs = Date.now(); + const startPos = pos.getBlocking(); + + let offset = 0; + while (offset < numEmbedTokens && !isCancelled.getBlocking()) { + const seqLen = Math.min(numEmbedTokens - offset, maxSeqLen); + const chunk = embeddings.subarray(offset * hiddenDim, (offset + seqLen) * hiddenDim); + + if (startPos + offset + seqLen > maxContextLen) { + throw RnExecuTorchError( + 'EXECUTION_FAILED', + `prefill: Context length exceeded (${startPos + offset + seqLen} > ${maxContextLen}).` + ); + } + + const positions = BigInt64Array.from( + { length: seqLen }, // prettier-ignore + (_, i) => BigInt(startPos + offset + i) + ); + + // Media embeddings (vision / audio) do not carry per-layer token embeddings; + // zero tensor is passed for ple_tok. + const zeroPle = new Float32Array(1 * seqLen * numLayers * pleDim); + + const tensors = [ + tensor('float32', [1, seqLen, hiddenDim], chunk), + tensor('float32', [1, seqLen, numLayers, pleDim], zeroPle), + tensor('int64', [seqLen], positions), + ] as const; + + const [tChunk, tPleTok, tPositions] = tensors; + + try { + model.execute('text_decoder', [tChunk, tPleTok, tPositions], [tLogits]); + } finally { + tensors.forEach((t) => t.dispose()); + } + + pos.setBlocking(startPos + offset + seqLen); + offset += seqLen; + } + + const durationMs = Date.now() - startMs; + const numTokens = numEmbedTokens; + const tokensPerSecond = durationMs > 0 ? (numTokens / durationMs) * 1000 : 0; + + return { numTokens, durationMs, tokensPerSecond }; + }; + + const prefillImage = (prompt: Extract): LLMPrefillStats => { + 'worklet'; + isCancelled.setBlocking(false); + + if (prompt.image.dtype !== 'float32') { + throw RnExecuTorchError( + 'INVALID_ARGUMENT', + `Expected image tensor of dtype 'float32', received '${prompt.image.dtype}'.` + ); + } + if (imgShape.some((d, i) => d !== prompt.image.shape[i])) { + throw RnExecuTorchError( + 'INVALID_ARGUMENT', + `Image tensor shape [${prompt.image.shape.join(', ')}] != [${imgShape.join(', ')}].` + ); + } + + const tVisionEmbed = tensor('float32', [1, visualTokens, hiddenDim]); + try { + model.execute('vision_encoder', [prompt.image], [tVisionEmbed]); + const embedding = tVisionEmbed.getData(new Float32Array(tVisionEmbed.numel)); + return prefillEmbeddings(embedding, visualTokens); + } finally { + tVisionEmbed.dispose(); + } + }; + + const prefillAudio = (prompt: Extract): LLMPrefillStats => { + 'worklet'; + isCancelled.setBlocking(false); + + if (prompt.audio.dtype !== 'float32') { + throw RnExecuTorchError( + 'INVALID_ARGUMENT', + `Expected audio tensor of dtype 'float32', received '${prompt.audio.dtype}'.` + ); + } + + const numSamples = prompt.audio.numel; + if (numSamples === 0) { + throw RnExecuTorchError('INVALID_ARGUMENT', 'Audio waveform is empty.'); + } + + // Pad waveform to multiple of AUDIO_SAMPLES_PER_BLOCK + const numBlocks = Math.ceil(numSamples / AUDIO_SAMPLES_PER_BLOCK); + const paddedSamples = numBlocks * AUDIO_SAMPLES_PER_BLOCK; + const numAudioTokens = numBlocks * 12; // TODO: magic number + + const rawData = prompt.audio.getData(new Float32Array(numSamples)); + const paddedWav = new Float32Array(paddedSamples); + paddedWav.set(rawData); + + const tWaveform = tensor('float32', [1, paddedSamples], paddedWav); + const tNumValid = tensor('int64', [], BigInt64Array.of(BigInt(numSamples))); // TODO: empty shape!!! + const tAudioEmbeds = tensor('float32', [1, numAudioTokens, hiddenDim]); + const tAudioMask = tensor('float32', [1, numAudioTokens]); + + try { + model.execute('audio_encoder', [tWaveform, tNumValid], [tAudioEmbeds, tAudioMask]); + const embedding = tAudioEmbeds.getData(new Float32Array(tAudioEmbeds.numel)); + return prefillEmbeddings(embedding, numAudioTokens); + } finally { + tWaveform.dispose(); + tNumValid.dispose(); + tAudioEmbeds.dispose(); + tAudioMask.dispose(); + } + }; + + const prefill = (prompt: Prompt): LLMPrefillStats => { + 'worklet'; + isCancelled.setBlocking(false); + + const segments = typeof prompt === 'string' ? [prompt] : prompt; + if (segments.length === 0) { + throw RnExecuTorchError('INVALID_ARGUMENT', 'prefill: Empty prompt provided.'); + } + + let totStats: LLMPrefillStats | undefined; + const combine = (st1: LLMPrefillStats, st2?: LLMPrefillStats): LLMPrefillStats => { + if (st2 === undefined) return st1; + const numTokens = st1.numTokens + st2.numTokens; + const durationMs = st1.durationMs + st2.durationMs; + const tokensPerSecond = durationMs > 0 ? (numTokens / durationMs) * 1000 : 0; + return { numTokens, durationMs, tokensPerSecond }; + }; + + for (const segment of segments) { + if (typeof segment === 'string') { + const stats = prefillText(segment); + totStats = combine(stats, totStats); + continue; + } + + switch (segment.kind) { + case 'image': { + const stats = prefillImage(segment); + totStats = combine(stats, totStats); + break; + } + case 'audio': { + const stats = prefillAudio(segment); + totStats = combine(stats, totStats); + break; + } + default: + throw RnExecuTorchError( + 'INVALID_ARGUMENT', + `prefill: Unsupported media input kind '${(segment as { kind: string }).kind}'.` // TODO: no need for as + ); + } + } + + return totStats!; + }; + + const generate = ( + prompt: Prompt, + config?: LLMGenerationConfig, + onToken?: (token: string) => void + ): LLMGenerationStats => { + 'worklet'; + isCancelled.setBlocking(false); + + const prefillStats = prefill(prompt); + + const generateStartMs = Date.now(); + const generatedTokens: number[] = []; + const maxNewTokens = config?.maxNewTokens ?? Infinity; + + const tCurPos = tensor('int64', [1]); + const tTokenId = tensor('int64', [1, 1]); + const tTokenEmbed = tensor('float32', [1, 1, hiddenDim]); + const tPleTok = tensor('float32', [1, 1, numLayers, pleDim]); + + let numTokens = 0; + let curPos = pos.getBlocking(); + + try { + while (numTokens < maxNewTokens && !isCancelled.getBlocking()) { + if (curPos >= maxContextLen) break; + + const genTokArray = Int32Array.from(generatedTokens); + const nextToken = sample(tLogits, { generatedTokens: genTokArray }, config); + + onToken?.(tokenizer.decode(Int32Array.of(nextToken))); + generatedTokens.push(nextToken); + + if (!config?.ignoreEos && eosIds.includes(nextToken)) break; + + // Embed single token + tTokenId.setData(BigInt64Array.of(BigInt(nextToken))); + model.execute('token_embedding', [tTokenId], [tTokenEmbed, tPleTok]); + + // Decode single token embedding + tCurPos.setData(BigInt64Array.of(BigInt(curPos))); + model.execute('text_decoder', [tTokenEmbed, tPleTok, tCurPos], [tLogits]); + + numTokens += 1; + curPos += 1; + pos.setBlocking(curPos); + } + } finally { + tTokenId.dispose(); + tTokenEmbed.dispose(); + tPleTok.dispose(); + tCurPos.dispose(); + } + + const durationMs = Date.now() - generateStartMs; + const tokensPerSecond = durationMs > 0 ? (numTokens / durationMs) * 1000 : 0; + + return { numTokens, durationMs, tokensPerSecond, prefill: prefillStats }; + }; + + return { + // Metadata + modelPath: model.path, + tokenizerPath: tokenizer.path, + modalities: modalities ?? ['image', 'audio'], + + // Lifecycle & Execution + dispose, + stop, + + // KV Cache + reset, + getKVCacheState, -export function createLLMGemmaRunner(model: Model, tokenizer: Tokenizer): LLMRunner { - throw new Error(`Not implemented ${model.path}, ${tokenizer.path}`); + // Generation + prefill, + generate, + } as unknown as LLMRunner; } diff --git a/packages/react-native-executorch/src/extensions/llm/utils/chatPreprocessor.ts b/packages/react-native-executorch/src/extensions/llm/utils/chatPreprocessor.ts index 5b74aafdd2..dc0add6d2e 100644 --- a/packages/react-native-executorch/src/extensions/llm/utils/chatPreprocessor.ts +++ b/packages/react-native-executorch/src/extensions/llm/utils/chatPreprocessor.ts @@ -22,7 +22,7 @@ import type { ToolDefinition, ToolCall } from './toolCalling'; */ export type ChatMediaInput = | { readonly kind: 'image'; readonly image: ImageBuffer } - | { readonly kind: 'audio'; readonly audio: unknown }; + | { readonly kind: 'audio'; readonly audio: Float32Array }; /** * Interleaved text and media content for a chat turn. @@ -356,7 +356,9 @@ export function createChatPreprocessor(config: ChatPreprocessorConfig): ChatPrep } if (media?.kind === 'audio') { - throw RnExecuTorchError('INVALID_ARGUMENT', 'Audio input not yet supported'); + const tAudio = tensor('float32', [1, media.audio.length], media.audio); + prompt.push({ kind: 'audio', audio: tAudio }); + tensors.push(tAudio); } lastIndex = regex.lastIndex; From 45c9c0fb076d8b51e46f55f8a24861298767d9c5 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Mon, 12 Oct 2026 16:04:40 +0200 Subject: [PATCH 49/83] update --- .../react-native-executorch/src/models.ts | 153 +++++++++++------- 1 file changed, 97 insertions(+), 56 deletions(-) diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index ba54cf27af..e5e090e876 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -2068,26 +2068,25 @@ const PPOCRV6_SMALL_VULKAN_FP16: PaddleOcrModel = { // ============================================================================= // LLMs // ============================================================================= -const LFM2_5_BASE_URL = `${BASE_URL}-lfm-2.5/${VERSION_TAG}`; -const SCRATCH_LLMS_BASE_URL = 'https://huggingface.co/bhanc/scratch/resolve/main/llms'; +const LFM2_5_BASE_URL = `${BASE_URL}-lfm-2.5/${NEXT_VERSION_TAG}`; const LFM2_5_1_2B_XNNPACK_8DA4W: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/lfm_2_5_1_2b_xnnpack_8da4w.pte`, + modelPath: `${LFM2_5_BASE_URL}/1_2b/xnnpack/lfm_2_5_1_2b_xnnpack_8da4w.pte`, tokenizerPath: `${LFM2_5_BASE_URL}/1_2b/tokenizer.json`, tokenizerConfigPath: `${LFM2_5_BASE_URL}/1_2b/tokenizer_config.json`, }; const LFM2_5_1_2B_MLX_INT4: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/lfm_2_5_1_2b_mlx_int4.pte`, + modelPath: `${LFM2_5_BASE_URL}/1_2b/mlx/lfm_2_5_1_2b_mlx_int4.pte`, tokenizerPath: `${LFM2_5_BASE_URL}/1_2b/tokenizer.json`, tokenizerConfigPath: `${LFM2_5_BASE_URL}/1_2b/tokenizer_config.json`, }; const LFM2_5_350M_XNNPACK_8DA4W: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/lfm_2_5_350m_xnnpack_8da4w.pte`, + modelPath: `${LFM2_5_BASE_URL}/350m/xnnpack/lfm_2_5_350m_xnnpack_8da4w.pte`, tokenizerPath: `${LFM2_5_BASE_URL}/350m/tokenizer.json`, tokenizerConfigPath: `${LFM2_5_BASE_URL}/350m/tokenizer_config.json`, }; const LFM2_5_350M_MLX_INT4: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/lfm_2_5_350m_mlx_int4.pte`, + modelPath: `${LFM2_5_BASE_URL}/350m/mlx/lfm_2_5_350m_mlx_int4.pte`, tokenizerPath: `${LFM2_5_BASE_URL}/350m/tokenizer.json`, tokenizerConfigPath: `${LFM2_5_BASE_URL}/350m/tokenizer_config.json`, }; @@ -2104,245 +2103,277 @@ const LFM2_5_VL_PREPROCESSOR_CONFIG = { }, }; const LFM2_5_VL_450M_XNNPACK_8DA4W: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/lfm_2_5_vl_450m_xnnpack_8da4w.pte`, + modelPath: `${LFM2_5_BASE_URL}/vl_450m/xnnpack/lfm_2_5_vl_450m_xnnpack_8da4w.pte`, tokenizerPath: `${LFM2_5_BASE_URL}/vl_450m/tokenizer.json`, tokenizerConfigPath: `${LFM2_5_BASE_URL}/vl_450m/tokenizer_config.json`, modalities: ['image'], preprocessorConfig: LFM2_5_VL_PREPROCESSOR_CONFIG, }; const LFM2_5_VL_450M_MLX_INT4: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/lfm_2_5_vl_450m_mlx_int4.pte`, + modelPath: `${LFM2_5_BASE_URL}/vl_450m/mlx/lfm_2_5_vl_450m_mlx_int4.pte`, tokenizerPath: `${LFM2_5_BASE_URL}/vl_450m/tokenizer.json`, tokenizerConfigPath: `${LFM2_5_BASE_URL}/vl_450m/tokenizer_config.json`, modalities: ['image'], preprocessorConfig: LFM2_5_VL_PREPROCESSOR_CONFIG, }; const LFM2_5_VL_1_6B_XNNPACK_8DA4W: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/lfm_2_5_vl_1_6b_xnnpack_8da4w.pte`, + modelPath: `${LFM2_5_BASE_URL}/vl_1_6b/xnnpack/lfm_2_5_vl_1_6b_xnnpack_8da4w.pte`, tokenizerPath: `${LFM2_5_BASE_URL}/vl_1_6b/tokenizer.json`, tokenizerConfigPath: `${LFM2_5_BASE_URL}/vl_1_6b/tokenizer_config.json`, modalities: ['image'], preprocessorConfig: LFM2_5_VL_PREPROCESSOR_CONFIG, }; const LFM2_5_VL_450M_VULKAN_8DA4W: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/lfm_2_5_vl_450m_vulkan_8da4w.pte`, + modelPath: `${LFM2_5_BASE_URL}/vl_450m/vulkan/lfm_2_5_vl_450m_vulkan_8da4w.pte`, tokenizerPath: `${LFM2_5_BASE_URL}/vl_450m/tokenizer.json`, tokenizerConfigPath: `${LFM2_5_BASE_URL}/vl_450m/tokenizer_config.json`, modalities: ['image'], preprocessorConfig: LFM2_5_VL_PREPROCESSOR_CONFIG, }; const LFM2_5_VL_1_6B_VULKAN_8DA4W: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/lfm_2_5_vl_1_6b_vulkan_8da4w.pte`, + modelPath: `${LFM2_5_BASE_URL}/vl_1_6b/vulkan/lfm_2_5_vl_1_6b_vulkan_8da4w.pte`, tokenizerPath: `${LFM2_5_BASE_URL}/vl_1_6b/tokenizer.json`, tokenizerConfigPath: `${LFM2_5_BASE_URL}/vl_1_6b/tokenizer_config.json`, modalities: ['image'], preprocessorConfig: LFM2_5_VL_PREPROCESSOR_CONFIG, }; const LFM2_5_VL_1_6B_MLX_INT4: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/lfm_2_5_vl_1_6b_mlx_int4.pte`, + modelPath: `${LFM2_5_BASE_URL}/vl_1_6b/mlx/lfm_2_5_vl_1_6b_mlx_int4.pte`, tokenizerPath: `${LFM2_5_BASE_URL}/vl_1_6b/tokenizer.json`, tokenizerConfigPath: `${LFM2_5_BASE_URL}/vl_1_6b/tokenizer_config.json`, modalities: ['image'], preprocessorConfig: LFM2_5_VL_PREPROCESSOR_CONFIG, }; const LFM2_5_VL_1_6B_MLX_INT8: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/lfm_2_5_vl_1_6b_mlx_int8.pte`, + modelPath: `${LFM2_5_BASE_URL}/vl_1_6b/mlx/lfm_2_5_vl_1_6b_mlx_int8.pte`, tokenizerPath: `${LFM2_5_BASE_URL}/vl_1_6b/tokenizer.json`, tokenizerConfigPath: `${LFM2_5_BASE_URL}/vl_1_6b/tokenizer_config.json`, modalities: ['image'], preprocessorConfig: LFM2_5_VL_PREPROCESSOR_CONFIG, }; -const BIELIK_V3_1_5B_BASE_URL = `${BASE_URL}-bielik-v3.0/${VERSION_TAG}`; +const BIELIK_V3_1_5B_BASE_URL = `${BASE_URL}-bielik-v3.0/${NEXT_VERSION_TAG}`; const BIELIK_V3_1_5B_XNNPACK_8DA4W: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/bielik_v3_0_1_5b_xnnpack_8da4w.pte`, + modelPath: `${BIELIK_V3_1_5B_BASE_URL}/xnnpack/bielik_v3_0_1_5b_xnnpack_8da4w.pte`, tokenizerPath: `${BIELIK_V3_1_5B_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${BIELIK_V3_1_5B_BASE_URL}/tokenizer_config.json`, }; -const LLAMA3_2_BASE_URL = `${BASE_URL}-llama-3.2/${VERSION_TAG}`; +const LLAMA3_2_BASE_URL = `${BASE_URL}-llama-3.2/${NEXT_VERSION_TAG}`; const LLAMA3_2_3B_SPINQUANT: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/llama_3_2_3b_xnnpack_spinquant.pte`, + modelPath: `${LLAMA3_2_BASE_URL}/3b/xnnpack/llama_3_2_3b_xnnpack_spinquant.pte`, tokenizerPath: `${LLAMA3_2_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${LLAMA3_2_BASE_URL}/tokenizer_config.json`, }; const LLAMA3_2_1B_SPINQUANT: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/llama_3_2_1b_xnnpack_spinquant.pte`, + modelPath: `${LLAMA3_2_BASE_URL}/1b/xnnpack/llama_3_2_1b_xnnpack_spinquant.pte`, tokenizerPath: `${LLAMA3_2_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${LLAMA3_2_BASE_URL}/tokenizer_config.json`, }; const LLAMA3_2_1B_MLX_INT4: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/llama_3_2_1b_mlx_int4.pte`, + modelPath: `${LLAMA3_2_BASE_URL}/1b/mlx/llama_3_2_1b_mlx_int4.pte`, tokenizerPath: `${LLAMA3_2_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${LLAMA3_2_BASE_URL}/tokenizer_config.json`, }; const LLAMA3_2_3B_MLX_INT4: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/llama_3_2_3b_mlx_int4.pte`, + modelPath: `${LLAMA3_2_BASE_URL}/3b/mlx/llama_3_2_3b_mlx_int4.pte`, tokenizerPath: `${LLAMA3_2_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${LLAMA3_2_BASE_URL}/tokenizer_config.json`, }; -const SMOLLM2_BASE_URL = `${BASE_URL}-smolLm-2/${VERSION_TAG}`; +const SMOLLM2_BASE_URL = `${BASE_URL}-smolLm-2/${NEXT_VERSION_TAG}`; const SMOLLM2_135M_8DA8W: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/smollm2_135m_xnnpack_8da8w.pte`, + modelPath: `${SMOLLM2_BASE_URL}/135m/xnnpack/smollm2_135m_xnnpack_8da8w.pte`, tokenizerPath: `${SMOLLM2_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${SMOLLM2_BASE_URL}/tokenizer_config.json`, }; const SMOLLM2_360M_8DA8W: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/smollm2_360m_xnnpack_8da8w.pte`, + modelPath: `${SMOLLM2_BASE_URL}/360m/xnnpack/smollm2_360m_xnnpack_8da8w.pte`, tokenizerPath: `${SMOLLM2_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${SMOLLM2_BASE_URL}/tokenizer_config.json`, }; const SMOLLM2_1_7B_8DA8W: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/smollm2_1_7b_xnnpack_8da8w.pte`, + modelPath: `${SMOLLM2_BASE_URL}/1_7b/xnnpack/smollm2_1_7b_xnnpack_8da8w.pte`, tokenizerPath: `${SMOLLM2_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${SMOLLM2_BASE_URL}/tokenizer_config.json`, }; const SMOLLM2_135M_MLX_INT8: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/smollm2_135m_mlx_int8.pte`, + modelPath: `${SMOLLM2_BASE_URL}/135m/mlx/smollm2_135m_mlx_int8.pte`, tokenizerPath: `${SMOLLM2_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${SMOLLM2_BASE_URL}/tokenizer_config.json`, }; const SMOLLM2_360M_MLX_INT8: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/smollm2_360m_mlx_int8.pte`, + modelPath: `${SMOLLM2_BASE_URL}/360m/mlx/smollm2_360m_mlx_int8.pte`, tokenizerPath: `${SMOLLM2_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${SMOLLM2_BASE_URL}/tokenizer_config.json`, }; const SMOLLM2_1_7B_MLX_INT8: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/smollm2_1_7b_mlx_int8.pte`, + modelPath: `${SMOLLM2_BASE_URL}/1_7b/mlx/smollm2_1_7b_mlx_int8.pte`, tokenizerPath: `${SMOLLM2_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${SMOLLM2_BASE_URL}/tokenizer_config.json`, }; -const HAMMER2_1_BASE_URL = `${BASE_URL}-hammer-2.1/${VERSION_TAG}`; +const HAMMER2_1_BASE_URL = `${BASE_URL}-hammer-2.1/${NEXT_VERSION_TAG}`; const HAMMER2_1_0_5B_XNNPACK_8DA4W: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/hammer_2_1_0_5b_xnnpack_8da4w.pte`, + modelPath: `${HAMMER2_1_BASE_URL}/0_5b/xnnpack/hammer_2_1_0_5b_xnnpack_8da4w.pte`, tokenizerPath: `${HAMMER2_1_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${HAMMER2_1_BASE_URL}/tokenizer_config.json`, }; const HAMMER2_1_1_5B_XNNPACK_8DA4W: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/hammer_2_1_1_5b_xnnpack_8da4w.pte`, + modelPath: `${HAMMER2_1_BASE_URL}/1_5b/xnnpack/hammer_2_1_1_5b_xnnpack_8da4w.pte`, tokenizerPath: `${HAMMER2_1_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${HAMMER2_1_BASE_URL}/tokenizer_config.json`, }; const HAMMER2_1_3B_XNNPACK_8DA4W: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/hammer_2_1_3b_xnnpack_8da4w.pte`, + modelPath: `${HAMMER2_1_BASE_URL}/3b/xnnpack/hammer_2_1_3b_xnnpack_8da4w.pte`, tokenizerPath: `${HAMMER2_1_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${HAMMER2_1_BASE_URL}/tokenizer_config.json`, }; const HAMMER2_1_0_5B_MLX_INT4: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/hammer_2_1_0_5b_mlx_int4.pte`, + modelPath: `${HAMMER2_1_BASE_URL}/0_5b/mlx/hammer_2_1_0_5b_mlx_int4.pte`, tokenizerPath: `${HAMMER2_1_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${HAMMER2_1_BASE_URL}/tokenizer_config.json`, }; const HAMMER2_1_1_5B_MLX_INT4: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/hammer_2_1_1_5b_mlx_int4.pte`, + modelPath: `${HAMMER2_1_BASE_URL}/1_5b/mlx/hammer_2_1_1_5b_mlx_int4.pte`, tokenizerPath: `${HAMMER2_1_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${HAMMER2_1_BASE_URL}/tokenizer_config.json`, }; const HAMMER2_1_3B_MLX_INT4: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/hammer_2_1_3b_mlx_int4.pte`, + modelPath: `${HAMMER2_1_BASE_URL}/3b/mlx/hammer_2_1_3b_mlx_int4.pte`, tokenizerPath: `${HAMMER2_1_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${HAMMER2_1_BASE_URL}/tokenizer_config.json`, }; -const PHI4_MINI_BASE_URL = `${BASE_URL}-phi-4-mini/${VERSION_TAG}`; +const PHI4_MINI_BASE_URL = `${BASE_URL}-phi-4-mini/${NEXT_VERSION_TAG}`; const PHI4_MINI_XNNPACK_8DA4W: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/phi_4_mini_xnnpack_8da4w.pte`, + modelPath: `${PHI4_MINI_BASE_URL}/xnnpack/phi_4_mini_xnnpack_8da4w.pte`, tokenizerPath: `${PHI4_MINI_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${PHI4_MINI_BASE_URL}/tokenizer_config.json`, }; const PHI4_MINI_MLX_INT4: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/phi_4_mini_mlx_int4.pte`, + modelPath: `${PHI4_MINI_BASE_URL}/mlx/phi_4_mini_mlx_int4.pte`, tokenizerPath: `${PHI4_MINI_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${PHI4_MINI_BASE_URL}/tokenizer_config.json`, }; -const QWEN2_5_BASE_URL = `${BASE_URL}-qwen-2.5/${VERSION_TAG}`; +const QWEN2_5_BASE_URL = `${BASE_URL}-qwen-2.5/${NEXT_VERSION_TAG}`; const QWEN2_5_0_5B_XNNPACK_8DA4W: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/qwen_2_5_0_5b_xnnpack_8da4w.pte`, + modelPath: `${QWEN2_5_BASE_URL}/0_5b/xnnpack/qwen_2_5_0_5b_xnnpack_8da4w.pte`, tokenizerPath: `${QWEN2_5_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${QWEN2_5_BASE_URL}/tokenizer_config.json`, }; const QWEN2_5_1_5B_XNNPACK_8DA4W: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/qwen_2_5_1_5b_xnnpack_8da4w.pte`, + modelPath: `${QWEN2_5_BASE_URL}/1_5b/xnnpack/qwen_2_5_1_5b_xnnpack_8da4w.pte`, tokenizerPath: `${QWEN2_5_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${QWEN2_5_BASE_URL}/tokenizer_config.json`, }; const QWEN2_5_3B_XNNPACK_8DA4W: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/qwen_2_5_3b_xnnpack_8da4w.pte`, + modelPath: `${QWEN2_5_BASE_URL}/3b/xnnpack/qwen_2_5_3b_xnnpack_8da4w.pte`, tokenizerPath: `${QWEN2_5_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${QWEN2_5_BASE_URL}/tokenizer_config.json`, }; const QWEN2_5_0_5B_MLX_INT4: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/qwen_2_5_0_5b_mlx_int4.pte`, + modelPath: `${QWEN2_5_BASE_URL}/0_5b/mlx/qwen_2_5_0_5b_mlx_int4.pte`, tokenizerPath: `${QWEN2_5_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${QWEN2_5_BASE_URL}/tokenizer_config.json`, }; const QWEN2_5_1_5B_MLX_INT4: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/qwen_2_5_1_5b_mlx_int4.pte`, + modelPath: `${QWEN2_5_BASE_URL}/1_5b/mlx/qwen_2_5_1_5b_mlx_int4.pte`, tokenizerPath: `${QWEN2_5_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${QWEN2_5_BASE_URL}/tokenizer_config.json`, }; const QWEN2_5_3B_MLX_INT4: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/qwen_2_5_3b_mlx_int4.pte`, + modelPath: `${QWEN2_5_BASE_URL}/3b/mlx/qwen_2_5_3b_mlx_int4.pte`, tokenizerPath: `${QWEN2_5_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${QWEN2_5_BASE_URL}/tokenizer_config.json`, }; -const GEMMA4_BASE_URL = `${BASE_URL}-gemma-4/${VERSION_TAG}`; +const GEMMA4_BASE_URL = `${BASE_URL}-gemma-4/${NEXT_VERSION_TAG}`; +// Gemma 4 omni (multimodal) is parked for now: its builds are not part of the +// v0.11.0 upstream migration, so the model is commented out until they are. +// const GEMMA4_OMNI_BASE_URL = `${BASE_URL}-gemma-4-multimodal/${NEXT_VERSION_TAG}`; +// +// const GEMMA4_OMNI_PREPROCESSOR_CONFIG = { +// image: { +// visionToken: { start: '<|image>', end: '' }, +// targetShape: [1, 3, 448, 640] as const, +// preprocessorOpts: { +// resizeMode: 'letterbox' as const, +// interpolation: 'linear' as const, +// normalizeOpts: { alpha: 1.0, beta: 0.0 }, +// }, +// }, +// audio: { +// audioToken: { start: '<|audio>', end: '' }, +// }, +// }; const GEMMA4_E2B_XNNPACK_8DA4W: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/gemma_4_e2b_xnnpack_8da4w.pte`, + modelPath: `${GEMMA4_BASE_URL}/e2b/xnnpack/gemma_4_e2b_xnnpack_8da4w.pte`, tokenizerPath: `${GEMMA4_BASE_URL}/e2b/tokenizer.json`, tokenizerConfigPath: `${GEMMA4_BASE_URL}/e2b/tokenizer_config.json`, }; const GEMMA4_E2B_VULKAN_8DA4W: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/gemma_4_e2b_vulkan_8da4w.pte`, + modelPath: `${GEMMA4_BASE_URL}/e2b/vulkan/gemma_4_e2b_vulkan_8da4w.pte`, tokenizerPath: `${GEMMA4_BASE_URL}/e2b/tokenizer.json`, tokenizerConfigPath: `${GEMMA4_BASE_URL}/e2b/tokenizer_config.json`, }; const GEMMA4_E2B_MLX_INT4: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/gemma4_e2b_mlx_int4.pte`, + modelPath: `${GEMMA4_BASE_URL}/e2b/mlx/gemma4_e2b_mlx_int4.pte`, tokenizerPath: `${GEMMA4_BASE_URL}/e2b/tokenizer.json`, tokenizerConfigPath: `${GEMMA4_BASE_URL}/e2b/tokenizer_config.json`, }; - -const QWEN3_BASE_URL = `${BASE_URL}-qwen-3/${VERSION_TAG}`; +// const GEMMA4_OMNI_E2B_MLX_INT4: LLMModel = { +// modelPath: `${GEMMA4_OMNI_BASE_URL}/e2b/mlx/gemma_4_omni_e2b_mlx_int4.pte`, +// tokenizerPath: `${GEMMA4_BASE_URL}/e2b/tokenizer.json`, +// tokenizerConfigPath: `${GEMMA4_BASE_URL}/e2b/tokenizer_config.json`, +// modalities: ['image', 'audio'], +// preprocessorConfig: GEMMA4_OMNI_PREPROCESSOR_CONFIG, +// }; +// const GEMMA4_OMNI_E2B_XNNPACK_8DA4W: LLMModel = { +// modelPath: `${GEMMA4_OMNI_BASE_URL}/e2b/xnnpack/gemma_4_e2b_xnnpack_8da4w.pte`, +// tokenizerPath: `${GEMMA4_BASE_URL}/e2b/tokenizer.json`, +// tokenizerConfigPath: `${GEMMA4_BASE_URL}/e2b/tokenizer_config.json`, +// modalities: ['image', 'audio'], +// preprocessorConfig: GEMMA4_OMNI_PREPROCESSOR_CONFIG, +// }; + +const QWEN3_BASE_URL = `${BASE_URL}-qwen-3/${NEXT_VERSION_TAG}`; const QWEN3_0_6B_XNNPACK_8DA4W: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/qwen_3_0_6b_xnnpack_8da4w.pte`, + modelPath: `${QWEN3_BASE_URL}/0_6b/xnnpack/qwen_3_0_6b_xnnpack_8da4w.pte`, tokenizerPath: `${QWEN3_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${QWEN3_BASE_URL}/tokenizer_config.json`, }; const QWEN3_1_7B_XNNPACK_8DA4W: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/qwen_3_1_7b_xnnpack_8da4w.pte`, + modelPath: `${QWEN3_BASE_URL}/1_7b/xnnpack/qwen_3_1_7b_xnnpack_8da4w.pte`, tokenizerPath: `${QWEN3_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${QWEN3_BASE_URL}/tokenizer_config.json`, }; const QWEN3_4B_XNNPACK_8DA4W: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/qwen_3_4b_xnnpack_8da4w.pte`, + modelPath: `${QWEN3_BASE_URL}/4b/xnnpack/qwen_3_4b_xnnpack_8da4w.pte`, tokenizerPath: `${QWEN3_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${QWEN3_BASE_URL}/tokenizer_config.json`, }; const QWEN3_0_6B_MLX_INT4: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/qwen_3_0_6b_mlx_int4.pte`, + modelPath: `${QWEN3_BASE_URL}/0_6b/mlx/qwen_3_0_6b_mlx_int4.pte`, tokenizerPath: `${QWEN3_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${QWEN3_BASE_URL}/tokenizer_config.json`, }; const QWEN3_1_7B_MLX_INT4: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/qwen_3_1_7b_mlx_int4.pte`, + modelPath: `${QWEN3_BASE_URL}/1_7b/mlx/qwen_3_1_7b_mlx_int4.pte`, tokenizerPath: `${QWEN3_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${QWEN3_BASE_URL}/tokenizer_config.json`, }; const QWEN3_4B_MLX_INT4: LLMModel = { - modelPath: `${SCRATCH_LLMS_BASE_URL}/qwen_3_4b_mlx_int4.pte`, + modelPath: `${QWEN3_BASE_URL}/4b/mlx/qwen_3_4b_mlx_int4.pte`, tokenizerPath: `${QWEN3_BASE_URL}/tokenizer.json`, tokenizerConfigPath: `${QWEN3_BASE_URL}/tokenizer_config.json`, }; @@ -3359,6 +3390,16 @@ export const models = { VULKAN_8DA4W: GEMMA4_E2B_VULKAN_8DA4W, MLX_INT4: GEMMA4_E2B_MLX_INT4, }), + // Gemma 4 omni (multimodal) is parked until its builds join the upstream + // migration; see the commented-out constants above. + // /** + // * Google Gemma 4 E2B multimodal (omni) language model supporting text, image, + // * and audio inputs. + // */ + // GEMMA4_OMNI_E2B: variants({ + // MLX_INT4: GEMMA4_OMNI_E2B_MLX_INT4, + // XNNPACK_8DA4W: GEMMA4_OMNI_E2B_XNNPACK_8DA4W, + // }), }, /** * Text embedding models mapping sentences and documents into dense vector From 2b3546624f5a599971a1fc83af206b1512ea3f09 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Mon, 12 Oct 2026 17:12:16 +0200 Subject: [PATCH 50/83] refactor(llm): use tensor array destructuring pattern in multimodal runner --- .../extensions/llm/runners/llmMultimodalRunner.ts | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts b/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts index a09886585a..f964afd2a2 100644 --- a/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts @@ -298,9 +298,13 @@ export function createLLMMultimodalRunner( const generatedTokens: number[] = []; const maxNewTokens = config?.maxNewTokens ?? Infinity; - const tCurPos = tensor('int64', [1]); - const tTokenId = tensor('int64', [1, 1]); - const tTokenEmbed = tensor('float32', [1, 1, hiddenDim]); + const tensors = [ + tensor('int64', [1]), + tensor('int64', [1, 1]), + tensor('float32', [1, 1, hiddenDim]), + ] as const; + + const [tCurPos, tTokenId, tTokenEmbed] = tensors; let numTokens = 0; let curPos = pos.getBlocking(); @@ -330,9 +334,7 @@ export function createLLMMultimodalRunner( pos.setBlocking(curPos); } } finally { - tTokenId.dispose(); - tTokenEmbed.dispose(); - tCurPos.dispose(); + tensors.forEach((t) => t.dispose()); } const durationMs = Date.now() - generateStartMs; From e74754b2c94c0648cb6714b621c604caa9e246a6 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Mon, 12 Oct 2026 17:32:47 +0200 Subject: [PATCH 51/83] feat(nlp-app): set tailored generation configs for each LLM model family --- apps/nlp/constants/llm.ts | 227 ++++++++++++++++++++++++++++++-------- 1 file changed, 180 insertions(+), 47 deletions(-) diff --git a/apps/nlp/constants/llm.ts b/apps/nlp/constants/llm.ts index e07ee0d63c..407d7bcad4 100644 --- a/apps/nlp/constants/llm.ts +++ b/apps/nlp/constants/llm.ts @@ -283,11 +283,144 @@ export function parseLlamaToolCalls(text: string): llm.ToolParserResult | undefi return undefined; } -// --- Common Configurations --- +// --- Model Family Generation Configurations --- + +/** + * Liquid AI LFM 2.5 recommended generation parameters. + * Sampling parameters are specified in the official Liquid AI GGUF model cards + * (under the llama-cli Quickstart: `--temp 0.1 --top-k 50 --repeat-penalty 1.05`). + * @see https://huggingface.co/LiquidAI/LFM2.5-1.2B-Instruct-GGUF + * @see https://huggingface.co/LiquidAI/LFM2.5-350M-GGUF + */ +export const LFM2_5_GENERATION_CONFIG: llm.LLMGenerationConfig = { + temperature: 0.1, + topP: 0.1, + topK: 50, + repetitionPenalty: 1.05, + maxNewTokens: 512, +}; + +/** + * Google Gemma 4 E2B generation configuration. + * Published directly in the upstream `generation_config.json`. + * @see https://huggingface.co/google/gemma-4-e2b + * @see https://huggingface.co/google/gemma-4-e2b/raw/main/generation_config.json + */ +export const GEMMA4_GENERATION_CONFIG: llm.LLMGenerationConfig = { + temperature: 1.0, + topP: 0.95, + topK: 64, + maxNewTokens: 512, +}; + +/** + * MadeAgents Hammer 2.1 tool-calling model generation configuration. + * Published in upstream `generation_config.json`. + * @see https://huggingface.co/MadeAgents/Hammer2.1-0.5b + * @see https://huggingface.co/MadeAgents/Hammer2.1-0.5b/raw/main/generation_config.json + * @see https://huggingface.co/MadeAgents/Hammer2.1-3b/raw/main/generation_config.json + */ +export const HAMMER2_1_GENERATION_CONFIG: llm.LLMGenerationConfig = { + temperature: 0.7, + topP: 0.8, + topK: 20, + repetitionPenalty: 1.05, + maxNewTokens: 512, +}; + +/** + * Meta Llama 3.2 1B & 3B generation configuration. + * Published in `generation_config.json` (mirrored from gated meta-llama upstream). + * @see https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct + * @see https://huggingface.co/unsloth/Llama-3.2-1B-Instruct/raw/main/generation_config.json + * @see https://huggingface.co/unsloth/Llama-3.2-3B-Instruct/raw/main/generation_config.json + */ +export const LLAMA3_2_GENERATION_CONFIG: llm.LLMGenerationConfig = { + temperature: 0.6, + topP: 0.9, + maxNewTokens: 512, +}; + +/** + * Hugging Face SmolLM2 135M, 360M & 1.7B generation configuration. + * Documented in the official Hugging Face model cards generation snippet + * (`model.generate(..., temperature=0.2, top_p=0.9, do_sample=True)`). + * @see https://huggingface.co/HuggingFaceTB/SmolLM2-1.7B-Instruct + * @see https://huggingface.co/HuggingFaceTB/SmolLM2-360M-Instruct + * @see https://huggingface.co/HuggingFaceTB/SmolLM2-135M-Instruct + */ +export const SMOLLM2_GENERATION_CONFIG: llm.LLMGenerationConfig = { + temperature: 0.2, + topP: 0.9, + maxNewTokens: 512, +}; -export const DEFAULT_GENERATION_CONFIG: llm.LLMGenerationConfig = { +/** + * Alibaba Qwen 2.5 0.5B & 1.5B generation configuration. + * Published in upstream `generation_config.json`. + * @see https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct/raw/main/generation_config.json + * @see https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct/raw/main/generation_config.json + */ +export const QWEN2_5_0_5B_1_5B_GENERATION_CONFIG: llm.LLMGenerationConfig = { temperature: 0.7, + topP: 0.8, + topK: 20, + repetitionPenalty: 1.1, + maxNewTokens: 512, +}; + +/** + * Alibaba Qwen 2.5 3B generation configuration. + * Published in upstream `generation_config.json`. + * @see https://huggingface.co/Qwen/Qwen2.5-3B-Instruct + * @see https://huggingface.co/Qwen/Qwen2.5-3B-Instruct/raw/main/generation_config.json + */ +export const QWEN2_5_3B_GENERATION_CONFIG: llm.LLMGenerationConfig = { + temperature: 0.7, + topP: 0.8, + topK: 20, + repetitionPenalty: 1.05, + maxNewTokens: 512, +}; + +/** + * Alibaba Qwen 3 (0.6B, 1.7B, 4B) generation configuration. + * Published in upstream `generation_config.json`. + * @see https://huggingface.co/Qwen/Qwen3-0.6B/raw/main/generation_config.json + * @see https://huggingface.co/Qwen/Qwen3-1.7B/raw/main/generation_config.json + * @see https://huggingface.co/Qwen/Qwen3-4B/raw/main/generation_config.json + */ +export const QWEN3_GENERATION_CONFIG: llm.LLMGenerationConfig = { + temperature: 0.6, + topP: 0.95, + topK: 20, + maxNewTokens: 512, +}; + +/** + * Microsoft Phi-4 Mini 3.8B generation configuration. + * Microsoft specifies greedy generation (`temperature: 0.0, do_sample: False`) + * in the official model card Python and vLLM inference snippets. + * @see https://huggingface.co/microsoft/Phi-4-mini-instruct + * @see https://huggingface.co/microsoft/Phi-4-mini-instruct/raw/main/README.md + */ +export const PHI4_MINI_GENERATION_CONFIG: llm.LLMGenerationConfig = { + temperature: 0.0, + maxNewTokens: 512, +}; + +/** + * SpeakLeash Bielik v3 1.5B generation configuration. + * Bielik 1.5B is fine-tuned from Qwen2.5-1.5B; SpeakLeash recommends a lower temperature + * (0.1–0.2 with top_p 0.95) to prevent hallucinations and maintain coherence in Polish. + * @see https://huggingface.co/speakleash/Bielik-1.5B-v3.0-Instruct + * @see https://huggingface.co/speakleash/Bielik-1.5B-v3.0-Instruct-GGUF + */ +export const BIELIK_V3_GENERATION_CONFIG: llm.LLMGenerationConfig = { + temperature: 0.2, + topP: 0.95, maxNewTokens: 512, + repetitionPenalty: 1.1, }; export const DEFAULT_SYSTEM_PROMPT = 'You are a helpful assistant. Your name is HAL-9000'; @@ -337,7 +470,7 @@ export const LLM_MODELS: LLMModelConfig[] = [ name: 'LFM 2.5 1.2B', backend: 'xnnpack', model: models.llm.LFM2_5_1_2B.XNNPACK_8DA4W, - generationConfig: DEFAULT_GENERATION_CONFIG, + generationConfig: LFM2_5_GENERATION_CONFIG, systemPrompt: DEFAULT_SYSTEM_PROMPT, stopRegex: undefined, toolOpts: undefined, @@ -347,7 +480,7 @@ export const LLM_MODELS: LLMModelConfig[] = [ name: 'LFM 2.5 1.2B', backend: 'mlx', model: models.llm.LFM2_5_1_2B.MLX_INT4, - generationConfig: DEFAULT_GENERATION_CONFIG, + generationConfig: LFM2_5_GENERATION_CONFIG, systemPrompt: DEFAULT_SYSTEM_PROMPT, stopRegex: undefined, toolOpts: undefined, @@ -358,7 +491,7 @@ export const LLM_MODELS: LLMModelConfig[] = [ name: 'LFM 2.5 350M', backend: 'xnnpack', model: models.llm.LFM2_5_350M.XNNPACK_8DA4W, - generationConfig: DEFAULT_GENERATION_CONFIG, + generationConfig: LFM2_5_GENERATION_CONFIG, systemPrompt: DEFAULT_SYSTEM_PROMPT, stopRegex: undefined, toolOpts: undefined, @@ -368,7 +501,7 @@ export const LLM_MODELS: LLMModelConfig[] = [ name: 'LFM 2.5 350M', backend: 'mlx', model: models.llm.LFM2_5_350M.MLX_INT4, - generationConfig: DEFAULT_GENERATION_CONFIG, + generationConfig: LFM2_5_GENERATION_CONFIG, systemPrompt: DEFAULT_SYSTEM_PROMPT, stopRegex: undefined, toolOpts: undefined, @@ -379,7 +512,7 @@ export const LLM_MODELS: LLMModelConfig[] = [ name: 'LFM 2.5 VL 450M', backend: 'xnnpack', model: models.llm.LFM2_5_VL_450M.XNNPACK_8DA4W, - generationConfig: DEFAULT_GENERATION_CONFIG, + generationConfig: LFM2_5_GENERATION_CONFIG, systemPrompt: DEFAULT_SYSTEM_PROMPT, stopRegex: undefined, toolOpts: undefined, @@ -389,7 +522,7 @@ export const LLM_MODELS: LLMModelConfig[] = [ name: 'LFM 2.5 VL 450M', backend: 'mlx', model: models.llm.LFM2_5_VL_450M.MLX_INT4, - generationConfig: DEFAULT_GENERATION_CONFIG, + generationConfig: LFM2_5_GENERATION_CONFIG, systemPrompt: DEFAULT_SYSTEM_PROMPT, stopRegex: undefined, toolOpts: undefined, @@ -400,7 +533,7 @@ export const LLM_MODELS: LLMModelConfig[] = [ name: 'LFM 2.5 VL 1.6B', backend: 'xnnpack', model: models.llm.LFM2_5_VL_1_6B.XNNPACK_8DA4W, - generationConfig: DEFAULT_GENERATION_CONFIG, + generationConfig: LFM2_5_GENERATION_CONFIG, systemPrompt: DEFAULT_SYSTEM_PROMPT, stopRegex: undefined, toolOpts: undefined, @@ -410,7 +543,7 @@ export const LLM_MODELS: LLMModelConfig[] = [ name: 'LFM 2.5 VL 1.6B', backend: 'mlx', model: models.llm.LFM2_5_VL_1_6B.MLX_INT4, - generationConfig: DEFAULT_GENERATION_CONFIG, + generationConfig: LFM2_5_GENERATION_CONFIG, systemPrompt: DEFAULT_SYSTEM_PROMPT, stopRegex: undefined, toolOpts: undefined, @@ -421,7 +554,7 @@ export const LLM_MODELS: LLMModelConfig[] = [ name: 'LFM 2.5 VL 1.6B (int8)', backend: 'mlx', model: models.llm.LFM2_5_VL_1_6B.MLX_INT8, - generationConfig: DEFAULT_GENERATION_CONFIG, + generationConfig: LFM2_5_GENERATION_CONFIG, systemPrompt: DEFAULT_SYSTEM_PROMPT, stopRegex: undefined, toolOpts: undefined, @@ -432,7 +565,7 @@ export const LLM_MODELS: LLMModelConfig[] = [ name: 'LFM 2.5 VL 450M', backend: 'vulkan', model: models.llm.LFM2_5_VL_450M.VULKAN_8DA4W, - generationConfig: DEFAULT_GENERATION_CONFIG, + generationConfig: LFM2_5_GENERATION_CONFIG, systemPrompt: DEFAULT_SYSTEM_PROMPT, stopRegex: undefined, toolOpts: undefined, @@ -442,7 +575,7 @@ export const LLM_MODELS: LLMModelConfig[] = [ name: 'LFM 2.5 VL 1.6B', backend: 'vulkan', model: models.llm.LFM2_5_VL_1_6B.VULKAN_8DA4W, - generationConfig: DEFAULT_GENERATION_CONFIG, + generationConfig: LFM2_5_GENERATION_CONFIG, systemPrompt: DEFAULT_SYSTEM_PROMPT, stopRegex: undefined, toolOpts: undefined, @@ -454,7 +587,7 @@ export const LLM_MODELS: LLMModelConfig[] = [ name: 'Gemma 4 E2B', backend: 'xnnpack', model: models.llm.GEMMA4_E2B.XNNPACK_8DA4W, - generationConfig: DEFAULT_GENERATION_CONFIG, + generationConfig: GEMMA4_GENERATION_CONFIG, systemPrompt: DEFAULT_SYSTEM_PROMPT, stopRegex: GEMMA_TOOL_STOP_REGEX, toolOpts: GEMMA_TOOL_OPTS, @@ -464,7 +597,7 @@ export const LLM_MODELS: LLMModelConfig[] = [ name: 'Gemma 4 E2B', backend: 'mlx', model: models.llm.GEMMA4_E2B.MLX_INT4, - generationConfig: DEFAULT_GENERATION_CONFIG, + generationConfig: GEMMA4_GENERATION_CONFIG, systemPrompt: DEFAULT_SYSTEM_PROMPT, stopRegex: GEMMA_TOOL_STOP_REGEX, toolOpts: GEMMA_TOOL_OPTS, @@ -475,7 +608,7 @@ export const LLM_MODELS: LLMModelConfig[] = [ name: 'Gemma 4 E2B', backend: 'vulkan', model: models.llm.GEMMA4_E2B.VULKAN_8DA4W, - generationConfig: DEFAULT_GENERATION_CONFIG, + generationConfig: GEMMA4_GENERATION_CONFIG, systemPrompt: DEFAULT_SYSTEM_PROMPT, stopRegex: GEMMA_TOOL_STOP_REGEX, toolOpts: GEMMA_TOOL_OPTS, @@ -487,7 +620,7 @@ export const LLM_MODELS: LLMModelConfig[] = [ name: 'Hammer 2.1 0.5B', backend: 'xnnpack', model: models.llm.HAMMER2_1_0_5B.XNNPACK_8DA4W, - generationConfig: DEFAULT_GENERATION_CONFIG, + generationConfig: HAMMER2_1_GENERATION_CONFIG, systemPrompt: undefined, stopRegex: HAMMER_TOOL_STOP_REGEX, toolOpts: HAMMER_TOOL_OPTS, @@ -497,7 +630,7 @@ export const LLM_MODELS: LLMModelConfig[] = [ name: 'Hammer 2.1 0.5B', backend: 'mlx', model: models.llm.HAMMER2_1_0_5B.MLX_INT4, - generationConfig: DEFAULT_GENERATION_CONFIG, + generationConfig: HAMMER2_1_GENERATION_CONFIG, systemPrompt: undefined, stopRegex: HAMMER_TOOL_STOP_REGEX, toolOpts: HAMMER_TOOL_OPTS, @@ -508,7 +641,7 @@ export const LLM_MODELS: LLMModelConfig[] = [ name: 'Hammer 2.1 1.5B', backend: 'xnnpack', model: models.llm.HAMMER2_1_1_5B.XNNPACK_8DA4W, - generationConfig: DEFAULT_GENERATION_CONFIG, + generationConfig: HAMMER2_1_GENERATION_CONFIG, systemPrompt: undefined, stopRegex: HAMMER_TOOL_STOP_REGEX, toolOpts: HAMMER_TOOL_OPTS, @@ -518,7 +651,7 @@ export const LLM_MODELS: LLMModelConfig[] = [ name: 'Hammer 2.1 1.5B', backend: 'mlx', model: models.llm.HAMMER2_1_1_5B.MLX_INT4, - generationConfig: DEFAULT_GENERATION_CONFIG, + generationConfig: HAMMER2_1_GENERATION_CONFIG, systemPrompt: undefined, stopRegex: HAMMER_TOOL_STOP_REGEX, toolOpts: HAMMER_TOOL_OPTS, @@ -529,7 +662,7 @@ export const LLM_MODELS: LLMModelConfig[] = [ name: 'Hammer 2.1 3B', backend: 'xnnpack', model: models.llm.HAMMER2_1_3B.XNNPACK_8DA4W, - generationConfig: DEFAULT_GENERATION_CONFIG, + generationConfig: HAMMER2_1_GENERATION_CONFIG, systemPrompt: undefined, stopRegex: HAMMER_TOOL_STOP_REGEX, toolOpts: HAMMER_TOOL_OPTS, @@ -539,7 +672,7 @@ export const LLM_MODELS: LLMModelConfig[] = [ name: 'Hammer 2.1 3B', backend: 'mlx', model: models.llm.HAMMER2_1_3B.MLX_INT4, - generationConfig: DEFAULT_GENERATION_CONFIG, + generationConfig: HAMMER2_1_GENERATION_CONFIG, systemPrompt: undefined, stopRegex: HAMMER_TOOL_STOP_REGEX, toolOpts: HAMMER_TOOL_OPTS, @@ -552,7 +685,7 @@ export const LLM_MODELS: LLMModelConfig[] = [ name: 'Llama 3.2 1B', backend: 'xnnpack', model: models.llm.LLAMA3_2_1B.XNNPACK_SPINQUANT, - generationConfig: DEFAULT_GENERATION_CONFIG, + generationConfig: LLAMA3_2_GENERATION_CONFIG, systemPrompt: DEFAULT_SYSTEM_PROMPT, stopRegex: LLAMA_TOOL_STOP_REGEX, toolOpts: LLAMA_TOOL_OPTS, @@ -562,7 +695,7 @@ export const LLM_MODELS: LLMModelConfig[] = [ name: 'Llama 3.2 1B', backend: 'mlx', model: models.llm.LLAMA3_2_1B.MLX_INT4, - generationConfig: DEFAULT_GENERATION_CONFIG, + generationConfig: LLAMA3_2_GENERATION_CONFIG, systemPrompt: DEFAULT_SYSTEM_PROMPT, stopRegex: LLAMA_TOOL_STOP_REGEX, toolOpts: LLAMA_TOOL_OPTS, @@ -573,7 +706,7 @@ export const LLM_MODELS: LLMModelConfig[] = [ name: 'Llama 3.2 3B', backend: 'xnnpack', model: models.llm.LLAMA3_2_3B.XNNPACK_SPINQUANT, - generationConfig: DEFAULT_GENERATION_CONFIG, + generationConfig: LLAMA3_2_GENERATION_CONFIG, systemPrompt: DEFAULT_SYSTEM_PROMPT, stopRegex: LLAMA_TOOL_STOP_REGEX, toolOpts: LLAMA_TOOL_OPTS, @@ -583,7 +716,7 @@ export const LLM_MODELS: LLMModelConfig[] = [ name: 'Llama 3.2 3B', backend: 'mlx', model: models.llm.LLAMA3_2_3B.MLX_INT4, - generationConfig: DEFAULT_GENERATION_CONFIG, + generationConfig: LLAMA3_2_GENERATION_CONFIG, systemPrompt: DEFAULT_SYSTEM_PROMPT, stopRegex: LLAMA_TOOL_STOP_REGEX, toolOpts: LLAMA_TOOL_OPTS, @@ -596,7 +729,7 @@ export const LLM_MODELS: LLMModelConfig[] = [ name: 'SmolLM2 135M', backend: 'xnnpack', model: models.llm.SMOLLM2_135M.XNNPACK_8DA8W, - generationConfig: DEFAULT_GENERATION_CONFIG, + generationConfig: SMOLLM2_GENERATION_CONFIG, systemPrompt: DEFAULT_SYSTEM_PROMPT, stopRegex: undefined, toolOpts: undefined, @@ -606,7 +739,7 @@ export const LLM_MODELS: LLMModelConfig[] = [ name: 'SmolLM2 135M', backend: 'mlx', model: models.llm.SMOLLM2_135M.MLX_INT8, - generationConfig: DEFAULT_GENERATION_CONFIG, + generationConfig: SMOLLM2_GENERATION_CONFIG, systemPrompt: DEFAULT_SYSTEM_PROMPT, stopRegex: undefined, toolOpts: undefined, @@ -617,7 +750,7 @@ export const LLM_MODELS: LLMModelConfig[] = [ name: 'SmolLM2 360M', backend: 'xnnpack', model: models.llm.SMOLLM2_360M.XNNPACK_8DA8W, - generationConfig: DEFAULT_GENERATION_CONFIG, + generationConfig: SMOLLM2_GENERATION_CONFIG, systemPrompt: DEFAULT_SYSTEM_PROMPT, stopRegex: undefined, toolOpts: undefined, @@ -627,7 +760,7 @@ export const LLM_MODELS: LLMModelConfig[] = [ name: 'SmolLM2 360M', backend: 'mlx', model: models.llm.SMOLLM2_360M.MLX_INT8, - generationConfig: DEFAULT_GENERATION_CONFIG, + generationConfig: SMOLLM2_GENERATION_CONFIG, systemPrompt: DEFAULT_SYSTEM_PROMPT, stopRegex: undefined, toolOpts: undefined, @@ -638,7 +771,7 @@ export const LLM_MODELS: LLMModelConfig[] = [ name: 'SmolLM2 1.7B', backend: 'xnnpack', model: models.llm.SMOLLM2_1_7B.XNNPACK_8DA8W, - generationConfig: DEFAULT_GENERATION_CONFIG, + generationConfig: SMOLLM2_GENERATION_CONFIG, systemPrompt: DEFAULT_SYSTEM_PROMPT, stopRegex: undefined, toolOpts: undefined, @@ -648,7 +781,7 @@ export const LLM_MODELS: LLMModelConfig[] = [ name: 'SmolLM2 1.7B', backend: 'mlx', model: models.llm.SMOLLM2_1_7B.MLX_INT8, - generationConfig: DEFAULT_GENERATION_CONFIG, + generationConfig: SMOLLM2_GENERATION_CONFIG, systemPrompt: DEFAULT_SYSTEM_PROMPT, stopRegex: undefined, toolOpts: undefined, @@ -661,7 +794,7 @@ export const LLM_MODELS: LLMModelConfig[] = [ name: 'Qwen 2.5 0.5B', backend: 'xnnpack', model: models.llm.QWEN2_5_0_5B.XNNPACK_8DA4W, - generationConfig: DEFAULT_GENERATION_CONFIG, + generationConfig: QWEN2_5_0_5B_1_5B_GENERATION_CONFIG, systemPrompt: DEFAULT_SYSTEM_PROMPT, stopRegex: QWEN_TOOL_STOP_REGEX, toolOpts: QWEN_TOOL_OPTS, @@ -671,7 +804,7 @@ export const LLM_MODELS: LLMModelConfig[] = [ name: 'Qwen 2.5 0.5B', backend: 'mlx', model: models.llm.QWEN2_5_0_5B.MLX_INT4, - generationConfig: DEFAULT_GENERATION_CONFIG, + generationConfig: QWEN2_5_0_5B_1_5B_GENERATION_CONFIG, systemPrompt: DEFAULT_SYSTEM_PROMPT, stopRegex: QWEN_TOOL_STOP_REGEX, toolOpts: QWEN_TOOL_OPTS, @@ -682,7 +815,7 @@ export const LLM_MODELS: LLMModelConfig[] = [ name: 'Qwen 2.5 1.5B', backend: 'xnnpack', model: models.llm.QWEN2_5_1_5B.XNNPACK_8DA4W, - generationConfig: DEFAULT_GENERATION_CONFIG, + generationConfig: QWEN2_5_0_5B_1_5B_GENERATION_CONFIG, systemPrompt: DEFAULT_SYSTEM_PROMPT, stopRegex: QWEN_TOOL_STOP_REGEX, toolOpts: QWEN_TOOL_OPTS, @@ -692,7 +825,7 @@ export const LLM_MODELS: LLMModelConfig[] = [ name: 'Qwen 2.5 1.5B', backend: 'mlx', model: models.llm.QWEN2_5_1_5B.MLX_INT4, - generationConfig: DEFAULT_GENERATION_CONFIG, + generationConfig: QWEN2_5_0_5B_1_5B_GENERATION_CONFIG, systemPrompt: DEFAULT_SYSTEM_PROMPT, stopRegex: QWEN_TOOL_STOP_REGEX, toolOpts: QWEN_TOOL_OPTS, @@ -703,7 +836,7 @@ export const LLM_MODELS: LLMModelConfig[] = [ name: 'Qwen 2.5 3B', backend: 'xnnpack', model: models.llm.QWEN2_5_3B.XNNPACK_8DA4W, - generationConfig: DEFAULT_GENERATION_CONFIG, + generationConfig: QWEN2_5_3B_GENERATION_CONFIG, systemPrompt: DEFAULT_SYSTEM_PROMPT, stopRegex: QWEN_TOOL_STOP_REGEX, toolOpts: QWEN_TOOL_OPTS, @@ -713,7 +846,7 @@ export const LLM_MODELS: LLMModelConfig[] = [ name: 'Qwen 2.5 3B', backend: 'mlx', model: models.llm.QWEN2_5_3B.MLX_INT4, - generationConfig: DEFAULT_GENERATION_CONFIG, + generationConfig: QWEN2_5_3B_GENERATION_CONFIG, systemPrompt: DEFAULT_SYSTEM_PROMPT, stopRegex: QWEN_TOOL_STOP_REGEX, toolOpts: QWEN_TOOL_OPTS, @@ -726,7 +859,7 @@ export const LLM_MODELS: LLMModelConfig[] = [ name: 'Qwen 3 0.6B', backend: 'xnnpack', model: models.llm.QWEN3_0_6B.XNNPACK_8DA4W, - generationConfig: DEFAULT_GENERATION_CONFIG, + generationConfig: QWEN3_GENERATION_CONFIG, systemPrompt: DEFAULT_SYSTEM_PROMPT, stopRegex: QWEN_TOOL_STOP_REGEX, toolOpts: QWEN_TOOL_OPTS, @@ -736,7 +869,7 @@ export const LLM_MODELS: LLMModelConfig[] = [ name: 'Qwen 3 0.6B', backend: 'mlx', model: models.llm.QWEN3_0_6B.MLX_INT4, - generationConfig: DEFAULT_GENERATION_CONFIG, + generationConfig: QWEN3_GENERATION_CONFIG, systemPrompt: DEFAULT_SYSTEM_PROMPT, stopRegex: QWEN_TOOL_STOP_REGEX, toolOpts: QWEN_TOOL_OPTS, @@ -747,7 +880,7 @@ export const LLM_MODELS: LLMModelConfig[] = [ name: 'Qwen 3 1.7B', backend: 'xnnpack', model: models.llm.QWEN3_1_7B.XNNPACK_8DA4W, - generationConfig: DEFAULT_GENERATION_CONFIG, + generationConfig: QWEN3_GENERATION_CONFIG, systemPrompt: DEFAULT_SYSTEM_PROMPT, stopRegex: QWEN_TOOL_STOP_REGEX, toolOpts: QWEN_TOOL_OPTS, @@ -757,7 +890,7 @@ export const LLM_MODELS: LLMModelConfig[] = [ name: 'Qwen 3 1.7B', backend: 'mlx', model: models.llm.QWEN3_1_7B.MLX_INT4, - generationConfig: DEFAULT_GENERATION_CONFIG, + generationConfig: QWEN3_GENERATION_CONFIG, systemPrompt: DEFAULT_SYSTEM_PROMPT, stopRegex: QWEN_TOOL_STOP_REGEX, toolOpts: QWEN_TOOL_OPTS, @@ -768,7 +901,7 @@ export const LLM_MODELS: LLMModelConfig[] = [ name: 'Qwen 3 4B', backend: 'xnnpack', model: models.llm.QWEN3_4B.XNNPACK_8DA4W, - generationConfig: DEFAULT_GENERATION_CONFIG, + generationConfig: QWEN3_GENERATION_CONFIG, systemPrompt: DEFAULT_SYSTEM_PROMPT, stopRegex: QWEN_TOOL_STOP_REGEX, toolOpts: QWEN_TOOL_OPTS, @@ -778,7 +911,7 @@ export const LLM_MODELS: LLMModelConfig[] = [ name: 'Qwen 3 4B', backend: 'mlx', model: models.llm.QWEN3_4B.MLX_INT4, - generationConfig: DEFAULT_GENERATION_CONFIG, + generationConfig: QWEN3_GENERATION_CONFIG, systemPrompt: DEFAULT_SYSTEM_PROMPT, stopRegex: QWEN_TOOL_STOP_REGEX, toolOpts: QWEN_TOOL_OPTS, @@ -791,7 +924,7 @@ export const LLM_MODELS: LLMModelConfig[] = [ name: 'Phi-4 Mini 3.8B', backend: 'xnnpack', model: models.llm.PHI4_MINI.XNNPACK_8DA4W, - generationConfig: DEFAULT_GENERATION_CONFIG, + generationConfig: PHI4_MINI_GENERATION_CONFIG, systemPrompt: DEFAULT_SYSTEM_PROMPT, stopRegex: undefined, toolOpts: undefined, @@ -801,7 +934,7 @@ export const LLM_MODELS: LLMModelConfig[] = [ name: 'Phi-4 Mini 3.8B', backend: 'mlx', model: models.llm.PHI4_MINI.MLX_INT4, - generationConfig: DEFAULT_GENERATION_CONFIG, + generationConfig: PHI4_MINI_GENERATION_CONFIG, systemPrompt: DEFAULT_SYSTEM_PROMPT, stopRegex: undefined, toolOpts: undefined, @@ -814,7 +947,7 @@ export const LLM_MODELS: LLMModelConfig[] = [ name: 'Bielik v3 1.5B', backend: 'xnnpack', model: models.llm.BIELIK_V3_1_5B.XNNPACK_8DA4W, - generationConfig: DEFAULT_GENERATION_CONFIG, + generationConfig: BIELIK_V3_GENERATION_CONFIG, // eslint-disable-next-line @cspell/spellchecker systemPrompt: 'Jesteś pomocnym asystentem AI mówiącym po polsku.', stopRegex: undefined, From 253391ab01b0e6b4708aef3f91864049ad7bf9c6 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Mon, 12 Oct 2026 18:46:33 +0200 Subject: [PATCH 52/83] refactor(llm): align runner reset bounds and clean up gemma runner TODOs --- .../extensions/llm/runners/llmGemmaRunner.ts | 57 +++++++++++-------- .../llm/runners/llmMultimodalRunner.ts | 5 +- .../extensions/llm/runners/llmTextRunner.ts | 5 +- 3 files changed, 38 insertions(+), 29 deletions(-) diff --git a/packages/react-native-executorch/src/extensions/llm/runners/llmGemmaRunner.ts b/packages/react-native-executorch/src/extensions/llm/runners/llmGemmaRunner.ts index 3a13b46e6b..60a71b43b9 100644 --- a/packages/react-native-executorch/src/extensions/llm/runners/llmGemmaRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/runners/llmGemmaRunner.ts @@ -50,19 +50,27 @@ export const LLM_GEMMA_SPEC = { ), ...method( 'audio_encoder', - [f32(1, Dyn('audioSamples')), i64()], + [f32(1, Dyn('audioSamples')), i64(1)], [f32(1, Dyn('audioTokens'), 'hiddenDim'), f32(1, Dyn('audioTokens'))] ), ...METADATA_SPEC, }; const AUDIO_SAMPLES_PER_BLOCK = 7680; +const TOKENS_PER_AUDIO_BLOCK = 12; export function createLLMGemmaRunner( model: Model, tokenizer: Tokenizer, modalities?: readonly Modality[] ): LLMRunner { + if (!modalities?.includes('image') || !modalities?.includes('audio')) { + throw RnExecuTorchError( + 'INVALID_ARGUMENT', + `Gemma runner requires both 'image' and 'audio' in modalities.` + ); + } + const { dims, dim } = validateSpec(model.schema, { default: LLM_GEMMA_SPEC }); const [imgH, imgW, vocabSize] = dims.constant('imgH', 'imgW', 'vocabSize'); @@ -74,8 +82,6 @@ export function createLLMGemmaRunner( const [maxContextLen] = model.execute('get_max_context_len', [], []) as [number]; const eosIds = model.execute('get_eos_ids', [], []) as number[]; - // TODO(bh): validate modalities - if (maxContextLen < maxSeqLen || !Number.isInteger(maxContextLen)) { throw RnExecuTorchError( 'SCHEMA_MISMATCH', @@ -102,11 +108,11 @@ export function createLLMGemmaRunner( const pos = createSynchronizable(0); const reset = (targetPos?: number): void => { - // TODO(bh): should be curPos not max ctx len - if (targetPos !== undefined && (targetPos < 0 || targetPos >= maxContextLen)) { + const curPos = pos.getBlocking(); + if (targetPos !== undefined && (targetPos < 0 || targetPos > curPos)) { throw RnExecuTorchError( 'INVALID_ARGUMENT', - `targetPos (${targetPos}) must be in range [0, ${maxContextLen}).` + `targetPos (${targetPos}) must be in range [0, ${curPos}].` ); } pos.setBlocking(targetPos ?? 0); @@ -158,7 +164,6 @@ export function createLLMGemmaRunner( (_, i) => BigInt(startPos + offset + i) ); - // TODO(bh): this pattern always when >= 3 tensors const tensors = [ // prettier-ignore tensor('int64', [1, seqLen], BigInt64Array.from(chunk, (v) => BigInt(v))), @@ -284,26 +289,27 @@ export function createLLMGemmaRunner( // Pad waveform to multiple of AUDIO_SAMPLES_PER_BLOCK const numBlocks = Math.ceil(numSamples / AUDIO_SAMPLES_PER_BLOCK); const paddedSamples = numBlocks * AUDIO_SAMPLES_PER_BLOCK; - const numAudioTokens = numBlocks * 12; // TODO: magic number + const numAudioTokens = numBlocks * TOKENS_PER_AUDIO_BLOCK; const rawData = prompt.audio.getData(new Float32Array(numSamples)); const paddedWav = new Float32Array(paddedSamples); paddedWav.set(rawData); - const tWaveform = tensor('float32', [1, paddedSamples], paddedWav); - const tNumValid = tensor('int64', [], BigInt64Array.of(BigInt(numSamples))); // TODO: empty shape!!! - const tAudioEmbeds = tensor('float32', [1, numAudioTokens, hiddenDim]); - const tAudioMask = tensor('float32', [1, numAudioTokens]); + const tensors = [ + tensor('float32', [1, paddedSamples], paddedWav), + tensor('int64', [1], BigInt64Array.of(BigInt(numSamples))), + tensor('float32', [1, numAudioTokens, hiddenDim]), + tensor('float32', [1, numAudioTokens]), + ] as const; + + const [tWaveform, tNumValid, tAudioEmbeds, tAudioMask] = tensors; try { model.execute('audio_encoder', [tWaveform, tNumValid], [tAudioEmbeds, tAudioMask]); const embedding = tAudioEmbeds.getData(new Float32Array(tAudioEmbeds.numel)); return prefillEmbeddings(embedding, numAudioTokens); } finally { - tWaveform.dispose(); - tNumValid.dispose(); - tAudioEmbeds.dispose(); - tAudioMask.dispose(); + tensors.forEach((t) => t.dispose()); } }; @@ -346,7 +352,7 @@ export function createLLMGemmaRunner( default: throw RnExecuTorchError( 'INVALID_ARGUMENT', - `prefill: Unsupported media input kind '${(segment as { kind: string }).kind}'.` // TODO: no need for as + `prefill: Unsupported media input kind '${(segment as MediaInput).kind}'.` ); } } @@ -368,10 +374,14 @@ export function createLLMGemmaRunner( const generatedTokens: number[] = []; const maxNewTokens = config?.maxNewTokens ?? Infinity; - const tCurPos = tensor('int64', [1]); - const tTokenId = tensor('int64', [1, 1]); - const tTokenEmbed = tensor('float32', [1, 1, hiddenDim]); - const tPleTok = tensor('float32', [1, 1, numLayers, pleDim]); + const tensors = [ + tensor('int64', [1]), + tensor('int64', [1, 1]), + tensor('float32', [1, 1, hiddenDim]), + tensor('float32', [1, 1, numLayers, pleDim]), + ] as const; + + const [tCurPos, tTokenId, tTokenEmbed, tPleTok] = tensors; let numTokens = 0; let curPos = pos.getBlocking(); @@ -401,10 +411,7 @@ export function createLLMGemmaRunner( pos.setBlocking(curPos); } } finally { - tTokenId.dispose(); - tTokenEmbed.dispose(); - tPleTok.dispose(); - tCurPos.dispose(); + tensors.forEach((t) => t.dispose()); } const durationMs = Date.now() - generateStartMs; diff --git a/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts b/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts index f964afd2a2..2f88ebd97d 100644 --- a/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts @@ -94,10 +94,11 @@ export function createLLMMultimodalRunner( const pos = createSynchronizable(0); const reset = (targetPos?: number): void => { - if (targetPos !== undefined && (targetPos < 0 || targetPos >= maxContextLen)) { + const curPos = pos.getBlocking(); + if (targetPos !== undefined && (targetPos < 0 || targetPos > curPos)) { throw RnExecuTorchError( 'INVALID_ARGUMENT', - `targetPos (${targetPos}) must be in range [0, ${maxContextLen}).` + `targetPos (${targetPos}) must be in range [0, ${curPos}].` ); } pos.setBlocking(targetPos ?? 0); diff --git a/packages/react-native-executorch/src/extensions/llm/runners/llmTextRunner.ts b/packages/react-native-executorch/src/extensions/llm/runners/llmTextRunner.ts index edd79ad0c4..6597cd378a 100644 --- a/packages/react-native-executorch/src/extensions/llm/runners/llmTextRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/runners/llmTextRunner.ts @@ -78,10 +78,11 @@ export function createLLMTextRunner( const pos = createSynchronizable(0); const reset = (targetPos?: number): void => { - if (targetPos !== undefined && (targetPos < 0 || targetPos >= maxContextLen)) { + const curPos = pos.getBlocking(); + if (targetPos !== undefined && (targetPos < 0 || targetPos > curPos)) { throw RnExecuTorchError( 'INVALID_ARGUMENT', - `targetPos (${targetPos}) must be in range [0, ${maxContextLen}).` + `targetPos (${targetPos}) must be in range [0, ${curPos}].` ); } pos.setBlocking(targetPos ?? 0); From 6f037653a8920b49a4642c307fc317a37f8caca3 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Tue, 13 Oct 2026 12:45:18 +0200 Subject: [PATCH 53/83] fix: remove llmBrand --- .../src/extensions/llm/llmRunner.ts | 8 -------- .../src/extensions/llm/runners/llmGemmaRunner.ts | 2 +- .../src/extensions/llm/runners/llmMultimodalRunner.ts | 2 +- .../src/extensions/llm/runners/llmTextRunner.ts | 2 +- 4 files changed, 3 insertions(+), 11 deletions(-) diff --git a/packages/react-native-executorch/src/extensions/llm/llmRunner.ts b/packages/react-native-executorch/src/extensions/llm/llmRunner.ts index 7eb8a51fba..e666604286 100644 --- a/packages/react-native-executorch/src/extensions/llm/llmRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/llmRunner.ts @@ -22,8 +22,6 @@ import { createLLMTextRunner, LLM_TEXT_SPEC } from './runners/llmTextRunner'; import { createLLMMultimodalRunner, LLM_MULTIMODAL_SPEC } from './runners/llmMultimodalRunner'; import { createLLMGemmaRunner, LLM_GEMMA_SPEC } from './runners/llmGemmaRunner'; -declare const llmRunnerBrand: unique symbol; - /** * Configuration options for LLM text generation. * @category LLM / Types @@ -155,12 +153,6 @@ export type LLMRunner = { config?: LLMGenerationConfig, onToken?: (token: string) => void ): LLMGenerationStats; - - /** - * Prevents plain JS objects from being cast as LLMRunners. - * @internal - */ - readonly [llmRunnerBrand]: never; }; /** diff --git a/packages/react-native-executorch/src/extensions/llm/runners/llmGemmaRunner.ts b/packages/react-native-executorch/src/extensions/llm/runners/llmGemmaRunner.ts index 60a71b43b9..163646b7ad 100644 --- a/packages/react-native-executorch/src/extensions/llm/runners/llmGemmaRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/runners/llmGemmaRunner.ts @@ -437,5 +437,5 @@ export function createLLMGemmaRunner( // Generation prefill, generate, - } as unknown as LLMRunner; + }; } diff --git a/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts b/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts index 2f88ebd97d..54edb4593c 100644 --- a/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts @@ -361,5 +361,5 @@ export function createLLMMultimodalRunner( // Generation prefill, generate, - } as unknown as LLMRunner; + }; } diff --git a/packages/react-native-executorch/src/extensions/llm/runners/llmTextRunner.ts b/packages/react-native-executorch/src/extensions/llm/runners/llmTextRunner.ts index 6597cd378a..0cf4cd6eb0 100644 --- a/packages/react-native-executorch/src/extensions/llm/runners/llmTextRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/runners/llmTextRunner.ts @@ -233,5 +233,5 @@ export function createLLMTextRunner( // Generation prefill, generate, - } as unknown as LLMRunner; + }; } From 7ad4a1d1055337a4c858d4fe49280c2317b934c8 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Tue, 13 Oct 2026 13:39:09 +0200 Subject: [PATCH 54/83] test(llm): add tests for multimodal runner, variant dispatch, sampler options and construction failure --- .../__tests__/extensions/sampler.test.ts | 65 ++++ .../__tests__/support/fakeJsi.ts | 1 + .../__tests__/support/fakeOps.ts | 18 +- .../tasks/constructionFailure.test.ts | 15 + .../__tests__/tasks/llmRunner.test.ts | 291 ++++++++++++++++++ 5 files changed, 389 insertions(+), 1 deletion(-) create mode 100644 packages/react-native-executorch/__tests__/extensions/sampler.test.ts create mode 100644 packages/react-native-executorch/__tests__/tasks/llmRunner.test.ts diff --git a/packages/react-native-executorch/__tests__/extensions/sampler.test.ts b/packages/react-native-executorch/__tests__/extensions/sampler.test.ts new file mode 100644 index 0000000000..fdcb175b1f --- /dev/null +++ b/packages/react-native-executorch/__tests__/extensions/sampler.test.ts @@ -0,0 +1,65 @@ +import { tensor } from '../../src/core/tensor'; +import { sample } from '../../src/extensions/llm/sampler'; +import { llm } from '../support/fakeOps'; + +describe('sample — options merging', () => { + it('passes resolved default sampling options to native JSI', () => { + const logits = tensor('float32', [1, 10]); + try { + sample(logits, { generatedTokens: new Int32Array(0) }); + + const options = llm.lastSampleOptions(); + expect(options).toBeDefined(); + expect(options).toMatchObject({ + temperature: 0.8, + repetitionPenalty: 1.0, + topK: 40, + topP: 0.95, + minP: 0.05, + }); + expect(options?.dry).toBeUndefined(); + expect(options?.xtc).toBeUndefined(); + } finally { + logits.dispose(); + } + }); + + it('overrides default sampling options and merges dry and xtc defaults', () => { + const logits = tensor('float32', [1, 10]); + try { + sample( + logits, + { generatedTokens: new Int32Array(0) }, + { + temperature: 0.2, + topK: 10, + dry: { multiplier: 0.8, allowedLength: 4 }, + xtc: { probability: 0.5, threshold: 0.3 }, + } + ); + + const options = llm.lastSampleOptions(); + expect(options).toBeDefined(); + expect(options).toMatchObject({ + temperature: 0.2, + topK: 10, + topP: 0.95, + minP: 0.05, + repetitionPenalty: 1.0, + dry: { + multiplier: 0.8, + base: 1.75, + allowedLength: 4, + penaltyLastN: -1, + sequenceBreakers: new Int32Array(0), + }, + xtc: { + probability: 0.5, + threshold: 0.3, + }, + }); + } finally { + logits.dispose(); + } + }); +}); diff --git a/packages/react-native-executorch/__tests__/support/fakeJsi.ts b/packages/react-native-executorch/__tests__/support/fakeJsi.ts index 1e68fe6d25..ccff078281 100644 --- a/packages/react-native-executorch/__tests__/support/fakeJsi.ts +++ b/packages/react-native-executorch/__tests__/support/fakeJsi.ts @@ -303,6 +303,7 @@ export const fakeJsi = { registeredBackends = ['XnnpackBackend', 'CoreMLBackend']; jsi.isEmulator = false; fakePhonemizer.reset(); + llm.reset(); tensorTracker.reset(); }, }; diff --git a/packages/react-native-executorch/__tests__/support/fakeOps.ts b/packages/react-native-executorch/__tests__/support/fakeOps.ts index 85ac5f8481..d12b765fc3 100644 --- a/packages/react-native-executorch/__tests__/support/fakeOps.ts +++ b/packages/react-native-executorch/__tests__/support/fakeOps.ts @@ -707,6 +707,8 @@ export const speech = { type FakeSamplingConstraints = { allowedTokens?: Int32Array; bannedTokens?: Int32Array }; +let lastSampleOptions: Record | undefined; + export const llm = { /** * Greedy stand-in for the native sampler: applies the `constraints` callback @@ -717,8 +719,12 @@ export const llm = { sample( logits: FakeTensor, ctx: { generatedTokens?: Int32Array }, - options: { constraints?: (ctx: unknown) => FakeSamplingConstraints | undefined } + options: { constraints?: (ctx: unknown) => FakeSamplingConstraints | undefined } & Record< + string, + unknown + > ): number { + lastSampleOptions = options; expectShape(logits, [1, logits.numel], 'sample: logits'); const { allowedTokens, bannedTokens } = options.constraints?.(ctx) ?? {}; const allowed = allowedTokens && new Set(allowedTokens); @@ -737,4 +743,14 @@ export const llm = { if (best === -1) throw new Error('sample: constraints left no token to choose'); return best; }, + + /** @returns The options object passed to the most recent `sample` call. */ + lastSampleOptions(): Record | undefined { + return lastSampleOptions; + }, + + /** Clears the recorded sample options. */ + reset(): void { + lastSampleOptions = undefined; + }, }; diff --git a/packages/react-native-executorch/__tests__/tasks/constructionFailure.test.ts b/packages/react-native-executorch/__tests__/tasks/constructionFailure.test.ts index f5a7f2dbfb..7a2fb895a9 100644 --- a/packages/react-native-executorch/__tests__/tasks/constructionFailure.test.ts +++ b/packages/react-native-executorch/__tests__/tasks/constructionFailure.test.ts @@ -36,12 +36,15 @@ import { createFsmnVoiceActivityDetector } from '../../src/extensions/speech/tas import { createKokoroTextToSpeech } from '../../src/extensions/speech/tasks/kokoroTextToSpeech'; import { createSupertonicTextToSpeech } from '../../src/extensions/speech/tasks/supertonicTextToSpeech'; import { createWhisperSpeechToText } from '../../src/extensions/speech/tasks/whisperSpeechToText'; +import { createLLMRunner } from '../../src/extensions/llm/llmRunner'; +import { createLLMChatSession } from '../../src/extensions/llm/tasks/llmChatSession'; import { fakeFs } from '../support/blobUtilMock'; import { fakeJsi } from '../support/fakeJsi'; import { STRETCH_PREPROCESSING, exported } from '../support/fixtures'; const MODEL_PATH = '/models/mismatched.pte'; const TOKENIZER_PATH = '/models/tokenizer.json'; +const TOKENIZER_CONFIG_PATH = '/models/tokenizer_config.json'; const CHARSET_PATH = '/models/charset.json'; const VOICE_PATH = '/models/voice.bin'; @@ -140,6 +143,13 @@ const factories: Record Promise> = { phonemizer: { lang: 'en-us' }, voices: { v: VOICE_PATH }, } as never), + llmRunner: () => createLLMRunner(MODEL_PATH, TOKENIZER_PATH), + llmChatSession: () => + createLLMChatSession({ + modelPath: MODEL_PATH, + tokenizerPath: TOKENIZER_PATH, + tokenizerConfigPath: TOKENIZER_CONFIG_PATH, + }), }; /** Everything the fake still considers allocated, as one comparable object. */ @@ -156,6 +166,11 @@ describe('create — schema validation failure', () => { beforeEach(() => { fakeJsi.registerModel(MODEL_PATH, { schema: MISMATCHED }); fakeJsi.registerTokenizer(TOKENIZER_PATH, { tokens: ['a', 'b'] }); + fakeFs.write( + TOKENIZER_CONFIG_PATH, + // eslint-disable-next-line camelcase + JSON.stringify({ chat_template: '{{ prompt }}', eos_token: '<|end|>' }) + ); fakeFs.write(CHARSET_PATH, JSON.stringify(['a', 'b', 'c'])); fakeFs.write(VOICE_PATH, new Uint8Array(1024)); }); diff --git a/packages/react-native-executorch/__tests__/tasks/llmRunner.test.ts b/packages/react-native-executorch/__tests__/tasks/llmRunner.test.ts new file mode 100644 index 0000000000..4ae7362f39 --- /dev/null +++ b/packages/react-native-executorch/__tests__/tasks/llmRunner.test.ts @@ -0,0 +1,291 @@ +import { f32, i64, method, RangeDim } from '../../src/core/schema'; +import { tensor } from '../../src/core/tensor'; +import { createLLMRunner } from '../../src/extensions/llm/llmRunner'; +import { createLLMMultimodalRunner } from '../../src/extensions/llm/runners/llmMultimodalRunner'; +import { loadModel } from '../../src/core/model'; +import { loadTokenizer } from '../../src/extensions/nlp'; +import { fakeJsi } from '../support/fakeJsi'; +import { exported } from '../support/fixtures'; +import { tracked } from '../support/lifetime'; + +const MODEL_PATH = '/models/test_llm.pte'; +const TOKENIZER_PATH = '/models/test_tokenizer.json'; + +const TEXT_SPEC = exported({ + ...method('forward', [i64(1, RangeDim(1, 128)), i64(1)], [f32(1, 100)]), + ...method('get_max_seq_len', [], [{ kind: 'Int' }]), + ...method('get_max_context_len', [], [{ kind: 'Int' }]), + ...method('get_vocab_size', [], [{ kind: 'Int' }]), + ...method('use_kv_cache', [], [{ kind: 'Bool' }]), + ...method('enable_dynamic_shape', [], [{ kind: 'Bool' }]), + ...method('get_eos_ids', [], [{ kind: 'Int' }]), +}); + +const MULTIMODAL_SPEC = exported({ + ...method('text_decoder', [f32(1, RangeDim(1, 128), 64), i64(RangeDim(1, 128))], [f32(1, 100)]), + ...method('vision_encoder', [f32(1, 3, 32, 32)], [f32(1, 4, 64)]), + ...method('token_embedding', [i64(1, RangeDim(1, 128))], [f32(1, RangeDim(1, 128), 64)]), + ...method('get_max_seq_len', [], [{ kind: 'Int' }]), + ...method('get_max_context_len', [], [{ kind: 'Int' }]), + ...method('get_vocab_size', [], [{ kind: 'Int' }]), + ...method('use_kv_cache', [], [{ kind: 'Bool' }]), + ...method('enable_dynamic_shape', [], [{ kind: 'Bool' }]), + ...method('get_eos_ids', [], [{ kind: 'Int' }]), +}); + +const GEMMA_SPEC = exported({ + ...method( + 'token_embedding', + [i64(1, RangeDim(1, 128))], + [f32(1, RangeDim(1, 128), 64), f32(1, RangeDim(1, 128), 2, 8)] + ), + ...method( + 'text_decoder', + [f32(1, RangeDim(1, 128), 64), f32(1, RangeDim(1, 128), 2, 8), i64(RangeDim(1, 128))], + [f32(1, 1, 100)] + ), + ...method('vision_encoder', [f32(1, 3, 32, 32)], [f32(1, 4, 64)]), + ...method( + 'audio_encoder', + [f32(1, RangeDim(1, 7680)), i64(1)], + [f32(1, RangeDim(1, 12), 64), f32(1, RangeDim(1, 12))] + ), + ...method('get_max_seq_len', [], [{ kind: 'Int' }]), + ...method('get_max_context_len', [], [{ kind: 'Int' }]), + ...method('get_vocab_size', [], [{ kind: 'Int' }]), + ...method('use_kv_cache', [], [{ kind: 'Bool' }]), + ...method('enable_dynamic_shape', [], [{ kind: 'Bool' }]), + ...method('get_eos_ids', [], [{ kind: 'Int' }]), +}); + +function registerModelWithSchema(schema: any, options: { maxContextLen?: number } = {}) { + const maxContextLen = options.maxContextLen ?? 512; + fakeJsi.registerModel(MODEL_PATH, { + schema, + execute: (methodName, _inputs, outputs) => { + switch (methodName) { + case 'get_max_seq_len': + return [128]; + case 'get_max_context_len': + return [maxContextLen]; + case 'get_vocab_size': + return [100]; + case 'use_kv_cache': + case 'enable_dynamic_shape': + return [true]; + case 'get_eos_ids': + return [1]; + case 'text_decoder': { + const out = outputs[0]; + if (out) { + // Write a greedy choice to index 2 + out.setElement(2, 10); + } + return undefined; + } + case 'token_embedding': + case 'vision_encoder': + case 'audio_encoder': + case 'forward': + default: + return undefined; + } + }, + }); +} + +beforeEach(() => { + fakeJsi.registerTokenizer(TOKENIZER_PATH, { + tokens: ['', '', 'hello', 'world', 'vision'], + open: true, + }); +}); + +describe('createLLMRunner — variant dispatch & modalities validation', () => { + it('dispatches to text runner when schema matches LLM_TEXT_SPEC', async () => { + registerModelWithSchema(TEXT_SPEC); + const runner = tracked(await createLLMRunner(MODEL_PATH, TOKENIZER_PATH)); + expect(runner).toBeDefined(); + expect(runner.modalities).toEqual([]); + }); + + it('rejects text runner if modalities are provided', async () => { + registerModelWithSchema(TEXT_SPEC); + await expect(createLLMRunner(MODEL_PATH, TOKENIZER_PATH, ['image'])).rejects.toThrow( + /Text-only model does not support modalities/ + ); + }); + + it('dispatches to vision multimodal runner when schema matches LLM_MULTIMODAL_SPEC', async () => { + registerModelWithSchema(MULTIMODAL_SPEC); + const runner = tracked(await createLLMRunner(MODEL_PATH, TOKENIZER_PATH, ['image'])); + expect(runner).toBeDefined(); + expect(runner.modalities).toEqual(['image']); + }); + + it('rejects multimodal runner if image modality is missing', async () => { + registerModelWithSchema(MULTIMODAL_SPEC); + await expect(createLLMRunner(MODEL_PATH, TOKENIZER_PATH)).rejects.toThrow( + /Multimodal vision-language model requires 'image' in modalities/ + ); + }); + + it('dispatches to gemma runner when schema matches LLM_GEMMA_SPEC', async () => { + registerModelWithSchema(GEMMA_SPEC); + const runner = tracked(await createLLMRunner(MODEL_PATH, TOKENIZER_PATH, ['image', 'audio'])); + expect(runner).toBeDefined(); + expect(runner.modalities).toEqual(['image', 'audio']); + }); + + it('rejects gemma runner if image or audio modality is missing', async () => { + registerModelWithSchema(GEMMA_SPEC); + await expect(createLLMRunner(MODEL_PATH, TOKENIZER_PATH, ['image'])).rejects.toThrow( + /Gemma runner requires both 'image' and 'audio' in modalities/ + ); + }); + + it('rejects with SCHEMA_MISMATCH on unsupported model schema', async () => { + fakeJsi.registerModel(MODEL_PATH, { + schema: exported({ + ...method('unsupported', [f32(1)], [f32(1)]), + }), + }); + await expect(createLLMRunner(MODEL_PATH, TOKENIZER_PATH)).rejects.toThrow( + /doesn't match any of the provided variants/ + ); + }); +}); + +describe('createLLMMultimodalRunner — execution & KV cache lifecycle', () => { + beforeEach(() => { + registerModelWithSchema(MULTIMODAL_SPEC, { maxContextLen: 256 }); + }); + + it('prefills text and updates KV cache position', async () => { + const model = await loadModel(MODEL_PATH); + const tokenizer = await loadTokenizer(TOKENIZER_PATH); + const runner = tracked(createLLMMultimodalRunner(model, tokenizer, ['image'])); + + expect(runner.getKVCacheState()).toEqual({ + pos: 0, + maxContextLen: 256, + remainingTokens: 256, + usageRatio: 0, + }); + + const stats = runner.prefill('hello world'); + expect(stats.numTokens).toBe(2); + expect(runner.getKVCacheState().pos).toBe(2); + expect(runner.getKVCacheState().remainingTokens).toBe(254); + }); + + it('prefills image and updates KV cache position by visual tokens count', async () => { + const model = await loadModel(MODEL_PATH); + const tokenizer = await loadTokenizer(TOKENIZER_PATH); + const runner = tracked(createLLMMultimodalRunner(model, tokenizer, ['image'])); + + const img = tensor('float32', [1, 3, 32, 32]); + try { + const stats = runner.prefill([{ kind: 'image', image: img }]); + expect(stats.numTokens).toBe(4); // visualTokens = 4 + expect(runner.getKVCacheState().pos).toBe(4); + } finally { + img.dispose(); + } + }); + + it('rejects invalid image tensor shape or dtype', async () => { + const model = await loadModel(MODEL_PATH); + const tokenizer = await loadTokenizer(TOKENIZER_PATH); + const runner = tracked(createLLMMultimodalRunner(model, tokenizer, ['image'])); + + const badDtypeImg = tensor('uint8', [1, 3, 32, 32]); + try { + expect(() => runner.prefill([{ kind: 'image', image: badDtypeImg }])).toThrow( + /Expected image tensor of dtype 'float32'/ + ); + } finally { + badDtypeImg.dispose(); + } + + const badShapeImg = tensor('float32', [1, 3, 64, 64]); + try { + expect(() => runner.prefill([{ kind: 'image', image: badShapeImg }])).toThrow( + /Image tensor shape/ + ); + } finally { + badShapeImg.dispose(); + } + }); + + it('handles interleaved text and image prompts', async () => { + const model = await loadModel(MODEL_PATH); + const tokenizer = await loadTokenizer(TOKENIZER_PATH); + const runner = tracked(createLLMMultimodalRunner(model, tokenizer, ['image'])); + + const img = tensor('float32', [1, 3, 32, 32]); + try { + const stats = runner.prefill(['hello', { kind: 'image', image: img }, 'world']); + expect(stats.numTokens).toBe(1 + 4 + 1); + expect(runner.getKVCacheState().pos).toBe(6); + } finally { + img.dispose(); + } + }); + + it('resets KV cache pos to 0 or valid targetPos', async () => { + const model = await loadModel(MODEL_PATH); + const tokenizer = await loadTokenizer(TOKENIZER_PATH); + const runner = tracked(createLLMMultimodalRunner(model, tokenizer, ['image'])); + + runner.prefill('hello world'); + expect(runner.getKVCacheState().pos).toBe(2); + + runner.reset(1); + expect(runner.getKVCacheState().pos).toBe(1); + + runner.reset(); + expect(runner.getKVCacheState().pos).toBe(0); + + expect(() => runner.reset(10)).toThrow(/must be in range/); + expect(() => runner.reset(-1)).toThrow(/must be in range/); + }); + + it('generates tokens and triggers onToken callback', async () => { + const model = await loadModel(MODEL_PATH); + const tokenizer = await loadTokenizer(TOKENIZER_PATH); + const runner = tracked(createLLMMultimodalRunner(model, tokenizer, ['image'])); + + const tokens: string[] = []; + const stats = runner.generate('hello', { maxNewTokens: 3 }, (tok) => tokens.push(tok)); + + expect(stats.numTokens).toBe(3); + expect(tokens.length).toBe(3); + expect(tokens).toEqual(['hello', 'hello', 'hello']); + }); + + it('stops generation early when stop() is called', async () => { + const model = await loadModel(MODEL_PATH); + const tokenizer = await loadTokenizer(TOKENIZER_PATH); + const runner = tracked(createLLMMultimodalRunner(model, tokenizer, ['image'])); + + let count = 0; + const stats = runner.generate('hello', { maxNewTokens: 10 }, () => { + count++; + if (count === 2) runner.stop(); + }); + + expect(stats.numTokens).toBe(2); + }); + + it('disposes underlying resources cleanly', async () => { + const model = await loadModel(MODEL_PATH); + const tokenizer = await loadTokenizer(TOKENIZER_PATH); + const runner = createLLMMultimodalRunner(model, tokenizer, ['image']); + + runner.dispose(); + expect(fakeJsi.liveModels()).toEqual([]); + expect(fakeJsi.liveTokenizers()).toEqual([]); + expect(fakeJsi.liveTensorDescriptions()).toEqual([]); + }); +}); From 4462dacdfa3a5dd9378d028e1644826c48b93ea5 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Tue, 13 Oct 2026 13:40:58 +0200 Subject: [PATCH 55/83] refactor(llm): initialize prefill totStats to zero stats and omit non-null assertion --- .../src/extensions/llm/runners/llmGemmaRunner.ts | 13 ++++++------- .../extensions/llm/runners/llmMultimodalRunner.ts | 11 +++++------ 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/packages/react-native-executorch/src/extensions/llm/runners/llmGemmaRunner.ts b/packages/react-native-executorch/src/extensions/llm/runners/llmGemmaRunner.ts index 163646b7ad..47d01d0b83 100644 --- a/packages/react-native-executorch/src/extensions/llm/runners/llmGemmaRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/runners/llmGemmaRunner.ts @@ -322,9 +322,8 @@ export function createLLMGemmaRunner( throw RnExecuTorchError('INVALID_ARGUMENT', 'prefill: Empty prompt provided.'); } - let totStats: LLMPrefillStats | undefined; - const combine = (st1: LLMPrefillStats, st2?: LLMPrefillStats): LLMPrefillStats => { - if (st2 === undefined) return st1; + let totStats: LLMPrefillStats = { numTokens: 0, durationMs: 0, tokensPerSecond: 0 }; + const combine = (st1: LLMPrefillStats, st2: LLMPrefillStats): LLMPrefillStats => { const numTokens = st1.numTokens + st2.numTokens; const durationMs = st1.durationMs + st2.durationMs; const tokensPerSecond = durationMs > 0 ? (numTokens / durationMs) * 1000 : 0; @@ -334,19 +333,19 @@ export function createLLMGemmaRunner( for (const segment of segments) { if (typeof segment === 'string') { const stats = prefillText(segment); - totStats = combine(stats, totStats); + totStats = combine(totStats, stats); continue; } switch (segment.kind) { case 'image': { const stats = prefillImage(segment); - totStats = combine(stats, totStats); + totStats = combine(totStats, stats); break; } case 'audio': { const stats = prefillAudio(segment); - totStats = combine(stats, totStats); + totStats = combine(totStats, stats); break; } default: @@ -357,7 +356,7 @@ export function createLLMGemmaRunner( } } - return totStats!; + return totStats; }; const generate = ( diff --git a/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts b/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts index 54edb4593c..cc68f90993 100644 --- a/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts @@ -252,9 +252,8 @@ export function createLLMMultimodalRunner( throw RnExecuTorchError('INVALID_ARGUMENT', 'prefill: Empty prompt provided.'); } - let totStats: LLMPrefillStats | undefined; - const combine = (st1: LLMPrefillStats, st2?: LLMPrefillStats): LLMPrefillStats => { - if (st2 === undefined) return st1; + let totStats: LLMPrefillStats = { numTokens: 0, durationMs: 0, tokensPerSecond: 0 }; + const combine = (st1: LLMPrefillStats, st2: LLMPrefillStats): LLMPrefillStats => { const numTokens = st1.numTokens + st2.numTokens; const durationMs = st1.durationMs + st2.durationMs; const tokensPerSecond = durationMs > 0 ? (numTokens / durationMs) * 1000 : 0; @@ -264,14 +263,14 @@ export function createLLMMultimodalRunner( for (const segment of segments) { if (typeof segment === 'string') { const stats = prefillText(segment); - totStats = combine(stats, totStats); + totStats = combine(totStats, stats); continue; } switch (segment.kind) { case 'image': { const stats = prefillImage(segment); - totStats = combine(stats, totStats); + totStats = combine(totStats, stats); break; } default: @@ -282,7 +281,7 @@ export function createLLMMultimodalRunner( } } - return totStats!; + return totStats; }; const generate = ( From 1d4b3b476ba499a5ffda28defd83758195efb343 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Tue, 13 Oct 2026 13:44:38 +0200 Subject: [PATCH 56/83] fix(core): reject non-finite numbers in double and float conversions --- .../react-native-executorch/cpp/core/conversions.cpp | 6 +++++- .../cpp/tests/core/ConversionsTest.cpp | 12 +++++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/packages/react-native-executorch/cpp/core/conversions.cpp b/packages/react-native-executorch/cpp/core/conversions.cpp index 395870c5e6..b0932e85ac 100644 --- a/packages/react-native-executorch/cpp/core/conversions.cpp +++ b/packages/react-native-executorch/cpp/core/conversions.cpp @@ -21,7 +21,11 @@ double asType(jsi::Runtime & /*rt*/, const std::string &ctx, const jsi:: if (!val.isNumber()) { throw error::InvalidArgument(ctx + " must be a number"); } - return val.asNumber(); + double d = val.asNumber(); + if (!std::isfinite(d)) { + throw error::InvalidArgument(ctx + " must be a finite number"); + } + return d; } template <> diff --git a/packages/react-native-executorch/cpp/tests/core/ConversionsTest.cpp b/packages/react-native-executorch/cpp/tests/core/ConversionsTest.cpp index 7e215f7274..969ef16b43 100644 --- a/packages/react-native-executorch/cpp/tests/core/ConversionsTest.cpp +++ b/packages/react-native-executorch/cpp/tests/core/ConversionsTest.cpp @@ -26,7 +26,7 @@ using ConversionsTest = JsiTestEnv; // through, so its range and type checks are what stop a bad JS call from // reaching a reinterpret_cast. Tested against real jsi::Values. -jsi::Value number(jsi::Runtime &rt, double v) { +jsi::Value number(jsi::Runtime & /*rt*/, double v) { return jsi::Value(v); } @@ -59,6 +59,16 @@ TEST_F(ConversionsTest, RejectsNaNAndInfinity) { const double nan = std::numeric_limits::quiet_NaN(); const double inf = std::numeric_limits::infinity(); + EXPECT_TRUE(throwsCoded([&] { conversions::asType(rt(), "ctx", number(rt(), nan)); }, kInvalidArgument, + "ctx must be a finite number")); + EXPECT_TRUE(throwsCoded([&] { conversions::asType(rt(), "ctx", number(rt(), inf)); }, kInvalidArgument, + "ctx must be a finite number")); + EXPECT_TRUE(throwsCoded([&] { conversions::asType(rt(), "ctx", number(rt(), -inf)); }, kInvalidArgument, + "ctx must be a finite number")); + EXPECT_TRUE(throwsCoded([&] { conversions::asType(rt(), "ctx", number(rt(), nan)); }, kInvalidArgument, + "ctx must be a finite number")); + EXPECT_TRUE(throwsCoded([&] { conversions::asType(rt(), "ctx", number(rt(), inf)); }, kInvalidArgument, + "ctx must be a finite number")); EXPECT_TRUE(throwsCoded([&] { conversions::asType(rt(), "ctx", number(rt(), nan)); }, kInvalidArgument)); EXPECT_TRUE(throwsCoded([&] { conversions::asType(rt(), "ctx", number(rt(), inf)); }, kInvalidArgument)); EXPECT_TRUE(throwsCoded([&] { conversions::asType(rt(), "ctx", number(rt(), -inf)); }, kInvalidArgument)); From 893c08f8c1a2442599e4fcaf80b40a3ec9c3cac3 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Tue, 13 Oct 2026 13:53:57 +0200 Subject: [PATCH 57/83] feat(llm): add penaltyLastN window and seed to sampler with full-chain pipeline tests --- .../__tests__/extensions/sampler.test.ts | 5 + .../cpp/extensions/llm/sampler.cpp | 28 ++++- .../cpp/tests/extensions/LlmOpsTest.cpp | 117 ++++++++++++++++++ .../src/extensions/llm/sampler.ts | 10 ++ 4 files changed, 156 insertions(+), 4 deletions(-) diff --git a/packages/react-native-executorch/__tests__/extensions/sampler.test.ts b/packages/react-native-executorch/__tests__/extensions/sampler.test.ts index fdcb175b1f..3bcc2eafea 100644 --- a/packages/react-native-executorch/__tests__/extensions/sampler.test.ts +++ b/packages/react-native-executorch/__tests__/extensions/sampler.test.ts @@ -13,6 +13,7 @@ describe('sample — options merging', () => { expect(options).toMatchObject({ temperature: 0.8, repetitionPenalty: 1.0, + penaltyLastN: -1, topK: 40, topP: 0.95, minP: 0.05, @@ -33,6 +34,8 @@ describe('sample — options merging', () => { { temperature: 0.2, topK: 10, + penaltyLastN: 64, + seed: 12345, dry: { multiplier: 0.8, allowedLength: 4 }, xtc: { probability: 0.5, threshold: 0.3 }, } @@ -46,6 +49,8 @@ describe('sample — options merging', () => { topP: 0.95, minP: 0.05, repetitionPenalty: 1.0, + penaltyLastN: 64, + seed: 12345, dry: { multiplier: 0.8, base: 1.75, diff --git a/packages/react-native-executorch/cpp/extensions/llm/sampler.cpp b/packages/react-native-executorch/cpp/extensions/llm/sampler.cpp index 5e4980beab..de8ba35abd 100644 --- a/packages/react-native-executorch/cpp/extensions/llm/sampler.cpp +++ b/packages/react-native-executorch/cpp/extensions/llm/sampler.cpp @@ -48,18 +48,27 @@ struct Candidate { * For each unique token previously generated: * - Positive logits are scaled down by dividing by the penalty factor. * - Non-positive logits are penalized by multiplying by the penalty factor. - * * @param logits Logits span to modify in place. * @param tokens History of previously generated token IDs. * @param penalty Multiplicative repetition penalty factor (> 1.0 penalizes repeats). + * @param penaltyLastN Maximum recent tokens window to scan (-1 for unlimited). */ -void applyRepetitionPenalty(std::span logits, const std::vector &tokens, float penalty) { +void applyRepetitionPenalty(std::span logits, + const std::vector &tokens, + float penalty, + int32_t penaltyLastN) { if (penalty == 1.0f || tokens.empty()) { return; } + const size_t totalTokens = tokens.size(); + const size_t windowStart = (penaltyLastN > 0 && std::cmp_greater(totalTokens, penaltyLastN)) + ? (totalTokens - static_cast(penaltyLastN)) + : 0; + std::unordered_set seen; - for (int32_t tok : tokens) { + for (size_t i = windowStart; i < totalTokens; ++i) { + const int32_t tok = tokens[i]; if (tok < 0 || !std::cmp_less(tok, logits.size()) || !seen.insert(tok).second) { continue; } @@ -523,7 +532,9 @@ void install_sample(jsi::Runtime &rt, jsi::Object &module) { const auto topP = getRequiredProperty(rt, "sample", opts, "topP"); const auto minP = getRequiredProperty(rt, "sample", opts, "minP"); + const auto seed = getOptionalProperty(rt, "sample", opts, "seed"); const auto temperature = getRequiredProperty(rt, "sample", opts, "temperature"); + const auto penaltyLastN = getOptionalProperty(rt, "sample", opts, "penaltyLastN").value_or(-1); const auto repetitionPenalty = getRequiredProperty(rt, "sample", opts, "repetitionPenalty"); const auto xtcOptions = getOptionalProperty(rt, "sample", opts, "xtc"); @@ -544,8 +555,17 @@ void install_sample(jsi::Runtime &rt, jsi::Object &module) { if (repetitionPenalty <= 0.0f) { throw error::InvalidArgument("sample: option 'repetitionPenalty' must be positive"); } + if (penaltyLastN < -1) { + throw error::InvalidArgument("sample: option 'penaltyLastN' must be -1 or non-negative"); + } static thread_local std::mt19937 rng(std::random_device{}()); + if (seed.has_value()) { + if (*seed > std::numeric_limits::max()) { + throw error::InvalidArgument("sample: option 'seed' must be a non-negative 32-bit integer"); + } + rng.seed(static_cast(*seed)); + } /** * The sampling pipeline follows the llama.cpp execution sequence: @@ -584,7 +604,7 @@ void install_sample(jsi::Runtime &rt, jsi::Object &module) { applyConstraintsCallback(rt, logits, constraintsFn.value(), args[1]); } - applyRepetitionPenalty(logits, genToks, repetitionPenalty); + applyRepetitionPenalty(logits, genToks, repetitionPenalty, penaltyLastN); if (dryOptions.has_value()) { const auto dryMultiplier = getRequiredProperty(rt, "sample: dry", *dryOptions, "multiplier"); diff --git a/packages/react-native-executorch/cpp/tests/extensions/LlmOpsTest.cpp b/packages/react-native-executorch/cpp/tests/extensions/LlmOpsTest.cpp index 092d37e594..da7fc31e3c 100644 --- a/packages/react-native-executorch/cpp/tests/extensions/LlmOpsTest.cpp +++ b/packages/react-native-executorch/cpp/tests/extensions/LlmOpsTest.cpp @@ -114,6 +114,16 @@ TEST_F(LlmOpsTest, ValidatesTheRangeOfEachCoreOption) { Case{"minP: 1", "'minP' must be in [0, 1)"}, Case{"temperature: -1", "'temperature' must be non-negative"}, Case{"repetitionPenalty: 0", "'repetitionPenalty' must be positive"}, + Case{"penaltyLastN: -2", "'penaltyLastN' must be -1 or non-negative"}, + Case{"seed: -1", "'seed' must be a non-negative integer"}, + Case{"seed: 4294967296", "'seed' must be a non-negative 32-bit integer"}, + Case{"seed: 1.5", "'seed' must be a non-negative integer"}, + Case{"temperature: NaN", "must be a finite number"}, + Case{"temperature: Infinity", "must be a finite number"}, + Case{"repetitionPenalty: NaN", "must be a finite number"}, + Case{"repetitionPenalty: Infinity", "must be a finite number"}, + Case{"topP: NaN", "must be a finite number"}, + Case{"minP: NaN", "must be a finite number"}, }; for (const auto &[override, message] : cases) { EXPECT_TRUE(isCodedError( @@ -274,6 +284,47 @@ TEST_F(LlmOpsTest, RepetitionPenaltyIgnoresOutOfRangeHistory) { 0); } +TEST_F(LlmOpsTest, RepetitionPenaltyHonoursWindow) { + // History [0, 1, 1]. Token 0 occurred 3 steps ago. + // With penaltyLastN: 2, only token 1 is in window: + // Token 0 logit remains 2.0 (unpenalised). + // Token 1 logit was 2.2, penalised by 1.5 -> 2.2 / 1.5 = 1.467. + // Token 0 (2.0) now beats token 1 (1.467). + EXPECT_EQ( + evalNumber(js("return sample(logits([2, 2.2]), ctx([0, 1, 1]), greedy({ repetitionPenalty: 1.5, penaltyLastN: 2 }));")), + 0); + + // With penaltyLastN: -1 or 3, token 0 is also penalised: 2.0 / 1.5 = 1.333, + // so token 1 (1.467) beats token 0 (1.333). + EXPECT_EQ( + evalNumber(js("return sample(logits([2, 2.2]), ctx([0, 1, 1]), greedy({ repetitionPenalty: 1.5, penaltyLastN: 3 }));")), + 1); +} + +TEST_F(LlmOpsTest, SeedProducesDeterministicDraws) { + auto draws1 = evalNumberArray(js(R"( + const t = logits([1, 1, 1, 1]); + const res = []; + for (let i = 0; i < 5; i++) { + t.setData(new Float32Array([1, 1, 1, 1])); + res.push(sample(t, ctx(), neutral({ seed: 42 + i, temperature: 1.0 }))); + } + return res; + )")); + + auto draws2 = evalNumberArray(js(R"( + const t = logits([1, 1, 1, 1]); + const res = []; + for (let i = 0; i < 5; i++) { + t.setData(new Float32Array([1, 1, 1, 1])); + res.push(sample(t, ctx(), neutral({ seed: 42 + i, temperature: 1.0 }))); + } + return res; + )")); + + EXPECT_EQ(draws1, draws2); +} + // --- DRY --------------------------------------------------------------------- // History 1 2 3 1 2: the suffix "1 2" already occurred, followed by 3, so @@ -404,5 +455,71 @@ TEST_F(LlmOpsTest, RewritesTheLogitsInPlace) { EXPECT_TRUE(std::isinf(result[2]) && result[2] < 0); } +// --- Full pipeline chain ----------------------------------------------------- + +TEST_F(LlmOpsTest, FullChainAppliesStagesInExactLlamaCppOrder) { + // Exercises constraints -> repetition penalty -> DRY -> top-k -> top-p -> min-p -> XTC -> temperature in one call. + // + // Initial logits: 6 tokens [0..5] with values: + // [10.0, 10.0, 8.0, 7.0, 6.0, 5.0] + // + // 1. Constraints: ban token 0 -> token 0 logit becomes -inf. + // Remaining unmasked candidates: tokens 1 (10.0), 2 (8.0), 3 (7.0), 4 (6.0), 5 (5.0). + // + // 2. Repetition penalty: token 1 was generated in history [1, 2]. + // With repetitionPenalty: 2.0, token 1 logit becomes 10.0 / 2.0 = 5.0. + // Logits: 2 (8.0), 3 (7.0), 4 (6.0), 5 (5.0), 1 (5.0). + // + // 3. DRY penalty: token 2 would extend suffix [1, 2]. + // With dry (base 2, multiplier 5, allowedLength 1), match length is 2: + // penalty = 5 * (2^(2 - 1)) = 10.0. + // Token 2 logit becomes 8.0 - 10.0 = -2.0. + // Logits: 3 (7.0), 4 (6.0), 5 (5.0), 1 (5.0), 2 (-2.0). + // + // 4. Top-K = 3: retains top 3 candidates: + // tokens 3 (7.0), 4 (6.0), 5 (5.0). + // Softmax over [7.0, 6.0, 5.0]: + // max = 7.0 -> exp(0) = 1.0, exp(-1) = 0.3679, exp(-2) = 0.1353. Sum = 1.5032. + // Probs: tok 3 ≈ 0.665, tok 4 ≈ 0.245, tok 5 ≈ 0.090. + // + // 5. Min-P = 0.2: min threshold is 0.2 * max_prob = 0.2 * 0.665 = 0.133. + // Tok 5 (0.090) < 0.133 is pruned! + // Surviving: tok 3 and tok 4. + // Normalized probs: tok 3 ≈ 0.731, tok 4 ≈ 0.269. + // + // 6. XTC (threshold: 0.1, probability: 1.0): + // Both tok 3 and tok 4 have p > 0.1. Multiple candidates meet threshold! + // With probability = 1.0, XTC removes the top candidate (tok 3). + // Only tok 4 survives -> returned token MUST be 4! + EXPECT_EQ(evalNumber(js(R"( + const t = logits([10.0, 10.0, 8.0, 7.0, 6.0, 5.0]); + const constraints = () => ({ bannedTokens: new Int32Array([0]) }); + const dry = { + multiplier: 5, + base: 2.0, + allowedLength: 1, + penaltyLastN: -1, + sequenceBreakers: new Int32Array(0), + }; + const xtc = { + threshold: 0.1, + probability: 1.0, + }; + return sample(t, ctx([1, 2]), { + temperature: 1.0, + topK: 3, + topP: 1.0, + minP: 0.2, + repetitionPenalty: 2.0, + penaltyLastN: -1, + seed: 42, + constraints, + dry, + xtc, + }); + )")), + 4); +} + } // namespace } // namespace rnexecutorch::tests diff --git a/packages/react-native-executorch/src/extensions/llm/sampler.ts b/packages/react-native-executorch/src/extensions/llm/sampler.ts index 2b7a6e1af6..955d80c511 100644 --- a/packages/react-native-executorch/src/extensions/llm/sampler.ts +++ b/packages/react-native-executorch/src/extensions/llm/sampler.ts @@ -109,6 +109,15 @@ export type SamplingConfig = { * discourage repetition. Defaults to 1.0 (disabled). */ readonly repetitionPenalty?: number; + /** + * Maximum number of recent tokens to consider for repetition penalty. + * Defaults to -1 (entire context). + */ + readonly penaltyLastN?: number; + /** + * Optional RNG seed for deterministic sampling. Must be a non-negative 32-bit integer. + */ + readonly seed?: number; /** Configuration for Don't Repeat Yourself (DRY) repetition penalty. */ readonly dry?: DryConfig; /** Configuration for Exclude Top Choices (XTC) sampling. */ @@ -129,6 +138,7 @@ const EMPTY_SEQUENCE_BREAKERS = new Int32Array(0); const DEFAULT_SAMPLING_OPTIONS = { temperature: 0.8, repetitionPenalty: 1.0, + penaltyLastN: -1, topK: 40, topP: 0.95, minP: 0.05, From 6d84a23deecba6aab5f330cab5a93569078fb031 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Tue, 13 Oct 2026 14:05:30 +0200 Subject: [PATCH 58/83] fix(build): run prepare before workspace typecheck --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 06f348a47e..30b0b8d049 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "scripts": { "prepare": "yarn workspaces foreach --all --exclude react-native-executorch-webrtc --topological-dev run prepare", "lint": "yarn workspaces foreach --all --exclude react-native-executorch-webrtc --parallel run lint", - "typecheck": "yarn workspaces foreach --all --exclude react-native-executorch-webrtc --parallel run typecheck" + "typecheck": "yarn prepare && yarn workspaces foreach --all --exclude react-native-executorch-webrtc --parallel run typecheck" }, "private": true, "devDependencies": { From 5312189fb15b9a742f52df029565e53ea4867902 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Tue, 13 Oct 2026 16:55:32 +0200 Subject: [PATCH 59/83] fix(llm): prevent EOS tokens reaching onToken and fix Hammer/Llama tool parsing - Check eosIds before emitting to onToken and pushing to generatedTokens across text, gemma, and multimodal runners - Support empty array [] tool call responses in parseHammerToolCalls - Strip <|python_tag|> in parseLlamaToolCalls --- apps/nlp/constants/llm.ts | 13 ++++++++++--- .../src/extensions/llm/runners/llmGemmaRunner.ts | 4 ++-- .../extensions/llm/runners/llmMultimodalRunner.ts | 4 ++-- .../src/extensions/llm/runners/llmTextRunner.ts | 4 ++-- 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/apps/nlp/constants/llm.ts b/apps/nlp/constants/llm.ts index 407d7bcad4..94bc5dfa2a 100644 --- a/apps/nlp/constants/llm.ts +++ b/apps/nlp/constants/llm.ts @@ -167,11 +167,17 @@ export const HAMMER_TOOL_STOP_REGEX = /(?:<\|im_end\|>)/; export function parseHammerToolCalls(text: string): llm.ToolParserResult | undefined { const jsonMatch = - text.match(/```(?:json)?\s*([\s\S]*?)\s*```/) ?? text.match(/\[\s*\{[\s\S]*\}\s*\]/); + text.match(/```(?:json)?\s*([\s\S]*?)\s*```/) ?? + text.match(/\[\s*\{[\s\S]*\}\s*\]/) ?? + text.match(/\[\s*\]/); const jsonStr = jsonMatch ? (jsonMatch[1] ?? jsonMatch[0]) : text.trim(); try { const parsed = JSON.parse(jsonStr); + if (Array.isArray(parsed) && parsed.length === 0) { + const textContent = text.replace(jsonMatch ? jsonMatch[0] : jsonStr, '').trim(); + return { toolCalls: [], textContent: textContent || undefined }; + } if (!Array.isArray(parsed) || parsed.length === 0) return undefined; const toolCalls: llm.ToolCall[] = []; @@ -239,7 +245,8 @@ export const LLAMA_TOOL_STOP_REGEX = /(?:<\|eot_id\|>|<\|eom_id\|>)/; export function parseLlamaToolCalls(text: string): llm.ToolParserResult | undefined { const cleanText = text - .replace(/<\|(?:eot_id|eom_id|start_header_id|end_header_id)\|>[\s\S]*?$/g, '') + .replace(/<\|(?:eot_id|eom_id|start_header_id|end_header_id|python_tag)\|>[\s\S]*?$/g, '') + .replace(/<\|python_tag\|>/g, '') .trim(); const jsonMatch = @@ -273,7 +280,7 @@ export function parseLlamaToolCalls(text: string): llm.ToolParserResult | undefi ]; const textContent = cleanText .replace(jsonMatch ? jsonMatch[0] : jsonStr, '') - .replace(/<\|(?:eot_id|eom_id)\|>/g, '') + .replace(/<\|(?:eot_id|eom_id|python_tag)\|>/g, '') .trim(); return { toolCalls, textContent: textContent || undefined }; } diff --git a/packages/react-native-executorch/src/extensions/llm/runners/llmGemmaRunner.ts b/packages/react-native-executorch/src/extensions/llm/runners/llmGemmaRunner.ts index 47d01d0b83..a464cdf124 100644 --- a/packages/react-native-executorch/src/extensions/llm/runners/llmGemmaRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/runners/llmGemmaRunner.ts @@ -392,11 +392,11 @@ export function createLLMGemmaRunner( const genTokArray = Int32Array.from(generatedTokens); const nextToken = sample(tLogits, { generatedTokens: genTokArray }, config); + if (!config?.ignoreEos && eosIds.includes(nextToken)) break; + onToken?.(tokenizer.decode(Int32Array.of(nextToken))); generatedTokens.push(nextToken); - if (!config?.ignoreEos && eosIds.includes(nextToken)) break; - // Embed single token tTokenId.setData(BigInt64Array.of(BigInt(nextToken))); model.execute('token_embedding', [tTokenId], [tTokenEmbed, tPleTok]); diff --git a/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts b/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts index cc68f90993..9595273aa8 100644 --- a/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts @@ -316,11 +316,11 @@ export function createLLMMultimodalRunner( const genTokArray = Int32Array.from(generatedTokens); const nextToken = sample(tLogits, { generatedTokens: genTokArray }, config); + if (!config?.ignoreEos && eosIds.includes(nextToken)) break; + onToken?.(tokenizer.decode(Int32Array.of(nextToken))); generatedTokens.push(nextToken); - if (!config?.ignoreEos && eosIds.includes(nextToken)) break; - // Embed single token tTokenId.setData(BigInt64Array.of(BigInt(nextToken))); model.execute('token_embedding', [tTokenId], [tTokenEmbed]); diff --git a/packages/react-native-executorch/src/extensions/llm/runners/llmTextRunner.ts b/packages/react-native-executorch/src/extensions/llm/runners/llmTextRunner.ts index 0cf4cd6eb0..0c636e1cef 100644 --- a/packages/react-native-executorch/src/extensions/llm/runners/llmTextRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/runners/llmTextRunner.ts @@ -191,11 +191,11 @@ export function createLLMTextRunner( const genTokArray = Int32Array.from(generatedTokens); const nextToken = sample(tLogits, { generatedTokens: genTokArray }, config); + if (!config?.ignoreEos && eosIds.includes(nextToken)) break; + onToken?.(tokenizer.decode(Int32Array.of(nextToken), /* skipSpecialTokens= */ false)); generatedTokens.push(nextToken); - if (!config?.ignoreEos && eosIds.includes(nextToken)) break; - tToken.setData(BigInt64Array.of(BigInt(nextToken))); tCurPos.setData(BigInt64Array.of(BigInt(curPos))); From 5b55ea820f00954d3956c3aed5e2f943d3a61637 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Wed, 14 Oct 2026 00:23:55 +0200 Subject: [PATCH 60/83] feat(llm): support Gemma Omni model and audio prefill chunking --- apps/nlp/app/llm/index.tsx | 260 +++++++++++++++++- apps/nlp/constants/llm.ts | 10 + apps/nlp/package.json | 1 + .../__tests__/tasks/llmRunner.test.ts | 28 +- .../cpp/extensions/llm/sampler.cpp | 2 +- .../cpp/tests/extensions/LlmOpsTest.cpp | 9 - .../extensions/llm/runners/llmGemmaRunner.ts | 109 +++++--- .../src/extensions/llm/sampler.ts | 2 +- .../react-native-executorch/src/models.ts | 72 ++--- yarn.lock | 1 + 10 files changed, 394 insertions(+), 100 deletions(-) diff --git a/apps/nlp/app/llm/index.tsx b/apps/nlp/app/llm/index.tsx index cab66b496e..d3c7124ba3 100644 --- a/apps/nlp/app/llm/index.tsx +++ b/apps/nlp/app/llm/index.tsx @@ -11,11 +11,13 @@ import { Platform, Alert, Image as RNImage, + Modal, } from 'react-native'; import { Skia } from '@shopify/react-native-skia'; import RNBlobUtil from 'react-native-blob-util'; import { useSafeAreaInsets } from 'react-native-safe-area-context'; import { useLLMChatSession, type llm, type cv } from 'react-native-executorch'; +import { decodeAudioData } from 'react-native-audio-api'; import ScreenWrapper from '../../components/ScreenWrapper'; import { ModelPicker, type ModelOption } from '../../components/ModelPicker'; import { Button } from '../../components/Button'; @@ -26,6 +28,7 @@ type Turn = { role: 'user' | 'assistant' | 'tool'; content: string; imageUri?: string; + audioName?: string; stats?: llm.LLMGenerationStats; toolCalls?: readonly llm.ToolCall[]; }; @@ -121,7 +124,18 @@ function LLMContent() { buffer: cv.ImageBuffer; } | null>(null); + const [attachedAudio, setAttachedAudio] = useState<{ + uri: string; + name: string; + samples: Float32Array; + } | null>(null); + + const [isAudioModalVisible, setIsAudioModalVisible] = useState(false); + const [audioUrl, setAudioUrl] = useState('https://models.silero.ai/vad_models/en.wav'); + const [isDecodingAudio, setIsDecodingAudio] = useState(false); + const supportsImages = Boolean(activeModel.model.modalities?.includes('image')); + const supportsAudio = Boolean(activeModel.model.modalities?.includes('audio')); const handleBackendChange = (newBackend: LLMBackend) => { if (newBackend === selectedBackend) return; @@ -136,6 +150,8 @@ function LLMContent() { setTurns([]); setStreamingResponse(null); setAttachedImage(null); + setAttachedAudio(null); + setIsAudioModalVisible(false); setInput(''); }; @@ -146,6 +162,8 @@ function LLMContent() { setTurns([]); setStreamingResponse(null); setAttachedImage(null); + setAttachedAudio(null); + setIsAudioModalVisible(false); setInput(''); }; @@ -177,6 +195,8 @@ function LLMContent() { setTurns([]); setStreamingResponse(null); setAttachedImage(null); + setAttachedAudio(null); + setIsAudioModalVisible(false); setInput(''); Alert.alert('Deleted', 'Model files deleted from device storage.'); } catch (err: any) { @@ -206,27 +226,62 @@ function LLMContent() { } }; + const handleAttachAudio = async () => { + const trimmed = audioUrl.trim(); + if (!trimmed) return; + setIsDecodingAudio(true); + try { + const audioBuffer = await decodeAudioData(trimmed, 16000); + const samples = audioBuffer.getChannelData(0); + const durationSec = (samples.length / 16000).toFixed(1); + const filename = trimmed.split('/').pop() || 'audio.wav'; + setAttachedAudio({ + uri: trimmed, + name: `${filename} (${durationSec}s)`, + samples, + }); + setIsAudioModalVisible(false); + } catch (err: any) { + Alert.alert('Error', err?.message || 'Failed to decode audio URL'); + } finally { + setIsDecodingAudio(false); + } + }; + const handleSend = async () => { const textMessage = input.trim(); - if ((!textMessage && !attachedImage) || !sendMessage || isGenerating) return; + if ((!textMessage && !attachedImage && !attachedAudio) || !sendMessage || isGenerating) return; const currentImage = attachedImage; + const currentAudio = attachedAudio; setAttachedImage(null); + setAttachedAudio(null); setInput(''); setStreamingResponse(''); setTurns((prev) => [ ...prev, - { role: 'user', content: textMessage, imageUri: currentImage?.uri }, + { + role: 'user', + content: textMessage, + imageUri: currentImage?.uri, + audioName: currentAudio?.name, + }, ]); try { let payload; - if (currentImage) { - payload = [ - { kind: 'image' as const, image: currentImage.buffer }, - textMessage || "What's in this image?", - ]; + if (currentImage || currentAudio) { + const mediaItems: (string | llm.ChatMediaInput)[] = []; + if (currentImage) { + mediaItems.push({ kind: 'image' as const, image: currentImage.buffer }); + } + if (currentAudio) { + mediaItems.push({ kind: 'audio' as const, audio: currentAudio.samples }); + } + const defaultPrompt = currentAudio ? "What's in this audio?" : "What's in this image?"; + mediaItems.push(textMessage || defaultPrompt); + payload = mediaItems; } else { payload = textMessage; } @@ -377,7 +432,14 @@ function LLMContent() { > {turns.length === 0 && streamingResponse === null && ( - Ask {activeModel.name} anything{supportsImages ? ' or attach an image.' : '.'} + Ask {activeModel.name} anything + {supportsImages && supportsAudio + ? ' or attach an image or audio.' + : supportsImages + ? ' or attach an image.' + : supportsAudio + ? ' or attach audio.' + : '.'} )} {turns.map((turn, idx) => ( @@ -395,6 +457,14 @@ function LLMContent() { {turn.imageUri && ( )} + {turn.audioName && ( + + 🎙️ + + {turn.audioName} + + + )} {turn.toolCalls && turn.toolCalls.length > 0 && ( 🛠️ Tool Call: @@ -446,6 +516,25 @@ function LLMContent() { )} + {attachedAudio && ( + + + 🎙️ + + + + {attachedAudio.name} + + + setAttachedAudio(null)} + > + ✕ + + + )} + {supportsImages && ( 📷 )} + {supportsAudio && ( + setIsAudioModalVisible(true)} + disabled={isGenerating} + > + 🎙️ + + )} Send )} + + { + if (!isDecodingAudio) setIsAudioModalVisible(false); + }} + > + + + Attach Audio URL + + Enter a direct URL to a WAV or audio file (will be decoded at 16kHz): + + + + setIsAudioModalVisible(false)} + disabled={isDecodingAudio} + > + Cancel + + + {isDecodingAudio ? ( + + ) : ( + Attach + )} + + + + + )} @@ -733,4 +883,94 @@ const styles = StyleSheet.create({ sendButtonDisabled: { backgroundColor: '#a3cdff' }, stopButton: { backgroundColor: '#e03131' }, sendButtonText: { color: '#fff', fontSize: 15, fontWeight: '600' }, + turnAudioBadge: { + flexDirection: 'row', + alignItems: 'center', + backgroundColor: 'rgba(255, 255, 255, 0.2)', + paddingHorizontal: 8, + paddingVertical: 4, + borderRadius: 8, + marginBottom: 6, + gap: 6, + }, + turnAudioIcon: { fontSize: 14 }, + turnAudioText: { color: '#fff', fontSize: 12, fontWeight: '500' }, + audioPreviewBadge: { + width: 40, + height: 40, + borderRadius: 6, + backgroundColor: '#e7f5ff', + justifyContent: 'center', + alignItems: 'center', + }, + audioIconText: { fontSize: 20 }, + modalOverlay: { + flex: 1, + backgroundColor: 'rgba(0, 0, 0, 0.5)', + justifyContent: 'center', + alignItems: 'center', + padding: 20, + }, + modalContent: { + width: '100%', + maxWidth: 400, + backgroundColor: '#fff', + borderRadius: 16, + padding: 20, + shadowColor: '#000', + shadowOffset: { width: 0, height: 2 }, + shadowOpacity: 0.25, + shadowRadius: 4, + elevation: 5, + }, + modalTitle: { + fontSize: 18, + fontWeight: '700', + color: '#212529', + marginBottom: 8, + }, + modalDescription: { + fontSize: 13, + color: '#6c757d', + marginBottom: 14, + lineHeight: 18, + }, + modalInput: { + backgroundColor: '#f1f3f5', + borderRadius: 10, + paddingHorizontal: 12, + paddingVertical: 10, + fontSize: 14, + color: '#212529', + marginBottom: 16, + }, + modalButtons: { + flexDirection: 'row', + justifyContent: 'flex-end', + gap: 10, + }, + modalButton: { + paddingHorizontal: 16, + paddingVertical: 10, + borderRadius: 8, + minWidth: 80, + alignItems: 'center', + justifyContent: 'center', + }, + modalButtonCancel: { + backgroundColor: '#e9ecef', + }, + modalButtonCancelText: { + color: '#495057', + fontWeight: '600', + fontSize: 14, + }, + modalButtonConfirm: { + backgroundColor: '#0070f3', + }, + modalButtonConfirmText: { + color: '#fff', + fontWeight: '600', + fontSize: 14, + }, }); diff --git a/apps/nlp/constants/llm.ts b/apps/nlp/constants/llm.ts index 94bc5dfa2a..00961584b9 100644 --- a/apps/nlp/constants/llm.ts +++ b/apps/nlp/constants/llm.ts @@ -589,6 +589,16 @@ export const LLM_MODELS: LLMModelConfig[] = [ }, // Gemma 4 family + { + id: 'gemma-4-omni-e2b-xnnpack', + name: 'Gemma 4 Omni E2B', + backend: 'xnnpack', + model: models.llm.GEMMA4_OMNI_E2B.XNNPACK_8DA4W, + generationConfig: GEMMA4_GENERATION_CONFIG, + systemPrompt: DEFAULT_SYSTEM_PROMPT, + stopRegex: GEMMA_TOOL_STOP_REGEX, + toolOpts: GEMMA_TOOL_OPTS, + }, { id: 'gemma-4-e2b-xnnpack', name: 'Gemma 4 E2B', diff --git a/apps/nlp/package.json b/apps/nlp/package.json index d065f9ef40..4c60c67a67 100644 --- a/apps/nlp/package.json +++ b/apps/nlp/package.json @@ -37,6 +37,7 @@ "expo-router": "~57.0.26", "react": "19.2.3", "react-native": "0.86.3", + "react-native-audio-api": "0.13.1", "react-native-blob-util": "^0.25.0", "react-native-drawer-layout": "^4.2.2", "react-native-executorch": "workspace:*", diff --git a/packages/react-native-executorch/__tests__/tasks/llmRunner.test.ts b/packages/react-native-executorch/__tests__/tasks/llmRunner.test.ts index 4ae7362f39..0416f1cdda 100644 --- a/packages/react-native-executorch/__tests__/tasks/llmRunner.test.ts +++ b/packages/react-native-executorch/__tests__/tasks/llmRunner.test.ts @@ -1,4 +1,4 @@ -import { f32, i64, method, RangeDim } from '../../src/core/schema'; +import { constraint, f32, i64, method, RangeDim } from '../../src/core/schema'; import { tensor } from '../../src/core/tensor'; import { createLLMRunner } from '../../src/extensions/llm/llmRunner'; import { createLLMMultimodalRunner } from '../../src/extensions/llm/runners/llmMultimodalRunner'; @@ -37,18 +37,38 @@ const GEMMA_SPEC = exported({ ...method( 'token_embedding', [i64(1, RangeDim(1, 128))], - [f32(1, RangeDim(1, 128), 64), f32(1, RangeDim(1, 128), 2, 8)] + [f32(1, RangeDim(1, 128), 64), f32(1, RangeDim(1, 128), 2, 8)], + [ + constraint.equality( + { paramSide: 'input', tensorIdx: 0, dimIdx: 1 }, + { paramSide: 'output', tensorIdx: 0, dimIdx: 1 }, + { paramSide: 'output', tensorIdx: 1, dimIdx: 1 } + ), + ] ), ...method( 'text_decoder', [f32(1, RangeDim(1, 128), 64), f32(1, RangeDim(1, 128), 2, 8), i64(RangeDim(1, 128))], - [f32(1, 1, 100)] + [f32(1, 1, 100)], + [ + constraint.equality( + { paramSide: 'input', tensorIdx: 0, dimIdx: 1 }, + { paramSide: 'input', tensorIdx: 1, dimIdx: 1 }, + { paramSide: 'input', tensorIdx: 2, dimIdx: 0 } + ), + ] ), ...method('vision_encoder', [f32(1, 3, 32, 32)], [f32(1, 4, 64)]), ...method( 'audio_encoder', [f32(1, RangeDim(1, 7680)), i64(1)], - [f32(1, RangeDim(1, 12), 64), f32(1, RangeDim(1, 12))] + [f32(1, RangeDim(1, 12), 64), f32(1, RangeDim(1, 12))], + [ + constraint.equality( + { paramSide: 'output', tensorIdx: 0, dimIdx: 1 }, + { paramSide: 'output', tensorIdx: 1, dimIdx: 1 } + ), + ] ), ...method('get_max_seq_len', [], [{ kind: 'Int' }]), ...method('get_max_context_len', [], [{ kind: 'Int' }]), diff --git a/packages/react-native-executorch/cpp/extensions/llm/sampler.cpp b/packages/react-native-executorch/cpp/extensions/llm/sampler.cpp index de8ba35abd..cd2fd95ef1 100644 --- a/packages/react-native-executorch/cpp/extensions/llm/sampler.cpp +++ b/packages/react-native-executorch/cpp/extensions/llm/sampler.cpp @@ -516,7 +516,7 @@ void install_sample(jsi::Runtime &rt, jsi::Object &module) { throw error::InvalidArgument("sample: Usage: sample(logits, ctx, options)"); } - auto src = tensor::fromJs(rt, "sample: logits", args[0], DType::float32, {1, "vocab_size"}); + auto src = tensor::fromJs(rt, "sample: logits", args[0], DType::float32, std::nullopt); auto lock = tensor::tryLockUnique(rt, "sample: logits", src); if (src->numel_ == 0) { diff --git a/packages/react-native-executorch/cpp/tests/extensions/LlmOpsTest.cpp b/packages/react-native-executorch/cpp/tests/extensions/LlmOpsTest.cpp index da7fc31e3c..db18f93786 100644 --- a/packages/react-native-executorch/cpp/tests/extensions/LlmOpsTest.cpp +++ b/packages/react-native-executorch/cpp/tests/extensions/LlmOpsTest.cpp @@ -63,15 +63,6 @@ TEST_F(LlmOpsTest, RejectsAWrongArgumentCount) { "Usage: sample(logits, ctx, options)")); } -TEST_F(LlmOpsTest, RejectsLogitsOfTheWrongRank) { - EXPECT_TRUE(isCodedError(evalThrowing(js(R"( - const t = createTensor([4], 'float32'); - sample(t, ctx(), greedy()); - )")), - "INVALID_ARGUMENT", - "sample: logits")); -} - TEST_F(LlmOpsTest, RejectsABatchOfLogits) { // The runner decodes one sequence; a [2, vocab] tensor is a caller bug. EXPECT_TRUE(isCodedError(evalThrowing(js(R"( diff --git a/packages/react-native-executorch/src/extensions/llm/runners/llmGemmaRunner.ts b/packages/react-native-executorch/src/extensions/llm/runners/llmGemmaRunner.ts index a464cdf124..7346e1ae1f 100644 --- a/packages/react-native-executorch/src/extensions/llm/runners/llmGemmaRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/runners/llmGemmaRunner.ts @@ -3,7 +3,14 @@ import { createSynchronizable } from 'react-native-worklets'; import { tensor } from '../../../core/tensor'; import type { Model } from '../../../core/model'; import { RnExecuTorchError } from '../../../core/error'; -import { f32, i64, method, DynamicDim as Dyn, validateSpec } from '../../../core/schema'; +import { + f32, + i64, + method, + DynamicDim as Dyn, + validateSpec, + constraint, +} from '../../../core/schema'; import type { Tokenizer } from '../../nlp'; import { sample } from '../sampler'; @@ -32,7 +39,14 @@ export const LLM_GEMMA_SPEC = { ...method( 'token_embedding', [i64(1, Dyn('seqLen'))], - [f32(1, Dyn('seqLen'), 'hiddenDim'), f32(1, Dyn('seqLen'), 'numLayers', 'pleDim')] + [f32(1, Dyn('seqLen'), 'hiddenDim'), f32(1, Dyn('seqLen'), 'numLayers', 'pleDim')], + [ + constraint.equality( + { paramSide: 'input', tensorIdx: 0, dimIdx: 1 }, + { paramSide: 'output', tensorIdx: 0, dimIdx: 1 }, + { paramSide: 'output', tensorIdx: 1, dimIdx: 1 } + ), + ] ), ...method( 'text_decoder', @@ -41,7 +55,14 @@ export const LLM_GEMMA_SPEC = { f32(1, Dyn('seqLen'), 'numLayers', 'pleDim'), i64(Dyn('seqLen')), ], - [f32(1, 1, 'vocabSize')] + [f32(1, 1, 'vocabSize')], + [ + constraint.equality( + { paramSide: 'input', tensorIdx: 0, dimIdx: 1 }, + { paramSide: 'input', tensorIdx: 1, dimIdx: 1 }, + { paramSide: 'input', tensorIdx: 2, dimIdx: 0 } + ), + ] ), ...method( 'vision_encoder', // prettier-ignore @@ -51,13 +72,29 @@ export const LLM_GEMMA_SPEC = { ...method( 'audio_encoder', [f32(1, Dyn('audioSamples')), i64(1)], - [f32(1, Dyn('audioTokens'), 'hiddenDim'), f32(1, Dyn('audioTokens'))] + [f32(1, Dyn('audioTokens'), 'hiddenDim'), f32(1, Dyn('audioTokens'))], + [ + constraint.equality( + { paramSide: 'output', tensorIdx: 0, dimIdx: 1 }, + { paramSide: 'output', tensorIdx: 1, dimIdx: 1 } + ), + ] ), ...METADATA_SPEC, }; const AUDIO_SAMPLES_PER_BLOCK = 7680; const TOKENS_PER_AUDIO_BLOCK = 12; +const MAX_AUDIO_BLOCKS = 62; +const MAX_AUDIO_CHUNK_SAMPLES = MAX_AUDIO_BLOCKS * AUDIO_SAMPLES_PER_BLOCK; + +function combineStats(st1: LLMPrefillStats, st2: LLMPrefillStats): LLMPrefillStats { + 'worklet'; + const numTokens = st1.numTokens + st2.numTokens; + const durationMs = st1.durationMs + st2.durationMs; + const tokensPerSecond = durationMs > 0 ? (numTokens / durationMs) * 1000 : 0; + return { numTokens, durationMs, tokensPerSecond }; +} export function createLLMGemmaRunner( model: Model, @@ -286,31 +323,43 @@ export function createLLMGemmaRunner( throw RnExecuTorchError('INVALID_ARGUMENT', 'Audio waveform is empty.'); } - // Pad waveform to multiple of AUDIO_SAMPLES_PER_BLOCK - const numBlocks = Math.ceil(numSamples / AUDIO_SAMPLES_PER_BLOCK); - const paddedSamples = numBlocks * AUDIO_SAMPLES_PER_BLOCK; - const numAudioTokens = numBlocks * TOKENS_PER_AUDIO_BLOCK; - const rawData = prompt.audio.getData(new Float32Array(numSamples)); - const paddedWav = new Float32Array(paddedSamples); - paddedWav.set(rawData); - const tensors = [ - tensor('float32', [1, paddedSamples], paddedWav), - tensor('int64', [1], BigInt64Array.of(BigInt(numSamples))), - tensor('float32', [1, numAudioTokens, hiddenDim]), - tensor('float32', [1, numAudioTokens]), - ] as const; + let totStats: LLMPrefillStats = { numTokens: 0, durationMs: 0, tokensPerSecond: 0 }; - const [tWaveform, tNumValid, tAudioEmbeds, tAudioMask] = tensors; + let offset = 0; + while (offset < numSamples && !isCancelled.getBlocking()) { + const chunkSamples = Math.min(numSamples - offset, MAX_AUDIO_CHUNK_SAMPLES); + const numBlocks = Math.ceil(chunkSamples / AUDIO_SAMPLES_PER_BLOCK); + const paddedSamples = numBlocks * AUDIO_SAMPLES_PER_BLOCK; + const numAudioTokens = numBlocks * TOKENS_PER_AUDIO_BLOCK; - try { - model.execute('audio_encoder', [tWaveform, tNumValid], [tAudioEmbeds, tAudioMask]); - const embedding = tAudioEmbeds.getData(new Float32Array(tAudioEmbeds.numel)); - return prefillEmbeddings(embedding, numAudioTokens); - } finally { - tensors.forEach((t) => t.dispose()); + const chunkData = rawData.subarray(offset, offset + chunkSamples); + const paddedWav = new Float32Array(paddedSamples); + paddedWav.set(chunkData); + + const tensors = [ + tensor('float32', [1, paddedSamples], paddedWav), + tensor('int64', [1], BigInt64Array.of(BigInt(chunkSamples))), + tensor('float32', [1, numAudioTokens, hiddenDim]), + tensor('float32', [1, numAudioTokens]), + ] as const; + + const [tWaveform, tNumValid, tAudioEmbeds, tAudioMask] = tensors; + + try { + model.execute('audio_encoder', [tWaveform, tNumValid], [tAudioEmbeds, tAudioMask]); + const embedding = tAudioEmbeds.getData(new Float32Array(tAudioEmbeds.numel)); + const chunkStats = prefillEmbeddings(embedding, numAudioTokens); + totStats = combineStats(totStats, chunkStats); + } finally { + tensors.forEach((t) => t.dispose()); + } + + offset += chunkSamples; } + + return totStats; }; const prefill = (prompt: Prompt): LLMPrefillStats => { @@ -323,29 +372,23 @@ export function createLLMGemmaRunner( } let totStats: LLMPrefillStats = { numTokens: 0, durationMs: 0, tokensPerSecond: 0 }; - const combine = (st1: LLMPrefillStats, st2: LLMPrefillStats): LLMPrefillStats => { - const numTokens = st1.numTokens + st2.numTokens; - const durationMs = st1.durationMs + st2.durationMs; - const tokensPerSecond = durationMs > 0 ? (numTokens / durationMs) * 1000 : 0; - return { numTokens, durationMs, tokensPerSecond }; - }; for (const segment of segments) { if (typeof segment === 'string') { const stats = prefillText(segment); - totStats = combine(totStats, stats); + totStats = combineStats(totStats, stats); continue; } switch (segment.kind) { case 'image': { const stats = prefillImage(segment); - totStats = combine(totStats, stats); + totStats = combineStats(totStats, stats); break; } case 'audio': { const stats = prefillAudio(segment); - totStats = combine(totStats, stats); + totStats = combineStats(totStats, stats); break; } default: diff --git a/packages/react-native-executorch/src/extensions/llm/sampler.ts b/packages/react-native-executorch/src/extensions/llm/sampler.ts index 955d80c511..3f9a159002 100644 --- a/packages/react-native-executorch/src/extensions/llm/sampler.ts +++ b/packages/react-native-executorch/src/extensions/llm/sampler.ts @@ -161,7 +161,7 @@ const DEFAULT_XTC_OPTIONS = { * **Important:** The logits buffer is written to in place (penalties, DRY and constraints are * applied directly to it), so its contents are undefined after this call. Copy * the tensor first if the original logits are still needed. - * @param logits 2D float32 logits Tensor of shape `[1, vocabSize]`. Clobbered by the call. + * @param logits Float32 logits Tensor. Clobbered by the call. * @param ctx Generation context containing emitted token history. * @param config Optional sampling configuration (defaults to `{}`). * @returns The sampled token id. diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index e5e090e876..7be45b2103 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -2296,24 +2296,22 @@ const QWEN2_5_3B_MLX_INT4: LLMModel = { }; const GEMMA4_BASE_URL = `${BASE_URL}-gemma-4/${NEXT_VERSION_TAG}`; -// Gemma 4 omni (multimodal) is parked for now: its builds are not part of the -// v0.11.0 upstream migration, so the model is commented out until they are. -// const GEMMA4_OMNI_BASE_URL = `${BASE_URL}-gemma-4-multimodal/${NEXT_VERSION_TAG}`; -// -// const GEMMA4_OMNI_PREPROCESSOR_CONFIG = { -// image: { -// visionToken: { start: '<|image>', end: '' }, -// targetShape: [1, 3, 448, 640] as const, -// preprocessorOpts: { -// resizeMode: 'letterbox' as const, -// interpolation: 'linear' as const, -// normalizeOpts: { alpha: 1.0, beta: 0.0 }, -// }, -// }, -// audio: { -// audioToken: { start: '<|audio>', end: '' }, -// }, -// }; +const GEMMA4_OMNI_BASE_URL = `${BASE_URL}-gemma-4-multimodal/${NEXT_VERSION_TAG}`; + +const GEMMA4_OMNI_PREPROCESSOR_CONFIG = { + image: { + visionToken: { start: '<|image>', end: '' }, + targetShape: [1, 3, 448, 640] as const, + preprocessorOpts: { + resizeMode: 'letterbox' as const, + interpolation: 'linear' as const, + normalizeOpts: { alpha: 1.0, beta: 0.0 }, + }, + }, + audio: { + audioToken: { start: '<|audio>', end: '' }, + }, +}; const GEMMA4_E2B_XNNPACK_8DA4W: LLMModel = { modelPath: `${GEMMA4_BASE_URL}/e2b/xnnpack/gemma_4_e2b_xnnpack_8da4w.pte`, @@ -2330,20 +2328,13 @@ const GEMMA4_E2B_MLX_INT4: LLMModel = { tokenizerPath: `${GEMMA4_BASE_URL}/e2b/tokenizer.json`, tokenizerConfigPath: `${GEMMA4_BASE_URL}/e2b/tokenizer_config.json`, }; -// const GEMMA4_OMNI_E2B_MLX_INT4: LLMModel = { -// modelPath: `${GEMMA4_OMNI_BASE_URL}/e2b/mlx/gemma_4_omni_e2b_mlx_int4.pte`, -// tokenizerPath: `${GEMMA4_BASE_URL}/e2b/tokenizer.json`, -// tokenizerConfigPath: `${GEMMA4_BASE_URL}/e2b/tokenizer_config.json`, -// modalities: ['image', 'audio'], -// preprocessorConfig: GEMMA4_OMNI_PREPROCESSOR_CONFIG, -// }; -// const GEMMA4_OMNI_E2B_XNNPACK_8DA4W: LLMModel = { -// modelPath: `${GEMMA4_OMNI_BASE_URL}/e2b/xnnpack/gemma_4_e2b_xnnpack_8da4w.pte`, -// tokenizerPath: `${GEMMA4_BASE_URL}/e2b/tokenizer.json`, -// tokenizerConfigPath: `${GEMMA4_BASE_URL}/e2b/tokenizer_config.json`, -// modalities: ['image', 'audio'], -// preprocessorConfig: GEMMA4_OMNI_PREPROCESSOR_CONFIG, -// }; +const GEMMA4_OMNI_E2B_XNNPACK_8DA4W: LLMModel = { + modelPath: `${GEMMA4_OMNI_BASE_URL}/e2b/xnnpack/gemma_4_e2b_xnnpack_8da4w.pte`, + tokenizerPath: `${GEMMA4_BASE_URL}/e2b/tokenizer.json`, + tokenizerConfigPath: `${GEMMA4_BASE_URL}/e2b/tokenizer_config.json`, + modalities: ['image', 'audio'], + preprocessorConfig: GEMMA4_OMNI_PREPROCESSOR_CONFIG, +}; const QWEN3_BASE_URL = `${BASE_URL}-qwen-3/${NEXT_VERSION_TAG}`; @@ -3390,16 +3381,13 @@ export const models = { VULKAN_8DA4W: GEMMA4_E2B_VULKAN_8DA4W, MLX_INT4: GEMMA4_E2B_MLX_INT4, }), - // Gemma 4 omni (multimodal) is parked until its builds join the upstream - // migration; see the commented-out constants above. - // /** - // * Google Gemma 4 E2B multimodal (omni) language model supporting text, image, - // * and audio inputs. - // */ - // GEMMA4_OMNI_E2B: variants({ - // MLX_INT4: GEMMA4_OMNI_E2B_MLX_INT4, - // XNNPACK_8DA4W: GEMMA4_OMNI_E2B_XNNPACK_8DA4W, - // }), + /** + * Google Gemma 4 E2B multimodal (omni) language model supporting text, image, + * and audio inputs. + */ + GEMMA4_OMNI_E2B: variants({ + XNNPACK_8DA4W: GEMMA4_OMNI_E2B_XNNPACK_8DA4W, + }), }, /** * Text embedding models mapping sentences and documents into dense vector diff --git a/yarn.lock b/yarn.lock index 948f235b36..c5356eb1cb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -15479,6 +15479,7 @@ __metadata: expo-router: "npm:~57.0.26" react: "npm:19.2.3" react-native: "npm:0.86.3" + react-native-audio-api: "npm:0.13.1" react-native-blob-util: "npm:^0.25.0" react-native-drawer-layout: "npm:^4.2.2" react-native-executorch: "workspace:*" From c75e6969d911b93c5dc47ce3bc8c8b017b35e68d Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Wed, 14 Oct 2026 00:26:56 +0200 Subject: [PATCH 61/83] test(llm): add test suite for createLLMGemmaRunner and remove TODO in runner --- .../__tests__/support/fakeOps.ts | 11 +- .../__tests__/tasks/llmRunner.test.ts | 134 ++++++++++++++++++ .../extensions/llm/runners/llmGemmaRunner.ts | 1 - 3 files changed, 144 insertions(+), 2 deletions(-) diff --git a/packages/react-native-executorch/__tests__/support/fakeOps.ts b/packages/react-native-executorch/__tests__/support/fakeOps.ts index d12b765fc3..a81f6d8562 100644 --- a/packages/react-native-executorch/__tests__/support/fakeOps.ts +++ b/packages/react-native-executorch/__tests__/support/fakeOps.ts @@ -725,7 +725,16 @@ export const llm = { > ): number { lastSampleOptions = options; - expectShape(logits, [1, logits.numel], 'sample: logits'); + const isAllowedShape = + (logits.shape.length === 1 && logits.shape[0] === logits.numel) || + (logits.shape.length === 2 && logits.shape[0] === 1 && logits.shape[1] === logits.numel) || + (logits.shape.length === 3 && + logits.shape[0] === 1 && + logits.shape[1] === 1 && + logits.shape[2] === logits.numel); + if (!isAllowedShape) { + throw new Error(`sample: logits: invalid shape [${logits.shape}]`); + } const { allowedTokens, bannedTokens } = options.constraints?.(ctx) ?? {}; const allowed = allowedTokens && new Set(allowedTokens); const banned = new Set(bannedTokens ?? []); diff --git a/packages/react-native-executorch/__tests__/tasks/llmRunner.test.ts b/packages/react-native-executorch/__tests__/tasks/llmRunner.test.ts index 0416f1cdda..1406ba834e 100644 --- a/packages/react-native-executorch/__tests__/tasks/llmRunner.test.ts +++ b/packages/react-native-executorch/__tests__/tasks/llmRunner.test.ts @@ -2,6 +2,7 @@ import { constraint, f32, i64, method, RangeDim } from '../../src/core/schema'; import { tensor } from '../../src/core/tensor'; import { createLLMRunner } from '../../src/extensions/llm/llmRunner'; import { createLLMMultimodalRunner } from '../../src/extensions/llm/runners/llmMultimodalRunner'; +import { createLLMGemmaRunner } from '../../src/extensions/llm/runners/llmGemmaRunner'; import { loadModel } from '../../src/core/model'; import { loadTokenizer } from '../../src/extensions/nlp'; import { fakeJsi } from '../support/fakeJsi'; @@ -309,3 +310,136 @@ describe('createLLMMultimodalRunner — execution & KV cache lifecycle', () => { expect(fakeJsi.liveTensorDescriptions()).toEqual([]); }); }); + +describe('createLLMGemmaRunner — execution & multimodal prefill', () => { + beforeEach(() => { + registerModelWithSchema(GEMMA_SPEC, { maxContextLen: 512 }); + }); + + it('prefills text and updates KV cache position', async () => { + const model = await loadModel(MODEL_PATH); + const tokenizer = await loadTokenizer(TOKENIZER_PATH); + const runner = tracked(createLLMGemmaRunner(model, tokenizer, ['image', 'audio'])); + + expect(runner.getKVCacheState()).toEqual({ + pos: 0, + maxContextLen: 512, + remainingTokens: 512, + usageRatio: 0, + }); + + const stats = runner.prefill('hello world'); + expect(stats.numTokens).toBe(2); + expect(runner.getKVCacheState().pos).toBe(2); + expect(runner.getKVCacheState().remainingTokens).toBe(510); + }); + + it('prefills image and updates KV cache position by visual tokens count', async () => { + const model = await loadModel(MODEL_PATH); + const tokenizer = await loadTokenizer(TOKENIZER_PATH); + const runner = tracked(createLLMGemmaRunner(model, tokenizer, ['image', 'audio'])); + + const img = tensor('float32', [1, 3, 32, 32]); + try { + const stats = runner.prefill([{ kind: 'image', image: img }]); + expect(stats.numTokens).toBe(4); // visualTokens = 4 + expect(runner.getKVCacheState().pos).toBe(4); + } finally { + img.dispose(); + } + }); + + it('prefills audio and chunks properly into block tokens', async () => { + const model = await loadModel(MODEL_PATH); + const tokenizer = await loadTokenizer(TOKENIZER_PATH); + const runner = tracked(createLLMGemmaRunner(model, tokenizer, ['image', 'audio'])); + + // 7680 samples = 1 block = 12 tokens + const wav = tensor('float32', [1, 7680]); + try { + const stats = runner.prefill([{ kind: 'audio', audio: wav }]); + expect(stats.numTokens).toBe(12); + expect(runner.getKVCacheState().pos).toBe(12); + } finally { + wav.dispose(); + } + }); + + it('rejects invalid audio tensor dtype or empty audio', async () => { + const model = await loadModel(MODEL_PATH); + const tokenizer = await loadTokenizer(TOKENIZER_PATH); + const runner = tracked(createLLMGemmaRunner(model, tokenizer, ['image', 'audio'])); + + const badDtypeWav = tensor('int32', [1, 7680]); + try { + expect(() => runner.prefill([{ kind: 'audio', audio: badDtypeWav }])).toThrow( + /Expected audio tensor of dtype 'float32'/ + ); + } finally { + badDtypeWav.dispose(); + } + + const emptyWav = { + dtype: 'float32', + shape: [0], + numel: 0, + } as any; + expect(() => runner.prefill([{ kind: 'audio', audio: emptyWav }])).toThrow( + /Audio waveform is empty/ + ); + }); + + it('handles interleaved text, image, and audio prompts', async () => { + const model = await loadModel(MODEL_PATH); + const tokenizer = await loadTokenizer(TOKENIZER_PATH); + const runner = tracked(createLLMGemmaRunner(model, tokenizer, ['image', 'audio'])); + + const img = tensor('float32', [1, 3, 32, 32]); + const wav = tensor('float32', [1, 7680]); + try { + const stats = runner.prefill([ + 'describe this:', + { kind: 'image', image: img }, + 'and listen:', + { kind: 'audio', audio: wav }, + ]); + // 2 text + 4 image + 2 text + 12 audio = 20 tokens + expect(stats.numTokens).toBe(2 + 4 + 2 + 12); + expect(runner.getKVCacheState().pos).toBe(20); + } finally { + img.dispose(); + wav.dispose(); + } + }); + + it('generates tokens and triggers onToken callback', async () => { + const model = await loadModel(MODEL_PATH); + const tokenizer = await loadTokenizer(TOKENIZER_PATH); + const runner = tracked(createLLMGemmaRunner(model, tokenizer, ['image', 'audio'])); + + const tokens: string[] = []; + const stats = runner.generate('hello', { maxNewTokens: 3 }, (tok) => tokens.push(tok)); + + expect(stats.numTokens).toBe(3); + expect(tokens.length).toBe(3); + expect(tokens).toEqual(['hello', 'hello', 'hello']); + }); + + it('resets KV cache and disposes cleanly', async () => { + const model = await loadModel(MODEL_PATH); + const tokenizer = await loadTokenizer(TOKENIZER_PATH); + const runner = createLLMGemmaRunner(model, tokenizer, ['image', 'audio']); + + runner.prefill('hello world'); + expect(runner.getKVCacheState().pos).toBe(2); + + runner.reset(1); + expect(runner.getKVCacheState().pos).toBe(1); + runner.reset(); + expect(runner.getKVCacheState().pos).toBe(0); + + runner.dispose(); + expect(fakeJsi.liveModels()).toEqual([]); + expect(fakeJsi.liveTokenizers()).toEqual([]); + }); +}); diff --git a/packages/react-native-executorch/src/extensions/llm/runners/llmGemmaRunner.ts b/packages/react-native-executorch/src/extensions/llm/runners/llmGemmaRunner.ts index 7346e1ae1f..ede46b3b17 100644 --- a/packages/react-native-executorch/src/extensions/llm/runners/llmGemmaRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/runners/llmGemmaRunner.ts @@ -32,7 +32,6 @@ const METADATA_SPEC = { ...method('get_vocab_size', [], [{ kind: 'Int' }]), ...method('use_kv_cache', [], [{ kind: 'Bool' }]), ...method('enable_dynamic_shape', [], [{ kind: 'Bool' }]), - // TODO: add `has_ple` metadata }; export const LLM_GEMMA_SPEC = { From 43763304a4df0e5e7013f892736b4a4931c93792 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Wed, 14 Oct 2026 00:40:19 +0200 Subject: [PATCH 62/83] test(llm): allow arbitrary shapes in sample and test flattening --- .../__tests__/support/fakeOps.ts | 10 ---------- .../cpp/tests/extensions/LlmOpsTest.cpp | 12 ++++++------ 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/packages/react-native-executorch/__tests__/support/fakeOps.ts b/packages/react-native-executorch/__tests__/support/fakeOps.ts index a81f6d8562..2ef9d5d8fa 100644 --- a/packages/react-native-executorch/__tests__/support/fakeOps.ts +++ b/packages/react-native-executorch/__tests__/support/fakeOps.ts @@ -725,16 +725,6 @@ export const llm = { > ): number { lastSampleOptions = options; - const isAllowedShape = - (logits.shape.length === 1 && logits.shape[0] === logits.numel) || - (logits.shape.length === 2 && logits.shape[0] === 1 && logits.shape[1] === logits.numel) || - (logits.shape.length === 3 && - logits.shape[0] === 1 && - logits.shape[1] === 1 && - logits.shape[2] === logits.numel); - if (!isAllowedShape) { - throw new Error(`sample: logits: invalid shape [${logits.shape}]`); - } const { allowedTokens, bannedTokens } = options.constraints?.(ctx) ?? {}; const allowed = allowedTokens && new Set(allowedTokens); const banned = new Set(bannedTokens ?? []); diff --git a/packages/react-native-executorch/cpp/tests/extensions/LlmOpsTest.cpp b/packages/react-native-executorch/cpp/tests/extensions/LlmOpsTest.cpp index db18f93786..82097b184c 100644 --- a/packages/react-native-executorch/cpp/tests/extensions/LlmOpsTest.cpp +++ b/packages/react-native-executorch/cpp/tests/extensions/LlmOpsTest.cpp @@ -63,14 +63,14 @@ TEST_F(LlmOpsTest, RejectsAWrongArgumentCount) { "Usage: sample(logits, ctx, options)")); } -TEST_F(LlmOpsTest, RejectsABatchOfLogits) { - // The runner decodes one sequence; a [2, vocab] tensor is a caller bug. - EXPECT_TRUE(isCodedError(evalThrowing(js(R"( +TEST_F(LlmOpsTest, AcceptsMultiDimensionalLogitsByFlattening) { + // Anything goes and is flattened: a multi-dimensional logits tensor like [2, 4] is treated as a flat logits array. + EXPECT_EQ(evalNumber(js(R"( const t = createTensor([2, 4], 'float32'); - sample(t, ctx(), greedy()); + t.setData(new Float32Array([0.1, 0.2, 0.3, 0.4, 0.5, 3.0, 0.7, 0.8])); + return sample(t, ctx(), greedy()); )")), - "INVALID_ARGUMENT", - "sample: logits")); + 5); } TEST_F(LlmOpsTest, RejectsNonFloatLogits) { From 87fc502253e4f2ceeda1d00f8100005a1fa4dc55 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Wed, 14 Oct 2026 00:49:00 +0200 Subject: [PATCH 63/83] fix --- .../src/extensions/llm/runners/llmGemmaRunner.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/react-native-executorch/src/extensions/llm/runners/llmGemmaRunner.ts b/packages/react-native-executorch/src/extensions/llm/runners/llmGemmaRunner.ts index ede46b3b17..21d241a653 100644 --- a/packages/react-native-executorch/src/extensions/llm/runners/llmGemmaRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/runners/llmGemmaRunner.ts @@ -436,7 +436,7 @@ export function createLLMGemmaRunner( if (!config?.ignoreEos && eosIds.includes(nextToken)) break; - onToken?.(tokenizer.decode(Int32Array.of(nextToken))); + onToken?.(tokenizer.decode(Int32Array.of(nextToken), /* skipSpecialTokens= */ false)); generatedTokens.push(nextToken); // Embed single token From a32fce17e9e8c0cb2437cfb9e5c5027d74962779 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Wed, 14 Oct 2026 16:12:50 +0200 Subject: [PATCH 64/83] fix(llm): mark reset and getKVCacheState as worklet in runners --- .../__tests__/api/workletDirective.test.ts | 46 +++++++++++++++++++ .../extensions/llm/runners/llmGemmaRunner.ts | 2 + .../llm/runners/llmMultimodalRunner.ts | 2 + .../extensions/llm/runners/llmTextRunner.ts | 2 + 4 files changed, 52 insertions(+) diff --git a/packages/react-native-executorch/__tests__/api/workletDirective.test.ts b/packages/react-native-executorch/__tests__/api/workletDirective.test.ts index 325f3278c2..f1b564e37f 100644 --- a/packages/react-native-executorch/__tests__/api/workletDirective.test.ts +++ b/packages/react-native-executorch/__tests__/api/workletDirective.test.ts @@ -167,6 +167,52 @@ describe('worklet directive', () => { expect(missing).toEqual([]); }); + + it('marks reset and getKVCacheState as worklets in LLM runners', () => { + const runnerFiles = [ + 'extensions/llm/runners/llmTextRunner.ts', + 'extensions/llm/runners/llmMultimodalRunner.ts', + 'extensions/llm/runners/llmGemmaRunner.ts', + ]; + + const methodsToCheck = ['reset', 'getKVCacheState']; + + for (const file of runnerFiles) { + const source = ts.createSourceFile( + file, + readFileSync(join(SRC, file), 'utf8'), + ts.ScriptTarget.Latest, + true + ); + + const found: Record = {}; + + const visit = (node: ts.Node) => { + if ( + ts.isVariableDeclaration(node) && + ts.isIdentifier(node.name) && + methodsToCheck.includes(node.name.text) && + node.initializer && + (ts.isArrowFunction(node.initializer) || ts.isFunctionExpression(node.initializer)) && + node.initializer.body && + ts.isBlock(node.initializer.body) + ) { + found[node.name.text] = hasWorkletDirective(node.initializer.body); + } + ts.forEachChild(node, visit); + }; + + visit(source); + + for (const method of methodsToCheck) { + expect({ file, method, isWorklet: found[method] }).toEqual({ + file, + method, + isWorklet: true, + }); + } + } + }); }); describe('architecture boundaries', () => { diff --git a/packages/react-native-executorch/src/extensions/llm/runners/llmGemmaRunner.ts b/packages/react-native-executorch/src/extensions/llm/runners/llmGemmaRunner.ts index 21d241a653..0596f444df 100644 --- a/packages/react-native-executorch/src/extensions/llm/runners/llmGemmaRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/runners/llmGemmaRunner.ts @@ -144,6 +144,7 @@ export function createLLMGemmaRunner( const pos = createSynchronizable(0); const reset = (targetPos?: number): void => { + 'worklet'; const curPos = pos.getBlocking(); if (targetPos !== undefined && (targetPos < 0 || targetPos > curPos)) { throw RnExecuTorchError( @@ -155,6 +156,7 @@ export function createLLMGemmaRunner( }; const getKVCacheState = (): LLMKVCacheState => { + 'worklet'; const curPos = pos.getBlocking(); return { pos: curPos, diff --git a/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts b/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts index 9595273aa8..0ecfb1520a 100644 --- a/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/runners/llmMultimodalRunner.ts @@ -94,6 +94,7 @@ export function createLLMMultimodalRunner( const pos = createSynchronizable(0); const reset = (targetPos?: number): void => { + 'worklet'; const curPos = pos.getBlocking(); if (targetPos !== undefined && (targetPos < 0 || targetPos > curPos)) { throw RnExecuTorchError( @@ -105,6 +106,7 @@ export function createLLMMultimodalRunner( }; const getKVCacheState = (): LLMKVCacheState => { + 'worklet'; const curPos = pos.getBlocking(); return { pos: curPos, diff --git a/packages/react-native-executorch/src/extensions/llm/runners/llmTextRunner.ts b/packages/react-native-executorch/src/extensions/llm/runners/llmTextRunner.ts index 0c636e1cef..3244562c30 100644 --- a/packages/react-native-executorch/src/extensions/llm/runners/llmTextRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/runners/llmTextRunner.ts @@ -78,6 +78,7 @@ export function createLLMTextRunner( const pos = createSynchronizable(0); const reset = (targetPos?: number): void => { + 'worklet'; const curPos = pos.getBlocking(); if (targetPos !== undefined && (targetPos < 0 || targetPos > curPos)) { throw RnExecuTorchError( @@ -89,6 +90,7 @@ export function createLLMTextRunner( }; const getKVCacheState = (): LLMKVCacheState => { + 'worklet'; const curPos = pos.getBlocking(); return { pos: curPos, From 1dbf87fb92b9a76b74dd0a96bf1797160d450268 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Wed, 14 Oct 2026 16:13:34 +0200 Subject: [PATCH 65/83] feat(llm): add Gemma Omni Vulkan model variant --- apps/nlp/app/llm/index.tsx | 5 +++++ apps/nlp/constants/llm.ts | 10 ++++++++++ packages/react-native-executorch/src/models.ts | 8 ++++++++ 3 files changed, 23 insertions(+) diff --git a/apps/nlp/app/llm/index.tsx b/apps/nlp/app/llm/index.tsx index d3c7124ba3..cb02669735 100644 --- a/apps/nlp/app/llm/index.tsx +++ b/apps/nlp/app/llm/index.tsx @@ -371,6 +371,11 @@ function LLMContent() { 📷 Vision Multimodal )} + {supportsAudio && ( + + 🎙️ Audio Multimodal + + )}