diff --git a/.cspell-wordlist.txt b/.cspell-wordlist.txt
index 2e3e47e2a5..dabdb3c665 100644
--- a/.cspell-wordlist.txt
+++ b/.cspell-wordlist.txt
@@ -400,6 +400,7 @@ skiplist
skiplisted
startoftext
maxsim
+multinomial
adb
benchmarkable
benchprobe
diff --git a/apps/nlp/app/_layout.tsx b/apps/nlp/app/_layout.tsx
index 4db20c304f..7d637faa15 100644
--- a/apps/nlp/app/_layout.tsx
+++ b/apps/nlp/app/_layout.tsx
@@ -55,6 +55,13 @@ export default function Layout() {
title: 'LLM',
}}
/>
+
);
}
diff --git a/apps/nlp/app/constrained-decoding/index.tsx b/apps/nlp/app/constrained-decoding/index.tsx
new file mode 100644
index 0000000000..ac34fc26ce
--- /dev/null
+++ b/apps/nlp/app/constrained-decoding/index.tsx
@@ -0,0 +1,431 @@
+import React, { useEffect, useState } from 'react';
+import { Platform, Pressable, ScrollView, StyleSheet, Switch, Text, View } from 'react-native';
+import {
+ models,
+ RnExecuTorchError,
+ useLLMChatSession,
+ useTokenizer,
+ type llm,
+} from 'react-native-executorch';
+import { Button } from '../../components/Button';
+import { ModelStatus } from '../../components/ModelStatus';
+import ScreenWrapper from '../../components/ScreenWrapper';
+import { theme } from '../../theme';
+
+const MODEL = models.llm.LFM2_5_350M.XNNPACK_8DA4W;
+const COLORS = { amber: '#eea532', jade: '#2baf87', violet: '#9465db' };
+const OPTIONS = {
+ color: Object.keys(COLORS),
+ effect: ['healing', 'flight', 'invisibility'],
+ rarity: ['common', 'rare', 'legendary'],
+};
+type Field = keyof typeof OPTIONS;
+const FIELDS: Field[] = ['color', 'effect', 'rarity'];
+// A finite choice grammar: each complete recipe has one canonical token path.
+const RECIPES = OPTIONS.color.flatMap((color) =>
+ OPTIONS.effect.flatMap((effect) => OPTIONS.rarity.map((rarity) => ({ color, effect, rarity })))
+);
+
+function validateOutput(output: string, selected: Record): string[] {
+ if (!output.trim()) return ['The model returned no text.'];
+ let parsed: unknown;
+ try {
+ parsed = JSON.parse(output);
+ } catch (error) {
+ return [`Invalid JSON: ${error instanceof Error ? error.message : String(error)}`];
+ }
+ if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
+ return ['Expected a JSON object with color, effect, and rarity fields.'];
+ }
+ const record = parsed as Record;
+ const issues: string[] = [];
+ const keys = Object.keys(record);
+ for (const field of FIELDS) {
+ if (!Object.prototype.hasOwnProperty.call(record, field)) {
+ issues.push(`Missing field: ${field}.`);
+ } else if (!selected[field].includes(record[field] as string)) {
+ issues.push(
+ `${field}: received ${JSON.stringify(record[field])}; allowed: ${selected[field].join(', ')}.`
+ );
+ }
+ }
+ const extra = keys.filter((key) => !FIELDS.includes(key as Field));
+ if (extra.length) issues.push(`Unexpected fields: ${extra.join(', ')}.`);
+ if (!issues.length && keys.join(',') !== FIELDS.join(',')) {
+ issues.push('Fields must appear in color, effect, rarity order.');
+ }
+ return issues;
+}
+
+function PotionLab() {
+ const [loadStarted, setLoadStarted] = useState(false);
+ const [selected, setSelected] = useState>({
+ color: ['amber', 'jade'],
+ effect: ['healing', 'flight'],
+ rarity: ['rare'],
+ });
+ const [showRecipes, setShowRecipes] = useState(false);
+ const [constrained, setConstrained] = useState(true);
+ const [paths, setPaths] = useState([]);
+ const [output, setOutput] = useState('');
+ const [busy, setBusy] = useState(false);
+ const [result, setResult] = useState<'idle' | 'complete' | 'stopped' | 'failed'>('idle');
+ const [runError, setRunError] = useState('');
+ const session = useLLMChatSession(MODEL, {
+ preventLoad: !loadStarted,
+ resetOnTurn: true,
+ stopRegex: /\}$/,
+ initialMessages: [
+ {
+ role: 'system',
+ content:
+ 'You are a fantasy alchemist. Reply with only a JSON object: ' +
+ '"color": "amber", "jade", or "violet"; ' +
+ '"effect": "healing", "flight", or "invisibility"; ' +
+ '"rarity": "common", "rare", or "legendary". ' +
+ 'Use exactly these three keys in that order, without markdown or explanation.',
+ },
+ ],
+ });
+ const {
+ encode,
+ decode,
+ error: tokenizerError,
+ } = useTokenizer(session.resource?.tokenizerPath ?? MODEL.tokenizerPath, {
+ preventLoad: !session.resource,
+ });
+
+ useEffect(() => {
+ if (!encode || !decode) return;
+ let cancelled = false;
+ const prepare = async () => {
+ const empty = await encode('');
+ const compiled = await Promise.all(
+ RECIPES.map(async (recipe) => {
+ const json = JSON.stringify(recipe);
+ const encoded = await encode(json);
+ // Remove tokenizer-added BOS/EOS tokens before constraining generated output.
+ let start = 0;
+ let end = encoded.length;
+ while (start < end && start < empty.length && encoded[start] === empty[start]) start++;
+ let suffix = empty.length - 1;
+ while (end > start && suffix >= start && encoded[end - 1] === empty[suffix]) {
+ end--;
+ suffix--;
+ }
+ const tokens = encoded.slice(start, end);
+ if (!tokens.length || (await decode(tokens, true)) !== json) {
+ throw RnExecuTorchError('INVALID_STATE', 'Cannot tokenize potion recipes losslessly.');
+ }
+ return Array.from(tokens);
+ })
+ );
+ if (!cancelled) setPaths(compiled);
+ };
+ prepare().catch((error: unknown) => {
+ if (!cancelled) setRunError(error instanceof Error ? error.message : String(error));
+ });
+ return () => {
+ cancelled = true;
+ };
+ }, [encode, decode]);
+
+ const permittedRecipes = RECIPES.filter((recipe) =>
+ FIELDS.every((field) => selected[field].includes(recipe[field]))
+ );
+ const allowedPaths = paths.filter((_, index) =>
+ FIELDS.every((field) => selected[field].includes(RECIPES[index]![field]))
+ );
+ const toggleOption = (field: Field, value: string) => {
+ const values = selected[field];
+ if (busy || (values.length === 1 && values.includes(value))) return;
+ setSelected({
+ ...selected,
+ [field]: values.includes(value)
+ ? values.filter((item) => item !== value)
+ : [...values, value],
+ });
+ setOutput('');
+ setResult('idle');
+ setRunError('');
+ };
+
+ const brew = async () => {
+ if (!session.sendMessage || busy || !allowedPaths.length) return;
+ const recipePaths = allowedPaths;
+ setBusy(true);
+ setOutput('');
+ setResult('idle');
+ setRunError('');
+ const constraints: llm.SamplingConstraintsCallback = ({ generatedTokens }) => {
+ 'worklet';
+ // Hermes TypedArray.from expects array-like input; a Set can produce an
+ // empty typed array even when Set.size > 0, masking the entire vocabulary.
+ const allowed: number[] = [];
+ for (const path of recipePaths) {
+ if (path.length <= generatedTokens.length) continue;
+ let matches = true;
+ for (let i = 0; i < generatedTokens.length; i++) {
+ if (path[i] !== generatedTokens[i]) {
+ matches = false;
+ break;
+ }
+ }
+ const next = path[generatedTokens.length]!;
+ if (matches && !allowed.includes(next)) allowed.push(next);
+ }
+ if (!allowed.length) {
+ throw RnExecuTorchError('INVALID_STATE', 'No valid potion continuation.');
+ }
+ return { allowedTokens: Int32Array.from(allowed) };
+ };
+ try {
+ await session.sendMessage(
+ `Create a potion. Choose one value per field from these allowed options: ${JSON.stringify(selected)}. ` +
+ 'Reply with only the JSON object, in color, effect, rarity order.',
+ (token) => setOutput((text) => text + token),
+ {
+ temperature: 0.9,
+ maxNewTokens: constrained ? Math.max(...recipePaths.map((path) => path.length)) : 160,
+ constraints: constrained ? constraints : undefined,
+ }
+ );
+ setResult((current) => (current === 'stopped' ? current : 'complete'));
+ } catch (error) {
+ setResult('failed');
+ setRunError(error instanceof Error ? error.message : String(error));
+ } finally {
+ setBusy(false);
+ }
+ };
+ const issues = result === 'complete' ? validateOutput(output, selected) : [];
+ const potion =
+ result === 'complete' && !issues.length
+ ? permittedRecipes.find(
+ (recipe) => JSON.stringify(recipe) === JSON.stringify(JSON.parse(output))
+ )
+ : undefined;
+
+ return (
+
+ 🧪 Potion Lab
+ Model: LFM2.5 350M · XNNPACK · 8DA4W
+
+ Choose which values each JSON field may contain, then let the model pick a potion.
+
+ {!loadStarted ? (
+
+
+ ) : (
+ 0}
+ downloadProgress={session.downloadProgress}
+ error={session.error?.message ?? tokenizerError?.message}
+ />
+ )}
+
+ Allowed values
+
+ Select one or more options per field. Keep just one to lock that field.
+
+ {FIELDS.map((field) => (
+
+ {field}
+
+ {OPTIONS[field].map((value) => {
+ const checked = selected[field].includes(value);
+ const disabled = busy || (checked && selected[field].length === 1);
+ return (
+ toggleOption(field, value)}
+ >
+
+ {checked ? '✓ ' : ''}
+ {value}
+
+
+ );
+ })}
+
+
+ ))}
+
+
+ Constrain output
+
+ Compare the same prompt with the token filter on or off.
+
+
+ {
+ setConstrained(value);
+ setResult('idle');
+ setOutput('');
+ setRunError('');
+ }}
+ />
+
+
+
+ {busy && (
+
+
+
+ What is the grammar?
+
+ A grammar defines which outputs are allowed. Here it is a finite list of JSON recipes:
+ exactly three fields, in color → effect → rarity order, with each value taken from your
+ selections. No extra fields or explanation.
+
+ {JSON.stringify(selected, null, 2)}
+
+ These selections allow {permittedRecipes.length}{' '}
+ {permittedRecipes.length === 1 ? 'recipe' : 'recipes'}. The model chooses between them. At
+ each step, the sampler only allows tokens that can still complete one of these recipes.
+ This demo uses a finite recipe list, not a general JSON grammar parser.
+
+ setShowRecipes((value) => !value)}
+ >
+
+ {showRecipes ? 'Hide' : 'Show'} permitted recipes ({permittedRecipes.length})
+
+
+ {showRecipes && (
+
+ {permittedRecipes.map((recipe) => JSON.stringify(recipe)).join('\n\n')}
+
+ )}
+
+
+ Live output
+
+ {constrained
+ ? 'Token filter ON — only the permitted recipes can be generated.'
+ : 'Token filter OFF — your selections are prompt instructions only. The model may ignore them; the result is still validated below.'}
+
+
+ {output ||
+ (result === 'complete' ? '(No text returned.)' : 'Your potion’s JSON appears here…')}
+
+ {potion && (
+
+ ✦ {potion.color} · {potion.rarity} potion of {potion.effect}
+
+ )}
+ {!busy && result === 'complete' && (
+
+
+ {issues.length
+ ? 'Generation completed — output failed validation.'
+ : 'Output passed validation.'}
+
+
+ {constrained
+ ? 'The sampler enforced the selected recipe list.'
+ : issues.length
+ ? 'With the token filter off, the model did not follow the requested grammar. This is an output validation failure, not an inference error.'
+ : 'The model followed the prompt this time. With the token filter off, this is not guaranteed.'}
+
+ {issues.map((issue) => (
+
+ {issue}
+
+ ))}
+
+ )}
+ {!busy && result === 'stopped' && (
+
+ Generation was stopped. The partial output has not been validated.
+
+ )}
+ {!!runError && (
+
+ {result === 'failed' ? 'Generation failed: ' : ''}
+ {runError}
+
+ )}
+
+
+ );
+}
+
+export default function ConstrainedDecodingScreen() {
+ return (
+
+
+
+ );
+}
+
+const styles = StyleSheet.create({
+ content: { padding: 20, paddingBottom: 40, gap: 16, backgroundColor: theme.colors.background },
+ title: { color: theme.colors.primary, fontSize: 30, fontWeight: '700' },
+ heading: { color: theme.colors.primary, fontSize: 16, fontWeight: '600' },
+ description: { color: theme.colors.textMuted, lineHeight: 21 },
+ card: { padding: 16, borderRadius: 16, backgroundColor: '#fff', gap: 16 },
+ row: { flexDirection: 'row', alignItems: 'center', gap: 12 },
+ copy: { flex: 1, gap: 4 },
+ field: { gap: 8 },
+ options: { flexDirection: 'row', flexWrap: 'wrap', gap: 8 },
+ chip: {
+ paddingVertical: 10,
+ paddingHorizontal: 14,
+ borderRadius: 16,
+ borderWidth: 1,
+ borderColor: theme.colors.lightBorder,
+ },
+ selectedChip: { backgroundColor: '#f0e9fa', borderColor: '#734cb0' },
+ selectedText: { color: '#6034a3', fontWeight: '600' },
+ code: {
+ padding: 16,
+ minHeight: 100,
+ borderRadius: 8,
+ backgroundColor: '#191d32',
+ color: '#d6e6c5',
+ fontFamily: Platform.OS === 'ios' ? 'Menlo' : 'monospace',
+ lineHeight: 22,
+ },
+ potion: { fontSize: 20, fontWeight: '700', textTransform: 'capitalize' },
+ error: { color: theme.colors.errorText },
+ success: { color: '#257854', fontWeight: '600' },
+ validOutput: { backgroundColor: '#eaf7ef', padding: 12, borderRadius: 8, gap: 8 },
+ invalidOutput: {
+ gap: 8,
+ color: theme.colors.errorText,
+ backgroundColor: theme.colors.errorBackground,
+ padding: 12,
+ borderRadius: 8,
+ fontWeight: '600',
+ lineHeight: 21,
+ },
+});
diff --git a/apps/nlp/app/index.tsx b/apps/nlp/app/index.tsx
index 7580909abd..7531cc1f30 100644
--- a/apps/nlp/app/index.tsx
+++ b/apps/nlp/app/index.tsx
@@ -26,6 +26,12 @@ export default function Home() {
router.navigate('llm/')}>
LLM
+ router.navigate('constrained-decoding/')}
+ >
+ Potion Lab · Constrained Decoding
+
);
diff --git a/apps/nlp/app/llm/index.tsx b/apps/nlp/app/llm/index.tsx
index 658435e62a..cb02669735 100644
--- a/apps/nlp/app/llm/index.tsx
+++ b/apps/nlp/app/llm/index.tsx
@@ -11,56 +11,81 @@ import {
Platform,
Alert,
Image as RNImage,
+ Modal,
} from 'react-native';
import { Skia } from '@shopify/react-native-skia';
import RNBlobUtil from 'react-native-blob-util';
import { useSafeAreaInsets } from 'react-native-safe-area-context';
import { useLLMChatSession, type llm, type cv } from 'react-native-executorch';
+import { decodeAudioData } from 'react-native-audio-api';
import ScreenWrapper from '../../components/ScreenWrapper';
import { ModelPicker, type ModelOption } from '../../components/ModelPicker';
import { Button } from '../../components/Button';
import { getImage, skImageToBuffer } from '../../utils';
-import { LLM_MODELS, type LLMModelConfig } from '../../constants/llm';
+import { LLM_MODELS, type LLMModelConfig, type LLMBackend } from '../../constants/llm';
type Turn = {
role: 'user' | 'assistant' | 'tool';
content: string;
imageUri?: string;
+ audioName?: string;
stats?: llm.LLMGenerationStats;
toolCalls?: readonly llm.ToolCall[];
};
function formatStats(stats: llm.LLMGenerationStats): string {
- const decodeMs = stats.inferenceEndMs - stats.firstTokenMs;
- const tokensPerSec = decodeMs > 0 ? (stats.numGeneratedTokens / decodeMs) * 1000 : 0;
- const decodeTtftMs = stats.firstTokenMs - stats.inferenceStartMs;
- const totalTtftMs = decodeTtftMs + (stats.prefillDurationMs ?? 0);
- const totalMs = stats.inferenceEndMs - stats.inferenceStartMs + (stats.prefillDurationMs ?? 0);
+ const totalMs = stats.durationMs + stats.prefill.durationMs;
return (
- `${stats.numGeneratedTokens} tokens · ` +
- `${tokensPerSec.toFixed(1)} tok/s · ` +
- `${totalTtftMs.toFixed(0)}ms ttft · ` +
+ `${stats.numTokens} tokens · ` +
+ `${stats.tokensPerSecond.toFixed(1)} tok/s · ` +
+ `${stats.prefill.durationMs.toFixed(0)}ms prefill · ` +
`${(totalMs / 1000).toFixed(2)}s`
);
}
function LLMContent() {
const insets = useSafeAreaInsets();
- const [selectedModelId, setSelectedModelId] = useState(LLM_MODELS[0]!.id);
+ const [selectedBackend, setSelectedBackend] = useState('xnnpack');
+
+ const availableBackends: ModelOption[] = useMemo(() => {
+ const list: { label: string; value: LLMBackend }[] = [{ label: 'XNNPACK', value: 'xnnpack' }];
+ if (Platform.OS === 'ios') {
+ list.push({ label: 'MLX (Apple GPU)', value: 'mlx' });
+ }
+ if (Platform.OS === 'android') {
+ list.push({ label: 'Vulkan (Android GPU)', value: 'vulkan' });
+ }
+ return list;
+ }, []);
+
+ const modelsForBackend = useMemo(
+ () =>
+ LLM_MODELS.filter(
+ (m) => m.backend === selectedBackend && (!m.iosOnly || Platform.OS === 'ios')
+ ),
+ [selectedBackend]
+ );
+
+ const [selectedModelId, setSelectedModelId] = useState(
+ () => modelsForBackend[0]?.id ?? LLM_MODELS[0]!.id
+ );
const [isDownloadStarted, setIsDownloadStarted] = useState(false);
const activeModel: LLMModelConfig = useMemo(
- () => LLM_MODELS.find((m) => m.id === selectedModelId) ?? LLM_MODELS[0]!,
- [selectedModelId]
+ () =>
+ modelsForBackend.find((m) => m.id === selectedModelId) ??
+ modelsForBackend[0] ??
+ LLM_MODELS[0]!,
+ [modelsForBackend, selectedModelId]
);
const modelOptions: ModelOption[] = useMemo(
() =>
- LLM_MODELS.filter((m) => !m.iosOnly || Platform.OS === 'ios').map((m) => ({
+ modelsForBackend.map((m) => ({
label: m.name,
value: m.id,
})),
- []
+ [modelsForBackend]
);
const initialMessages: llm.ChatMessage[] = useMemo(
@@ -99,7 +124,36 @@ function LLMContent() {
buffer: cv.ImageBuffer;
} | null>(null);
+ const [attachedAudio, setAttachedAudio] = useState<{
+ uri: string;
+ name: string;
+ samples: Float32Array;
+ } | null>(null);
+
+ const [isAudioModalVisible, setIsAudioModalVisible] = useState(false);
+ const [audioUrl, setAudioUrl] = useState('https://models.silero.ai/vad_models/en.wav');
+ const [isDecodingAudio, setIsDecodingAudio] = useState(false);
+
const supportsImages = Boolean(activeModel.model.modalities?.includes('image'));
+ const supportsAudio = Boolean(activeModel.model.modalities?.includes('audio'));
+
+ const handleBackendChange = (newBackend: LLMBackend) => {
+ if (newBackend === selectedBackend) return;
+ setSelectedBackend(newBackend);
+ const firstForBackend = LLM_MODELS.find(
+ (m) => m.backend === newBackend && (!m.iosOnly || Platform.OS === 'ios')
+ );
+ if (firstForBackend) {
+ setSelectedModelId(firstForBackend.id);
+ }
+ setIsDownloadStarted(false);
+ setTurns([]);
+ setStreamingResponse(null);
+ setAttachedImage(null);
+ setAttachedAudio(null);
+ setIsAudioModalVisible(false);
+ setInput('');
+ };
const handleModelChange = (newModelId: string) => {
if (newModelId === selectedModelId) return;
@@ -108,6 +162,8 @@ function LLMContent() {
setTurns([]);
setStreamingResponse(null);
setAttachedImage(null);
+ setAttachedAudio(null);
+ setIsAudioModalVisible(false);
setInput('');
};
@@ -139,6 +195,8 @@ function LLMContent() {
setTurns([]);
setStreamingResponse(null);
setAttachedImage(null);
+ setAttachedAudio(null);
+ setIsAudioModalVisible(false);
setInput('');
Alert.alert('Deleted', 'Model files deleted from device storage.');
} catch (err: any) {
@@ -168,27 +226,62 @@ function LLMContent() {
}
};
+ const handleAttachAudio = async () => {
+ const trimmed = audioUrl.trim();
+ if (!trimmed) return;
+ setIsDecodingAudio(true);
+ try {
+ const audioBuffer = await decodeAudioData(trimmed, 16000);
+ const samples = audioBuffer.getChannelData(0);
+ const durationSec = (samples.length / 16000).toFixed(1);
+ const filename = trimmed.split('/').pop() || 'audio.wav';
+ setAttachedAudio({
+ uri: trimmed,
+ name: `${filename} (${durationSec}s)`,
+ samples,
+ });
+ setIsAudioModalVisible(false);
+ } catch (err: any) {
+ Alert.alert('Error', err?.message || 'Failed to decode audio URL');
+ } finally {
+ setIsDecodingAudio(false);
+ }
+ };
+
const handleSend = async () => {
const textMessage = input.trim();
- if ((!textMessage && !attachedImage) || !sendMessage || isGenerating) return;
+ if ((!textMessage && !attachedImage && !attachedAudio) || !sendMessage || isGenerating) return;
const currentImage = attachedImage;
+ const currentAudio = attachedAudio;
setAttachedImage(null);
+ setAttachedAudio(null);
setInput('');
setStreamingResponse('');
setTurns((prev) => [
...prev,
- { role: 'user', content: textMessage, imageUri: currentImage?.uri },
+ {
+ role: 'user',
+ content: textMessage,
+ imageUri: currentImage?.uri,
+ audioName: currentAudio?.name,
+ },
]);
try {
let payload;
- if (currentImage) {
- payload = [
- { kind: 'image' as const, image: currentImage.buffer },
- textMessage || "What's in this image?",
- ];
+ if (currentImage || currentAudio) {
+ const mediaItems: (string | llm.ChatMediaInput)[] = [];
+ if (currentImage) {
+ mediaItems.push({ kind: 'image' as const, image: currentImage.buffer });
+ }
+ if (currentAudio) {
+ mediaItems.push({ kind: 'audio' as const, audio: currentAudio.samples });
+ }
+ const defaultPrompt = currentAudio ? "What's in this audio?" : "What's in this image?";
+ mediaItems.push(textMessage || defaultPrompt);
+ payload = mediaItems;
} else {
payload = textMessage;
}
@@ -236,6 +329,14 @@ function LLMContent() {
keyboardVerticalOffset={Platform.OS === 'ios' ? 90 : 0}
>
+ {availableBackends.length > 1 && (
+
+ )}
📷 Vision Multimodal
)}
+ {supportsAudio && (
+
+ 🎙️ Audio Multimodal
+
+ )}