From 011820f8b0718b4ac6f2789ce9883f5c0b319ec1 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Mon, 19 Oct 2026 16:22:03 +0200 Subject: [PATCH 1/9] feat: add depth estimation pipeline, hook, and hosted models --- .../scripts/download-libs.js | 4 +- .../extensions/cv/tasks/depthEstimation.ts | 174 ++++++++++++++++++ .../src/hooks/useDepthEstimator.ts | 37 ++++ packages/react-native-executorch/src/index.ts | 2 + .../react-native-executorch/src/models.ts | 44 +++++ 5 files changed, 260 insertions(+), 1 deletion(-) create mode 100644 packages/react-native-executorch/src/extensions/cv/tasks/depthEstimation.ts create mode 100644 packages/react-native-executorch/src/hooks/useDepthEstimator.ts diff --git a/packages/react-native-executorch/scripts/download-libs.js b/packages/react-native-executorch/scripts/download-libs.js index 45732133b8..f223505ea7 100644 --- a/packages/react-native-executorch/scripts/download-libs.js +++ b/packages/react-native-executorch/scripts/download-libs.js @@ -44,7 +44,7 @@ * libs: opencv, phonemis * features: llm, multimodalLLM, speechToText, textToSpeech, vad, privacyFilter, * textEmbeddings, imageEmbeddings, - * classification, objectDetection, semanticSegmentation, instanceSegmentation, + * classification, objectDetection, depthEstimation, semanticSegmentation, instanceSegmentation, * ocr, verticalOCR, keypointDetection, styleTransfer, textToImage, segmentAnything, * tokenizer * @@ -164,6 +164,8 @@ const FEATURE_MAP = { backends: ['xnnpack', 'coreml', 'mlx', 'vulkan'], libs: ['opencv'], }, + // Depth Anything V2 ships xnnpack, coreml and vulkan. + depthEstimation: { backends: ['xnnpack', 'coreml', 'vulkan'], libs: ['opencv'] }, // DeepLab/FCN/LR-ASPP/selfie all ship xnnpack + coreml. semanticSegmentation: { backends: ['xnnpack', 'coreml'], libs: ['opencv'] }, // YOLO-seg xnnpack-only, rf_detr-seg/fastsam add coreml, and all three ship diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/depthEstimation.ts b/packages/react-native-executorch/src/extensions/cv/tasks/depthEstimation.ts new file mode 100644 index 0000000000..02c2099655 --- /dev/null +++ b/packages/react-native-executorch/src/extensions/cv/tasks/depthEstimation.ts @@ -0,0 +1,174 @@ +/** + * Depth estimation task pipeline with image preprocessing and floating-point + * depth map resizing. + */ +import type { WorkletRuntime } from 'react-native-worklets'; + +import { createResourceScope } from '../../../core/lifetime'; +import { loadModel } from '../../../core/model'; +import { wrapAsync } from '../../../core/runtime'; +import { validateSpec, method, f32 } from '../../../core/schema'; +import { tensor } from '../../../core/tensor'; +import type { ImageBuffer } from '../image'; +import { resize, type InterpolationMethod } from '../ops/image'; +import { createImagePreprocessor, type ImagePreprocessorOptions } from '../utils/imagePreprocessor'; + +/** + * Result structure representing the output of a depth estimation task. + * @category CV / Types + */ +export type DepthMap = { + /** + * Raw model values, indexed as `data[y * width + x]`. Each result owns its + * buffer. Values are not normalized or converted to display pixels. + * Interpretation and units depend on the loaded model. + */ + readonly data: Float32Array; + /** Width of the depth map in pixels. */ + readonly width: number; + /** Height of the depth map in pixels. */ + readonly height: number; +}; + +/** + * Options for configuring a depth estimator preprocessor and output + * interpolation. + * @category CV / Types + */ +export type DepthEstimatorOptions = Omit & { + /** Resize mode for input images. Must be `'stretch'`. */ + readonly resizeMode: 'stretch'; + /** Interpolation method used when resizing depth maps back to input image dimensions. */ + readonly outInterpolation: InterpolationMethod; +}; + +/** + * Model configuration required to instantiate a depth estimator task runner. + * @category CV / Types + */ +export type DepthEstimatorModel = { + /** Local path or remote URL of the `.pte` model file. */ + readonly modelPath: string; + /** + * Input preprocessing and output depth map interpolation. `resizeMode` is + * fixed to `'stretch'`. See {@link DepthEstimatorOptions}. + */ + readonly modelOpts: DepthEstimatorOptions; +}; + +/** + * Depth estimation task runner. + * @category CV / Types + */ +export type DepthEstimator = { + /** + * Releases all allocated native resources. + */ + readonly dispose: () => void; + + /** + * Asynchronously performs depth estimation on the input image. + * + * The full input image is stretched to the model's input size, and the + * floating-point depth map is resized back to the input image dimensions. + * Depth values retain the model's original scale. + * @param input The input image buffer. + * @returns A promise resolving to the depth map at the input image resolution. + * @throws {RnExecuTorchError} With code `RESOURCE_BUSY` if the model is in + * use, or `RESOURCE_DISPOSED` if disposed. + */ + readonly estimateDepth: (input: ImageBuffer) => Promise; + + /** + * Synchronous version of {@link estimateDepth} to be executed directly on the + * caller or worklet thread. + */ + readonly estimateDepthWorklet: (input: ImageBuffer) => DepthMap; +}; + +/** + * Creates a depth estimator runner for executing local depth estimation models. + * + * It validates the model inputs and outputs requirements, pre-allocates the + * necessary static execution tensors, sets up an image preprocessor, and + * registers clean disposal hooks to clear all native memory. + * + * Accepts batched or unbatched CHW RGB inputs and matching depth outputs. + * Output spatial dimensions may differ from the input tensor. Preprocessing + * and output interpolation follow `modelOpts`; depth values retain the model's + * original scale. + * @category CV / Tasks + * @param config Depth estimator task configuration containing path and options. + * See {@link DepthEstimatorModel}. + * @param runtime Optional worklet runtime thread on which to run the model + * execution. + * @returns A promise resolving to the instantiated {@link DepthEstimator} runner. + * @throws {RnExecuTorchError} With code `LOAD_FAILED` if model fails to load, + * or `SCHEMA_MISMATCH` if model schema does not match depth estimation spec. + */ +export async function createDepthEstimator( + config: DepthEstimatorModel, + runtime?: WorkletRuntime +): Promise { + const scope = createResourceScope(); + const dispose = scope.dispose; + + try { + const { modelPath, modelOpts } = config; + const model = scope.track(await wrapAsync(loadModel, runtime)(modelPath)); + + const { variant, dims } = validateSpec(model.schema, { + batched: method( + 'forward', // prettier-ignore + [f32(1, 3, 'H', 'W')], + [f32(1, 'OH', 'OW')] + ), + unbatched: method( + 'forward', // prettier-ignore + [f32(3, 'H', 'W')], + [f32('OH', 'OW')] + ), + }); + + const [H, W, OH, OW] = dims.constant('H', 'W', 'OH', 'OW'); + const inpShape = { batched: [1, 3, H, W], unbatched: [3, H, W] }[variant]; + const outShape = { batched: [1, OH, OW], unbatched: [OH, OW] }[variant]; + + const tensors = [ + scope.track(tensor('float32', outShape)), + scope.track(tensor('float32', [OH, OW, 1])), + ] as const; + + const [tOutput, tDepth] = tensors; + const preprocessor = scope.track(createImagePreprocessor(modelOpts, inpShape)); + + const estimateDepthWorklet = (input: ImageBuffer): DepthMap => { + 'worklet'; + const tInput = preprocessor.process(input); + model.execute('forward', [tInput], [tOutput]); + + const tResize = tensor('float32', [input.height, input.width, 1]); + try { + const data = tOutput + .copyTo(tDepth) + .through(resize, tResize, { mode: 'stretch', interpolation: modelOpts.outInterpolation }) + .getData(new Float32Array(input.width * input.height)); + + return { data, width: input.width, height: input.height }; + } finally { + tResize.dispose(); + } + }; + + const estimateDepth = wrapAsync(estimateDepthWorklet, runtime); + + return { + dispose, + estimateDepth, + estimateDepthWorklet, + }; + } catch (error) { + dispose(); + throw error; + } +} diff --git a/packages/react-native-executorch/src/hooks/useDepthEstimator.ts b/packages/react-native-executorch/src/hooks/useDepthEstimator.ts new file mode 100644 index 0000000000..53cac0963d --- /dev/null +++ b/packages/react-native-executorch/src/hooks/useDepthEstimator.ts @@ -0,0 +1,37 @@ +import { useModel } from './useModel'; +import { useResourceDownload, type ResourceOptions } from './useResourceDownload'; +import { + createDepthEstimator, + type DepthEstimatorModel, +} from '../extensions/cv/tasks/depthEstimation'; + +/** + * React hook to load and run a depth estimation model. + * + * This hook manages downloading (if remote URLs are provided) and loading the + * model assets, compiling them, tracking download progress and load errors, and + * releasing native memory when the component unmounts or the configuration + * changes. + * + * For imperative usage, see {@link createDepthEstimator}. + * @category Hooks + * @param config The depth estimation model configuration. + * See {@link DepthEstimatorModel}. + * @param options Load and caching options. See {@link ResourceOptions}. + * @returns The same object as {@link DepthEstimator} (without `dispose`), + * combined with loading state and download progress. + * @see {@link DepthEstimator} + */ +export function useDepthEstimator(config: DepthEstimatorModel, options?: ResourceOptions) { + const { resource, downloadProgress, downloadError } = useResourceDownload(config, options); + const { model, error } = useModel(createDepthEstimator, resource); + + return { + isReady: !!model, + error: downloadError || error, + downloadProgress, + resource, + estimateDepth: model?.estimateDepth, + estimateDepthWorklet: model?.estimateDepthWorklet, + }; +} diff --git a/packages/react-native-executorch/src/index.ts b/packages/react-native-executorch/src/index.ts index b9fff6b01a..9a1040aada 100644 --- a/packages/react-native-executorch/src/index.ts +++ b/packages/react-native-executorch/src/index.ts @@ -46,6 +46,7 @@ // Hooks — primary API for app developers export * from './hooks/useClassifier'; export * from './hooks/useStyleTransfer'; +export * from './hooks/useDepthEstimator'; export * from './hooks/useSemanticSegmenter'; export * from './hooks/useInstanceSegmenter'; export * from './hooks/useKeypointDetector'; @@ -74,6 +75,7 @@ export * from './constants'; // Task APIs — for developers needing manual lifetime/disposal control export * from './extensions/cv/tasks/classification'; export * from './extensions/cv/tasks/styleTransfer'; +export * from './extensions/cv/tasks/depthEstimation'; export * from './extensions/cv/tasks/semanticSegmentation'; export * from './extensions/cv/tasks/instanceSegmentation'; export * from './extensions/cv/tasks/keypointDetection'; diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index de37ec051d..5a803a605d 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -22,6 +22,7 @@ import type { StyleTransferModel } from './extensions/cv/tasks/styleTransfer'; import type { SemanticSegmenterModel } from './extensions/cv/tasks/semanticSegmentation'; import type { KeypointDetectorModel } from './extensions/cv/tasks/keypointDetection'; import type { InstanceSegmenterModel } from './extensions/cv/tasks/instanceSegmentation'; +import type { DepthEstimatorModel } from './extensions/cv/tasks/depthEstimation'; import type { ImageEmbedderModel } from './extensions/cv/tasks/imageEmbedding'; import type { SdxsTextToImageModel } from './extensions/cv/tasks/sdxsTextToImage'; import type { TextEmbedderModel } from './extensions/nlp/tasks/textEmbedding'; @@ -278,6 +279,33 @@ const STYLE_TRANSFER_UDNIE_VULKAN_FP16: StyleTransferModel = { modelOpts: STYLE_TRANSFER_OPTS, }; +// ============================================================================= +// Depth Estimation +// ============================================================================= + +const DEPTH_ANYTHING_V2_OPTS = { + resizeMode: 'stretch' as const, + interpolation: 'cubic' as const, + normalizeOpts: IMAGENET_NORM, + outInterpolation: 'linear' as const, +}; +const DEPTH_ANYTHING_V2_SMALL_XNNPACK_INT8: DepthEstimatorModel = { + modelPath: `${BASE_URL}-depth-anything-v2/${NEXT_VERSION_TAG}/small/xnnpack/depth_anything_v2_small_xnnpack_int8.pte`, + modelOpts: DEPTH_ANYTHING_V2_OPTS, +}; +const DEPTH_ANYTHING_V2_SMALL_XNNPACK_FP32: DepthEstimatorModel = { + modelPath: `${BASE_URL}-depth-anything-v2/${NEXT_VERSION_TAG}/small/xnnpack/depth_anything_v2_small_xnnpack_fp32.pte`, + modelOpts: DEPTH_ANYTHING_V2_OPTS, +}; +const DEPTH_ANYTHING_V2_SMALL_COREML_FP16: DepthEstimatorModel = { + modelPath: `${BASE_URL}-depth-anything-v2/${NEXT_VERSION_TAG}/small/coreml/depth_anything_v2_small_coreml_fp16.pte`, + modelOpts: DEPTH_ANYTHING_V2_OPTS, +}; +const DEPTH_ANYTHING_V2_SMALL_VULKAN_FP16: DepthEstimatorModel = { + modelPath: `${BASE_URL}-depth-anything-v2/${NEXT_VERSION_TAG}/small/vulkan/depth_anything_v2_small_vulkan_fp16.pte`, + modelOpts: DEPTH_ANYTHING_V2_OPTS, +}; + // ============================================================================= // Semantic Segmentation // ============================================================================= @@ -2372,6 +2400,22 @@ export const models = { }), }, + /** + * Depth estimation models that predict a floating-point depth value per pixel. + */ + depthEstimation: { + /** + * Depth Anything V2 Small with a ViT-S/14 backbone. Predicts relative inverse + * depth from a single image; larger values indicate closer regions. + */ + DEPTH_ANYTHING_V2_SMALL: variants({ + XNNPACK_INT8: DEPTH_ANYTHING_V2_SMALL_XNNPACK_INT8, + XNNPACK_FP32: DEPTH_ANYTHING_V2_SMALL_XNNPACK_FP32, + COREML_FP16: DEPTH_ANYTHING_V2_SMALL_COREML_FP16, + VULKAN_FP16: DEPTH_ANYTHING_V2_SMALL_VULKAN_FP16, + }), + }, + /** * Semantic segmentation models that classify each pixel into target object or * background classes. From 13df400066205fdac91c30faa60a3e13448f6073 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Mon, 19 Oct 2026 16:22:40 +0200 Subject: [PATCH 2/9] docs: document depth estimation usage and models --- .../computer-vision/12-depth-estimation.md | 178 ++++++++++++++++++ .../08-native-libraries.md | 1 + 2 files changed, 179 insertions(+) create mode 100644 docs/docs/02-extensions/computer-vision/12-depth-estimation.md diff --git a/docs/docs/02-extensions/computer-vision/12-depth-estimation.md b/docs/docs/02-extensions/computer-vision/12-depth-estimation.md new file mode 100644 index 0000000000..b8997b1f63 --- /dev/null +++ b/docs/docs/02-extensions/computer-vision/12-depth-estimation.md @@ -0,0 +1,178 @@ +--- +title: Depth Estimation +slug: /extensions/depth-estimation +description: 'Estimate pixel-level relative depth from a single image with Depth Anything V2 for portrait effects, scene understanding, and photo animation.' +keywords: + [ + react native, + depth estimation, + depth anything, + depth map, + portrait effects, + mobile ml, + on-device ai, + ] +--- + +# Depth Estimation + +Depth estimation predicts a depth value for every pixel in an image. The pipeline returns a floating-point depth map matching the original image dimensions, which can be used for portrait blur, photo animation, and scene understanding. + +Depth Anything V2 predicts **relative inverse depth**: larger values indicate closer surfaces. Its values are not distances in meters, and their scale is not calibrated across images. + +## Quick Start + +The [`useDepthEstimator`](../../06-api-reference/functions/useDepthEstimator.md) hook manages model downloading, initialization, and lifecycle: + +```tsx +import { models, useDepthEstimator } from 'react-native-executorch'; +import type { ImageBuffer } from 'react-native-executorch/cv'; + +function MyComponent() { + const estimator = useDepthEstimator(models.depthEstimation.DEPTH_ANYTHING_V2_SMALL.DEFAULT); + + // Hook state: + // estimator.isReady — true once model is downloaded and loaded in memory + // estimator.downloadProgress — 0 to 100 download progress + // estimator.error — Error instance if download or load failed + // estimator.resource — resolved config with all URLs replaced by local file paths + + const handleEstimate = async (imageBuffer: ImageBuffer) => { + if (!estimator.isReady || !estimator.estimateDepth) return; + + // Run inference on a background thread + const depth = await estimator.estimateDepth(imageBuffer); + const x = Math.floor(depth.width / 2); + const y = Math.floor(depth.height / 2); + console.log('Center depth:', depth.data[y * depth.width + x]); + }; + + // Trigger handleEstimate from an image picker, button press, or camera frame +} +``` + +:::tip Full Interactive Example +See [`apps/computer-vision/app/depthEstimation/index.tsx`](https://github.com/software-mansion/react-native-executorch/blob/main/apps/computer-vision/app/depthEstimation/index.tsx) for a complete screen with model selection, photo capture, depth visualization, interactive photo effects, and latency tracking. +::: + +## Output Format + +[`estimateDepth()`](../../06-api-reference/type-aliases/DepthEstimator.md#estimatedepth) returns a [`DepthMap`](../../06-api-reference/type-aliases/DepthMap.md) object: + +```typescript +type DepthMap = { + /** Raw depth values in row-major order: data[y * width + x] */ + readonly data: Float32Array; + /** Width of the depth map in pixels */ + readonly width: number; + /** Height of the depth map in pixels */ + readonly height: number; +}; +``` + +The map contains one float per source image pixel. Each result owns its buffer and remains valid after subsequent inference or pipeline disposal. + +Values retain the model's original scale. For visualization, normalize a copy of the data and apply a color map in your application. + +## Configuration & Options + +The [`DepthEstimatorOptions`](../../06-api-reference/type-aliases/DepthEstimatorOptions.md) in `modelOpts` configure input preprocessing and output resizing. Hosted presets include these settings: + +| Option | Depth Anything V2 preset | Description | +| :----------------- | :----------------------- | :---------------------------------------------------------------------------------------- | +| `resizeMode` | `'stretch'` | Resize the full image to the model input dimensions. Currently only stretch is supported. | +| `interpolation` | `'cubic'` | Interpolation used to resize the input image. | +| `normalizeOpts` | `IMAGENET_NORM` | Input channel scaling and normalization. | +| `outInterpolation` | `'linear'` | Interpolation used to resize the float depth map to the source image dimensions. | + +There are no per-inference options. To change preprocessing for a different model, supply a custom model configuration. + +## Imperative API + +For background processing or manual lifecycle management outside React components, create the estimator using [`createDepthEstimator`](../../06-api-reference/functions/createDepthEstimator.md): + +```typescript +import { createDepthEstimator, download, models } from 'react-native-executorch'; + +// Download and cache model assets before creating the pipeline +const model = await download(models.depthEstimation.DEPTH_ANYTHING_V2_SMALL.DEFAULT); +const estimator = await createDepthEstimator(model); + +try { + const depth = await estimator.estimateDepth(imageBuffer); + console.log('Depth map dimensions:', depth.width, depth.height); +} finally { + // Always release native resources when finished + estimator.dispose(); +} +``` + +## Synchronous Execution + +For worklet-based processing, [`createDepthEstimator`](../../06-api-reference/functions/createDepthEstimator.md) exposes a synchronous [`estimateDepthWorklet`](../../06-api-reference/type-aliases/DepthEstimator.md#estimatedepthworklet) function: + +```typescript +// Called synchronously on the worklet thread +const depth = estimator.estimateDepthWorklet(frameBuffer); +``` + +See [Worklets & Threading](../../03-core-and-advanced/06-worklets-and-threading.md) for details on worklet execution contexts. + +## Available Models + +Ready-to-use models are available in [`models.depthEstimation`](../../06-api-reference/variables/models.md#depthestimation): + +| Model Family | Variants | Size Range | Supported Backends | Notes | +| :-------------------------- | :--------------------------------------------------------------------------------------- | :---------------- | :----------------------------------------------- | :-------------------------------------------------------------------------- | +| **Depth Anything V2 Small** | [See](../../06-api-reference/variables/models.md#depthestimationdepth_anything_v2_small) | 35.5 MB – 99.0 MB | XNNPACK (CPU), Core ML (Apple), Vulkan (Android) | ViT-S/14 backbone with a 518 × 518 input and relative inverse depth output. | + +Use `DEFAULT` for automatic backend selection, or choose `XNNPACK_INT8`, `XNNPACK_FP32`, `COREML_FP16`, or `VULKAN_FP16` explicitly. The exports are hosted in the [Depth Anything V2 Hugging Face repository](https://huggingface.co/software-mansion/react-native-executorch-depth-anything-v2). + +:::tip Using Custom Models +Pass a [`DepthEstimatorModel`](../../06-api-reference/type-aliases/DepthEstimatorModel.md) configuration to [`useDepthEstimator`](../../06-api-reference/functions/useDepthEstimator.md) or download it before calling [`createDepthEstimator`](../../06-api-reference/functions/createDepthEstimator.md): + +```typescript +import { createDepthEstimator, download, IMAGENET_NORM } from 'react-native-executorch'; + +const model = await download({ + modelPath: 'https://example.com/my-depth-model.pte', + modelOpts: { + resizeMode: 'stretch' as const, + interpolation: 'cubic' as const, + normalizeOpts: IMAGENET_NORM, + outInterpolation: 'linear' as const, + }, +}); +const estimator = await createDepthEstimator(model); +``` + +Choose normalization and interpolation settings for your model. The pipeline accepts static float32 RGB inputs `[1, 3, H, W]` with outputs `[1, OH, OW]`, or unbatched inputs `[3, H, W]` with outputs `[OH, OW]`. Input and output spatial dimensions may differ. Output values retain the custom model's meaning and units. + +The pipeline verifies the exported input and output shapes automatically. See [Exporting Custom Models](../../03-core-and-advanced/07-exporting-custom-models.md#using-a-built-in-pipeline) for export requirements. +::: + +## API Reference + +### Hooks & Pipelines + +- [`useDepthEstimator()`](../../06-api-reference/functions/useDepthEstimator.md) — React hook for model downloading, state, and lifecycle. +- [`createDepthEstimator()`](../../06-api-reference/functions/createDepthEstimator.md) — Imperative factory for depth estimation pipelines. + +### Types & Options + +- [`DepthEstimator`](../../06-api-reference/type-aliases/DepthEstimator.md) — Runner interface with `estimateDepth` and `estimateDepthWorklet`. +- [`DepthMap`](../../06-api-reference/type-aliases/DepthMap.md) — Floating-point depth data with width and height. +- [`DepthEstimatorModel`](../../06-api-reference/type-aliases/DepthEstimatorModel.md) — Model configuration spec. +- [`DepthEstimatorOptions`](../../06-api-reference/type-aliases/DepthEstimatorOptions.md) — Input preprocessing and output interpolation options. +- [`ImageBuffer`](../../06-api-reference/react-native-executorch/namespaces/cv/type-aliases/ImageBuffer.md) — Input image buffer structure. + +### Model Presets & Constants + +- [`models.depthEstimation`](../../06-api-reference/variables/models.md#depthestimation) — Pre-configured depth estimation models registry. +- [`IMAGENET_NORM`](../../06-api-reference/variables/IMAGENET_NORM.md) — Input normalization used by Depth Anything V2 presets. + +:::info Source Code +View the implementation on GitHub: + +- [`src/extensions/cv/tasks/depthEstimation.ts` ↗](https://github.com/software-mansion/react-native-executorch/blob/main/packages/react-native-executorch/src/extensions/cv/tasks/depthEstimation.ts) + ::: diff --git a/docs/docs/03-core-and-advanced/08-native-libraries.md b/docs/docs/03-core-and-advanced/08-native-libraries.md index 0dd37bba93..258e0e38a3 100644 --- a/docs/docs/03-core-and-advanced/08-native-libraries.md +++ b/docs/docs/03-core-and-advanced/08-native-libraries.md @@ -93,6 +93,7 @@ Specifying a task under `features` is shorthand: it automatically expands to the | `textEmbeddings` | xnnpack, coreml, mlx, vulkan | — | | `imageEmbeddings` | xnnpack, coreml, mlx, vulkan | opencv | | `classification` | xnnpack, coreml | opencv | +| `depthEstimation` | xnnpack, coreml, vulkan | opencv | | `objectDetection` | xnnpack, coreml, vulkan | opencv | | `keypointDetection` | xnnpack, coreml, mlx, vulkan | opencv | | `semanticSegmentation` | xnnpack, coreml | opencv | From b8eab4cb615df1fd590fd74a16f2883e30685dbb Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Mon, 19 Oct 2026 16:22:48 +0200 Subject: [PATCH 3/9] test: cover depth estimation pipeline and registry --- .../api/__snapshots__/apiSurface.test.ts.snap | 3 + .../__tests__/api/modelVariants.test.ts | 1 + .../tasks/constructionFailure.test.ts | 2 + .../__tests__/tasks/depthEstimation.test.ts | 187 ++++++++++++++++++ 4 files changed, 193 insertions(+) create mode 100644 packages/react-native-executorch/__tests__/tasks/depthEstimation.test.ts diff --git a/packages/react-native-executorch/__tests__/api/__snapshots__/apiSurface.test.ts.snap b/packages/react-native-executorch/__tests__/api/__snapshots__/apiSurface.test.ts.snap index 99804c6b0f..a6c6fa9291 100644 --- a/packages/react-native-executorch/__tests__/api/__snapshots__/apiSurface.test.ts.snap +++ b/packages/react-native-executorch/__tests__/api/__snapshots__/apiSurface.test.ts.snap @@ -4,6 +4,7 @@ exports[`public API surface exports one hook per user-facing task 1`] = ` [ "useClassifier", "useColbertEmbedder", + "useDepthEstimator", "useImageEmbedder", "useInstanceSegmenter", "useKeypointDetector", @@ -133,6 +134,7 @@ exports[`public API surface matches the recorded export list 1`] = ` "WHISPER_SAMPLE_RATE_HZ", "createClassifier", "createColbertEmbedder", + "createDepthEstimator", "createFsmnVoiceActivityDetector", "createImageEmbedder", "createInstanceSegmenter", @@ -169,6 +171,7 @@ exports[`public API surface matches the recorded export list 1`] = ` "tensor", "useClassifier", "useColbertEmbedder", + "useDepthEstimator", "useImageEmbedder", "useInstanceSegmenter", "useKeypointDetector", diff --git a/packages/react-native-executorch/__tests__/api/modelVariants.test.ts b/packages/react-native-executorch/__tests__/api/modelVariants.test.ts index ef303485c6..0279433e71 100644 --- a/packages/react-native-executorch/__tests__/api/modelVariants.test.ts +++ b/packages/react-native-executorch/__tests__/api/modelVariants.test.ts @@ -336,6 +336,7 @@ describe('feature map', () => { // added without an entry here fails the coverage case below. const FEATURE_OF_CATEGORY: Record = { classification: 'classification', + depthEstimation: 'depthEstimation', styleTransfer: 'styleTransfer', semanticSegmentation: 'semanticSegmentation', objectDetection: 'objectDetection', diff --git a/packages/react-native-executorch/__tests__/tasks/constructionFailure.test.ts b/packages/react-native-executorch/__tests__/tasks/constructionFailure.test.ts index d2edbf4358..f4520e350a 100644 --- a/packages/react-native-executorch/__tests__/tasks/constructionFailure.test.ts +++ b/packages/react-native-executorch/__tests__/tasks/constructionFailure.test.ts @@ -28,6 +28,7 @@ import { createPaddleOcr } from '../../src/extensions/cv/tasks/paddleOcr'; import { createSdxsTextToImage } from '../../src/extensions/cv/tasks/sdxsTextToImage'; import { createSemanticSegmenter } from '../../src/extensions/cv/tasks/semanticSegmentation'; import { createStyleTransfer } from '../../src/extensions/cv/tasks/styleTransfer'; +import { createDepthEstimator } from '../../src/extensions/cv/tasks/depthEstimation'; import { createPrivacyFilter } from '../../src/extensions/nlp/tasks/privacyFilter'; import { createTextEmbedder } from '../../src/extensions/nlp/tasks/textEmbedding'; import { createColbertEmbedder } from '../../src/extensions/nlp/tasks/colbertEmbedding'; @@ -86,6 +87,7 @@ const factories: Record Promise> = { objectDetector: () => createObjectDetector(cv), semanticSegmenter: () => createSemanticSegmenter(cv), styleTransfer: () => createStyleTransfer(cv), + depthEstimator: () => createDepthEstimator(cv), imageEmbedder: () => createImageEmbedder(cv), instanceSegmenter: () => createInstanceSegmenter(cv), keypointDetector: () => createKeypointDetector(cv), diff --git a/packages/react-native-executorch/__tests__/tasks/depthEstimation.test.ts b/packages/react-native-executorch/__tests__/tasks/depthEstimation.test.ts new file mode 100644 index 0000000000..ccb7609f5d --- /dev/null +++ b/packages/react-native-executorch/__tests__/tasks/depthEstimation.test.ts @@ -0,0 +1,187 @@ +import { IMAGENET_NORM } from '../../src/constants'; +import { f32, i32, method } from '../../src/core/schema'; +import { createDepthEstimator } from '../../src/extensions/cv/tasks/depthEstimation'; +import { fakeJsi } from '../support/fakeJsi'; +import { tracked } from '../support/lifetime'; +import { exported, imageBuffer, writesOutputs } from '../support/fixtures'; +import type { FakeTensor } from '../support/fakeTensor'; + +const MODEL_PATH = '/models/depth.pte'; +const config = { + modelPath: MODEL_PATH, + modelOpts: { + resizeMode: 'stretch', + interpolation: 'cubic', + normalizeOpts: IMAGENET_NORM, + outInterpolation: 'linear', + }, +} as const; +const schema = exported(method('forward', [f32(1, 3, 2, 3)], [f32(1, 2, 3)])); + +describe('createDepthEstimator', () => { + beforeEach(() => { + fakeJsi.registerModel(MODEL_PATH, { + schema, + execute: writesOutputs([-1.25, 0.5, 2.75, 4, 10.125, 512.5]), + }); + }); + + it('preserves float values and row-major ordering without normalizing or clipping', async () => { + const estimator = tracked(await createDepthEstimator(config)); + const result = await estimator.estimateDepth(imageBuffer(3, 2)); + + expect(result).toEqual({ + width: 3, + height: 2, + data: Float32Array.from([-1.25, 0.5, 2.75, 4, 10.125, 512.5]), + }); + }); + + it('restores a constant depth field to portrait and landscape source dimensions', async () => { + fakeJsi.registerModel(MODEL_PATH, { schema, execute: writesOutputs(Array(6).fill(12.25)) }); + const estimator = tracked(await createDepthEstimator(config)); + for (const [width, height] of [ + [8, 4], + [4, 8], + ]) { + const result = await estimator.estimateDepth(imageBuffer(width!, height!)); + expect(result.width).toBe(width); + expect(result.height).toBe(height); + expect([...result.data]).toEqual(Array(width! * height!).fill(12.25)); + } + }); + + it('converts RGBA to RGB channels-first with ImageNet normalization', async () => { + fakeJsi.registerModel(MODEL_PATH, { + schema, + execute: (_method, inputs) => { + const input = inputs[0] as FakeTensor; + const means = [0.485, 0.456, 0.406]; + const deviations = [0.229, 0.224, 0.225]; + const pixels = [255, 128, 0]; + for (let channel = 0; channel < 3; channel++) { + for (let pixel = 0; pixel < 6; pixel++) { + expect(input.getElement(channel * 6 + pixel)).toBeCloseTo( + (pixels[channel]! / 255 - means[channel]!) / deviations[channel]!, + 5 + ); + } + } + }, + }); + const estimator = tracked(await createDepthEstimator(config)); + await estimator.estimateDepth({ + data: Uint8Array.from(Array(6).fill([255, 128, 0, 64]).flat()), + width: 3, + height: 2, + format: 'rgba', + layout: 'hwc', + }); + }); + + it('owns each result buffer across subsequent calls and disposal', async () => { + let call = 0; + fakeJsi.registerModel(MODEL_PATH, { + schema, + execute: (_method, _inputs, outputs) => { + outputs[0]!.setElement(0, ++call); + }, + }); + const estimator = tracked(await createDepthEstimator(config)); + const first = await estimator.estimateDepth(imageBuffer(3, 2)); + const second = estimator.estimateDepthWorklet(imageBuffer(3, 2)); + expect(first.data).not.toBe(second.data); + second.data[0] = 99; + estimator.dispose(); + expect(first.data[0]).toBe(1); + }); + + it('agrees between asynchronous and synchronous entry points', async () => { + const estimator = tracked(await createDepthEstimator(config)); + const input = imageBuffer(3, 2); + expect(await estimator.estimateDepth(input)).toEqual(estimator.estimateDepthWorklet(input)); + }); + + it.each([ + { input: [1, 3, 4, 6], output: [1, 2, 3] }, + { input: [3, 4, 6], output: [2, 3] }, + ])('accepts input $input and lower-resolution output $output', async ({ input, output }) => { + fakeJsi.registerModel(MODEL_PATH, { + schema: exported(method('forward', [f32(...input)], [f32(...output)])), + execute: writesOutputs(Array(6).fill(7.25)), + }); + const estimator = tracked(await createDepthEstimator(config)); + const result = await estimator.estimateDepth(imageBuffer(12, 8)); + expect(result.width).toBe(12); + expect(result.height).toBe(8); + expect([...result.data]).toEqual(Array(96).fill(7.25)); + }); + + it('uses model-specific normalization instead of assuming ImageNet inputs', async () => { + fakeJsi.registerModel(MODEL_PATH, { + schema, + execute: (_method, inputs) => { + const input = inputs[0] as FakeTensor; + expect(input.getElement(0)).toBeCloseTo(255 / 127.5 - 1); + expect(input.getElement(6)).toBeCloseTo(128 / 127.5 - 1); + expect(input.getElement(12)).toBeCloseTo(-1); + }, + }); + const estimator = tracked( + await createDepthEstimator({ + ...config, + modelOpts: { + resizeMode: 'stretch', + interpolation: 'nearest', + normalizeOpts: { alpha: 1 / 127.5, beta: -1 }, + outInterpolation: 'nearest', + }, + }) + ); + await estimator.estimateDepth({ + data: Uint8Array.from(Array(6).fill([255, 128, 0]).flat()), + width: 3, + height: 2, + format: 'rgb', + layout: 'hwc', + }); + }); + + it.each([[f32(1, 1, 2, 3)], [f32(1, 2, 2, 3)], [i32(1, 2, 3)], [f32(2, 2, 3)]])( + 'rejects incompatible depth output %j and releases the model', + async (output) => { + fakeJsi.registerModel(MODEL_PATH, { + schema: exported(method('forward', [f32(1, 3, 2, 3)], [output])), + }); + await expect(createDepthEstimator(config)).rejects.toMatchObject({ code: 'SCHEMA_MISMATCH' }); + expect(fakeJsi.liveModels()).toEqual([]); + expect(fakeJsi.liveTensors()).toBe(0); + } + ); + + it('releases per-call tensors and all resources on repeated disposal', async () => { + const estimator = tracked(await createDepthEstimator(config)); + const initialTensors = fakeJsi.liveTensors(); + for (const size of [4, 8, 16]) { + await estimator.estimateDepth(imageBuffer(size, size)); + expect(fakeJsi.liveTensors()).toBe(initialTensors); + } + estimator.dispose(); + estimator.dispose(); + expect(fakeJsi.liveTensors()).toBe(0); + expect(fakeJsi.liveModels()).toEqual([]); + }); + + it('releases transient preprocessing tensors when inference fails', async () => { + fakeJsi.registerModel(MODEL_PATH, { + schema, + execute: () => { + throw new Error('Inference failed'); + }, + }); + const estimator = tracked(await createDepthEstimator(config)); + const initialTensors = fakeJsi.liveTensors(); + await expect(estimator.estimateDepth(imageBuffer(8, 4))).rejects.toThrow('Inference failed'); + expect(fakeJsi.liveTensors()).toBe(initialTensors); + }); +}); From e65fd3ab68231f0d31bd39999325628fe6f06334 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Mon, 19 Oct 2026 16:23:11 +0200 Subject: [PATCH 4/9] feat(example): add interactive depth estimation screen --- apps/computer-vision/app/_layout.tsx | 7 + .../app/depthEstimation/index.tsx | 233 +++++++++++++++ apps/computer-vision/app/index.tsx | 3 + .../computer-vision/components/DepthPhoto.tsx | 278 ++++++++++++++++++ .../components/depthPhotoShader.ts | 64 ++++ apps/computer-vision/package.json | 1 + 6 files changed, 586 insertions(+) create mode 100644 apps/computer-vision/app/depthEstimation/index.tsx create mode 100644 apps/computer-vision/components/DepthPhoto.tsx create mode 100644 apps/computer-vision/components/depthPhotoShader.ts diff --git a/apps/computer-vision/app/_layout.tsx b/apps/computer-vision/app/_layout.tsx index 2ee132c0fb..db3837fe98 100644 --- a/apps/computer-vision/app/_layout.tsx +++ b/apps/computer-vision/app/_layout.tsx @@ -69,6 +69,13 @@ export default function Layout() { title: 'Keypoint Detection', }} /> + 0 && Number.isFinite(value) ? Math.round(((value - min) / range) * 255) : 0; + rgba[i * 4] = gray; + rgba[i * 4 + 1] = gray; + rgba[i * 4 + 2] = gray; + rgba[i * 4 + 3] = 255; + } + const image = Skia.Image.MakeImage( + { + width: depth.width, + height: depth.height, + colorType: ColorType.RGBA_8888, + alphaType: AlphaType.Opaque, + }, + Skia.Data.fromBytes(rgba), + depth.width * 4 + ); + if (!image) throw new Error('Failed to render the depth preview.'); + return { image, min, max }; +} + +function DepthDemo({ + model, + busy, + onBusyChange, +}: { + model: DepthEstimatorModel; + busy: boolean; + onBusyChange: (busy: boolean) => void; +}) { + const [imageUri, setImageUri] = useState(null); + const [preview, setPreview] = useState(null); + const [latency, setLatency] = useState(null); + const [error, setError] = useState(null); + const mounted = useRef(true); + useEffect(() => { + mounted.current = true; + return () => { + mounted.current = false; + }; + }, []); + + const image = useImage(imageUri, (failure) => setError(failure.message)); + const estimator = useDepthEstimator(model); + + const pickImage = async (camera: boolean) => { + onBusyChange(true); + setError(null); + try { + const uri = await getImage(camera); + if (uri && mounted.current) { + setImageUri(uri); + setPreview(null); + setLatency(null); + } + } catch (failure) { + if (mounted.current) setError(String(failure)); + } finally { + if (mounted.current) onBusyChange(false); + } + }; + + const runDepthEstimation = async (sync: boolean) => { + if (!image || !estimator.estimateDepth || !estimator.estimateDepthWorklet || busy) return; + onBusyChange(true); + setError(null); + try { + const input = skImageToBuffer(image); + const start = Date.now(); + const depth = sync + ? estimator.estimateDepthWorklet(input) + : await estimator.estimateDepth(input); + const elapsed = Date.now() - start; + if (!mounted.current) return; + setPreview(createDepthPreview(depth)); + setLatency(elapsed); + } catch (failure) { + if (mounted.current) setError(String(failure)); + } finally { + if (mounted.current) onBusyChange(false); + } + }; + + return ( + <> + + {preview && image ? ( + + ) : ( + { + if (!busy) pickImage(false); + }} + /> + )} + +