diff --git a/.github/workflows/face-recognition.yml b/.github/workflows/face-recognition.yml new file mode 100644 index 0000000000..2e4314e8d9 --- /dev/null +++ b/.github/workflows/face-recognition.yml @@ -0,0 +1,30 @@ +name: face-recognition + +on: + push: + branches: [master] + pull_request: + paths: + - rust/face-recognition/** + - .github/workflows/face-recognition.yml + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + rust-face-recognition: + runs-on: ubuntu-24.04 + container: ghcr.io/dfinity/icp-dev-env-rust:1.0.1 + env: + ICP_CLI_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + - name: Install build dependencies + run: apt-get update && apt-get install -y --no-install-recommends build-essential + - name: Deploy and test + working-directory: rust/face-recognition + run: | + icp network start -d + icp deploy + bash test.sh diff --git a/.github/workflows/rust-face-recognition-example.yaml b/.github/workflows/rust-face-recognition-example.yaml deleted file mode 100644 index 6c13bbc757..0000000000 --- a/.github/workflows/rust-face-recognition-example.yaml +++ /dev/null @@ -1,46 +0,0 @@ -name: rust-face-recognition -on: - push: - branches: - - master - pull_request: - paths: - - rust/face-recognition/** - - .github/workflows/provision-darwin.sh - - .github/workflows/provision-linux.sh - - .github/workflows/rust-face-recognition-example.yaml - - .ic-commit -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true -jobs: - rust-face-recognition-example-darwin: - runs-on: macos-15 - steps: - - uses: actions/checkout@50fbc622fc4ef5163becd7fab6573eac35f8462e # v1.2.0 - - name: Provision Darwin - run: bash .github/workflows/provision-darwin.sh - - name: Remove networks.json - run: rm -f ~/.config/dfx/networks.json - - name: Rust Face Recognition Darwin - run: | - dfx start --background - pushd rust/face-recognition - npm install - dfx deploy --yes - popd - rust-face-recognition-example-linux: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@50fbc622fc4ef5163becd7fab6573eac35f8462e # v1.2.0 - - name: Provision Linux - run: bash .github/workflows/provision-linux.sh - - name: Remove networks.json - run: rm -f ~/.config/dfx/networks.json - - name: Rust Face Recognition Linux - run: | - dfx start --background - pushd rust/face-recognition - npm install - dfx deploy --yes - popd diff --git a/rust/face-recognition/.cargo/config.toml b/rust/face-recognition/.cargo/config.toml deleted file mode 100644 index 067cfd27dc..0000000000 --- a/rust/face-recognition/.cargo/config.toml +++ /dev/null @@ -1,5 +0,0 @@ -[build] -target = ["wasm32-wasi"] - -[target.wasm32-wasi] -rustflags = ["-Ctarget-feature=+simd128"] diff --git a/rust/face-recognition/.gitignore b/rust/face-recognition/.gitignore index c3ed99af22..c7fc1f0666 100644 --- a/rust/face-recognition/.gitignore +++ b/rust/face-recognition/.gitignore @@ -1,4 +1,3 @@ -.dfx/ build/ node_modules/ dist/ @@ -8,6 +7,7 @@ _MACOSX target/ *.old.did .idea -src/backend/assets/version-RFB-320.onnx -src/backend/assets/facerec.onnx -.env +# ONNX model files are downloaded/generated at runtime — do not commit +*.onnx +frontend/src/bindings/ +.icp/cache/ diff --git a/rust/face-recognition/Cargo.toml b/rust/face-recognition/Cargo.toml index 4c16df46ee..d1e49e317a 100644 --- a/rust/face-recognition/Cargo.toml +++ b/rust/face-recognition/Cargo.toml @@ -1,3 +1,3 @@ [workspace] -members = ["src/backend"] +members = ["backend"] resolver = "2" diff --git a/rust/face-recognition/README.md b/rust/face-recognition/README.md index bf6ea0903c..7586a0a284 100644 --- a/rust/face-recognition/README.md +++ b/rust/face-recognition/README.md @@ -1,96 +1,91 @@ # ICP face recognition -This is an ICP smart contract runs face detection and face recognition of user's photo that can be uploaded either from a camera or a local file. +This example demonstrates running face detection and face recognition inside an ICP canister using the [Tract ONNX inference engine](https://github.com/sonos/tract). Users can upload photos from a camera or local file, detect faces, and identify people by name. -The smart contract consists of two canisters: +The example consists of two canisters: -- The backend canister embeds the [the Tract ONNX inference engine](https://github.com/sonos/tract) with two ONNX models. One model is used to detect a face in the photo and return its bounding box. Another model is used for computing face embeddings. -- The frontend canister contains the Web assets such as HTML, JS, CSS that are served to the browser. +- **backend** — embeds the Tract ONNX inference engine. Exposes endpoints for uploading ONNX model files in chunks, loading them into memory, detecting faces, computing face embeddings, and recognizing people. Also includes `run_detection` (query) and `run_recognition` (update) endpoints that run the models against a built-in test image — useful during development for capacity planning. Since query calls don't persist logs on ICP, only the update call produces a visible instruction count: -# Models + ```bash + icp canister call backend run_recognition '()' + icp canister logs backend + ``` +- **frontend** — serves the web UI (HTML/JS/CSS). -The smart contract uses two models: one for detecting the face and another for recognizing the face. +## Models -## Face detection +The backend uses two ONNX models that are too large to embed in the Wasm binary and must be uploaded after deployment. `icp deploy` handles this automatically via its sync phase: -A face detection model finds the bounding box of a face in the image. -You can download [Ultraface](https://github.com/onnx/models/tree/main/validated/vision/body_analysis/ultraface) - ultra-lightweight face detection model - [[here](https://github.com/onnx/models/blob/bec48b6a70e5e9042c0badbaafefe4454e072d08/validated/vision/body_analysis/ultraface/models/version-RFB-320.onnx)]. +- **Face detection** ([Ultraface](https://github.com/onnx/models/tree/main/validated/vision/body_analysis/ultraface)) — downloaded automatically. +- **Face recognition** ([facenet-pytorch](https://github.com/timesler/facenet-pytorch) InceptionResnetV1) — generated automatically if Python 3.9–3.12 is available (`facenet-pytorch`, `torch`, and `onnx` are installed via pip). If no compatible Python is found, place `face-recognition.onnx` in the project root manually and run `icp deploy` again. -Alternatively, you can run +Models are stored in stable memory and survive canister upgrades — they reload automatically without re-uploading. Note: the persons database (added via the frontend) is stored in heap memory and is cleared on upgrade. -``` -./download-face-detection-model.sh -``` - -## Face recognition - -A face recognition model computes a vector embedding of an image with a face. -You can obtain a pretrained model from [facenet-pytorch](https://github.com/timesler/facenet-pytorch) as follows. +## Build and deploy from the command line -- #### Step 1: Install `python` and `pip`: https://packaging.python.org/en/latest/tutorials/installing-packages/. +### Prerequisites -- #### Step 2: Install `facenet-pytorch` and `torch`: +**Required:** -``` -pip install facenet-pytorch -pip install torch -pip install onnx -``` +- [Node.js](https://nodejs.org/) v18+ +- [icp-cli](https://cli.internetcomputer.org/): `npm install -g @icp-sdk/icp-cli @icp-sdk/ic-wasm` +- [Rust](https://www.rust-lang.org/tools/install) v1.85+ with `wasm32-wasip1` target: `rustup target add wasm32-wasip1` +- [wasi2ic](https://github.com/wasm-forge/wasi2ic): `cargo install wasi2ic` -- #### Step 3: Export ONNX model. Start a python shell and run the following commands or create a python file and run it: +`wasm-opt` is installed automatically on first deploy if not already present. -``` -import torch -import facenet_pytorch -resnet = facenet_pytorch.InceptionResnetV1(pretrained='vggface2').eval() -input = torch.randn(1, 3, 160, 160) -torch.onnx.export(resnet, input, "face-recognition.onnx", verbose=False, opset_version=11) -``` +**Optional (for automatic face recognition model generation):** -- #### Step 4: This should produce `face-recognition.onnx`. Copy the file to the root of this repository. +- Python 3.9–3.12 with pip — the sync phase auto-installs `facenet-pytorch`, `torch`, and `onnx` and generates `face-recognition.onnx`. Python 3.13+ is not yet supported by torch. -## Prerequisites +### Install -- [x] Install the [IC - SDK](https://internetcomputer.org/docs/current/developer-docs/getting-started/install). For local testing, `dfx >= 0.22.0` is required. -- [x] Clone the example dapp project: `git clone https://github.com/dfinity/examples` -- [x] Install `wasi2ic`: Follow the steps in https://github.com/wasm-forge/wasi2ic and make sure that `wasi2ic` binary is in your `$PATH`. -- [x] Install `wasm-opt`: `cargo install wasm-opt` +```bash +git clone https://github.com/dfinity/examples +cd examples/rust/face-recognition +``` -## Build the application +### Deploy -``` -dfx start --background -dfx deploy +```bash +icp network start -d +icp deploy +icp network stop ``` -If the deployment is successful, the it will show the `frontend` URL. -Open that URL in browser to interact with the smart contract. +`icp deploy` runs three phases: +1. **Build** — compiles the Rust backend to WASM (via wasm32-wasip1 + wasi2ic). +2. **Deploy** — installs the backend and frontend canisters. +3. **Sync** — downloads the face detection model, generates the face recognition model (if Python 3.9–3.12 is available), and uploads both to the canister. Skipped on redeployment if models are already loaded. -## Chunk uploading of models +After deployment the CLI prints the frontend URL. Open it in a browser to interact with the canister. -Since the models are large, they cannot be embedded into the Wasm binary of the smart contract. -Instead they should be uploaded separately. +### Test -[DecideAI](https://decideai.xyz/) implemented a tool for incremental uploading of models: https://github.com/modclub-app/ic-file-uploader/tree/main. +```bash +bash test.sh +``` -You can install the tool with +`test.sh` exercises the model management API without requiring models to be loaded. The frontend shows a setup instruction if models are not yet uploaded. -``` -cargo install ic-file-uploader +For frontend development with hot reload: + +```bash +npm run dev --prefix frontend ``` -Afterwards, execute the `upload-models-to-canister.sh` script, which runs the following commands: +## Updating the Candid interface -``` -dfx canister call backend clear_face_detection_model_bytes -dfx canister call backend clear_face_recognition_model_bytes -ic-file-uploader backend append_face_detection_model_bytes version-RFB-320.onnx -ic-file-uploader backend append_face_recognition_model_bytes face-recognition.onnx -dfx canister call backend setup_models +Only needed if you change the backend endpoints. Requires `candid-extractor` (`cargo install candid-extractor`): + +```bash +icp build backend && candid-extractor ./target/wasm32-wasip1/release/backend.wasm > backend/backend.did ``` ## Credits -Thanks to [DecideAI](https://decideai.xyz/) for discussions and providing [ic-file-uploader](https://github.com/modclub-app/ic-file-uploader/tree/main). +Thanks to [DecideAI](https://decideai.xyz/) for discussions and providing [ic-file-uploader](https://github.com/decide-ai/ic-file-uploader). + +## Security considerations and best practices +See the [ICP security best practices](https://docs.internetcomputer.org/guides/security/overview). diff --git a/rust/face-recognition/src/backend/Cargo.toml b/rust/face-recognition/backend/Cargo.toml similarity index 71% rename from rust/face-recognition/src/backend/Cargo.toml rename to rust/face-recognition/backend/Cargo.toml index bf9676ca06..012aabf7b9 100644 --- a/rust/face-recognition/src/backend/Cargo.toml +++ b/rust/face-recognition/backend/Cargo.toml @@ -1,22 +1,21 @@ [package] -edition = "2021" name = "backend" version = "1.1.0" +edition = "2024" [lib] +# cdylib produces a .wasm binary suitable for deployment as an ICP canister. crate-type = ["cdylib"] [dependencies] anyhow = "1.0" bytes = "1.5.0" candid = "0.10" -colorgrad = "0.6" -ic-cdk = "0.14.0" -ic-stable-structures = "0.6" -ic-wasi-polyfill = "0.4.1" +ic-cdk = "0.20" +ic-stable-structures = "0.7" +ic-wasi-polyfill = "0.13" image = { version = "0.25.1", features = ["png"], default-features = false } prost = "0.11.0" prost-types = "0.11.0" serde = { version = "1.0", features = ["derive"] } tract-onnx = { git = "https://github.com/sonos/tract", rev = "2a2914ac29390cc08963301c9f3d437b52dd321a" } - diff --git a/rust/face-recognition/src/backend/assets/image.png b/rust/face-recognition/backend/assets/image.png similarity index 100% rename from rust/face-recognition/src/backend/assets/image.png rename to rust/face-recognition/backend/assets/image.png diff --git a/rust/face-recognition/src/backend/backend.did b/rust/face-recognition/backend/backend.did similarity index 85% rename from rust/face-recognition/src/backend/backend.did rename to rust/face-recognition/backend/backend.did index 944ed2256f..2b3eb7581e 100644 --- a/rust/face-recognition/src/backend/backend.did +++ b/rust/face-recognition/backend/backend.did @@ -38,12 +38,15 @@ service : { "recognize": (image: blob) -> (Recognition); "add": (label: text, image: blob) -> (Addition); + // Returns true once both ONNX models have been loaded via setup_models(). + "models_ready": () -> (bool) query; + // These endpoints are used for incremental uploading of model files. "clear_face_detection_model_bytes": () -> (); "clear_face_recognition_model_bytes": () -> (); "append_face_detection_model_bytes": (bytes: blob) -> (); "append_face_recognition_model_bytes": (bytes: blob) -> (); - "setup_models": () -> (); + "setup_models": () -> (variant { Ok: null; Err: text }); // These endpoints are used only for testing and benchmarking. "run_detection": () -> (Detection) query; diff --git a/rust/face-recognition/src/backend/src/benchmarking.rs b/rust/face-recognition/backend/src/benchmarking.rs similarity index 68% rename from rust/face-recognition/src/backend/src/benchmarking.rs rename to rust/face-recognition/backend/src/benchmarking.rs index 63373151bb..c5a148d907 100644 --- a/rust/face-recognition/src/backend/src/benchmarking.rs +++ b/rust/face-recognition/backend/src/benchmarking.rs @@ -1,10 +1,18 @@ -// The code below is used for testing and benchmarking. +// Benchmarking and smoke-test endpoints. +// +// run_detection() and run_recognition() run the respective models against a +// hardcoded test image embedded in the Wasm binary. They log the IC instruction +// count (visible via `icp canister logs backend`) so you can measure the +// computational cost of each inference call. +// +// run_detection is a query (fast, single-node execution). +// run_recognition is an update (replicated execution, consistent with recognize()). use crate::{onnx, Detection, Error, Recognition}; const IMAGE: &'static [u8] = include_bytes!("../assets/image.png"); -/// Formats thousands for the specified `u64` integer (helper function). +/// Formats a u64 with thousands separators, e.g. 1_234_567. fn fmt(n: u64) -> String { n.to_string() .as_bytes() diff --git a/rust/face-recognition/src/backend/src/lib.rs b/rust/face-recognition/backend/src/lib.rs similarity index 79% rename from rust/face-recognition/src/backend/src/lib.rs rename to rust/face-recognition/backend/src/lib.rs index 21202ee358..e2abd226c5 100644 --- a/rust/face-recognition/src/backend/src/lib.rs +++ b/rust/face-recognition/backend/src/lib.rs @@ -119,10 +119,26 @@ fn append_face_recognition_model_bytes(bytes: Vec) { storage::append_bytes(FACE_RECOGNITION_FILE, bytes); } +/// Returns true if both ONNX models have been loaded via setup_models(). +/// The frontend uses this on page load to show a setup warning if models are missing. +#[ic_cdk::query] +fn models_ready() -> bool { + onnx::models_ready() +} + /// Once the model files have been incrementally uploaded, /// this function loads them into in-memory models. #[ic_cdk::update] fn setup_models() -> Result<(), String> { + if std::fs::metadata(FACE_DETECTION_FILE).is_err() + || std::fs::metadata(FACE_RECOGNITION_FILE).is_err() + { + return Err( + "Model files not found — upload them first using append_face_detection_model_bytes \ + and append_face_recognition_model_bytes" + .to_string(), + ); + } setup( storage::bytes(FACE_DETECTION_FILE), storage::bytes(FACE_RECOGNITION_FILE), @@ -140,4 +156,18 @@ fn init() { fn post_upgrade() { let wasi_memory = MEMORY_MANAGER.with(|m| m.borrow().get(WASI_MEMORY_ID)); ic_wasi_polyfill::init_with_memory(&[0u8; 32], &[], wasi_memory); + // Reload models from stable memory if they were uploaded before this upgrade. + // Guard with metadata check first — storage::bytes() panics on missing files. + if std::fs::metadata(FACE_DETECTION_FILE).is_ok() + && std::fs::metadata(FACE_RECOGNITION_FILE).is_ok() + { + if let Err(err) = setup( + storage::bytes(FACE_DETECTION_FILE), + storage::bytes(FACE_RECOGNITION_FILE), + ) { + ic_cdk::println!("post_upgrade: model reload failed: {}", err); + } + } } + +ic_cdk::export_candid!(); diff --git a/rust/face-recognition/src/backend/src/onnx.rs b/rust/face-recognition/backend/src/onnx.rs similarity index 90% rename from rust/face-recognition/src/backend/src/onnx.rs rename to rust/face-recognition/backend/src/onnx.rs index 0df91951d5..05dbad2bd9 100644 --- a/rust/face-recognition/src/backend/src/onnx.rs +++ b/rust/face-recognition/backend/src/onnx.rs @@ -15,6 +15,8 @@ type Model = SimplePlan, Graph> = RefCell::new(None); static FACE_RECOGNITION: RefCell> = RefCell::new(None); + // Stored in heap memory — cleared on canister upgrade. For a production + // system, persist DB to stable memory via ic-stable-structures. static DB: RefCell> = RefCell::new(vec![]); } @@ -89,10 +91,14 @@ pub fn setup(facedetect: Bytes, facerec: Bytes) -> TractResult<()> { setup_facerec(facerec) } +pub fn models_ready() -> bool { + FACE_DETECTION.with_borrow(|m| m.is_some()) && FACE_RECOGNITION.with_borrow(|m| m.is_some()) +} + /// Returns a bounding box around the face detected in the given image. pub fn detect(image: Vec) -> Result<(BoundingBox, f32), anyhow::Error> { FACE_DETECTION.with_borrow(|model| { - let model = model.as_ref().unwrap(); + let model = model.as_ref().ok_or_else(|| anyhow::anyhow!("Face detection model not loaded — call setup_models() first"))?; let image = image::load_from_memory(&image)?.to_rgb8(); // The model accepts an image of size 320x240px. @@ -129,7 +135,7 @@ pub fn detect(image: Vec) -> Result<(BoundingBox, f32), anyhow::Error> { /// Computes a face embedding corresponding to the given image of a face. pub fn embedding(image: Vec) -> Result { FACE_RECOGNITION.with_borrow(|model| { - let model = model.as_ref().unwrap(); + let model = model.as_ref().ok_or_else(|| anyhow::anyhow!("Face recognition model not loaded — call setup_models() first"))?; let image = image::load_from_memory(&image)?.to_rgb8(); // The model accepts an image of size 160x160px. diff --git a/rust/face-recognition/src/backend/src/storage.rs b/rust/face-recognition/backend/src/storage.rs similarity index 100% rename from rust/face-recognition/src/backend/src/storage.rs rename to rust/face-recognition/backend/src/storage.rs diff --git a/rust/face-recognition/build.sh b/rust/face-recognition/build.sh deleted file mode 100755 index 1578306038..0000000000 --- a/rust/face-recognition/build.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -set -ex - -export RUSTFLAGS=$RUSTFLAGS' -C target-feature=+simd128' -cargo build --release --target=wasm32-wasi -wasi2ic ./target/wasm32-wasi/release/backend.wasm ./target/wasm32-wasi/release/backend-ic.wasm -wasm-opt -Os --enable-simd --enable-bulk-memory -o ./target/wasm32-wasi/release/backend-ic.wasm \ - ./target/wasm32-wasi/release/backend-ic.wasm diff --git a/rust/face-recognition/dfx.json b/rust/face-recognition/dfx.json deleted file mode 100644 index 59d45e4c17..0000000000 --- a/rust/face-recognition/dfx.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "canisters": { - "backend": { - "metadata": [ - { - "name": "candid:service" - } - ], - "candid": "src/backend/backend.did", - "package": "backend", - "type": "custom", - "wasm": "target/wasm32-wasi/release/backend-ic.wasm", - "build": [ "bash build.sh" ] - }, - "frontend": { - "dependencies": [ - "backend" - ], - "frontend": { - "entrypoint": "src/frontend/src/index.html" - }, - "source": [ - "src/frontend/assets", - "dist/frontend/" - ], - "type": "assets" - } - }, - "defaults": { - "build": { - "args": "", - "packtool": "" - } - }, - "output_env_file": ".env", - "version": 1 -} diff --git a/rust/face-recognition/download-face-detection-model.sh b/rust/face-recognition/download-face-detection-model.sh index 3aea8a061f..e7262278ba 100755 --- a/rust/face-recognition/download-face-detection-model.sh +++ b/rust/face-recognition/download-face-detection-model.sh @@ -11,9 +11,9 @@ if [ -s "${ULTRAFACE_TGT}" ]; then fi if which wget >/dev/null; then - wget $ULTRAFACE_URL -O $ULTRAFACE_TGT + wget --inet4-only $ULTRAFACE_URL -O $ULTRAFACE_TGT elif which curl >/dev/null; then - curl -vL $ULTRAFACE_URL -o $ULTRAFACE_TGT + curl -4 -L $ULTRAFACE_URL -o $ULTRAFACE_TGT else echo "Couldn't find wget or curl." echo "Please download manually from \"$ULTRAFACE_URL\" and save the file in $ULTRAFACE_TGT." diff --git a/rust/face-recognition/src/frontend/assets/default.png b/rust/face-recognition/frontend/default.png similarity index 100% rename from rust/face-recognition/src/frontend/assets/default.png rename to rust/face-recognition/frontend/default.png diff --git a/rust/face-recognition/src/frontend/assets/favicon.ico b/rust/face-recognition/frontend/favicon.ico similarity index 100% rename from rust/face-recognition/src/frontend/assets/favicon.ico rename to rust/face-recognition/frontend/favicon.ico diff --git a/rust/face-recognition/src/frontend/src/index.html b/rust/face-recognition/frontend/index.html similarity index 87% rename from rust/face-recognition/src/frontend/src/index.html rename to rust/face-recognition/frontend/index.html index 3c11059ebc..7f121382ca 100644 --- a/rust/face-recognition/src/frontend/src/index.html +++ b/rust/face-recognition/frontend/index.html @@ -4,16 +4,17 @@ - Onchain ICP face recognition + ICP face recognition +
-

Onchain ICP face recognition

+

ICP face recognition