Assisted vision

.github/workflows/CICD.yml

@@ -11,11 +11,13 @@ on:
       - '**/*.md'
       - 'public/dist/*.js'
       - 'public/dist/**/*.js'
+      - 'ai-vision-service/**'
   pull_request:
     paths-ignore:
       - '**/*.md'
       - 'public/dist/*.js'
       - 'public/dist/**/*.js'
+      - 'ai-vision-service/**'
   # Allow manually triggering the workflow.
   workflow_dispatch:
 

.github/workflows/dependency-review.yml

@@ -25,3 +25,8 @@ jobs:
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
       - name: 'Dependency Review'
         uses: actions/dependency-review-action@2031cfc080254a8a887f58cffee85186f0e49e48 # v4.9.0
+        with:
+          # No fix available yet
+          # Note that the model is directly baked into the inage
+          # So the risk is limited.
+          allow-ghsas: GHSA-hqmj-h5c6-369m

.github/workflows/python_ai_vision.yml

@@ -0,0 +1,154 @@
+name: Python AI Vision Service
+
+on:
+  push:
+    branches:
+      - master
+      - assisted-vision
+    paths:
+      - 'ai-vision-service/**'
+  pull_request:
+    paths:
+      - 'ai-vision-service/**'
+  workflow_dispatch:
+
+# Declare default permissions as read only.
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ !contains(github.ref, 'master') && !startsWith(github.ref, 'refs/tags/') }}
+
+defaults:
+  run:
+    working-directory: ai-vision-service
+
+jobs:
+  # ---------------------------------------------------------------------------
+  # Lint – formatting and style
+  # ---------------------------------------------------------------------------
+  lint:
+    name: Lint (ruff)
+    runs-on: ubuntu-latest
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
+        with:
+          egress-policy: audit
+
+      - name: Checkout code
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
+        with:
+          enable-cache: true
+
+      - name: Install dev dependencies
+        run: uv sync --frozen
+
+      - name: Check formatting
+        run: uv run ruff format --check app/ tests/
+
+      - name: Lint
+        run: uv run ruff check app/ tests/
+
+  # ---------------------------------------------------------------------------
+  # Type check
+  # ---------------------------------------------------------------------------
+  typecheck:
+    name: Type check (ty)
+    runs-on: ubuntu-latest
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
+        with:
+          egress-policy: audit
+
+      - name: Checkout code
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
+        with:
+          enable-cache: true
+
+      - name: Install dev dependencies
+        run: uv sync --frozen
+
+      - name: Type check
+        run: uv run ty check app/
+
+  # ---------------------------------------------------------------------------
+  # Test matrix – Python 3.13 and 3.14
+  # ---------------------------------------------------------------------------
+  test:
+    name: Tests (Python ${{ matrix.python-version }})
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version:
+          - "3.13"
+          - "3.14"
+
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
+        with:
+          egress-policy: audit
+
+      - name: Checkout code
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
+        with:
+          enable-cache: true
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dev dependencies
+        run: uv sync --frozen
+
+      - name: Run tests
+        run: uv run pytest --cov=app --cov-report=xml -v
+
+      - name: Upload coverage
+        if: matrix.python-version == '3.13'
+        uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2
+        with:
+          files: ai-vision-service/coverage.xml
+          flags: ai-vision-service
+        continue-on-error: true
+
+  # ---------------------------------------------------------------------------
+  # Docker build verification
+  # ---------------------------------------------------------------------------
+  docker-build:
+    name: Docker build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0
+        with:
+          egress-policy: audit
+
+      - name: Checkout code
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd #v4.0.0
+
+      - name: Build Docker image (no model bake in CI to save time)
+        uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0
+        with:
+          context: ai-vision-service
+          push: false
+          load: true
+          tags: lychee-ai-vision:ci
+          # Override the model-bake step by targeting the builder stage
+          # to avoid downloading 300MB of model weights in CI.
+          target: builder
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

.gitignore

@@ -38,6 +38,8 @@ clover.xml
 .NO_AUTO_COMPOSER_MIGRATE
 storage/bootstrap/cache/*
 storage/image-jobs/*
+**/__pycache__/**
+.coverage
 
 # used by Vite
 public/hot

ai-vision-service/Dockerfile

@@ -0,0 +1,54 @@
+# Multi-stage build: keep the runtime image lean.
+
+# ---------------------------------------------------------------------------
+# Stage 1 – builder: install dependencies and bake the model weights.
+# ---------------------------------------------------------------------------
+FROM python:3.13-slim@sha256:739e7213785e88c0f702dcdc12c0973afcbd606dbf021a589cab77d6b00b579d AS builder
+
+# Install uv from the official image.
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv
+
+# Build tools required to compile insightface's Cython extension (mesh_core_cython).
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        g++ \
+        libgl1 \
+        libglib2.0-0 \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+# Install dependencies only (no source code) so layer cache is reused when
+# only application code changes.
+COPY pyproject.toml uv.lock README.md ./
+RUN uv sync --frozen --no-dev
+
+# Bake buffalo_l model weights into the image at build time (~300 MB download).
+# The resulting image is ~1 GB larger but starts instantly and works in
+# airgapped environments. Model updates require an image rebuild.
+RUN uv run python -c \
+    "from insightface.app import FaceAnalysis; \
+     a = FaceAnalysis(name='buffalo_l', root='/root/.insightface', providers=['CPUExecutionProvider']); \
+     a.prepare(ctx_id=-1); \
+     print('buffalo_l model downloaded.')"
+
+# ---------------------------------------------------------------------------
+# Stage 2 – runtime: copy only what's needed to run.
+# ---------------------------------------------------------------------------
+FROM python:3.13-slim@sha256:739e7213785e88c0f702dcdc12c0973afcbd606dbf021a589cab77d6b00b579d AS runtime
+
+WORKDIR /app
+
+# Copy the pre-built virtualenv and baked model weights from the builder stage.
+COPY --from=builder /app/.venv /app/.venv
+COPY --from=builder /root/.insightface /root/.insightface
+
+# Copy application source.
+COPY app/ ./app/
+
+ENV PATH="/app/.venv/bin:$PATH"
+
+EXPOSE 8000
+
+# Use a shell-form CMD so that the ${VISION_FACE_WORKERS:-1} variable is
+# expanded at container startup, not at image build time.
+CMD ["sh", "-c", "uvicorn app.main:app --host 0.0.0.0 --port 8000 --workers ${VISION_FACE_WORKERS:-1}"]

ai-vision-service/README.md

@@ -0,0 +1,118 @@
+# Lychee AI Vision Service
+
+Facial recognition microservice for [Lychee](https://github.com/LycheeOrg/Lychee).
+
+Detects faces in photos, stores embeddings, and supports selfie-based person
+claiming via a REST API consumed by the Lychee PHP backend.
+
+## Tech stack
+
+| Concern | Library |
+|---------|---------|
+| Web framework | FastAPI + Uvicorn |
+| Face detection & recognition | InsightFace (`buffalo_l`) + ONNX Runtime |
+| Face crop generation | Pillow |
+| Embedding clustering | scikit-learn (DBSCAN) |
+| Embedding storage | SQLite + sqlite-vec (default) / PostgreSQL + pgvector |
+| HTTP client (callbacks) | httpx |
+| Config | Pydantic BaseSettings |
+
+## Directory layout
+
+```
+ai-vision-service/
+├── app/
+│   ├── __init__.py
+│   ├── config.py          # AppSettings (Pydantic BaseSettings)
+│   ├── main.py            # FastAPI app factory & lifespan handler
+│   ├── api/
+│   │   ├── __init__.py
+│   │   ├── dependencies.py  # API key auth dependency
+│   │   ├── routes.py        # /detect, /match, /health
+│   │   └── schemas.py       # Pydantic request/response models
+│   ├── detection/
+│   │   ├── __init__.py
+│   │   ├── detector.py    # InsightFace wrapper
+│   │   └── cropper.py     # 150×150 JPEG crop generator
+│   ├── embeddings/
+│   │   ├── __init__.py
+│   │   ├── store.py         # Abstract EmbeddingStore protocol
+│   │   ├── sqlite_store.py  # SQLite + sqlite-vec implementation
+│   │   └── pgvector_store.py # PostgreSQL + pgvector implementation
+│   ├── clustering/
+│   │   ├── __init__.py
+│   │   └── clusterer.py   # DBSCAN clustering
+│   └── matching/
+│       ├── __init__.py
+│       └── matcher.py     # Selfie similarity matching
+├── tests/
+│   └── __init__.py
+├── Dockerfile
+├── pyproject.toml
+└── README.md
+```
+
+## Environment variables
+
+All variables are prefixed `VISION_FACE_`.
+
+| Variable | Required | Default | Description |
+|----------|----------|---------|-------------|
+| `VISION_FACE_LYCHEE_API_URL` | Yes | — | Lychee base URL for callbacks |
+| `VISION_FACE_API_KEY` | Yes | — | Shared API key: validated on inbound requests from Lychee, and sent on outbound callbacks to Lychee |
+| `VISION_FACE_MODEL_NAME` | No | `buffalo_l` | InsightFace model pack |
+| `VISION_FACE_DETECTION_THRESHOLD` | No | `0.5` | Confidence filter for detected faces |
+| `VISION_FACE_MATCH_THRESHOLD` | No | `0.5` | Cosine-similarity cutoff for selfie matching |
+| `VISION_FACE_RESCAN_IOU_THRESHOLD` | No | `0.5` | IoU threshold for bounding-box matching on re-scan |
+| `VISION_FACE_MAX_FACES_PER_PHOTO` | No | `10` | Maximum faces included in a callback payload |
+| `VISION_FACE_THREAD_POOL_SIZE` | No | `1` | Inference thread-pool size |
+| `VISION_FACE_STORAGE_BACKEND` | No | `sqlite` | `sqlite` or `pgvector` |
+| `VISION_FACE_STORAGE_PATH` | No | `/data/embeddings` | SQLite DB directory |
+| `VISION_FACE_PG_HOST` | No* | `localhost` | PostgreSQL host (*required with pgvector) |
+| `VISION_FACE_PG_PORT` | No | `5432` | PostgreSQL port |
+| `VISION_FACE_PG_DATABASE` | No* | `ai_vision` | PostgreSQL database (*required with pgvector) |
+| `VISION_FACE_PG_USER` | No* | `ai_vision` | PostgreSQL user (*required with pgvector) |
+| `VISION_FACE_PG_PASSWORD` | No* | `` | PostgreSQL password (*required with pgvector) |
+| `VISION_FACE_PHOTOS_PATH` | No | `/data/photos` | Shared volume mount for photo files |
+| `VISION_FACE_WORKERS` | No | `1` | Number of Uvicorn worker processes |
+| `VISION_FACE_LOG_LEVEL` | No | `info` | Log level |
+
+## Development
+
+```bash
+# Install uv (https://docs.astral.sh/uv/getting-started/installation/)
+curl -LsSf https://astral.sh/uv/install.sh | sh
+
+# Install all dependencies (including dev)
+uv sync
+
+# Lint and format
+uv run ruff format --check
+uv run ruff check
+
+# Type check
+uv run ty check
+
+# Run tests
+uv run pytest
+```
+
+## Docker
+
+```bash
+# Build (bakes buffalo_l model into the image – ~1 GB download on first build)
+docker build -t lychee-ai-vision .
+
+# Run
+docker run --rm \
+  -e VISION_FACE_LYCHEE_API_URL=http://lychee \
+  -e VISION_FACE_API_KEY=changeme \
+  -v /path/to/photos:/data/photos:ro \
+  -v ai-vision-embeddings:/data/embeddings \
+  -p 8000:8000 \
+  lychee-ai-vision
+```
+
+---
+
+*Last updated: 2026-03-21*

ai-vision-service/app/__init__.py

@@ -0,0 +1 @@
+"""Lychee AI Vision Service."""

ai-vision-service/app/api/__init__.py

@@ -0,0 +1 @@
+"""FastAPI routes, dependencies, and schemas."""