diff --git a/AI_ARCHITECTURE.md b/AI_ARCHITECTURE.md index 2dfb36c..95336cb 100644 --- a/AI_ARCHITECTURE.md +++ b/AI_ARCHITECTURE.md @@ -177,8 +177,11 @@ return ProtectedVideoTrack(source=track, privacy_filter=privacy_filter, session_ 5. 프레임 큐 `_queue`와 신원 대조 큐 `_identity_queue`를 만든다. 6. `privacy-blur-gpu-batcher`와 `privacy-blur-identity-worker` 두 데몬 스레드를 시작한다. -두 워커는 같은 `_processing_lock`을 공유한다. 프레임 배치 처리, 신원 대조 처리, -기준 얼굴 준비/적용이 같은 모델 상태를 만지기 때문에 동시에 실행되지 않게 막는다. +프레임 워커는 라이브 YOLO detector를 `_detector_lock`으로 보호한다. 기준 얼굴 +등록은 별도의 CPU detector 인스턴스로 얼굴을 먼저 찾기 때문에 업로드 준비가 +라이브 detector 락을 점유하지 않는다. SFace 신원 대조와 기준 얼굴 feature 적용은 +`_identity_lock`으로 보호하지만, 프레임 워커는 기존 추적/feature 상태로 계속 +프레임을 반환한다. ### 8.1 프레임 배치 워커 @@ -311,7 +314,7 @@ data/reference_face.jpg `register_reference_faces()`는 바로 최종 파일을 쓰지 않는다. 1. `.reference_face.{face_id}.tmp` 임시 파일을 쓴다. -2. 런타임이 살아 있으면 `prepare_identity_update()`로 임시 파일까지 포함해 기준 얼굴 임베딩을 미리 만든다. +2. 런타임이 살아 있으면 `prepare_identity_update()`로 임시 파일까지 포함해 기준 얼굴 임베딩을 미리 만든다. 이때 기준 얼굴 감지는 등록 전용 CPU detector로 수행해 라이브 스트림 detector를 막지 않는다. 3. 검증이 성공하면 `os.replace(tmp, final)`로 최종 파일을 발행한다. 4. `replace_existing=True`인 경우 기존 파일을 삭제한다. 5. `commit_identity_update()`로 공유 런타임과 현재 살아 있는 필터들에 새 config/reference feature를 반영한다. @@ -421,7 +424,9 @@ threshold 이상이면 해당 `FaceTrack.is_me=True`가 되고, `finish_frame()` 새 reference embedding은 `prepare_identity_update()`에서 먼저 준비한다. 준비가 성공한 뒤에만 파일을 최종 경로로 이동하고 `commit_identity_update()`를 호출한다. -따라서 잘못된 업로드가 기존 라이브 설정을 덮어쓰지 않는다. +준비 중에는 등록 전용 CPU detector가 기준 얼굴을 찾고, 기존 스트림은 현재 설정과 +feature로 계속 프레임을 처리한다. 따라서 잘못된 업로드가 기존 라이브 설정을 +덮어쓰지 않고, 업로드 준비가 라이브 detector 락을 점유하지 않는다. ### Q. AI가 실패하면 원본이 노출되는가? diff --git a/privacy_blur/runtime.py b/privacy_blur/runtime.py index 7cffa69..2502393 100644 --- a/privacy_blur/runtime.py +++ b/privacy_blur/runtime.py @@ -3,6 +3,7 @@ import asyncio import atexit from collections import defaultdict +from contextlib import nullcontext from concurrent.futures import Future from dataclasses import dataclass import logging @@ -267,7 +268,29 @@ def _load_reference_features( config: PrivacyBlurConfig, *, sface_session: ort.InferenceSession | None = None, + detector: Any | None = None, + device: str | None = None, + half: bool | None = None, ) -> torch.Tensor: + reference_faces = self._reference_faces_from_config( + config, + detector=detector, + device=device, + half=half, + ) + return self._reference_features_from_faces( + reference_faces, + sface_session=sface_session, + ) + + def _reference_faces_from_config( + self, + config: PrivacyBlurConfig, + *, + detector: Any | None = None, + device: str | None = None, + half: bool | None = None, + ) -> list[tuple[np.ndarray, np.ndarray, np.ndarray | None]]: references: list[tuple[Path, np.ndarray]] = [] for reference_path in config.identity_image_paths(): reference = cv2.imread(str(reference_path)) @@ -280,6 +303,9 @@ def _load_reference_features( detections_by_reference = self._predict_yolo( [reference for _, reference in references], config=config, + detector=detector, + device=device, + half=half, ) reference_faces: list[ tuple[np.ndarray, np.ndarray, np.ndarray | None] @@ -306,7 +332,14 @@ def _load_reference_features( if not reference_faces: raise ReferenceFaceNotDetectedError("No reference images configured") + return reference_faces + def _reference_features_from_faces( + self, + reference_faces: list[tuple[np.ndarray, np.ndarray, np.ndarray | None]], + *, + sface_session: ort.InferenceSession | None = None, + ) -> torch.Tensor: embeddings, valid = self._face_embeddings_from_landmarks( reference_faces, sface_session=sface_session, @@ -448,14 +481,28 @@ def _identity_worker_loop(self) -> None: def prepare_identity(self, config: PrivacyBlurConfig) -> _PreparedIdentity: """Build a reference embedding without pausing live frame batches.""" + if self._closed: + raise RuntimeError("The shared inference runtime is closed") if not config.enable_identity_exclusion: raise ValueError("identity exclusion must be enabled") for reference_path in config.identity_image_paths(): _require_file(reference_path) + _require_file(config.face_model_path) _require_file(config.face_recognition_model_path) + # Use an isolated CPU detector so uploads do not take the live detector lock. + reference_detector = YOLO(str(config.face_model_path)) + reference_faces = self._reference_faces_from_config( + config, + detector=reference_detector, + device="cpu", + half=False, + ) + with self._identity_lock: + if self._closed: + raise RuntimeError("The shared inference runtime is closed") sface_session = ( self.sface_session if self._sface_model_path == config.face_recognition_model_path @@ -465,8 +512,8 @@ def prepare_identity(self, config: PrivacyBlurConfig) -> _PreparedIdentity: sface_session = self._create_ort_session( config.face_recognition_model_path, ) - reference_features = self._load_reference_features( - config, + reference_features = self._reference_features_from_faces( + reference_faces, sface_session=sface_session, ) return _PreparedIdentity( @@ -591,22 +638,31 @@ def _predict_yolo( frames: list[np.ndarray], *, config: PrivacyBlurConfig | None = None, + detector: Any | None = None, + device: str | None = None, + half: bool | None = None, ) -> list[list[FaceDetection]]: from .core import FaceDetection if not frames: return [] config = config or self.config - with self._detector_lock: - results = self.detector.predict( + active_detector = detector if detector is not None else self.detector + detector_context = ( + nullcontext() if detector is not None else self._detector_lock + ) + predict_device = device if device is not None else str(self.device) + predict_half = half if half is not None else self.half_enabled + with detector_context: + results = active_detector.predict( source=frames, imgsz=config.face_imgsz, conf=config.face_conf_threshold, iou=config.face_iou_threshold, max_det=config.face_max_detections, verbose=False, - device=str(self.device), - half=self.half_enabled, + device=predict_device, + half=predict_half, batch=len(frames), ) diff --git a/tests/test_identity_performance.py b/tests/test_identity_performance.py index 8c212e0..beccc21 100644 --- a/tests/test_identity_performance.py +++ b/tests/test_identity_performance.py @@ -1,4 +1,6 @@ -from unittest.mock import MagicMock +import threading +from types import SimpleNamespace +from unittest.mock import MagicMock, patch import numpy as np import torch @@ -188,3 +190,88 @@ def test_missing_landmarks_are_not_identity_matched() -> None: assert scores == [-1.0] runtime._run_sface.assert_not_called() + + +class _ForbiddenLock: + def __enter__(self): + raise AssertionError("live detector lock should not be used") + + def __exit__(self, *args): + return False + + +def test_external_reference_detector_does_not_take_live_detector_lock() -> None: + runtime = SharedGpuRuntime.__new__(SharedGpuRuntime) + runtime._detector_lock = _ForbiddenLock() + runtime.detector = MagicMock(name="live_detector") + runtime.device = torch.device("cpu") + runtime.half_enabled = True + reference_detector = MagicMock(name="reference_detector") + reference_detector.predict.return_value = [ + SimpleNamespace(boxes=None, keypoints=None) + ] + frame = np.zeros((32, 32, 3), dtype=np.uint8) + + detections = runtime._predict_yolo( + [frame], + config=PrivacyBlurConfig(enable_identity_exclusion=False), + detector=reference_detector, + device="cpu", + half=False, + ) + + assert detections == [[]] + runtime.detector.predict.assert_not_called() + reference_detector.predict.assert_called_once() + assert reference_detector.predict.call_args.kwargs["device"] == "cpu" + assert reference_detector.predict.call_args.kwargs["half"] is False + + +def test_prepare_identity_uses_isolated_cpu_detector(tmp_path) -> None: + face_model = tmp_path / "face.pt" + sface_model = tmp_path / "sface.onnx" + reference_image = tmp_path / "reference.jpg" + for path in (face_model, sface_model, reference_image): + path.write_bytes(b"model") + config = PrivacyBlurConfig( + face_model_path=face_model, + face_recognition_model_path=sface_model, + me_image_path=reference_image, + enable_identity_exclusion=True, + ) + runtime = SharedGpuRuntime.__new__(SharedGpuRuntime) + runtime._closed = False + runtime._identity_lock = threading.RLock() + runtime.sface_session = object() + runtime._sface_model_path = sface_model + runtime._create_ort_session = MagicMock(name="create_ort_session") + reference_faces = [ + ( + np.zeros((32, 32, 3), dtype=np.uint8), + np.array([1, 1, 20, 20], dtype=np.float32), + np.ones((5, 2), dtype=np.float32), + ) + ] + reference_features = torch.ones((1, 128), dtype=torch.float32) + runtime._reference_faces_from_config = MagicMock(return_value=reference_faces) + runtime._reference_features_from_faces = MagicMock( + return_value=reference_features + ) + + with patch("privacy_blur.runtime.YOLO") as yolo: + prepared = runtime.prepare_identity(config) + + yolo.assert_called_once_with(str(face_model)) + runtime._reference_faces_from_config.assert_called_once_with( + config, + detector=yolo.return_value, + device="cpu", + half=False, + ) + runtime._reference_features_from_faces.assert_called_once_with( + reference_faces, + sface_session=runtime.sface_session, + ) + runtime._create_ort_session.assert_not_called() + assert prepared.sface_session is runtime.sface_session + assert prepared.reference_features is reference_features