diff --git a/apps/android/InnoLive/app/src/main/assets/owl.json b/apps/android/InnoLive/app/src/main/assets/owl.json new file mode 100644 index 000000000..fa57d82ff --- /dev/null +++ b/apps/android/InnoLive/app/src/main/assets/owl.json @@ -0,0 +1,392 @@ +{ + "id": "owl", + "version": 2, + "image": "owl.png", + "width": 2240, + "height": 928, + "coordinateSystem": "normalized-top-left", + "cellFaceRect": "normalized within each cell; excludes ears and tufts", + "cellLandmarks": "ArcFace 5-point template on the head, YuNet order, normalized within each cell; align by similarity transform", + "yawConvention": "positive yaw turns the muzzle toward image right; mirror the cell for negative yaw", + "mirrorNegativeYaw": true, + "faceRect": [ + 0.08727630972862246, + 0.158038765705865, + 0.8254470601677895, + 0.7895690079393058 + ], + "views": [ + { + "yaw": 0, + "cell": [ + 0, + 0, + 448, + 464 + ], + "faceRect": [ + 0.08727630972862246, + 0.158038765705865, + 0.8254470601677895, + 0.7895690079393058 + ], + "landmarks": [ + [ + 0.361785708793572, + 0.4594094695459152 + ], + [ + 0.6382142912064279, + 0.4594094695459152 + ], + [ + 0.5024878571358775, + 0.6120742996191156 + ], + [ + 0.3858349988503116, + 0.7682087739993785 + ], + [ + 0.6147178539208004, + 0.7668742907458338 + ] + ] + }, + { + "yaw": 10, + "cell": [ + 448, + 0, + 448, + 464 + ], + "faceRect": [ + 0.07408981344529564, + 0.158038765705865, + 0.8321293496659823, + 0.7895690079393058 + ], + "landmarks": [ + [ + 0.40678336817239014, + 0.4594094695459152 + ], + [ + 0.6790124183254582, + 0.4594094695459152 + ], + [ + 0.5502989497035742, + 0.6120742996191156 + ], + [ + 0.4228486051516874, + 0.7682087739993785 + ], + [ + 0.648360600695014, + 0.7668742907458338 + ] + ] + }, + { + "yaw": 20, + "cell": [ + 896, + 0, + 448, + 464 + ], + "faceRect": [ + 0.06420661189726425, + 0.158038765705865, + 0.8603636750153132, + 0.7895690079393058 + ], + "landmarks": [ + [ + 0.4546133710869721, + 0.4594094695459152 + ], + [ + 0.7143713336970124, + 0.4594094695459152 + ], + [ + 0.59658172460539, + 0.6120742996191156 + ], + [ + 0.46220640651881695, + 0.7682087739993785 + ], + [ + 0.6774955000728369, + 0.7668742907458338 + ] + ] + }, + { + "yaw": 30, + "cell": [ + 1344, + 0, + 448, + 464 + ], + "faceRect": [ + 0.0570419292364802, + 0.158038765705865, + 0.8781102970242499, + 0.7895690079393058 + ], + "landmarks": [ + [ + 0.5038224280412708, + 0.4594094695459152 + ], + [ + 0.7432166827576501, + 0.4594094695459152 + ], + [ + 0.6399299220315048, + 0.6120742996191156 + ], + [ + 0.5027125576244933, + 0.7682087739993785 + ], + [ + 0.7012372873723507, + 0.7668742907458338 + ] + ] + }, + { + "yaw": 40, + "cell": [ + 1792, + 0, + 448, + 464 + ], + "faceRect": [ + 0.05215570756367278, + 0.158038765705865, + 0.8805889008300644, + 0.7895690079393058 + ], + "landmarks": [ + [ + 0.5529153376285519, + 0.4594094695459152 + ], + [ + 0.7646720111370087, + 0.4594094695459152 + ], + [ + 0.6790264118462801, + 0.6120742996191156 + ], + [ + 0.5431362693863255, + 0.7682087739993785 + ], + [ + 0.7188645697065762, + 0.7668742907458338 + ] + ] + }, + { + "yaw": 50, + "cell": [ + 0, + 464, + 448, + 464 + ], + "faceRect": [ + 0.05013899292264669, + 0.158038765705865, + 0.8701175823807716, + 0.7895690079393058 + ], + "landmarks": [ + [ + 0.6004004340086665, + 0.4594094695459152 + ], + [ + 0.778085470199585, + 0.4594094695459152 + ], + [ + 0.7126833008868354, + 0.6120742996191156 + ], + [ + 0.5822493271636111, + 0.7682087739993785 + ], + [ + 0.7298417831105846, + 0.7668742907458338 + ] + ] + }, + { + "yaw": 60, + "cell": [ + 448, + 464, + 448, + 464 + ], + "faceRect": [ + 0.0501214180673872, + 0.158038765705865, + 0.8587373601538794, + 0.7895690079393058 + ], + "landmarks": [ + [ + 0.644834931407656, + 0.4594094695459152 + ], + [ + 0.7830493828015668, + 0.4594094695459152 + ], + [ + 0.7398778950529439, + 0.6120742996191156 + ], + [ + 0.6188633000212056, + 0.7682087739993785 + ], + [ + 0.7338353485933372, + 0.7668742907458338 + ] + ] + }, + { + "yaw": 70, + "cell": [ + 896, + 464, + 448, + 464 + ], + "faceRect": [ + 0.0499627409236772, + 0.158038765705865, + 0.845377678317683, + 0.7895690079393058 + ], + "landmarks": [ + [ + 0.6848686781844923, + 0.4594094695459152 + ], + [ + 0.7794130125216076, + 0.4594094695459152 + ], + [ + 0.7597839129822594, + 0.6120742996191156 + ], + [ + 0.6518656286810125, + 0.7682087739993785 + ], + [ + 0.7307239584624767, + 0.7668742907458338 + ] + ] + }, + { + "yaw": 80, + "cell": [ + 1344, + 464, + 448, + 464 + ], + "faceRect": [ + 0.05131737781422482, + 0.158038765705865, + 0.8197144764874662, + 0.7895690079393058 + ], + "landmarks": [ + [ + 0.7192852906882763, + 0.4594094695459152 + ], + [ + 0.767286797187158, + 0.4594094695459152 + ], + [ + 0.7717965096235275, + 0.6120742996191156 + ], + [ + 0.6802535967103073, + 0.7682087739993785 + ], + [ + 0.7206021233328751, + 0.7668742907458338 + ] + ] + }, + { + "yaw": 90, + "cell": [ + 1792, + 464, + 448, + 464 + ], + "faceRect": [ + 0.05080358790499827, + 0.158038765705865, + 0.8136273750237055, + 0.7895690079393058 + ], + "landmarks": [ + [ + 0.7470390307051795, + 0.4594094695459152 + ], + [ + 0.7470392137765884, + 0.4594094695459152 + ], + [ + 0.7755507550069264, + 0.6120742996191156 + ], + [ + 0.7031646860497338, + 0.7682087739993785 + ], + [ + 0.7037774260554994, + 0.7668742907458338 + ] + ] + } + ], + "fullFace": true, + "scope": "on-device-only" +} diff --git a/apps/android/InnoLive/app/src/main/assets/owl.png b/apps/android/InnoLive/app/src/main/assets/owl.png new file mode 100644 index 000000000..c97cb0dae Binary files /dev/null and b/apps/android/InnoLive/app/src/main/assets/owl.png differ diff --git a/apps/android/InnoLive/app/src/main/assets/rabbit.json b/apps/android/InnoLive/app/src/main/assets/rabbit.json new file mode 100644 index 000000000..a4f5169fd --- /dev/null +++ b/apps/android/InnoLive/app/src/main/assets/rabbit.json @@ -0,0 +1,392 @@ +{ + "id": "rabbit", + "version": 2, + "image": "rabbit.png", + "width": 2200, + "height": 1360, + "coordinateSystem": "normalized-top-left", + "cellFaceRect": "normalized within each cell; excludes ears and tufts", + "cellLandmarks": "ArcFace 5-point template on the head, YuNet order, normalized within each cell; align by similarity transform", + "yawConvention": "positive yaw turns the muzzle toward image right; mirror the cell for negative yaw", + "mirrorNegativeYaw": true, + "faceRect": [ + 0.08954505378549751, + 0.3972397373003118, + 0.820909612829035, + 0.5719661348006305 + ], + "views": [ + { + "yaw": 0, + "cell": [ + 0, + 0, + 440, + 680 + ], + "faceRect": [ + 0.08954505378549751, + 0.3972397373003118, + 0.820909612829035, + 0.5719661348006305 + ], + "landmarks": [ + [ + 0.3592727216807279, + 0.6373470544815063 + ], + [ + 0.6407272783192721, + 0.6373470544815063 + ], + [ + 0.5025330909019844, + 0.741518354065278 + ], + [ + 0.3837592715566809, + 0.8480571718776927 + ], + [ + 0.6168036330829967, + 0.8471465833046857 + ] + ] + }, + { + "yaw": 10, + "cell": [ + 440, + 0, + 440, + 680 + ], + "faceRect": [ + 0.07615375302054667, + 0.3972397373003118, + 0.8178246931596236, + 0.5719661348006305 + ], + "landmarks": [ + [ + 0.4030502132394097, + 0.6373470544815063 + ], + [ + 0.68022888248617, + 0.6373470544815063 + ], + [ + 0.5491962671279907, + 0.741518354065278 + ], + [ + 0.41959883841601286, + 0.8480571718776927 + ], + [ + 0.6492075221105056, + 0.8471465833046857 + ] + ] + }, + { + "yaw": 20, + "cell": [ + 880, + 0, + 440, + 680 + ], + "faceRect": [ + 0.06619165268811314, + 0.3972397373003118, + 0.8196260159665888, + 0.5719661348006305 + ], + "landmarks": [ + [ + 0.449773471057415, + 0.6373470544815063 + ], + [ + 0.7142542882399125, + 0.6373470544815063 + ], + [ + 0.5943646345626225, + 0.741518354065278 + ], + [ + 0.45788134011355314, + 0.8480571718776927 + ], + [ + 0.677077825773846, + 0.8471465833046857 + ] + ] + }, + { + "yaw": 30, + "cell": [ + 1320, + 0, + 440, + 680 + ], + "faceRect": [ + 0.05924955823204736, + 0.3972397373003118, + 0.8208307190374894, + 0.5719661348006305 + ], + "landmarks": [ + [ + 0.49802284376187755, + 0.6373470544815063 + ], + [ + 0.7417697212912819, + 0.6373470544815063 + ], + [ + 0.6366657858545129, + 0.741518354065278 + ], + [ + 0.4974436058239503, + 0.8480571718776927 + ], + [ + 0.6995677173137664, + 0.8471465833046857 + ] + ] + }, + { + "yaw": 40, + "cell": [ + 1760, + 0, + 440, + 680 + ], + "faceRect": [ + 0.05454132773659448, + 0.3972397373003118, + 0.8119241784919392, + 0.5719661348006305 + ], + "landmarks": [ + [ + 0.5463322896849026, + 0.6373470544815063 + ], + [ + 0.7619390845298767, + 0.6373470544815063 + ], + [ + 0.6748144057664004, + 0.741518354065278 + ], + [ + 0.5370835304260254, + 0.8480571718776927 + ], + [ + 0.7159938194534995, + 0.8471465833046857 + ] + ] + }, + { + "yaw": 50, + "cell": [ + 0, + 680, + 440, + 680 + ], + "faceRect": [ + 0.052754031528126126, + 0.3972397373003118, + 0.8101360900835558, + 0.5719661348006305 + ], + "landmarks": [ + [ + 0.593233938108791, + 0.6373470544815063 + ], + [ + 0.774149588021365, + 0.6373470544815063 + ], + [ + 0.7076514184474945, + 0.741518354065278 + ], + [ + 0.5755967020988464, + 0.8480571718776927 + ], + [ + 0.7258570811965249, + 0.8471465833046857 + ] + ] + }, + { + "yaw": 60, + "cell": [ + 440, + 680, + 440, + 680 + ], + "faceRect": [ + 0.05318042148243299, + 0.3972397373003118, + 0.8076980016448281, + 0.5719661348006305 + ], + "landmarks": [ + [ + 0.637302761186253, + 0.6373470544815063 + ], + [ + 0.7780302026055076, + 0.6373470544815063 + ], + [ + 0.7341790015047247, + 0.741518354065278 + ], + [ + 0.6118129367178137, + 0.8480571718776927 + ], + [ + 0.72885779467496, + 0.8471465833046857 + ] + ] + }, + { + "yaw": 70, + "cell": [ + 880, + 680, + 440, + 680 + ], + "faceRect": [ + 0.05324053547599101, + 0.3972397373003118, + 0.8038689965551549, + 0.5719661348006305 + ], + "landmarks": [ + [ + 0.6771996614607898, + 0.6373470544815063 + ], + [ + 0.7734629836949435, + 0.6373470544815063 + ], + [ + 0.753591184724461, + 0.741518354065278 + ], + [ + 0.644631740450859, + 0.8480571718776927 + ], + [ + 0.7249047636985779, + 0.8471465833046857 + ] + ] + }, + { + "yaw": 80, + "cell": [ + 1320, + 680, + 440, + 680 + ], + "faceRect": [ + 0.05475428971377288, + 0.3972397373003118, + 0.8099361295049841, + 0.5719661348006305 + ], + "landmarks": [ + [ + 0.7117124449123036, + 0.6373470544815063 + ], + [ + 0.7605867060748014, + 0.6373470544815063 + ], + [ + 0.7652981286699122, + 0.741518354065278 + ], + [ + 0.6730559899048372, + 0.8480571718776927 + ], + [ + 0.7141181230545044, + 0.8471465833046857 + ] + ] + }, + { + "yaw": 90, + "cell": [ + 1760, + 680, + 440, + 680 + ], + "faceRect": [ + 0.05436364195563578, + 0.3972397373003118, + 0.8166640736839987, + 0.5719661348006305 + ], + "landmarks": [ + [ + 0.7397924835031683, + 0.6373470544815063 + ], + [ + 0.7397926699031483, + 0.6373470544815063 + ], + [ + 0.7689441355791958, + 0.741518354065278 + ], + [ + 0.6962220473722978, + 0.8480571718776927 + ], + [ + 0.6968256105076183, + 0.8471465833046857 + ] + ] + } + ], + "fullFace": true, + "scope": "on-device-only" +} diff --git a/apps/android/InnoLive/app/src/main/assets/rabbit.png b/apps/android/InnoLive/app/src/main/assets/rabbit.png new file mode 100644 index 000000000..a184a08ef Binary files /dev/null and b/apps/android/InnoLive/app/src/main/assets/rabbit.png differ diff --git a/apps/android/InnoLive/app/src/main/java/com/framework/innolive/app/MainActivity.kt b/apps/android/InnoLive/app/src/main/java/com/framework/innolive/app/MainActivity.kt index 4be5c3b4e..1ba30a928 100644 --- a/apps/android/InnoLive/app/src/main/java/com/framework/innolive/app/MainActivity.kt +++ b/apps/android/InnoLive/app/src/main/java/com/framework/innolive/app/MainActivity.kt @@ -539,7 +539,7 @@ fun AppNavigation( val token = checkNotNull(authenticationSession.session.value).accessToken isAdministrator = youtubeCoordinator.isAdministrator(token, ::refreshCurrentAccessToken) isAdministratorChecked = true - // 관리자가 아닌 계정은 합성을 요청할 수 없으므로 이전 계정이 켠 선택을 지웁니다. + // 관리자가 아닌 계정은 실험실 방식(합성·동물)을 쓸 수 없으므로 이전 계정이 켠 선택을 지웁니다. if (!isAdministrator) webRtcSession.clearFaceSwapSelection(context) } catch (exception: CancellationException) { throw exception @@ -1013,8 +1013,8 @@ fun AppNavigation( webRtcSession.selectAIProcessing(context, onDevice) }, isAdministrator = isAdministrator, - faceSwapEnabled = webRtcSession.selectedFaceSwap, - onFaceSwapChanged = { enabled -> webRtcSession.selectFaceSwap(context, enabled) }, + faceAnonymization = webRtcSession.selectedFaceAnonymization, + onFaceAnonymizationChanged = { method -> webRtcSession.selectFaceAnonymization(context, method) }, canReplayBroadcastGuide = webRtcSession.sessionSnapshot == null && webRtcSession.broadcastState == BroadcastState.IDLE && !webRtcSession.isPreparingBroadcast, diff --git a/apps/android/InnoLive/app/src/main/java/com/framework/innolive/feature/live/CameraFrameAnalyzer.kt b/apps/android/InnoLive/app/src/main/java/com/framework/innolive/feature/live/CameraFrameAnalyzer.kt index 2bb3461e7..d8e8fd784 100644 --- a/apps/android/InnoLive/app/src/main/java/com/framework/innolive/feature/live/CameraFrameAnalyzer.kt +++ b/apps/android/InnoLive/app/src/main/java/com/framework/innolive/feature/live/CameraFrameAnalyzer.kt @@ -44,6 +44,8 @@ class CameraFrameAnalyzer( private val keepLatestProtectedFrame:Boolean = true, /** 미리보기·송출·가공본 녹화로 보낼 때 가운데를 확대하는 배율입니다. 원본 녹화와 얼굴 등록 입력은 자르지 않습니다. */ private val outputZoom: Float = 1f, + /** 실험실에서 고른 동물(`rabbit`/`owl`)입니다. 얼굴 합성처럼 세션을 만들 때 고정됩니다. */ + private val animal: String? = null, ) : ImageAnalysis.Analyzer, AutoCloseable { private val captureLock = Any() private val settings = AtomicReference(BroadcastVideoQualitySettings()) @@ -94,7 +96,7 @@ class CameraFrameAnalyzer( private var localProcessor: PrivacyFrameProcessor? = if (initialOnDevice && !batchTwoOptimizations) PrivacyFrameProcessor(checkNotNull(applicationContext), sharedEglContext,directGpuInput=directGpuInput, nativePostprocessing=nativePostprocessing,batchTwoOptimizations=false, - faceCoordinatorFactory=faceCoordinatorFactory) else null + faceCoordinatorFactory=faceCoordinatorFactory,animal=animal) else null private val route = PrivacyFrameRoute( when { !initialOnDevice -> PrivacyFrameMode.SERVER @@ -106,7 +108,7 @@ class CameraFrameAnalyzer( private fun processor():PrivacyFrameProcessor = localProcessor ?: PrivacyFrameProcessor( checkNotNull(applicationContext),sharedEglContext,directGpuInput=directGpuInput, nativePostprocessing=nativePostprocessing,batchTwoOptimizations=batchTwoOptimizations, - faceCoordinatorFactory=faceCoordinatorFactory, + faceCoordinatorFactory=faceCoordinatorFactory,animal=animal, ).also {localProcessor=it} private fun schedulePreparation() { diff --git a/apps/android/InnoLive/app/src/main/java/com/framework/innolive/feature/live/FaceAnonymizationPreference.kt b/apps/android/InnoLive/app/src/main/java/com/framework/innolive/feature/live/FaceAnonymizationPreference.kt index 8e04ffc16..a9cdab38e 100644 --- a/apps/android/InnoLive/app/src/main/java/com/framework/innolive/feature/live/FaceAnonymizationPreference.kt +++ b/apps/android/InnoLive/app/src/main/java/com/framework/innolive/feature/live/FaceAnonymizationPreference.kt @@ -3,16 +3,34 @@ package com.framework.innolive.feature.live import android.content.Context /** - * 실험실의 `블러 대신 얼굴 합성 사용하기` 선택입니다. 관리자 전용 실험 기능이며 기본값은 블러입니다. - * 세션을 만들 때 고정되므로 다음 방송 준비부터 적용합니다. + * 실험실의 비식별화 방법입니다. 합성은 서버 AI 처리에서, 토끼·부엉이는 온디바이스 처리에서만 쓰고 그 밖에는 블러입니다. + */ +enum class FaceAnonymizationMethod(val storageValue: String) { + BLUR("blur"), FACE_SWAP("face_swap"), RABBIT("rabbit"), OWL("owl"); + + /** 온디바이스 처리에서 얼굴에 씌우는 동물 자산 이름입니다. */ + val animal: String? get() = storageValue.takeIf { this == RABBIT || this == OWL } + + companion object { + fun from(value: String?): FaceAnonymizationMethod? = entries.firstOrNull { it.storageValue == value } + } +} + +/** + * 실험실의 비식별화 방법 선택입니다. 관리자 전용 실험 기능이며 기본값은 블러입니다. + * 세션을 만들 때 고정되므로 다음 방송 준비부터 적용합니다. 이전 버전의 `face_swap` 값도 읽습니다. */ internal class FaceAnonymizationPreference(context: Context) { private val preferences = context.applicationContext .getSharedPreferences("live_face_anonymization", Context.MODE_PRIVATE) - var faceSwap: Boolean - get() = preferences.getBoolean("face_swap", false) - set(value) { preferences.edit().putBoolean("face_swap", value).apply() } + var method: FaceAnonymizationMethod + get() = FaceAnonymizationMethod.from(preferences.getString("method", null)) + ?: if (preferences.getBoolean("face_swap", false)) FaceAnonymizationMethod.FACE_SWAP else FaceAnonymizationMethod.BLUR + set(value) { + preferences.edit().putString("method", value.storageValue) + .putBoolean("face_swap", value == FaceAnonymizationMethod.FACE_SWAP).apply() + } } internal const val FaceAnonymizationFaceSwap = "face_swap" @@ -23,3 +41,7 @@ internal const val FaceAnonymizationFaceSwap = "face_swap" */ internal fun sessionFaceAnonymization(faceSwapSelected: Boolean, onDeviceProcessing: Boolean): String? = FaceAnonymizationFaceSwap.takeIf { faceSwapSelected && !onDeviceProcessing } + +/** 세션에 고정할 동물입니다. 동물은 온디바이스 처리에서만 씁니다. */ +internal fun sessionAnimal(method: FaceAnonymizationMethod, onDeviceProcessing: Boolean): String? = + method.animal.takeIf { onDeviceProcessing } diff --git a/apps/android/InnoLive/app/src/main/java/com/framework/innolive/feature/live/LiveBroadcastTransport.kt b/apps/android/InnoLive/app/src/main/java/com/framework/innolive/feature/live/LiveBroadcastTransport.kt index ea9c0a834..ea2132b70 100644 --- a/apps/android/InnoLive/app/src/main/java/com/framework/innolive/feature/live/LiveBroadcastTransport.kt +++ b/apps/android/InnoLive/app/src/main/java/com/framework/innolive/feature/live/LiveBroadcastTransport.kt @@ -113,6 +113,8 @@ internal class LiveBroadcastTransportRequest( val previewOnly: Boolean = false, /** 실험실에서 얼굴 합성을 켰습니다. 서버 중계 세션을 서버 AI로 만들 때만 씁니다. */ val faceSwap: Boolean = false, + /** 실험실에서 고른 동물입니다. 온디바이스 처리에서만 씁니다. */ + val animal: String? = null, ) { val context: Context = context.applicationContext } @@ -135,6 +137,7 @@ internal fun createLiveBroadcastTransport(request: LiveBroadcastTransportRequest provider = request.provider, broadcastResolution = request.directPlan.resolution, faceAnonymization = sessionFaceAnonymization(request.faceSwap, request.initialOnDeviceProcessing), + animal = request.animal.takeIf { request.initialOnDeviceProcessing }, preferredAudioInput = request.preferredAudioInput, onStateChanged = onStateChanged, onRemoteTrackChanged = onRemoteTrackChanged, diff --git a/apps/android/InnoLive/app/src/main/java/com/framework/innolive/feature/live/WebRtcConnection.kt b/apps/android/InnoLive/app/src/main/java/com/framework/innolive/feature/live/WebRtcConnection.kt index 5cd809430..24a02856c 100644 --- a/apps/android/InnoLive/app/src/main/java/com/framework/innolive/feature/live/WebRtcConnection.kt +++ b/apps/android/InnoLive/app/src/main/java/com/framework/innolive/feature/live/WebRtcConnection.kt @@ -96,6 +96,8 @@ internal class WebRtcConnection( private val broadcastResolution: DirectPublishResolution = DirectPublishResolution.HD_720, /** 세션을 만들 때 서버에 요청하는 얼굴 가림 방식입니다. null이면 서버 기본값(블러)입니다. */ private val faceAnonymization: String? = null, + /** 온디바이스 처리에서 얼굴에 씌우는 동물입니다. 세션을 만들 때 고정합니다. */ + private val animal: String? = null, private var preferredAudioInput: AudioDeviceInfo?, private val onStateChanged: (WebRtcConnectionState, ConnectionFailure?) -> Unit, private val onRemoteTrackChanged: (VideoTrack?) -> Unit, @@ -396,6 +398,7 @@ internal class WebRtcConnection( initialOnDeviceProcessing, initialAnonymizationEnabled, sharedEglContext = checkNotNull(eglBase).eglBaseContext, + animal = animal, onProcessingFailure = { executeOnOwner { fail(ConnectionFailure.GENERIC) } }, onProtectedFrameSent = { executeOnOwner { diff --git a/apps/android/InnoLive/app/src/main/java/com/framework/innolive/feature/live/WebRtcSessionViewModel.kt b/apps/android/InnoLive/app/src/main/java/com/framework/innolive/feature/live/WebRtcSessionViewModel.kt index 7862ec001..3890854e8 100644 --- a/apps/android/InnoLive/app/src/main/java/com/framework/innolive/feature/live/WebRtcSessionViewModel.kt +++ b/apps/android/InnoLive/app/src/main/java/com/framework/innolive/feature/live/WebRtcSessionViewModel.kt @@ -268,9 +268,10 @@ class WebRtcSessionViewModel internal constructor( return selectInitialAIProcessing(context, onDevice) } - /** 실험실의 얼굴 합성 선택입니다. 다음 세션 생성부터 적용합니다. */ - var selectedFaceSwap by mutableStateOf(false) + /** 실험실의 비식별화 방법 선택입니다. 다음 세션 생성부터 적용합니다. */ + var selectedFaceAnonymization by mutableStateOf(FaceAnonymizationMethod.BLUR) private set + val selectedFaceSwap: Boolean get() = selectedFaceAnonymization == FaceAnonymizationMethod.FACE_SWAP private var isFaceSwapSelectionLoaded = false /** 다음·현재 세션이 얼굴 합성인지입니다. 서버가 합성 세션의 설명 앞에 안내문을 붙여 설명 한도가 줄어듭니다. */ @@ -279,28 +280,34 @@ class WebRtcSessionViewModel internal constructor( fun restoreFaceSwapSelection(context: Context) { if (isFaceSwapSelectionLoaded) return - selectedFaceSwap = FaceAnonymizationPreference(context).faceSwap + selectedFaceAnonymization = FaceAnonymizationPreference(context).method isFaceSwapSelectionLoaded = true } - /** 관리자가 아닌 계정으로 바뀌면 합성 선택을 지웁니다. 서버가 관리자가 아닌 계정의 합성을 403으로 거절합니다. */ + /** + * 관리자가 아닌 계정으로 바뀌면 실험실 선택(합성·동물)을 지웁니다. 서버가 관리자가 아닌 계정의 합성을 403으로 + * 거절하며, 동물도 같은 관리자 전용 실험 기능입니다. + */ fun clearFaceSwapSelection(context: Context) { - FaceAnonymizationPreference(context).faceSwap = false - selectedFaceSwap = false + FaceAnonymizationPreference(context).method = FaceAnonymizationMethod.BLUR + selectedFaceAnonymization = FaceAnonymizationMethod.BLUR isFaceSwapSelectionLoaded = true } /** - * 얼굴 가림 방식은 세션 생성 때 고정되므로 처리 방식과 같은 때에만 바꿉니다. 서버 AI의 준비 전 연결이 남아 - * 있으면 닫아 다음 방송 준비에서 새 방식으로 세션을 만듭니다. 온디바이스 연결은 합성을 쓰지 않아 유지합니다. + * 얼굴 가림 방식은 세션 생성 때 고정되므로 처리 방식과 같은 때에만 바꿉니다. 준비 전 연결이 쓰는 방식이 + * 바뀌면(서버 AI의 합성, 온디바이스의 동물) 연결을 닫아 다음 방송 준비에서 새 방식으로 세션을 만듭니다. */ - fun selectFaceSwap(context: Context, enabled: Boolean): Boolean { + fun selectFaceAnonymization(context: Context, method: FaceAnonymizationMethod): Boolean { if (!canChangeAIProcessing) return false restoreFaceSwapSelection(context) - if (selectedFaceSwap == enabled) return true - if (!selectedOnDeviceProcessing && connectionState != WebRtcConnectionState.IDLE) close() - FaceAnonymizationPreference(context).faceSwap = enabled - selectedFaceSwap = enabled + val previous = selectedFaceAnonymization + if (previous == method) return true + val changesConnection = if (selectedOnDeviceProcessing) previous.animal != method.animal + else (previous == FaceAnonymizationMethod.FACE_SWAP) != (method == FaceAnonymizationMethod.FACE_SWAP) + if (changesConnection && connectionState != WebRtcConnectionState.IDLE) close() + FaceAnonymizationPreference(context).method = method + selectedFaceAnonymization = method return true } @@ -394,6 +401,7 @@ class WebRtcSessionViewModel internal constructor( val initialOnDevice = selectedOnDeviceProcessing restoreFaceSwapSelection(context) val initialFaceSwap = selectedFaceSwap + val initialAnimal = sessionAnimal(selectedFaceAnonymization, initialOnDevice) isOnDevicePreviewOnly = previewOnly serverError = null preparationSettings = null @@ -558,6 +566,7 @@ class WebRtcSessionViewModel internal constructor( callbacks = callbacks, previewOnly = previewOnly, faceSwap = initialFaceSwap, + animal = initialAnimal, ), ) if (!isCurrentGeneration(generation)) { diff --git a/apps/android/InnoLive/app/src/main/java/com/framework/innolive/feature/live/direct/broadcast/AndroidDirectBroadcastMedia.kt b/apps/android/InnoLive/app/src/main/java/com/framework/innolive/feature/live/direct/broadcast/AndroidDirectBroadcastMedia.kt index 1b95f0bb8..4c1745015 100644 --- a/apps/android/InnoLive/app/src/main/java/com/framework/innolive/feature/live/direct/broadcast/AndroidDirectBroadcastMedia.kt +++ b/apps/android/InnoLive/app/src/main/java/com/framework/innolive/feature/live/direct/broadcast/AndroidDirectBroadcastMedia.kt @@ -109,6 +109,7 @@ private class AndroidDirectMediaPipeline( private val scope: SessionRecoveryScope, ) : DirectMediaPipeline { private val context: Context = request.context + private val animal: String? = request.animal private val callbacks = request.callbacks private val initialAnonymizationEnabled = request.initialAnonymizationEnabled private val audioManager = context.getSystemService(AudioManager::class.java) @@ -141,6 +142,7 @@ private class AndroidDirectMediaPipeline( onProcessingFailure = events::onProcessingFailure, sharedEglContext = egl.eglBaseContext, outputZoom = ON_DEVICE_EDGE_ZOOM, + animal = animal, ).also { it.faceCoordinatorFactory = { PrivacyFaceCoordinator(context, scope.storageKey) } } analyzer = created created.setDirectPublishSink { frame -> diff --git a/apps/android/InnoLive/app/src/main/java/com/framework/innolive/feature/live/privacy/PrivacyAnimalSprite.kt b/apps/android/InnoLive/app/src/main/java/com/framework/innolive/feature/live/privacy/PrivacyAnimalSprite.kt new file mode 100644 index 000000000..57b08a555 --- /dev/null +++ b/apps/android/InnoLive/app/src/main/java/com/framework/innolive/feature/live/privacy/PrivacyAnimalSprite.kt @@ -0,0 +1,385 @@ +package com.framework.innolive.feature.live.privacy + +import android.content.Context +import android.graphics.Bitmap +import android.graphics.BitmapFactory +import android.graphics.Canvas +import android.graphics.Color +import android.graphics.Matrix +import android.graphics.Paint +import android.graphics.PorterDuff +import android.graphics.PorterDuffXfermode +import android.graphics.Rect +import android.graphics.RectF +import android.util.Log +import org.json.JSONArray +import org.json.JSONObject +import kotlin.math.abs +import kotlin.math.atan2 +import kotlin.math.hypot +import kotlin.math.max +import kotlin.math.min + +/** One protected face whose YuNet landmarks (x/y pairs, upright frame pixels) passed the face-swap checks. */ +internal class PrivacyAnimalFace(val landmarks: FloatArray, val yaw: Float) { + init { require(landmarks.size == 10) } +} + +internal class PrivacyAnimalOverlay(val image: Bitmap, val atlas: PrivacyAnimalAtlas, val faces: List) + +/** + * Blender-rendered yaw views of one animal. Every cell is the same fixed orthographic camera frame. Each view carries + * the ArcFace 5-point template projected onto the animal head, so a face aligns like the server's InSwapper crop. + */ +internal class PrivacyAnimalAtlas(val width: Int, val height: Int, val views: List, val mirrorNegativeYaw: Boolean) { + /** Atlas pixels. */ + data class Cell(val x: Int, val y: Int, val width: Int, val height: Int) + /** [landmarks] are x/y pairs normalized to the cell, top-left origin, in YuNet order. */ + class View(val yaw: Float, val cell: Cell, val landmarks: FloatArray) + + /** + * Similarity transform from mirrored-canvas pixels to frame pixels: frame = [a -b; b a] * canvas + t. + * The two neighbouring yaw views share the canvas and cross-fade by [blend]. + */ + class Placement(val from: View, val to: View, val blend: Float, val mirrored: Boolean, + val a: Float, val b: Float, val tx: Float, val ty: Float) + + init { + require(views.isNotEmpty() && views.first().yaw == 0f && views.zipWithNext().all { (a, b) -> a.yaw < b.yaw }) + require(views.all { it.cell.width == views.first().cell.width && it.cell.height == views.first().cell.height }) + require(views.all { it.landmarks.size == 10 }) + } + + fun place(face: PrivacyAnimalFace): Placement? { + if (!face.yaw.isFinite() || !face.landmarks.all(Float::isFinite)) return null + val mirrored = mirrorNegativeYaw && face.yaw < 0f + val yaw = (if (mirrorNegativeYaw) abs(face.yaw) else face.yaw).coerceIn(0f, views.last().yaw) + val lower = views.indexOfLast { it.yaw <= yaw } + val upper = (lower + 1).coerceAtMost(views.lastIndex) + val t = if (upper == lower) 0f else (yaw - views[lower].yaw) / (views[upper].yaw - views[lower].yaw) + // One crisp view almost everywhere; a wide cross-fade would ghost both views' eyes at the midpoint. + val blend = smoothstep(.45f, .55f, t) + val cell = views[lower].cell + val reference = FloatArray(10) { index -> + val value = views[lower].landmarks[index] + (views[upper].landmarks[index] - views[lower].landmarks[index]) * blend + if (index % 2 == 0) value * cell.width else value * cell.height + } + // A mirrored cell swaps which eye and mouth corner lies on the image-left side. + val source = if (!mirrored) reference else FloatArray(10) { index -> + val point = MIRRORED_ORDER[index / 2] * 2 + index % 2 + if (index % 2 == 0) cell.width - reference[point] else reference[point] + } + val (a, b, tx, ty) = similarity(source, face.landmarks) ?: return null + return Placement(views[lower], views[upper], blend, mirrored, a, b, tx, ty) + } + + companion object { + private val MIRRORED_ORDER = intArrayOf(1, 0, 2, 4, 3) + + private fun smoothstep(edge0: Float, edge1: Float, x: Float): Float { + val t = ((x - edge0) / (edge1 - edge0)).coerceIn(0f, 1f) + return t * t * (3f - 2f * t) + } + + /** Least-squares similarity (rotation, uniform scale, translation) mapping [source] onto [target] points. */ + internal fun similarity(source: FloatArray, target: FloatArray): FloatArray? { + val count = source.size / 2 + var sx = 0.0; var sy = 0.0; var dx = 0.0; var dy = 0.0 + for (i in 0 until count) { sx += source[2*i]; sy += source[2*i+1]; dx += target[2*i]; dy += target[2*i+1] } + sx /= count; sy /= count; dx /= count; dy /= count + var norm = 0.0; var dot = 0.0; var cross = 0.0 + for (i in 0 until count) { + val px = source[2*i] - sx; val py = source[2*i+1] - sy + val qx = target[2*i] - dx; val qy = target[2*i+1] - dy + norm += px*px + py*py; dot += px*qx + py*qy; cross += px*qy - py*qx + } + if (norm < 1e-6) return null + val a = dot / norm; val b = cross / norm + if (hypot(a, b) < 1e-6 || !a.isFinite() || !b.isFinite()) return null + return floatArrayOf(a.toFloat(), b.toFloat(), (dx - (a*sx - b*sy)).toFloat(), (dy - (b*sx + a*sy)).toFloat()) + } + + private fun floats(values: JSONArray, size: Int): FloatArray { + require(values.length() == size) + return FloatArray(size) { values.getDouble(it).toFloat().also { v -> require(v.isFinite() && v in 0f..1f) } } + } + + fun parse(json: JSONObject, imageWidth: Int, imageHeight: Int): PrivacyAnimalAtlas { + require(json.getInt("width") == imageWidth && json.getInt("height") == imageHeight && json.getInt("version") >= 2) + val array = json.getJSONArray("views") + val views = List(array.length()) { index -> + val view = array.getJSONObject(index) + val c = view.getJSONArray("cell").let { cell -> require(cell.length() == 4); List(4) { cell.getInt(it) } } + require(c[0] >= 0 && c[1] >= 0 && c[2] > 0 && c[3] > 0 && c[0] + c[2] <= imageWidth && c[1] + c[3] <= imageHeight) + val points = view.getJSONArray("landmarks").let { list -> + require(list.length() == 5) + FloatArray(10) { list.getJSONArray(it / 2).let { p -> require(p.length() == 2); floats(p, 2)[it % 2] } } + } + View(view.getDouble("yaw").toFloat(), Cell(c[0], c[1], c[2], c[3]), points) + } + return PrivacyAnimalAtlas(imageWidth, imageHeight, views, json.optBoolean("mirrorNegativeYaw", false)) + } + } +} + +/** Head yaw from YuNet's five landmarks; only used to pick the animal's yaw view. */ +internal object PrivacyAnimalPose { + /** Nose tip depth in front of the eye line, in interocular distances, for an average adult face. */ + private const val NOSE_DEPTH = .55f + + /** + * [landmarks] are x/y pairs in YuNet order: right eye, left eye, nose, right and left mouth corners. The nose shifts + * along the eye line by depth * sin(yaw) while the eye distance shrinks by cos(yaw), so their ratio is depth * tan(yaw). + * Positive yaw turns the nose toward image right. + */ + fun yaw(landmarks: FloatArray): Float? { + require(landmarks.size == 10) + if (!landmarks.all(Float::isFinite)) return null + var axisX = landmarks[2] - landmarks[0]; var axisY = landmarks[3] - landmarks[1] + // Measure along the eye line pointing toward image right, whichever eye YuNet listed first. + if (axisX < 0f) { axisX = -axisX; axisY = -axisY } + val distance = hypot(axisX, axisY) + if (distance < 1e-3f) return null + val noseX = landmarks[4] - (landmarks[0] + landmarks[2]) / 2 + val noseY = landmarks[5] - (landmarks[1] + landmarks[3]) / 2 + val along = (noseX * axisX + noseY * axisY) / distance + return Math.toDegrees(atan2(along / distance, NOSE_DEPTH).toDouble()).toFloat() + } +} + +/** Face-swap target rules (innolive-ai `InSwapperRenderer.swap`) applied to on-device animal overlays. */ +internal object PrivacyAnimalTargets { + /** YOLO box padding on each side, relative to its longer side. */ + const val ROI_PADDING = .3f + const val MIN_IOU = .25f + const val MIN_SCORE = .65f + /** `face_too_small`: a bbox whose shorter side is under 16 px stays blurred. */ + const val MIN_FACE = 16f + + /** Boxes are (left, top, right, bottom) frame pixels. */ + class Box(val left: Float, val top: Float, val right: Float, val bottom: Float) { + val width get() = right - left + val height get() = bottom - top + } + + /** Padded integer ROI clipped to the frame, or null when the face is too small or outside the frame. */ + fun roi(box: Box, width: Int, height: Int): IntArray? { + if (min(box.width, box.height) < MIN_FACE) return null + val pad = max(box.width, box.height) * ROI_PADDING + val roi = intArrayOf(max(0, (box.left - pad).toInt()), max(0, (box.top - pad).toInt()), + min(width, (box.right + pad).toInt()), min(height, (box.bottom + pad).toInt())) + return roi.takeIf { it[2] > it[0] && it[3] > it[1] } + } + + /** Landmarks of the candidate overlapping the tracked box by more than [MIN_IOU], or null. */ + fun match(box: Box, candidates: List>): FloatArray? { + var best: FloatArray? = null; var bestIou = MIN_IOU + for ((candidate, landmarks) in candidates) { + val w = max(0f, min(box.right, candidate.right) - max(box.left, candidate.left)) + val h = max(0f, min(box.bottom, candidate.bottom) - max(box.top, candidate.top)) + val intersection = w * h + val iou = intersection / max(box.width * box.height + candidate.width * candidate.height - intersection, 1f) + if (iou > bestIou) { best = landmarks; bestIou = iou } + } + return best + } +} + +/** + * Per-track landmarks refreshed every [refreshFrames] processed frames, counted like the server's 30-frame metadata + * cache. Between refreshes the landmarks follow the tracked box; a failed refresh keeps the blur until the next one. + */ +internal class PrivacyAnimalLandmarkCache(private val refreshFrames: Int = REFRESH_FRAMES) { + private class Entry(val refreshedAt: Long, val relative: FloatArray?) + private val entries = HashMap() + private var frame = 0L + + /** Starts a frame; tracks no longer present, including held ones, drop their landmarks. */ + fun beginFrame(liveTracks: Set) { frame++; entries.keys.retainAll(liveTracks) } + + /** Frame-pixel landmarks for [track] in the box (left, top, right, bottom), or null to keep the blur. */ + fun landmarks(track: Int, left: Float, top: Float, right: Float, bottom: Float, estimate: () -> FloatArray?): FloatArray? { + val width = right - left; val height = bottom - top + val cached = entries[track] + val relative = if (cached != null && frame - cached.refreshedAt < refreshFrames) cached.relative + else estimate()?.let { points -> FloatArray(10) { i -> if (i % 2 == 0) (points[i] - left) / width else (points[i] - top) / height } } + .also { entries[track] = Entry(frame, it) } + return relative?.let { r -> FloatArray(10) { i -> if (i % 2 == 0) left + r[i] * width else top + r[i] * height } } + } + + fun reset() { entries.clear(); frame = 0L } + + companion object { const val REFRESH_FRAMES = 3 } +} + +/** + * Worker-owned compositor for one session's animal, fixed when the session was created. Like face swap, an animal + * replaces a face only inside its protected mask; unmatched, untracked, held or tiny faces keep the blur. + */ +internal class PrivacyAnimalSprite(private val context: Context, private val animal: String?) { + private var unavailable = false + private var cached: Pair? = null + private var detector: PrivacyYuNetModel? = null + private var detectorUnavailable = false + private val landmarkCache = PrivacyAnimalLandmarkCache() + private val poseScratch by lazy { + Bitmap.createBitmap(POSE_INPUT, POSE_INPUT, Bitmap.Config.ARGB_8888).apply { setHasAlpha(false) } + } + private val poseTransform = Matrix() + private val posePaint = Paint(Paint.FILTER_BITMAP_FLAG) + /** This frame's landmark cost for [PrivacyAnimalStats]: crop/readback and YuNet, summed over faces. */ + var lastCropNs = 0L + private set + var lastLandmarkNs = 0L + private set + + val mode: String get() = if (asset() == null) "blur" else checkNotNull(animal) + + fun beginFrame() { lastCropNs = 0L; lastLandmarkNs = 0L } + + fun asset(): Pair? { + if (animal !in ANIMALS || unavailable) return null + cached?.let { return it } + return runCatching { + val json = context.assets.open("$animal.json").bufferedReader().use { JSONObject(it.readText()) } + val image = context.assets.open("$animal.png").use { checkNotNull(BitmapFactory.decodeStream(it)) } + try { image to PrivacyAnimalAtlas.parse(json, image.width, image.height) } + catch (error: Throwable) { image.recycle(); throw error } + }.onFailure { unavailable = true }.getOrNull()?.also { cached = it } + } + + /** + * Faces that get an animal this frame. [crop] returns a [POSE_INPUT]-square, black-padded copy of the given frame + * square; the bitmap may be reused by the next call. + */ + fun faces(faces: List, layout: PrivacySegmentation.Letterbox, + crop: ((RectF) -> Bitmap)?): List { + if (crop == null || detectorUnavailable) return emptyList() + landmarkCache.beginFrame(faces.map { it.trackId }.toSet()) + return faces.mapNotNull { detection -> + // Untracked faces and held masks keep the blur, as face swap does. + if (detection.trackId == PrivacySegmentation.NO_TRACK || detection.held) return@mapNotNull null + val b = detection.box + val box = PrivacyAnimalTargets.Box((b.left - layout.left) * layout.sourceWidth / layout.resizedWidth, + (b.top - layout.top) * layout.sourceHeight / layout.resizedHeight, + (b.right - layout.left) * layout.sourceWidth / layout.resizedWidth, + (b.bottom - layout.top) * layout.sourceHeight / layout.resizedHeight) + val roi = PrivacyAnimalTargets.roi(box, layout.sourceWidth, layout.sourceHeight) ?: return@mapNotNull null + val landmarks = landmarkCache.landmarks(detection.trackId, box.left, box.top, box.right, box.bottom) { + landmarks(box, roi, crop) } ?: return@mapNotNull null + PrivacyAnimalFace(landmarks, PrivacyAnimalPose.yaw(landmarks) ?: return@mapNotNull null) + } + } + + private fun landmarks(box: PrivacyAnimalTargets.Box, roi: IntArray, crop: (RectF) -> Bitmap): FloatArray? = try { + val model = detector ?: PrivacyYuNetModel(context.applicationContext).also { detector = it } + // The ROI sits top-left in a square input scaled to POSE_INPUT; the rest is frame pixels or black padding. + val left = roi[0].toFloat(); val top = roi[1].toFloat() + val side = max(roi[2] - roi[0], roi[3] - roi[1]).toFloat() + val scale = side / POSE_INPUT + val started = System.nanoTime() + val input = crop(RectF(left, top, left + side, top + side)) + val cropped = System.nanoTime() + try { + PrivacyAnimalTargets.match(box, model.faces(input).filter { it.score >= PrivacyAnimalTargets.MIN_SCORE }.map { face -> + PrivacyAnimalTargets.Box(left + face.box.left * scale, top + face.box.top * scale, + left + face.box.right * scale, top + face.box.bottom * scale) to + FloatArray(10) { index -> face.landmarks[index / 2].let { p -> + if (index % 2 == 0) left + p.x * scale else top + p.y * scale } } + }) + } finally { lastCropNs += cropped - started; lastLandmarkNs += System.nanoTime() - cropped } + } catch (error: Exception) { + // `swapper_unavailable`: every face keeps the blur; frames are never delivered without protection. + detectorUnavailable = true + Log.w("PrivacyAnimal", "landmarks_unavailable type=${error.javaClass.simpleName}") + null + } + + /** CPU pose input from an upright frame; the returned bitmap is reused by the next call. */ + fun crop(upright: Bitmap, square: RectF): Bitmap { + val canvas = Canvas(poseScratch) + canvas.drawColor(Color.BLACK) + poseTransform.setRectToRect(square, RectF(0f, 0f, POSE_INPUT.toFloat(), POSE_INPUT.toFloat()), Matrix.ScaleToFit.FILL) + canvas.drawBitmap(upright, poseTransform, posePaint) + return poseScratch + } + + /** CPU fallback: animals over the already blurred frame, clipped to the protected [mask] like the GPU pass. */ + fun composite(protected: Bitmap, faces: List, mask: ByteArray, + layout: PrivacySegmentation.Letterbox, edgeCells: Int): Bitmap { + if (faces.isEmpty()) return protected + val (image, atlas) = asset() ?: return protected + val output = protected.copy(Bitmap.Config.ARGB_8888, true) ?: return protected + val canvas = Canvas(output) + val layer = canvas.saveLayer(null, null) + val paint = Paint(Paint.ANTI_ALIAS_FLAG or Paint.FILTER_BITMAP_FLAG) + faces.forEach { face -> + val place = atlas.place(face) ?: return@forEach + // The CPU fallback draws the nearest view instead of the GPU cross-fade. + val view = if (place.blend < .5f) place.from else place.to + val cell = view.cell + val transform = Matrix().apply { + if (place.mirrored) { setScale(-1f, 1f); postTranslate(cell.width.toFloat(), 0f) } + postConcat(Matrix().apply { setValues(floatArrayOf(place.a, -place.b, place.tx, place.b, place.a, place.ty, 0f, 0f, 1f)) }) + } + val save = canvas.save() + canvas.concat(transform) + canvas.drawBitmap(image, Rect(cell.x, cell.y, cell.x + cell.width, cell.y + cell.height), + RectF(0f, 0f, cell.width.toFloat(), cell.height.toFloat()), paint) + canvas.restoreToCount(save) + } + val expanded = PrivacyMaskRenderer.expandedMask(mask, edgeCells) + val size = PrivacySegmentation.MASK_SIZE + val alpha = Bitmap.createBitmap(size, size, Bitmap.Config.ALPHA_8) + alpha.copyPixelsFromBuffer(java.nio.ByteBuffer.wrap(expanded)) + val scaleX = 640f / size * layout.sourceWidth / layout.resizedWidth + val scaleY = 640f / size * layout.sourceHeight / layout.resizedHeight + canvas.drawBitmap(alpha, Matrix().apply { + setScale(scaleX, scaleY) + postTranslate(-layout.left * layout.sourceWidth.toFloat() / layout.resizedWidth, + -layout.top * layout.sourceHeight.toFloat() / layout.resizedHeight) + }, Paint(Paint.FILTER_BITMAP_FLAG).apply { xfermode = PorterDuffXfermode(PorterDuff.Mode.DST_IN) }) + canvas.restoreToCount(layer) + alpha.recycle() + return output + } + + fun reset() = landmarkCache.reset() + + fun close() { + cached?.first?.recycle(); cached = null + detector?.close(); detector = null + } + + companion object { + const val POSE_INPUT = 160 + val ANIMALS = setOf("rabbit", "owl") + } +} + +/** Debug-only 5 s summary of what an animal overlay adds per frame, logged per mode for blur/animal A/B runs. */ +internal class PrivacyAnimalStats { + private var mode = "" + private var startedNs = System.nanoTime() + private var faces = 0 + private val frame = ArrayList(); private val crop = ArrayList() + private val landmarks = ArrayList(); private val drawCpu = ArrayList() + private val drawGpu = ArrayList() + + fun record(mode: String, frameMs: Double, faceCount: Int, cropNs: Long, landmarkNs: Long, drawCpuNs: Long, drawGpuNs: Long?) { + val now = System.nanoTime() + if (mode != this.mode) { clear(); this.mode = mode; startedNs = now } + frame += frameMs; faces += faceCount + crop += cropNs / 1e6; landmarks += landmarkNs / 1e6; drawCpu += drawCpuNs / 1e6 + drawGpuNs?.let { drawGpu += it / 1e6 } + if (now - startedNs < 5_000_000_000L) return + fun p(values: List, q: Double) = values.sorted().let { if (it.isEmpty()) -1.0 else it[((it.size - 1) * q).toInt()] } + fun f(values: List) = "%.2f/%.2f".format(p(values, .5), p(values, .95)) + Log.i("PrivacyAnimal", "mode=$mode frames=${frame.size} faces_avg=${"%.2f".format(faces.toDouble() / frame.size)} " + + "frame_ms=${f(frame)} crop_ms=${f(crop)} yunet_ms=${f(landmarks)} draw_cpu_ms=${f(drawCpu)} " + + "draw_gpu_ms=${f(drawGpu)} gpu_samples=${drawGpu.size}") + clear(); startedNs = now + } + + private fun clear() { frame.clear(); crop.clear(); landmarks.clear(); drawCpu.clear(); drawGpu.clear(); faces = 0 } +} diff --git a/apps/android/InnoLive/app/src/main/java/com/framework/innolive/feature/live/privacy/PrivacyFrameProcessor.kt b/apps/android/InnoLive/app/src/main/java/com/framework/innolive/feature/live/privacy/PrivacyFrameProcessor.kt index ef75af07b..503d70a14 100644 --- a/apps/android/InnoLive/app/src/main/java/com/framework/innolive/feature/live/privacy/PrivacyFrameProcessor.kt +++ b/apps/android/InnoLive/app/src/main/java/com/framework/innolive/feature/live/privacy/PrivacyFrameProcessor.kt @@ -17,12 +17,13 @@ internal class PrivacyFrameProcessor(context: Context, private val sharedContext private val nativePostprocessing: Boolean = true, private val batchTwoOptimizations: Boolean = true, private val asyncFaceReadback: Boolean = false, - faceCoordinatorFactory: (() -> PrivacyFaceCoordinator)? = null) : AutoCloseable { + faceCoordinatorFactory: (() -> PrivacyFaceCoordinator)? = null, + animal: String? = null) : AutoCloseable { private var imageGpu: PrivacyGpuFramePipeline? = null private var imageGpuUnavailable = !allowGpuImages private val gpuRetry = PrivacyGpuRetryPolicy(allowGpuImages) private val model = PrivacyOnnxModel(context.applicationContext,directGpuInput=directGpuInput, - nativePostprocessing=nativePostprocessing,batchTwoOptimizations=batchTwoOptimizations) + nativePostprocessing=nativePostprocessing,batchTwoOptimizations=batchTwoOptimizations,animal=animal) private val pixels = PrivacyPixelConverter() private val faces = (faceCoordinatorFactory?.invoke() ?: PrivacyFaceCoordinator(context.applicationContext, optimized=nativePostprocessing)).also { coordinator -> @@ -36,6 +37,7 @@ internal class PrivacyFrameProcessor(context: Context, private val sharedContext var lastCameraCopiedPlanes: Int = 3 private set private var lastLogNs = System.nanoTime() + private val animalStats = if (BuildConfig.DEBUG || BuildConfig.PRIVACY_PERF_DIAGNOSTICS) PrivacyAnimalStats() else null private val sensorPixels = BitmapScratch() private val uprightPixels = BitmapScratch() private val restoredPixels = BitmapScratch() @@ -169,13 +171,16 @@ internal class PrivacyFrameProcessor(context: Context, private val sharedContext if (asyncFaceReadback && graph.supportsAsyncCrop) graph::cropAsync else null) else faces.exceptions(box.sourceWidth,box.sourceHeight,objects,box,timestampNs) }, - renderOnGpu=true,gpuGraph=graph) {mask -> graph.finish(mask,layout,sensorWidth,sensorHeight, - model.lastMaskPixels,outputUpright=true, + renderOnGpu=true,gpuGraph=graph) {mask -> graph.measureAnimalGpu=animalStats!=null; graph.finish(mask,layout,sensorWidth,sensorHeight, + model.lastMaskPixels,outputUpright=true,animalOverlay=model.animalOverlay(), blurSigma=mosaic.sigma(layout.sourceWidth,layout.sourceHeight), edgeCells=mosaic.edgeCells(layout.sourceWidth,layout.sourceHeight))} val completed = System.nanoTime() lastTimings = PrivacyFrameTimings((converted - started) / 1e6, checkNotNull(model.lastTimings), 0.0, (completed - started) / 1e6) + animalStats?.let { stats -> val sprite=model.animalCost + stats.record(sprite.mode,(completed-started)/1e6,model.lastAnalysis?.faces ?: 0,sprite.lastCropNs, + sprite.lastLandmarkNs,graph.lastAnimalDrawCpuNs,graph.lastAnimalDrawGpuNs) } if (BuildConfig.DEBUG && completed - lastLogNs >= 5_000_000_000L) { lastLogNs = completed Log.i("PrivacyPipeline", "image_gpu=true texture_output=true size=${sensorWidth}x${sensorHeight} camera_copied_planes=$lastCameraCopiedPlanes $lastTimings") diff --git a/apps/android/InnoLive/app/src/main/java/com/framework/innolive/feature/live/privacy/PrivacyGpuFramePipeline.kt b/apps/android/InnoLive/app/src/main/java/com/framework/innolive/feature/live/privacy/PrivacyGpuFramePipeline.kt index 890e62aae..85f302ae2 100644 --- a/apps/android/InnoLive/app/src/main/java/com/framework/innolive/feature/live/privacy/PrivacyGpuFramePipeline.kt +++ b/apps/android/InnoLive/app/src/main/java/com/framework/innolive/feature/live/privacy/PrivacyGpuFramePipeline.kt @@ -3,11 +3,13 @@ package com.framework.innolive.feature.live.privacy import android.graphics.Bitmap import android.graphics.Matrix import android.graphics.Rect +import android.graphics.RectF import android.opengl.GLES20.* import android.opengl.GLUtils import android.opengl.GLES30.GL_UNPACK_ROW_LENGTH import android.opengl.GLES30.GL_R8 import android.opengl.GLES30.GL_RED +import android.opengl.GLES30 import android.util.Log import android.content.Context import android.os.Handler @@ -56,6 +58,8 @@ internal class PrivacyGpuFramePipeline(sharedContext: EglBase.Context?, useGles3 private set private val maskPixels = ByteBuffer.allocateDirect(160 * 160 * if(compactMask) 1 else 4) private var cropPixels: ByteBuffer? = null + private var posePixels: ByteBuffer? = null + private var poseBitmap: Bitmap? = null private var faceReadbackPool: PrivacyFaceReadbackPool? = null private val outputPool = mutableListOf() private val leased = mutableSetOf() @@ -262,6 +266,24 @@ internal class PrivacyGpuFramePipeline(sharedContext: EglBase.Context?, useGles3 } } + /** Square head-pose input; [square] may extend past the frame, where it is padded with black. Reused per call. */ + internal fun poseCrop(square: RectF, size: Int): Bitmap = onGl { + require(square.width() > 0f && square.height() > 0f) + val cropped = target("pose", size, size) + draw("pose", POSE_CROP, cropped, listOf(targets.getValue("upright").texture)) { p -> + glUniform4f(p.getUniformLocation("box"), square.left / width, square.top / height, + square.width() / width, square.height() / height) + } + val pixels = posePixels?.takeIf { it.capacity() == size * size * 4 } + ?: ByteBuffer.allocateDirect(size * size * 4).also { posePixels = it } + readPixels(cropped, pixels) + val bitmap = poseBitmap?.takeIf { it.width == size } + ?: Bitmap.createBitmap(size, size, Bitmap.Config.ARGB_8888).also { poseBitmap?.recycle(); poseBitmap = it } + bitmap.setHasAlpha(false) + pixels.rewind(); bitmap.copyPixelsFromBuffer(pixels) + bitmap + } + /** Diagnostic asynchronous crop; all original pixels and crop coordinates are preserved. */ internal fun cropAsync(bounds: Rect): PrivacyFaceReadback = onGl { check(!closing && rowLengthSupported) @@ -411,9 +433,22 @@ internal class PrivacyGpuFramePipeline(sharedContext: EglBase.Context?, useGles3 readModelInput() } + private var uploadedAnimal: Bitmap? = null + /** CPU submit time of the last animal draw, and its GPU time from a timer query a few frames later. */ + internal var lastAnimalDrawCpuNs = 0L + private set + internal var lastAnimalDrawGpuNs: Long? = null + private set + internal var measureAnimalGpu = false + private val animalQueries = IntArray(3) + private val animalQueryPending = BooleanArray(3) + private var animalQueryIndex = 0 + private val timerQuerySupported by lazy { glGetString(GL_EXTENSIONS)?.contains("GL_EXT_disjoint_timer_query") == true && rowLengthSupported } + fun finish(mask: ByteArray, layout: PrivacySegmentation.Letterbox, sensorWidth: Int, sensorHeight: Int, maskPixels: Int = -1, outputUpright: Boolean = false, blurSigma: Int? = null, + animalOverlay: PrivacyAnimalOverlay? = null, edgeCells: Int = PrivacyMaskRenderer.PROTECTED_EDGE_CELLS): VideoFrame.TextureBuffer = onGl { check(!closing) require(layout.sourceWidth == width && layout.sourceHeight == height && mask.size == 160 * 160) @@ -428,8 +463,9 @@ internal class PrivacyGpuFramePipeline(sharedContext: EglBase.Context?, useGles3 Target(outputWidth, outputHeight).also { outputPool.add(it) } } val original = targets.getValue("upright") + var protectedMask: Target? = null if (if (maskPixels >= 0) maskPixels > 0 else mask.any { it.toInt() != 0 }) { - val alpha = uploadMask(mask, edgeCells) + val alpha = uploadMask(mask, edgeCells).also { protectedMask = it } val quarterW = PrivacyBroadcastBlur.reducedSize(width) val quarterH = PrivacyBroadcastBlur.reducedSize(height) val reduced = target("reduced", quarterW, quarterH) @@ -459,6 +495,41 @@ internal class PrivacyGpuFramePipeline(sharedContext: EglBase.Context?, useGles3 glUniform1i(p.getUniformLocation("rotation"), outputRotation) } } + lastAnimalDrawCpuNs = 0L; lastAnimalDrawGpuNs = null + val timing = measureAnimalGpu && timerQuerySupported && outputUpright && animalOverlay != null + if (timing) lastAnimalDrawGpuNs = beginAnimalQuery() + val drawStarted = System.nanoTime() + val animalMask = protectedMask + if (outputUpright && animalOverlay != null && animalMask != null) { + val bitmap = animalOverlay.image; val atlas = animalOverlay.atlas + val sprite = target("animal-sprite", bitmap.width, bitmap.height) + if (uploadedAnimal !== bitmap) { + glBindTexture(GL_TEXTURE_2D, sprite.texture) + GLUtils.texSubImage2D(GL_TEXTURE_2D, 0, 0, 0, bitmap) + uploadedAnimal = bitmap + } + fun cell(view: PrivacyAnimalAtlas.View) = floatArrayOf(view.cell.x.toFloat() / atlas.width, + view.cell.y.toFloat() / atlas.height, view.cell.width.toFloat() / atlas.width, + view.cell.height.toFloat() / atlas.height) + animalOverlay.faces.forEach { face -> + val place = atlas.place(face) ?: return@forEach + draw("animal", ANIMAL, output, listOf(sprite.texture, animalMask.texture)) { p -> + glEnable(GL_BLEND); glBlendFunc(GL_ONE, GL_ONE_MINUS_SRC_ALPHA) + glUniform2f(p.getUniformLocation("frame"), width.toFloat(), height.toFloat()) + glUniform4f(p.getUniformLocation("similarity"), place.a, place.b, place.tx, place.ty) + glUniform2f(p.getUniformLocation("cellSize"), place.from.cell.width.toFloat(), place.from.cell.height.toFloat()) + glUniform4f(p.getUniformLocation("box"), layout.left / 640f, layout.top / 640f, + layout.resizedWidth / 640f, layout.resizedHeight / 640f) + glUniform4fv(p.getUniformLocation("cellA"), 1, cell(place.from), 0) + glUniform4fv(p.getUniformLocation("cellB"), 1, cell(place.to), 0) + glUniform1f(p.getUniformLocation("blend"), place.blend) + glUniform1f(p.getUniformLocation("mirrored"), if (place.mirrored) 1f else 0f) + } + } + glDisable(GL_BLEND) + } + lastAnimalDrawCpuNs = System.nanoTime() - drawStarted + if (timing) GLES30.glEndQuery(TIME_ELAPSED) GlUtil.checkNoGLES2Error("privacy GPU output") val fence = if (useFence) PrivacyNativeGpuFence.create() else 0L lastFenceUsed = fence != 0L @@ -478,6 +549,26 @@ internal class PrivacyGpuFramePipeline(sharedContext: EglBase.Context?, useGles3 /** The Gaussian for the full-resolution sigma, run at the reduced resolution. One program per sigma. */ private val blurShaders = mutableMapOf() + /** Starts this frame's timer query and returns the oldest finished one, without waiting for the GPU. */ + private fun beginAnimalQuery(): Long? { + if (animalQueries[0] == 0) GLES30.glGenQueries(animalQueries.size, animalQueries, 0) + val slot = animalQueryIndex + animalQueryIndex = (animalQueryIndex + 1) % animalQueries.size + var result: Long? = null + if (animalQueryPending[slot]) { + val values = IntArray(1) + GLES30.glGetQueryObjectuiv(animalQueries[slot], GLES30.GL_QUERY_RESULT_AVAILABLE, values, 0) + val disjoint = IntArray(1).also { glGetIntegerv(GPU_DISJOINT, it, 0) }[0] != 0 + if (values[0] != 0) { + GLES30.glGetQueryObjectuiv(animalQueries[slot], GLES30.GL_QUERY_RESULT, values, 0) + if (!disjoint) result = values[0].toLong() and 0xffffffffL + } + } + GLES30.glBeginQuery(TIME_ELAPSED, animalQueries[slot]) + animalQueryPending[slot] = true + return result + } + private fun blurShader(name: String, sigma: Int): String = blurShaders.getOrPut(name) { val reduced = sigma.toDouble() / PrivacyBroadcastBlur.REDUCTION val radius = PrivacyBroadcastBlur.kernelRadius(reduced) @@ -603,7 +694,8 @@ internal class PrivacyGpuFramePipeline(sharedContext: EglBase.Context?, useGles3 glDeleteTextures(3, planeTextures, 0) glDeleteTextures(1, maskTexture, 0) shaders.values.forEach { it.release() }; shaders.clear() - modelBitmap.recycle() + modelBitmap.recycle(); poseBitmap?.recycle(); poseBitmap = null + if (animalQueries[0] != 0) GLES30.glDeleteQueries(animalQueries.size, animalQueries, 0) outputPool.filter { it !in leased }.toList().forEach { it.close(); outputPool.remove(it) } releaseIfIdle() } @@ -647,6 +739,9 @@ internal class PrivacyGpuFramePipeline(sharedContext: EglBase.Context?, useGles3 companion object { private val SAMPLERS = arrayOf("tex0", "tex1", "tex2") + /** GL_EXT_disjoint_timer_query. */ + private const val TIME_ELAPSED = 0x88BF + private const val GPU_DISJOINT = 0x8FBB private fun textureParameters(filter: Int) { glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, filter) glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, filter) @@ -672,6 +767,21 @@ internal class PrivacyGpuFramePipeline(sharedContext: EglBase.Context?, useGles3 void main(){ vec2 p=(uv-box.xy)/box.zw; gl_FragColor=(p.x<0.0||p.y<0.0||p.x>1.0||p.y>1.0)?vec4(vec3(114.0/255.0),1.0):texture2D(tex0,p); } """ private const val CROP = "uniform sampler2D tex0; uniform vec4 box; void main(){ gl_FragColor=texture2D(tex0,box.xy+uv*box.zw); }" + private const val POSE_CROP = "uniform sampler2D tex0; uniform vec4 box; void main(){ vec2 p=box.xy+uv*box.zw; " + + "gl_FragColor=(p.x<0.0||p.y<0.0||p.x>1.0||p.y>1.0)?vec4(0.0,0.0,0.0,1.0):texture2D(tex0,p); }" + /** + * One placed animal: the inverse landmark similarity takes frame pixels into the (mirrored) cell, two + * neighbouring yaw cells of the premultiplied atlas cross-fade, and the protected mask clips it like face swap. + */ + private const val ANIMAL = """ + uniform sampler2D tex0,tex1; uniform vec2 frame,cellSize; uniform vec4 similarity,cellA,cellB,box; uniform float blend,mirrored; + void main(){ vec2 d=uv*frame-similarity.zw; float n=dot(similarity.xy,similarity.xy); + vec2 q=vec2(similarity.x*d.x+similarity.y*d.y,-similarity.y*d.x+similarity.x*d.y)/n/cellSize; + if(q.x<0.0||q.y<0.0||q.x>1.0||q.y>1.0) discard; + if(mirrored>0.5) q.x=1.0-q.x; + float inside=texture2D(tex1,box.xy+uv*box.zw).r; + gl_FragColor=mix(texture2D(tex0,cellA.xy+q*cellA.zw),texture2D(tex0,cellB.xy+q*cellB.zw),blend)*inside; } + """ private const val RESTORE = "uniform sampler2D tex0; uniform int rotation; " + FORWARD + "void main(){ gl_FragColor=texture2D(tex0,upright(uv)); }" // Four bilinear taps one source texel away from the quarter-pixel centre average an exact // 4x4 block when the target is a quarter of the source; a single fetch would skip texels. diff --git a/apps/android/InnoLive/app/src/main/java/com/framework/innolive/feature/live/privacy/PrivacyOnnxModel.kt b/apps/android/InnoLive/app/src/main/java/com/framework/innolive/feature/live/privacy/PrivacyOnnxModel.kt index cb07d21a6..b4f807c72 100644 --- a/apps/android/InnoLive/app/src/main/java/com/framework/innolive/feature/live/privacy/PrivacyOnnxModel.kt +++ b/apps/android/InnoLive/app/src/main/java/com/framework/innolive/feature/live/privacy/PrivacyOnnxModel.kt @@ -25,7 +25,9 @@ internal class PrivacyOnnxModel(private val context: Context, private val nativePostprocessing: Boolean = true, private val batchTwoOptimizations: Boolean = true, /** Detect every N frames; BoT-SORT predicts the objects of the frames between. */ - private val detectionInterval: Int = DETECTION_INTERVAL) : AutoCloseable { + private val detectionInterval: Int = DETECTION_INTERVAL, + /** Session animal (`rabbit`/`owl`) fixed at session creation; null keeps blur only. */ + animal: String? = null) : AutoCloseable { private val environment = OrtEnvironment.getEnvironment() private var session: OrtSession? = null private val blur = PrivacyBitmapBlur() @@ -79,6 +81,11 @@ internal class PrivacyOnnxModel(private val context: Context, internal val retainsCpuSession: Boolean get() = session != null + private val animalSprite = PrivacyAnimalSprite(context, animal) + private var animalFaces: List = emptyList() + fun animalOverlay() = animalSprite.asset()?.let { PrivacyAnimalOverlay(it.first, it.second, animalFaces) } + internal val animalCost get() = animalSprite + fun process( upright: Bitmap, timestampNs: Long = System.nanoTime(), @@ -96,10 +103,13 @@ internal class PrivacyOnnxModel(private val context: Context, (layout.top + layout.resizedHeight).toFloat()), Paint(Paint.FILTER_BITMAP_FLAG), ) - return processPrepared(inputBitmap, layout, timestampNs, exemptFaces) { mask -> + return processPrepared(inputBitmap, layout, timestampNs, exemptFaces, poseSource = upright) { mask -> val sigma = mosaic.sigma(upright.width, upright.height) - PrivacyMaskRenderer.render(upright, mask, layout, { blur.apply(it, sigma) }, + val protected = PrivacyMaskRenderer.render(upright, mask, layout, { blur.apply(it, sigma) }, mosaic.edgeCells(upright.width, upright.height)) + val output = animalSprite.composite(protected, animalFaces, mask, layout, mosaic.edgeCells(upright.width, upright.height)) + if (output !== protected) protected.recycle() + output } } @@ -111,6 +121,7 @@ internal class PrivacyOnnxModel(private val context: Context, exemptFaces: (List, PrivacySegmentation.Letterbox) -> Set, renderOnGpu: Boolean = false, gpuGraph: PrivacyGpuFramePipeline? = null, + poseSource: Bitmap? = null, render: (ByteArray) -> T, ): T { require(preparedInput.width == 640 && preparedInput.height == 640) @@ -181,6 +192,12 @@ internal class PrivacyOnnxModel(private val context: Context, PrivacySegmentation.protectedDetections(objects, exempt), prototypes) stabilizer.apply(instances, timestampNs / 1_000_000_000.0).also { lastMaskPixels = -1 } } + // Head pose reads the current upright frame: the CPU bitmap, or a small crop of the GPU graph's texture. + animalSprite.beginFrame() + animalFaces = if (animalSprite.asset() == null) emptyList() else animalSprite.faces( + PrivacySegmentation.protectedDetections(objects, exempt).filter { it.classId == 0 }, layout, + poseSource?.let { upright -> { square: RectF -> animalSprite.crop(upright, square) } } + ?: gpuGraph?.let { graph -> { square: RectF -> graph.poseCrop(square, PrivacyAnimalSprite.POSE_INPUT) } }) val masked = System.nanoTime() val output = render(mask) val rendered = System.nanoTime() @@ -203,6 +220,7 @@ internal class PrivacyOnnxModel(private val context: Context, } fun resetTemporalState() { + animalFaces = emptyList(); animalSprite.reset() stabilizer.reset(); tracking.reset(); skippedDetections = 0; lastPrototypes = null if (postprocessorDelegate.isInitialized()) postprocessor.reset() } @@ -266,6 +284,7 @@ internal class PrivacyOnnxModel(private val context: Context, } override fun close() { + animalSprite.close() if (postprocessorDelegate.isInitialized()) postprocessor.close() releaseAccelerators() blur.close() diff --git a/apps/android/InnoLive/app/src/main/java/com/framework/innolive/feature/live/privacy/PrivacyYuNetModel.kt b/apps/android/InnoLive/app/src/main/java/com/framework/innolive/feature/live/privacy/PrivacyYuNetModel.kt index 9e7bc68b3..c3ac64431 100644 --- a/apps/android/InnoLive/app/src/main/java/com/framework/innolive/feature/live/privacy/PrivacyYuNetModel.kt +++ b/apps/android/InnoLive/app/src/main/java/com/framework/innolive/feature/live/privacy/PrivacyYuNetModel.kt @@ -174,6 +174,17 @@ internal class PrivacyYuNetModel(context: Context, private val reuseInputs: Bool } fun oneFace(image: Bitmap, enrollment: Boolean): YuNetFace? { + val faces = detect(image, enrollment) + if (faces.size > 1) throw AmbiguousFaceSampleException() + val restored = faces.singleOrNull() ?: return null + if (min(restored.box.width(), restored.box.height()) < if (enrollment) 40f else 24f) return null + return restored + } + + /** Every live-query face in [image] coordinates, for callers that pick one by overlap. */ + fun faces(image: Bitmap): List = detect(image, enrollment = false) + + private fun detect(image: Bitmap, enrollment: Boolean): List { val scale = if (enrollment) 1f else min(1f, 320f / max(image.width, image.height)) val resizedWidth = max(1, (image.width * scale).roundToInt()) val resizedHeight = max(1, (image.height * scale).roundToInt()) @@ -204,18 +215,13 @@ internal class PrivacyYuNetModel(context: Context, private val reuseInputs: Bool ?: FloatArray(buffer.remaining()).also { outputsScratch[name]=it } else FloatArray(buffer.remaining())).also(buffer::get) } - val faces = PrivacyYuNetDecoding.decode(outputs, width, height) - .filter { it.score >= if (enrollment) 0.9f else 0.6f } - if (faces.size > 1) throw AmbiguousFaceSampleException() - if (faces.isEmpty()) return null - val face = faces.single() val restoreX = image.width.toFloat() / resizedWidth val restoreY = image.height.toFloat() / resizedHeight - val restored = YuNetFace(RectF(face.box.left * restoreX, face.box.top * restoreY, - face.box.right * restoreX, face.box.bottom * restoreY), - face.landmarks.map { PointF(it.x * restoreX, it.y * restoreY) }, face.score) - if (min(restored.box.width(), restored.box.height()) < if (enrollment) 40f else 24f) return null - return restored + return PrivacyYuNetDecoding.decode(outputs, width, height) + .filter { it.score >= if (enrollment) 0.9f else 0.6f } + .map { face -> YuNetFace(RectF(face.box.left * restoreX, face.box.top * restoreY, + face.box.right * restoreX, face.box.bottom * restoreY), + face.landmarks.map { PointF(it.x * restoreX, it.y * restoreY) }, face.score) } } } finally { if (!reuseInputs) tensor.close() } } finally { diff --git a/apps/android/InnoLive/app/src/main/java/com/framework/innolive/feature/settings/Settings.kt b/apps/android/InnoLive/app/src/main/java/com/framework/innolive/feature/settings/Settings.kt index 4f86cd82d..04ffb539a 100644 --- a/apps/android/InnoLive/app/src/main/java/com/framework/innolive/feature/settings/Settings.kt +++ b/apps/android/InnoLive/app/src/main/java/com/framework/innolive/feature/settings/Settings.kt @@ -46,6 +46,7 @@ import androidx.compose.ui.res.stringResource import androidx.compose.ui.res.painterResource import androidx.compose.ui.unit.dp import com.framework.innolive.R +import com.framework.innolive.feature.live.FaceAnonymizationMethod import com.framework.innolive.feature.settings.camera.CameraStabilizationControl /** @@ -197,24 +198,26 @@ private fun AIProcessingSettings(props: SettingsScreenProps) { } /** - * 관리자 전용 실험 기능입니다. iOS와 같이 비식별화 방법(블러·합성)을 고릅니다. 얼굴 가림 방식은 세션 생성 때 - * 고정되므로 처리 방식과 같은 때에만 바꿉니다. + * 관리자 전용 실험 기능입니다. iOS와 같이 비식별화 방법(블러·합성·토끼·부엉이)을 고릅니다. 얼굴 가림 방식은 + * 세션 생성 때 고정되므로 처리 방식과 같은 때에만 바꿉니다. */ @Composable private fun LabSettings(props: SettingsScreenProps) { val enabled = props.canChangeAIProcessing && !props.isDeletingAccount && !props.isAccountDeletionPending var expanded by rememberSaveable { mutableStateOf(false) } - val options = listOf(false to R.string.lab_face_anonymization_blur, true to R.string.lab_face_anonymization_face_swap) + val options = listOf( + FaceAnonymizationMethod.BLUR to R.string.lab_face_anonymization_blur, + FaceAnonymizationMethod.FACE_SWAP to R.string.lab_face_anonymization_face_swap, + FaceAnonymizationMethod.RABBIT to R.string.lab_face_anonymization_rabbit, + FaceAnonymizationMethod.OWL to R.string.lab_face_anonymization_owl, + ) Column(modifier = Modifier.fillMaxWidth().padding(vertical = 8.dp)) { SettingsSectionLabel(stringResource(R.string.settings_section_lab)) Box { SettingsValueRow( label = stringResource(R.string.lab_face_anonymization), - value = stringResource( - if (props.faceSwapEnabled) R.string.lab_face_anonymization_face_swap - else R.string.lab_face_anonymization_blur, - ), + value = stringResource(options.first { it.first == props.faceAnonymization }.second), enabled = enabled, onClick = { expanded = true }, ) @@ -223,15 +226,15 @@ private fun LabSettings(props: SettingsScreenProps) { onDismissRequest = { expanded = false }, modifier = Modifier.testTag("lab-face-anonymization"), ) { - options.forEach { (faceSwap, label) -> + options.forEach { (method, label) -> DropdownMenuItem( text = { Text(stringResource(label)) }, - trailingIcon = if (props.faceSwapEnabled == faceSwap) { + trailingIcon = if (props.faceAnonymization == method) { { Icon(Icons.Outlined.Check, contentDescription = null) } } else null, onClick = { expanded = false - if (props.faceSwapEnabled != faceSwap) props.onFaceSwapChanged(faceSwap) + if (props.faceAnonymization != method) props.onFaceAnonymizationChanged(method) }, ) } diff --git a/apps/android/InnoLive/app/src/main/java/com/framework/innolive/feature/settings/SettingsScreenProps.kt b/apps/android/InnoLive/app/src/main/java/com/framework/innolive/feature/settings/SettingsScreenProps.kt index 33243dd48..ffc09f605 100644 --- a/apps/android/InnoLive/app/src/main/java/com/framework/innolive/feature/settings/SettingsScreenProps.kt +++ b/apps/android/InnoLive/app/src/main/java/com/framework/innolive/feature/settings/SettingsScreenProps.kt @@ -2,6 +2,7 @@ package com.framework.innolive.feature.settings import androidx.compose.runtime.Immutable import com.framework.innolive.feature.live.BroadcastVideoQualitySettings +import com.framework.innolive.feature.live.FaceAnonymizationMethod import com.framework.innolive.feature.live.VideoQualityCaptureState import com.framework.innolive.feature.login.oauth.google.SignInMethod import com.framework.innolive.ui.text.UiText @@ -41,8 +42,8 @@ data class SettingsScreenProps( val onSelectAIProcessing: (Boolean) -> Unit = {}, /** `GET /admin/me`로 확인한 관리자 계정입니다. 관리자에게만 실험실을 보여 줍니다. */ val isAdministrator: Boolean = false, - val faceSwapEnabled: Boolean = false, - val onFaceSwapChanged: (Boolean) -> Unit = {}, + val faceAnonymization: FaceAnonymizationMethod = FaceAnonymizationMethod.BLUR, + val onFaceAnonymizationChanged: (FaceAnonymizationMethod) -> Unit = {}, val planUsage: PlanUsage? = null, val isPlanLoading: Boolean = false, val planError: UiText? = null, diff --git a/apps/android/InnoLive/app/src/main/res/values-en/strings.xml b/apps/android/InnoLive/app/src/main/res/values-en/strings.xml index fd693b012..4a0327361 100644 --- a/apps/android/InnoLive/app/src/main/res/values-en/strings.xml +++ b/apps/android/InnoLive/app/src/main/res/values-en/strings.xml @@ -35,7 +35,9 @@ Anonymization method Blur Face swap - Applies from the next broadcast preparation. Face swap is used only with server AI processing. + Rabbit + Owl + Applies from the next broadcast preparation. Face swap is used only with server AI processing, and rabbit and owl only with on-device processing. Cancel broadcast preparation to change the anonymization method. On-device AI On-device faces diff --git a/apps/android/InnoLive/app/src/main/res/values-ja/strings.xml b/apps/android/InnoLive/app/src/main/res/values-ja/strings.xml index 99f2bad3a..e2e7b9c2b 100644 --- a/apps/android/InnoLive/app/src/main/res/values-ja/strings.xml +++ b/apps/android/InnoLive/app/src/main/res/values-ja/strings.xml @@ -35,7 +35,9 @@ 匿名化方式 ぼかし 顔合成 - 次の配信準備から適用されます。顔合成はサーバー AI 処理でのみ使用されます。 + うさぎ + ふくろう + 次の配信準備から適用されます。顔合成はサーバー AI 処理、うさぎ・ふくろうはオンデバイス処理でのみ使用されます。 配信準備をキャンセルすると匿名化方式を変更できます。 オンデバイスAI 端末内の顔管理 diff --git a/apps/android/InnoLive/app/src/main/res/values/strings.xml b/apps/android/InnoLive/app/src/main/res/values/strings.xml index 2ca673d3a..cf3c2ec15 100644 --- a/apps/android/InnoLive/app/src/main/res/values/strings.xml +++ b/apps/android/InnoLive/app/src/main/res/values/strings.xml @@ -279,7 +279,9 @@ 비식별화 방법 블러 합성 - 다음 방송 준비부터 적용됩니다. 합성은 서버 AI 처리에서만 사용됩니다. + 토끼 + 부엉이 + 다음 방송 준비부터 적용됩니다. 합성은 서버 AI 처리, 토끼·부엉이는 온디바이스 처리에서만 사용됩니다. 방송 준비를 취소한 뒤 비식별화 방법을 변경할 수 있습니다. 온디바이스 AI 기기 얼굴 관리 diff --git a/apps/android/InnoLive/app/src/test/java/com/framework/innolive/feature/live/privacy/PrivacyAnimalSpriteTest.kt b/apps/android/InnoLive/app/src/test/java/com/framework/innolive/feature/live/privacy/PrivacyAnimalSpriteTest.kt new file mode 100644 index 000000000..86dc4c5d9 --- /dev/null +++ b/apps/android/InnoLive/app/src/test/java/com/framework/innolive/feature/live/privacy/PrivacyAnimalSpriteTest.kt @@ -0,0 +1,166 @@ +package com.framework.innolive.feature.live.privacy + +import com.framework.innolive.feature.live.FaceAnonymizationMethod +import com.framework.innolive.feature.live.sessionAnimal +import org.json.JSONObject +import org.junit.Assert.assertArrayEquals +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test +import java.io.File +import kotlin.math.PI +import kotlin.math.cos +import kotlin.math.sin + +class PrivacyAnimalSpriteTest { + private fun atlas(name: String): PrivacyAnimalAtlas { + val json = JSONObject(File("src/main/assets/$name.json").readText()) + return PrivacyAnimalAtlas.parse(json, json.getInt("width"), json.getInt("height")) + } + + /** ArcFace-like 5 points: eyes 100 px apart at (cx, cy), rotated by [roll]. */ + private fun face(cx: Float, cy: Float, scale: Float = 100f, roll: Double = 0.0): FloatArray { + val template = floatArrayOf(-.5f, 0f, .5f, 0f, 0f, .57f, -.41f, 1.16f, .41f, 1.16f) + return FloatArray(10) { i -> + val x = template[i - i % 2]; val y = template[i - i % 2 + 1] + if (i % 2 == 0) cx + scale * (x * cos(roll) - y * sin(roll)).toFloat() + else cy + scale * (x * sin(roll) + y * cos(roll)).toFloat() + } + } + + private fun apply(place: PrivacyAnimalAtlas.Placement, x: Float, y: Float) = + floatArrayOf(place.a * x - place.b * y + place.tx, place.b * x + place.a * y + place.ty) + + @Test + fun bundledAtlasesCarryYawViewsWithTemplateLandmarks() { + for (name in listOf("rabbit", "owl")) { + val atlas = atlas(name) + assertTrue(atlas.mirrorNegativeYaw) + assertEquals((0..90 step 10).map(Int::toFloat), atlas.views.map { it.yaw }) + val front = atlas.views.first().landmarks + // Front view: eyes level and symmetric, nose between them, mouth below. + assertEquals(front[1], front[3], 1e-3f) + assertEquals(.5f, (front[0] + front[2]) / 2, 1e-2f) + assertTrue(front[5] > front[1] && front[7] > front[5]) + } + } + + @Test + fun similarityRecoversRotationScaleAndTranslation() { + val source = floatArrayOf(0f, 0f, 10f, 0f, 5f, 6f, 2f, 11f, 8f, 11f) + val angle = .3; val scale = 2.5f + val target = FloatArray(10) { i -> + val x = source[i - i % 2]; val y = source[i - i % 2 + 1] + if (i % 2 == 0) 40f + scale * (x * cos(angle) - y * sin(angle)).toFloat() + else -7f + scale * (x * sin(angle) + y * cos(angle)).toFloat() + } + val (a, b, tx, ty) = checkNotNull(PrivacyAnimalAtlas.similarity(source, target)) + assertEquals(scale * cos(angle).toFloat(), a, 1e-4f) + assertEquals(scale * sin(angle).toFloat(), b, 1e-4f) + assertEquals(40f, tx, 1e-3f); assertEquals(-7f, ty, 1e-3f) + assertNull(PrivacyAnimalAtlas.similarity(FloatArray(10) { 3f }, target)) + } + + @Test + fun frontFaceLandmarksLandOnTheAnimalTemplate() { + val atlas = atlas("rabbit") + val landmarks = face(400f, 300f, roll = .2) + val place = checkNotNull(atlas.place(PrivacyAnimalFace(landmarks, 0f))) + val cell = place.from.cell + val eye = apply(place, place.from.landmarks[0] * cell.width, place.from.landmarks[1] * cell.height) + assertEquals(landmarks[0], eye[0], 6f); assertEquals(landmarks[1], eye[1], 6f) + // Roll follows the face: the similarity rotation equals the eye-line angle. + assertEquals(.2f, kotlin.math.atan2(place.b, place.a), .03f) + } + + @Test + fun negativeYawMirrorsTheCellAndSwapsSides() { + val atlas = atlas("owl") + val right = checkNotNull(atlas.place(PrivacyAnimalFace(face(300f, 300f), 50f))) + val left = checkNotNull(atlas.place(PrivacyAnimalFace(face(300f, 300f), -50f))) + assertFalse(right.mirrored); assertTrue(left.mirrored) + assertEquals(right.from.yaw, left.from.yaw) + } + + @Test + fun crossFadeOnlyNearTheMidpointBetweenViews() { + val atlas = atlas("rabbit") + fun blend(yaw: Float) = checkNotNull(atlas.place(PrivacyAnimalFace(face(200f, 200f), yaw))).blend + assertEquals(0f, blend(24.4f), 0f) + assertEquals(.5f, blend(25f), 1e-5f) + assertEquals(1f, blend(25.6f), 0f) + assertEquals(90f, checkNotNull(atlas.place(PrivacyAnimalFace(face(200f, 200f), 120f))).from.yaw) + } + + @Test + fun yawRecoversSignFromLandmarks() { + for (yaw in listOf(-70.0, -30.0, 0.0, 20.0, 60.0)) { + val r = yaw * PI / 180 + val points = listOf(-.5 to 0.0, .5 to 0.0, 0.0 to .55, -.35 to .3, .35 to .3) + val heights = listOf(0.0, 0.0, .4, .8, .8) + val landmarks = points.zip(heights).flatMap { (p, y) -> + listOf(100 * (p.first * cos(r) + p.second * sin(r)), 100 * y) }.map(Double::toFloat).toFloatArray() + assertEquals(yaw.toFloat(), checkNotNull(PrivacyAnimalPose.yaw(landmarks)), .5f) + } + assertNull(PrivacyAnimalPose.yaw(FloatArray(10))) + } + + @Test + fun faceSwapTargetRulesForRoiAndOverlap() { + val box = PrivacyAnimalTargets.Box(100f, 100f, 200f, 220f) + // 0.3 x longer side (120) = 36 px padding, clipped to the 1280 x 250 frame. + assertArrayEquals(intArrayOf(64, 64, 236, 250), PrivacyAnimalTargets.roi(box, 1280, 250)) + assertNull(PrivacyAnimalTargets.roi(PrivacyAnimalTargets.Box(0f, 0f, 15f, 40f), 100, 100)) + val near = floatArrayOf(1f, 1f, 1f, 1f, 1f, 1f, 1f, 1f, 1f, 1f) + val far = FloatArray(10) + assertEquals(near, PrivacyAnimalTargets.match(box, listOf( + PrivacyAnimalTargets.Box(300f, 100f, 400f, 220f) to far, + PrivacyAnimalTargets.Box(105f, 110f, 205f, 225f) to near))) + // IoU must exceed 0.25, as in InSwapperRenderer.swap. + assertNull(PrivacyAnimalTargets.match(box, listOf(PrivacyAnimalTargets.Box(170f, 100f, 270f, 220f) to near))) + } + + @Test + fun landmarkCacheRefreshesEveryThreeFramesAndFollowsTheBox() { + val cache = PrivacyAnimalLandmarkCache() + var runs = 0 + val estimate = { runs++; FloatArray(10) { if (it % 2 == 0) 10f else 20f } } + repeat(7) { frame -> + cache.beginFrame(setOf(1)) + // The box moves 5 px right each frame; cached landmarks move with it. + val left = 5f * frame + val points = checkNotNull(cache.landmarks(1, left, 0f, left + 40f, 40f, estimate)) + if (frame < 3) assertEquals(10f + 5f * frame, points[0], 1e-4f) + } + assertEquals(3, runs) // frames 1, 4, 7 + cache.beginFrame(emptySet()) + cache.beginFrame(setOf(1)) + cache.landmarks(1, 0f, 0f, 40f, 40f, estimate) + assertEquals(4, runs) // an ended track starts over + } + + @Test + fun failedRefreshKeepsBlurUntilTheNextRefresh() { + val cache = PrivacyAnimalLandmarkCache() + var runs = 0 + val results = mutableListOf() + repeat(4) { + cache.beginFrame(setOf(7)) + results += cache.landmarks(7, 0f, 0f, 10f, 10f) { runs++; if (runs == 1) null else FloatArray(10) { 1f } } + } + assertArrayEquals(arrayOf(null, null, null), results.take(3).toTypedArray()) + assertTrue(results[3] != null) + assertEquals(2, runs) + } + + @Test + fun animalsAreFixedPerSessionOnlyForOnDeviceProcessing() { + assertEquals("rabbit", sessionAnimal(FaceAnonymizationMethod.RABBIT, onDeviceProcessing = true)) + assertNull(sessionAnimal(FaceAnonymizationMethod.OWL, onDeviceProcessing = false)) + assertNull(sessionAnimal(FaceAnonymizationMethod.FACE_SWAP, onDeviceProcessing = true)) + assertEquals(FaceAnonymizationMethod.OWL, FaceAnonymizationMethod.from("owl")) + assertNull(FaceAnonymizationMethod.from("cat")) + } +} diff --git a/apps/ios/InnoLive/InnoLive/Features/OnDevicePrivacy/PrivacyAnimalSprite.swift b/apps/ios/InnoLive/InnoLive/Features/OnDevicePrivacy/PrivacyAnimalSprite.swift new file mode 100644 index 000000000..4e804fd95 --- /dev/null +++ b/apps/ios/InnoLive/InnoLive/Features/OnDevicePrivacy/PrivacyAnimalSprite.swift @@ -0,0 +1,277 @@ +import CoreImage +import Foundation + +/// On-device animal for the lab anonymization method. The lab picker is locked while a session exists, +/// so the animal stays fixed for the session like face swap. +nonisolated enum PrivacyAnimal: String, CaseIterable, Identifiable { + case blur, rabbit, owl + var id: String { rawValue } + static var selected: Self { + guard AIProcessingMode.selected == .onDevice else { return .blur } + switch FaceAnonymizationMode.selected { + case .rabbit: return .rabbit + case .owl: return .owl + case .blur, .faceSwap: return .blur + } + } +} + +/// Blender-rendered yaw views of one animal. Every cell is the same fixed orthographic camera frame. Each view +/// carries the ArcFace 5-point template projected onto the animal head, so a face aligns like the server's InSwapper crop. +nonisolated struct PrivacyAnimalAtlas { + /// `cell` is atlas pixels; `landmarks` are normalized to the cell, both top-left, in YuNet order. + struct View { let yaw: Double; let cell: CGRect; let landmarks: [CGPoint] } + /// Similarity from (mirrored) top-left cell pixels to top-left frame pixels: frame = [a -b; b a] * cell + t. + struct Placement { + let from: View, to: View, blend: Double, mirrored: Bool + let a: Double, b: Double, tx: Double, ty: Double + } + + let size: CGSize + let views: [View] + let mirrorNegativeYaw: Bool + + private struct Metadata: Decodable { + struct Entry: Decodable { let yaw: Double; let cell: [Double]; let landmarks: [[Double]] } + let version: Int; let width: Int; let height: Int; let views: [Entry]; let mirrorNegativeYaw: Bool? + } + + init?(json: Data, imageSize: CGSize) { + guard let metadata = try? JSONDecoder().decode(Metadata.self, from: json), metadata.version >= 2, + CGFloat(metadata.width) == imageSize.width, CGFloat(metadata.height) == imageSize.height else { return nil } + var views: [View] = [] + for entry in metadata.views { + let c = entry.cell + guard c.count == 4, c[0] >= 0, c[1] >= 0, c[2] > 0, c[3] > 0, c[0] + c[2] <= imageSize.width, + c[1] + c[3] <= imageSize.height, entry.landmarks.count == 5, + entry.landmarks.allSatisfy({ $0.count == 2 && $0.allSatisfy { $0.isFinite && $0 >= 0 && $0 <= 1 } }) else { return nil } + views.append(View(yaw: entry.yaw, cell: CGRect(x: c[0], y: c[1], width: c[2], height: c[3]), + landmarks: entry.landmarks.map { CGPoint(x: $0[0], y: $0[1]) })) + } + guard let first = views.first, first.yaw == 0, zip(views, views.dropFirst()).allSatisfy({ $0.yaw < $1.yaw }), + views.allSatisfy({ $0.cell.size == first.cell.size }) else { return nil } + size = imageSize + self.views = views + mirrorNegativeYaw = metadata.mirrorNegativeYaw == true + } + + /// `landmarks` are top-left frame pixels in YuNet order. + func place(landmarks: [CGPoint], yaw: Double) -> Placement? { + guard landmarks.count == 5, yaw.isFinite, let last = views.last else { return nil } + let mirrored = mirrorNegativeYaw && yaw < 0 + let magnitude = min(max(mirrorNegativeYaw ? abs(yaw) : yaw, 0), last.yaw) + let lower = views.lastIndex(where: { $0.yaw <= magnitude }) ?? 0 + let upper = min(lower + 1, views.count - 1) + let t = upper == lower ? 0 : (magnitude - views[lower].yaw) / (views[upper].yaw - views[lower].yaw) + // One crisp view almost everywhere; a wide cross-fade would ghost both views' eyes at the midpoint. + let s = min(max((t - 0.45) / 0.1, 0), 1), blend = s * s * (3 - 2 * s) + let cell = views[lower].cell.size + var reference: [CGPoint] = [] + for (a, b) in zip(views[lower].landmarks, views[upper].landmarks) { + let x: CGFloat = (a.x + (b.x - a.x) * blend) * cell.width + let y: CGFloat = (a.y + (b.y - a.y) * blend) * cell.height + reference.append(CGPoint(x: x, y: y)) + } + // A mirrored cell swaps which eye and mouth corner lies on the image-left side. + if mirrored { + let source = reference + reference = [1, 0, 2, 4, 3].map { (index: Int) -> CGPoint in CGPoint(x: cell.width - source[index].x, y: source[index].y) } + } + guard let (a, b, tx, ty) = Self.similarity(reference, landmarks) else { return nil } + return Placement(from: views[lower], to: views[upper], blend: blend, mirrored: mirrored, a: a, b: b, tx: tx, ty: ty) + } + + /// Least-squares similarity (rotation, uniform scale, translation) mapping `source` onto `target`. + static func similarity(_ source: [CGPoint], _ target: [CGPoint]) -> (Double, Double, Double, Double)? { + guard source.count == target.count, !source.isEmpty else { return nil } + let n = Double(source.count) + let sx = source.reduce(0) { $0 + Double($1.x) } / n, sy = source.reduce(0) { $0 + Double($1.y) } / n + let dx = target.reduce(0) { $0 + Double($1.x) } / n, dy = target.reduce(0) { $0 + Double($1.y) } / n + var norm = 0.0, dot = 0.0, cross = 0.0 + for (p, q) in zip(source, target) { + let px = Double(p.x) - sx, py = Double(p.y) - sy, qx = Double(q.x) - dx, qy = Double(q.y) - dy + norm += px * px + py * py; dot += px * qx + py * qy; cross += px * qy - py * qx + } + guard norm >= 1e-6 else { return nil } + let a = dot / norm, b = cross / norm + guard a.isFinite, b.isFinite, hypot(a, b) >= 1e-6 else { return nil } + return (a, b, dx - (a * sx - b * sy), dy - (b * sx + a * sy)) + } +} + +/// Head yaw from YuNet's five landmarks; only used to pick the animal's yaw view. +nonisolated enum PrivacyAnimalPose { + /// Nose tip depth in front of the eye line, in interocular distances, for an average adult face. + static let noseDepth = 0.55 + + /// Top-left landmarks in YuNet order. The nose shifts along the eye line by depth * sin(yaw) while the eye + /// distance shrinks by cos(yaw), so their ratio is depth * tan(yaw). Positive yaw turns the nose toward image right. + static func yaw(_ points: [CGPoint]) -> Double? { + guard points.count == 5, points.allSatisfy({ $0.x.isFinite && $0.y.isFinite }) else { return nil } + var axis = CGVector(dx: points[1].x - points[0].x, dy: points[1].y - points[0].y) + // Measure along the eye line pointing toward image right, whichever eye YuNet listed first. + if axis.dx < 0 { axis = CGVector(dx: -axis.dx, dy: -axis.dy) } + let distance = hypot(axis.dx, axis.dy) + guard distance >= 1e-3 else { return nil } + let nose = CGVector(dx: points[2].x - (points[0].x + points[1].x) / 2, dy: points[2].y - (points[0].y + points[1].y) / 2) + let along = (nose.dx * axis.dx + nose.dy * axis.dy) / distance + return atan2(Double(along / distance), noseDepth) * 180 / .pi + } +} + +/// Face-swap target rules (innolive-ai `InSwapperRenderer.swap`) applied to on-device animal overlays. +nonisolated enum PrivacyAnimalTargets { + static let roiPadding: CGFloat = 0.3 + static let minIoU: CGFloat = 0.25 + static let minScore: Float = 0.65 + /// `face_too_small`: a bbox whose shorter side is under 16 px stays blurred. + static let minFace: CGFloat = 16 + + /// Padded integer ROI clipped to the frame (top-left), or nil when the face is too small or outside it. + static func roi(_ box: CGRect, frame: CGSize) -> CGRect? { + guard min(box.width, box.height) >= minFace else { return nil } + let pad = max(box.width, box.height) * roiPadding + let left = max(0, (box.minX - pad).rounded(.towardZero)), top = max(0, (box.minY - pad).rounded(.towardZero)) + let right = min(frame.width, (box.maxX + pad).rounded(.towardZero)), bottom = min(frame.height, (box.maxY + pad).rounded(.towardZero)) + guard right > left, bottom > top else { return nil } + return CGRect(x: left, y: top, width: right - left, height: bottom - top) + } + + /// Landmarks of the candidate overlapping the tracked box by more than `minIoU`, or nil. + static func match(_ box: CGRect, candidates: [(box: CGRect, landmarks: [CGPoint])]) -> [CGPoint]? { + var best: [CGPoint]?, bestIoU = minIoU + for candidate in candidates { + let overlap = box.intersection(candidate.box) + let intersection = overlap.isNull ? 0 : overlap.width * overlap.height + let iou = intersection / max(box.width * box.height + candidate.box.width * candidate.box.height - intersection, 1) + if iou > bestIoU { best = candidate.landmarks; bestIoU = iou } + } + return best + } +} + +/// Per-track landmarks refreshed every `refreshFrames` processed frames, counted like the server's 30-frame +/// metadata cache. Between refreshes they follow the tracked box; a failed refresh keeps the blur until the next one. +nonisolated final class PrivacyAnimalLandmarkCache { + static let refreshFrames = 3 + private struct Entry { let refreshedAt: Int; let relative: [CGPoint]? } + private var entries: [Int: Entry] = [:] + private var frame = 0 + private let refreshFrames: Int + + init(refreshFrames: Int = PrivacyAnimalLandmarkCache.refreshFrames) { self.refreshFrames = refreshFrames } + + /// Starts a frame; tracks no longer present drop their landmarks. + func beginFrame(liveTracks: Set) { frame += 1; entries = entries.filter { liveTracks.contains($0.key) } } + + /// Top-left frame landmarks for `track` in `box`, or nil to keep the blur. + func landmarks(track: Int, box: CGRect, estimate: () -> [CGPoint]?) -> [CGPoint]? { + let relative: [CGPoint]? + if let cached = entries[track], frame - cached.refreshedAt < refreshFrames { + relative = cached.relative + } else { + relative = estimate()?.map { CGPoint(x: ($0.x - box.minX) / box.width, y: ($0.y - box.minY) / box.height) } + entries[track] = Entry(refreshedAt: frame, relative: relative) + } + return relative?.map { CGPoint(x: box.minX + $0.x * box.width, y: box.minY + $0.y * box.height) } + } + + func reset() { entries.removeAll(); frame = 0 } +} + +/// Queue-owned compositor. Like face swap, an animal replaces a face only inside its protected mask; untracked, +/// tiny or unmatched faces keep the blur. Missing assets always retain the already protected image. +nonisolated final class PrivacyAnimalSprite { + static let poseInput: CGFloat = 160 + private var unavailable: Set = [] + private var cached: [PrivacyAnimal: (CIImage, PrivacyAnimalAtlas)] = [:] + private let cache = PrivacyAnimalLandmarkCache() + private var detector: PrivacyYuNetDetector? + private var detectorUnavailable = false + + private func asset(_ animal: PrivacyAnimal) -> (CIImage, PrivacyAnimalAtlas)? { + if let asset = cached[animal] { return asset } + guard animal != .blur, !unavailable.contains(animal) else { return nil } + unavailable.insert(animal) + guard let png = Bundle.main.url(forResource: animal.rawValue, withExtension: "png"), + let json = Bundle.main.url(forResource: animal.rawValue, withExtension: "json"), + let data = try? Data(contentsOf: json), let image = CIImage(contentsOf: png), + let atlas = PrivacyAnimalAtlas(json: data, imageSize: image.extent.size) else { return nil } + unavailable.remove(animal) + cached[animal] = (image, atlas) + return (image, atlas) + } + + /// `faces` are bottom-left Core Image boxes with their confirmed BoT-SORT track (nil when untracked). + /// `source` is the unblurred frame for landmarks; `mask` is the protected-region mask used for the blur. + func composite(_ protected: CIImage, faces: [(box: CGRect, track: Int?)], source: CIImage, mask: CIImage, + animal: PrivacyAnimal = .selected) -> CIImage { + guard animal != .blur, !faces.isEmpty, !detectorUnavailable, let (sprite, atlas) = asset(animal) else { return protected } + let frame = source.extent.size + cache.beginFrame(liveTracks: Set(faces.compactMap(\.track))) + let animals = faces.reduce(protected) { output, face -> CIImage in + // Untracked faces keep the blur, as face swap does. + guard let track = face.track else { return output } + let box = CGRect(x: face.box.minX, y: frame.height - face.box.maxY, width: face.box.width, height: face.box.height) + guard let roi = PrivacyAnimalTargets.roi(box, frame: frame), + let landmarks = cache.landmarks(track: track, box: box, estimate: { self.landmarks(source, box: box, roi: roi) }), + let yaw = PrivacyAnimalPose.yaw(landmarks), + let place = atlas.place(landmarks: landmarks, yaw: yaw) else { return output } + return layer(sprite, atlas, place, frameHeight: frame.height).composited(over: output) + } + guard animals !== protected else { return protected } + // Inside the protected mask only, like face swap's polygon; uncovered mask pixels keep the blur. + return animals.applyingFilter("CIBlendWithMask", parameters: [kCIInputBackgroundImageKey: protected, + kCIInputMaskImageKey: mask]) + .cropped(to: protected.extent) + } + + func reset() { cache.reset() } + + private func layer(_ sprite: CIImage, _ atlas: PrivacyAnimalAtlas, _ place: PrivacyAnimalAtlas.Placement, + frameHeight: CGFloat) -> CIImage { + func cell(_ view: PrivacyAnimalAtlas.View) -> CIImage { + let rect = CGRect(x: view.cell.minX, y: atlas.size.height - view.cell.maxY, width: view.cell.width, height: view.cell.height) + return sprite.cropped(to: rect).transformed(by: CGAffineTransform(translationX: -rect.minX, y: -rect.minY)) + } + let size = place.from.cell.size + var canvas = cell(place.from) + if place.blend > 0 { + canvas = canvas.applyingFilter("CIDissolveTransition", parameters: [kCIInputTargetImageKey: cell(place.to), + kCIInputTimeKey: place.blend]) + } + // Core Image cell (y-up) -> top-left cell -> mirror -> landmark similarity -> top-left frame -> Core Image frame. + var transform = CGAffineTransform(a: 1, b: 0, c: 0, d: -1, tx: 0, ty: size.height) + if place.mirrored { transform = transform.concatenating(CGAffineTransform(a: -1, b: 0, c: 0, d: 1, tx: size.width, ty: 0)) } + transform = transform + .concatenating(CGAffineTransform(a: place.a, b: place.b, c: -place.b, d: place.a, tx: place.tx, ty: place.ty)) + .concatenating(CGAffineTransform(a: 1, b: 0, c: 0, d: -1, tx: 0, ty: frameHeight)) + return canvas.transformed(by: transform) + } + + private func landmarks(_ source: CIImage, box: CGRect, roi: CGRect) -> [CGPoint]? { + // The top-left ROI sits at the top-left of a square input scaled to poseInput; the rest is frame or black. + let frameHeight = source.extent.height + let side = max(roi.width, roi.height), scale = side / Self.poseInput + let square = CGRect(x: roi.minX, y: frameHeight - roi.minY - side, width: side, height: side) + let crop = source.cropped(to: square).composited(over: CIImage(color: .black).cropped(to: square)) + .transformed(by: CGAffineTransform(translationX: -square.minX, y: -square.minY) + .concatenating(CGAffineTransform(scaleX: 1 / scale, y: 1 / scale))) + do { + let model = try detector ?? PrivacyYuNetDetector() + detector = model + let candidates = try model.faces(in: crop, size: CGSize(width: Self.poseInput, height: Self.poseInput)) + .filter { $0.score >= PrivacyAnimalTargets.minScore } + .map { face in + (box: CGRect(x: roi.minX + face.box.minX * scale, y: roi.minY + face.box.minY * scale, + width: face.box.width * scale, height: face.box.height * scale), + landmarks: face.landmarks.map { CGPoint(x: roi.minX + $0.x * scale, y: roi.minY + $0.y * scale) }) + } + return PrivacyAnimalTargets.match(box, candidates: candidates) + } catch { + // `swapper_unavailable`: every face keeps the blur. + detectorUnavailable = true + return nil + } + } +} diff --git a/apps/ios/InnoLive/InnoLive/Features/OnDevicePrivacy/PrivacyCamera.swift b/apps/ios/InnoLive/InnoLive/Features/OnDevicePrivacy/PrivacyCamera.swift index b54ebb5f7..7b2c3cfe2 100644 --- a/apps/ios/InnoLive/InnoLive/Features/OnDevicePrivacy/PrivacyCamera.swift +++ b/apps/ios/InnoLive/InnoLive/Features/OnDevicePrivacy/PrivacyCamera.swift @@ -104,7 +104,9 @@ nonisolated final class PrivacyCamera: NSObject, AVCaptureVideoDataOutputSampleB guard session.isRunning, let model, let buffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return } autoreleasepool { do { + let animalSelection = PrivacyAnimal.selected let (image, count, timings, faces) = try model.process(buffer) + guard PrivacyAnimal.selected == animalSelection else { return } let completed = ProcessInfo.processInfo.systemUptime let fps = lastCompleted == 0 ? 0 : 1 / (completed - lastCompleted) lastCompleted = completed diff --git a/apps/ios/InnoLive/InnoLive/Features/OnDevicePrivacy/PrivacyModel.swift b/apps/ios/InnoLive/InnoLive/Features/OnDevicePrivacy/PrivacyModel.swift index ae4ff6a80..455bbd730 100644 --- a/apps/ios/InnoLive/InnoLive/Features/OnDevicePrivacy/PrivacyModel.swift +++ b/apps/ios/InnoLive/InnoLive/Features/OnDevicePrivacy/PrivacyModel.swift @@ -22,6 +22,7 @@ nonisolated final class PrivacyModel { private let detectionsName: String private let prototypesName: String private let stabilizer = PrivacyMaskStabilizer() + private let animalSprite = PrivacyAnimalSprite() private let tracker = PrivacyBoTSORT() private let cameraMotion = PrivacyCameraMotion() private var previousExtent: CGRect? @@ -59,6 +60,7 @@ nonisolated final class PrivacyModel { stabilizer.reset() tracker.reset() cameraMotion.reset() + animalSprite.reset() faces.reset() previousExtent = nil previousTime = nil @@ -140,8 +142,20 @@ nonisolated final class PrivacyModel { y: size.height / layout.resized.height)) .cropped(to: original.extent) let blurred = try broadcastStyleBlur(original) - let result = blurred.applyingFilter("CIBlendWithMask", parameters: [kCIInputBackgroundImageKey: original, + var result = blurred.applyingFilter("CIBlendWithMask", parameters: [kCIInputBackgroundImageKey: original, kCIInputMaskImageKey: mask]) + if PrivacyAnimal.selected != .blur { + let animalFaces = protectedIndices.filter { objects[$0].classID == 0 && !allowed.contains($0) }.map { index in + let box = objects[index].box + return (box: CGRect(x: (box.minX - layout.left) * size.width / layout.resized.width, + y: (640 - box.maxY - layout.bottom) * size.height / layout.resized.height, + width: box.width * size.width / layout.resized.width, + height: box.height * size.height / layout.resized.height), + track: trackIDs[index]) + } + // Landmarks read the unblurred frame; the animal stays inside the protected mask like face swap. + result = animalSprite.composite(result, faces: animalFaces, source: original, mask: mask) + } // Frame only after all original-coordinate AI work and mask composition. Both // local CGImage preview and direct CVPixelBuffer uplink share this same output. let rendered = try render(PrivacyOutputFraming.apply(to: result.cropped(to: original.extent))) diff --git a/apps/ios/InnoLive/InnoLive/Features/OnDevicePrivacy/PrivacyUplinkProcessor.swift b/apps/ios/InnoLive/InnoLive/Features/OnDevicePrivacy/PrivacyUplinkProcessor.swift index b656beed1..e220f6d29 100644 --- a/apps/ios/InnoLive/InnoLive/Features/OnDevicePrivacy/PrivacyUplinkProcessor.swift +++ b/apps/ios/InnoLive/InnoLive/Features/OnDevicePrivacy/PrivacyUplinkProcessor.swift @@ -108,15 +108,16 @@ nonisolated final class PrivacyUplinkProcessor: @unchecked Sendable { func submit(_ frame: LKRTCVideoFrame, deliver: @escaping @Sendable (LKRTCVideoFrame) -> Void) { let payload = FramePayload(frame: frame) let libraryRevision = PrivacyFaceLibrary.revision + let animalSelection = PrivacyAnimal.selected let admission = gate.enqueue { [self] ticket in PrivacyPipelineProbe.count("gate_admitted") - process(payload, ticket: ticket, libraryRevision: libraryRevision, deliver: deliver) + process(payload, ticket: ticket, libraryRevision: libraryRevision, animalSelection: animalSelection, deliver: deliver) } if admission == .replaced || admission == .stopped { PrivacyPipelineProbe.count("gate_rejected") } if admission == .queued { PrivacyPipelineProbe.count("gate_queued") } } - private func process(_ payload: FramePayload, ticket: PrivacyFrameGate.Ticket, libraryRevision: Int, + private func process(_ payload: FramePayload, ticket: PrivacyFrameGate.Ticket, libraryRevision: Int, animalSelection: PrivacyAnimal, deliver: @escaping @Sendable (LKRTCVideoFrame) -> Void) { queue.async { [self] in autoreleasepool { @@ -156,6 +157,7 @@ nonisolated final class PrivacyUplinkProcessor: @unchecked Sendable { result = payload.frame } gate.finish(ticket) { + guard PrivacyAnimal.selected == animalSelection else { return } guard !ticket.enabled || libraryRevision == PrivacyFaceLibrary.revision else { return } #if DEBUG PrivacyUplinkValidationMetrics.recordProcessed(enabled: ticket.enabled) diff --git a/apps/ios/InnoLive/InnoLive/Features/OnDevicePrivacy/PrivacyYuNetDetector.swift b/apps/ios/InnoLive/InnoLive/Features/OnDevicePrivacy/PrivacyYuNetDetector.swift index 59b6552e0..4a0ad37ba 100644 --- a/apps/ios/InnoLive/InnoLive/Features/OnDevicePrivacy/PrivacyYuNetDetector.swift +++ b/apps/ios/InnoLive/InnoLive/Features/OnDevicePrivacy/PrivacyYuNetDetector.swift @@ -114,7 +114,25 @@ nonisolated final class PrivacyYuNetDetector { } func face(in image: CGImage, enrollment: Bool) throws -> PrivacyYuNetDecoding.Face { - let layout = PrivacyYuNetDecoding.InputLayout(size: CGSize(width: image.width, height: image.height), enrollment: enrollment) + try face(in: CIImage(cgImage: image), size: CGSize(width: image.width, height: image.height), enrollment: enrollment) + } + + /// `image` covers (0, 0, size); returned coordinates are top-left pixels of that image. + func face(in image: CIImage, size: CGSize, enrollment: Bool) throws -> PrivacyYuNetDecoding.Face { + let faces = try detect(in: image, size: size, enrollment: enrollment) + guard faces.count <= 1 else { throw PrivacyFaceSampleError.multipleFaces } + guard let face = faces.first else { throw PrivacyFaceSampleError.faceCount } + guard min(face.box.width, face.box.height) >= (enrollment ? 40 : 24) else { throw PrivacyFaceSampleError.size } + return face + } + + /// Every live-query face, for callers that pick one by overlap like the server's face swap. + func faces(in image: CIImage, size: CGSize) throws -> [PrivacyYuNetDecoding.Face] { + try detect(in: image, size: size, enrollment: false) + } + + private func detect(in image: CIImage, size: CGSize, enrollment: Bool) throws -> [PrivacyYuNetDecoding.Face] { + let layout = PrivacyYuNetDecoding.InputLayout(size: size, enrollment: enrollment) let width = layout.width, height = layout.height guard width <= 2048, height <= 2048 else { throw PrivacyFaceSampleError.size } if input == nil || CVPixelBufferGetWidth(input!) != width || CVPixelBufferGetHeight(input!) != height { @@ -127,9 +145,9 @@ nonisolated final class PrivacyYuNetDetector { } guard let input else { throw PrivacyModelError.imageBuffer } let bounds = CGRect(x: 0, y: 0, width: width, height: height) - let topAligned = CIImage(cgImage: image) - .transformed(by: CGAffineTransform(scaleX: layout.resized.width / CGFloat(image.width), - y: layout.resized.height / CGFloat(image.height))) + let topAligned = image + .transformed(by: CGAffineTransform(scaleX: layout.resized.width / size.width, + y: layout.resized.height / size.height)) .transformed(by: CGAffineTransform(translationX: 0, y: CGFloat(height) - layout.resized.height)) context.render(topAligned.composited(over: CIImage(color: .black).cropped(to: bounds)), to: input, bounds: bounds, colorSpace: CGColorSpaceCreateDeviceRGB()) @@ -143,12 +161,7 @@ nonisolated final class PrivacyYuNetDetector { } } // Server runs NMS at 0.6, then applies the higher enrollment score threshold. - let faces = try PrivacyYuNetDecoding.decode(outputs: values, width: width, height: height) - .filter { $0.score >= (enrollment ? 0.9 : 0.6) } - guard faces.count <= 1 else { throw PrivacyFaceSampleError.multipleFaces } - guard let detected = faces.first else { throw PrivacyFaceSampleError.faceCount } - let face = layout.restore(detected) - guard min(face.box.width, face.box.height) >= (enrollment ? 40 : 24) else { throw PrivacyFaceSampleError.size } - return face + return try PrivacyYuNetDecoding.decode(outputs: values, width: width, height: height) + .filter { $0.score >= (enrollment ? 0.9 : 0.6) }.map(layout.restore) } } diff --git a/apps/ios/InnoLive/InnoLive/Features/Settings/AIProcessingMode.swift b/apps/ios/InnoLive/InnoLive/Features/Settings/AIProcessingMode.swift index 963af68e1..d3c1ac882 100644 --- a/apps/ios/InnoLive/InnoLive/Features/Settings/AIProcessingMode.swift +++ b/apps/ios/InnoLive/InnoLive/Features/Settings/AIProcessingMode.swift @@ -47,11 +47,13 @@ nonisolated enum OnDeviceTransport: String, CaseIterable, Codable, Sendable, Ide } } -/// 서버 AI 세션의 얼굴 비식별화 방법이다. 합성은 관리자 전용 실험 기능이라 실험실에서만 고른다. -/// 세션 생성 때 고정되므로 다음 방송 준비부터 적용한다. +/// 얼굴 비식별화 방법이다. 합성·동물은 관리자 전용 실험 기능이라 실험실에서만 고른다. 합성은 서버 AI, +/// 토끼·부엉이는 온디바이스 처리에서만 쓰고 나머지는 블러다. 세션 생성 때 고정되므로 다음 방송 준비부터 적용한다. nonisolated enum FaceAnonymizationMode: String, CaseIterable, Codable, Sendable, Identifiable { case blur case faceSwap = "face_swap" + case rabbit + case owl static let storageKey = "com.framework.innolive.face-anonymization-mode" static var selected: FaceAnonymizationMode { @@ -62,6 +64,8 @@ nonisolated enum FaceAnonymizationMode: String, CaseIterable, Codable, Sendable, switch self { case .blur: String(localized: "블러") case .faceSwap: String(localized: "합성") + case .rabbit: String(localized: "토끼") + case .owl: String(localized: "부엉이") } } } diff --git a/apps/ios/InnoLive/InnoLive/Features/Settings/SettingsView.swift b/apps/ios/InnoLive/InnoLive/Features/Settings/SettingsView.swift index 0eaa23c83..33a9bb012 100644 --- a/apps/ios/InnoLive/InnoLive/Features/Settings/SettingsView.swift +++ b/apps/ios/InnoLive/InnoLive/Features/Settings/SettingsView.swift @@ -121,7 +121,7 @@ struct SettingsView: View { Text(youtube.session != nil ? String(localized: "방송 준비를 취소한 뒤 비식별화 방법을 변경할 수 있습니다.") - : String(localized: "다음 방송 준비부터 적용됩니다. 합성은 서버 AI 처리에서만 사용됩니다.")) + : String(localized: "다음 방송 준비부터 적용됩니다. 합성은 서버 AI 처리, 토끼·부엉이는 온디바이스 처리에서만 사용됩니다.")) .font(.footnote) .foregroundStyle(.secondary) } diff --git a/apps/ios/InnoLive/InnoLive/Resources/owl.json b/apps/ios/InnoLive/InnoLive/Resources/owl.json new file mode 100644 index 000000000..fa57d82ff --- /dev/null +++ b/apps/ios/InnoLive/InnoLive/Resources/owl.json @@ -0,0 +1,392 @@ +{ + "id": "owl", + "version": 2, + "image": "owl.png", + "width": 2240, + "height": 928, + "coordinateSystem": "normalized-top-left", + "cellFaceRect": "normalized within each cell; excludes ears and tufts", + "cellLandmarks": "ArcFace 5-point template on the head, YuNet order, normalized within each cell; align by similarity transform", + "yawConvention": "positive yaw turns the muzzle toward image right; mirror the cell for negative yaw", + "mirrorNegativeYaw": true, + "faceRect": [ + 0.08727630972862246, + 0.158038765705865, + 0.8254470601677895, + 0.7895690079393058 + ], + "views": [ + { + "yaw": 0, + "cell": [ + 0, + 0, + 448, + 464 + ], + "faceRect": [ + 0.08727630972862246, + 0.158038765705865, + 0.8254470601677895, + 0.7895690079393058 + ], + "landmarks": [ + [ + 0.361785708793572, + 0.4594094695459152 + ], + [ + 0.6382142912064279, + 0.4594094695459152 + ], + [ + 0.5024878571358775, + 0.6120742996191156 + ], + [ + 0.3858349988503116, + 0.7682087739993785 + ], + [ + 0.6147178539208004, + 0.7668742907458338 + ] + ] + }, + { + "yaw": 10, + "cell": [ + 448, + 0, + 448, + 464 + ], + "faceRect": [ + 0.07408981344529564, + 0.158038765705865, + 0.8321293496659823, + 0.7895690079393058 + ], + "landmarks": [ + [ + 0.40678336817239014, + 0.4594094695459152 + ], + [ + 0.6790124183254582, + 0.4594094695459152 + ], + [ + 0.5502989497035742, + 0.6120742996191156 + ], + [ + 0.4228486051516874, + 0.7682087739993785 + ], + [ + 0.648360600695014, + 0.7668742907458338 + ] + ] + }, + { + "yaw": 20, + "cell": [ + 896, + 0, + 448, + 464 + ], + "faceRect": [ + 0.06420661189726425, + 0.158038765705865, + 0.8603636750153132, + 0.7895690079393058 + ], + "landmarks": [ + [ + 0.4546133710869721, + 0.4594094695459152 + ], + [ + 0.7143713336970124, + 0.4594094695459152 + ], + [ + 0.59658172460539, + 0.6120742996191156 + ], + [ + 0.46220640651881695, + 0.7682087739993785 + ], + [ + 0.6774955000728369, + 0.7668742907458338 + ] + ] + }, + { + "yaw": 30, + "cell": [ + 1344, + 0, + 448, + 464 + ], + "faceRect": [ + 0.0570419292364802, + 0.158038765705865, + 0.8781102970242499, + 0.7895690079393058 + ], + "landmarks": [ + [ + 0.5038224280412708, + 0.4594094695459152 + ], + [ + 0.7432166827576501, + 0.4594094695459152 + ], + [ + 0.6399299220315048, + 0.6120742996191156 + ], + [ + 0.5027125576244933, + 0.7682087739993785 + ], + [ + 0.7012372873723507, + 0.7668742907458338 + ] + ] + }, + { + "yaw": 40, + "cell": [ + 1792, + 0, + 448, + 464 + ], + "faceRect": [ + 0.05215570756367278, + 0.158038765705865, + 0.8805889008300644, + 0.7895690079393058 + ], + "landmarks": [ + [ + 0.5529153376285519, + 0.4594094695459152 + ], + [ + 0.7646720111370087, + 0.4594094695459152 + ], + [ + 0.6790264118462801, + 0.6120742996191156 + ], + [ + 0.5431362693863255, + 0.7682087739993785 + ], + [ + 0.7188645697065762, + 0.7668742907458338 + ] + ] + }, + { + "yaw": 50, + "cell": [ + 0, + 464, + 448, + 464 + ], + "faceRect": [ + 0.05013899292264669, + 0.158038765705865, + 0.8701175823807716, + 0.7895690079393058 + ], + "landmarks": [ + [ + 0.6004004340086665, + 0.4594094695459152 + ], + [ + 0.778085470199585, + 0.4594094695459152 + ], + [ + 0.7126833008868354, + 0.6120742996191156 + ], + [ + 0.5822493271636111, + 0.7682087739993785 + ], + [ + 0.7298417831105846, + 0.7668742907458338 + ] + ] + }, + { + "yaw": 60, + "cell": [ + 448, + 464, + 448, + 464 + ], + "faceRect": [ + 0.0501214180673872, + 0.158038765705865, + 0.8587373601538794, + 0.7895690079393058 + ], + "landmarks": [ + [ + 0.644834931407656, + 0.4594094695459152 + ], + [ + 0.7830493828015668, + 0.4594094695459152 + ], + [ + 0.7398778950529439, + 0.6120742996191156 + ], + [ + 0.6188633000212056, + 0.7682087739993785 + ], + [ + 0.7338353485933372, + 0.7668742907458338 + ] + ] + }, + { + "yaw": 70, + "cell": [ + 896, + 464, + 448, + 464 + ], + "faceRect": [ + 0.0499627409236772, + 0.158038765705865, + 0.845377678317683, + 0.7895690079393058 + ], + "landmarks": [ + [ + 0.6848686781844923, + 0.4594094695459152 + ], + [ + 0.7794130125216076, + 0.4594094695459152 + ], + [ + 0.7597839129822594, + 0.6120742996191156 + ], + [ + 0.6518656286810125, + 0.7682087739993785 + ], + [ + 0.7307239584624767, + 0.7668742907458338 + ] + ] + }, + { + "yaw": 80, + "cell": [ + 1344, + 464, + 448, + 464 + ], + "faceRect": [ + 0.05131737781422482, + 0.158038765705865, + 0.8197144764874662, + 0.7895690079393058 + ], + "landmarks": [ + [ + 0.7192852906882763, + 0.4594094695459152 + ], + [ + 0.767286797187158, + 0.4594094695459152 + ], + [ + 0.7717965096235275, + 0.6120742996191156 + ], + [ + 0.6802535967103073, + 0.7682087739993785 + ], + [ + 0.7206021233328751, + 0.7668742907458338 + ] + ] + }, + { + "yaw": 90, + "cell": [ + 1792, + 464, + 448, + 464 + ], + "faceRect": [ + 0.05080358790499827, + 0.158038765705865, + 0.8136273750237055, + 0.7895690079393058 + ], + "landmarks": [ + [ + 0.7470390307051795, + 0.4594094695459152 + ], + [ + 0.7470392137765884, + 0.4594094695459152 + ], + [ + 0.7755507550069264, + 0.6120742996191156 + ], + [ + 0.7031646860497338, + 0.7682087739993785 + ], + [ + 0.7037774260554994, + 0.7668742907458338 + ] + ] + } + ], + "fullFace": true, + "scope": "on-device-only" +} diff --git a/apps/ios/InnoLive/InnoLive/Resources/owl.png b/apps/ios/InnoLive/InnoLive/Resources/owl.png new file mode 100644 index 000000000..c97cb0dae Binary files /dev/null and b/apps/ios/InnoLive/InnoLive/Resources/owl.png differ diff --git a/apps/ios/InnoLive/InnoLive/Resources/rabbit.json b/apps/ios/InnoLive/InnoLive/Resources/rabbit.json new file mode 100644 index 000000000..a4f5169fd --- /dev/null +++ b/apps/ios/InnoLive/InnoLive/Resources/rabbit.json @@ -0,0 +1,392 @@ +{ + "id": "rabbit", + "version": 2, + "image": "rabbit.png", + "width": 2200, + "height": 1360, + "coordinateSystem": "normalized-top-left", + "cellFaceRect": "normalized within each cell; excludes ears and tufts", + "cellLandmarks": "ArcFace 5-point template on the head, YuNet order, normalized within each cell; align by similarity transform", + "yawConvention": "positive yaw turns the muzzle toward image right; mirror the cell for negative yaw", + "mirrorNegativeYaw": true, + "faceRect": [ + 0.08954505378549751, + 0.3972397373003118, + 0.820909612829035, + 0.5719661348006305 + ], + "views": [ + { + "yaw": 0, + "cell": [ + 0, + 0, + 440, + 680 + ], + "faceRect": [ + 0.08954505378549751, + 0.3972397373003118, + 0.820909612829035, + 0.5719661348006305 + ], + "landmarks": [ + [ + 0.3592727216807279, + 0.6373470544815063 + ], + [ + 0.6407272783192721, + 0.6373470544815063 + ], + [ + 0.5025330909019844, + 0.741518354065278 + ], + [ + 0.3837592715566809, + 0.8480571718776927 + ], + [ + 0.6168036330829967, + 0.8471465833046857 + ] + ] + }, + { + "yaw": 10, + "cell": [ + 440, + 0, + 440, + 680 + ], + "faceRect": [ + 0.07615375302054667, + 0.3972397373003118, + 0.8178246931596236, + 0.5719661348006305 + ], + "landmarks": [ + [ + 0.4030502132394097, + 0.6373470544815063 + ], + [ + 0.68022888248617, + 0.6373470544815063 + ], + [ + 0.5491962671279907, + 0.741518354065278 + ], + [ + 0.41959883841601286, + 0.8480571718776927 + ], + [ + 0.6492075221105056, + 0.8471465833046857 + ] + ] + }, + { + "yaw": 20, + "cell": [ + 880, + 0, + 440, + 680 + ], + "faceRect": [ + 0.06619165268811314, + 0.3972397373003118, + 0.8196260159665888, + 0.5719661348006305 + ], + "landmarks": [ + [ + 0.449773471057415, + 0.6373470544815063 + ], + [ + 0.7142542882399125, + 0.6373470544815063 + ], + [ + 0.5943646345626225, + 0.741518354065278 + ], + [ + 0.45788134011355314, + 0.8480571718776927 + ], + [ + 0.677077825773846, + 0.8471465833046857 + ] + ] + }, + { + "yaw": 30, + "cell": [ + 1320, + 0, + 440, + 680 + ], + "faceRect": [ + 0.05924955823204736, + 0.3972397373003118, + 0.8208307190374894, + 0.5719661348006305 + ], + "landmarks": [ + [ + 0.49802284376187755, + 0.6373470544815063 + ], + [ + 0.7417697212912819, + 0.6373470544815063 + ], + [ + 0.6366657858545129, + 0.741518354065278 + ], + [ + 0.4974436058239503, + 0.8480571718776927 + ], + [ + 0.6995677173137664, + 0.8471465833046857 + ] + ] + }, + { + "yaw": 40, + "cell": [ + 1760, + 0, + 440, + 680 + ], + "faceRect": [ + 0.05454132773659448, + 0.3972397373003118, + 0.8119241784919392, + 0.5719661348006305 + ], + "landmarks": [ + [ + 0.5463322896849026, + 0.6373470544815063 + ], + [ + 0.7619390845298767, + 0.6373470544815063 + ], + [ + 0.6748144057664004, + 0.741518354065278 + ], + [ + 0.5370835304260254, + 0.8480571718776927 + ], + [ + 0.7159938194534995, + 0.8471465833046857 + ] + ] + }, + { + "yaw": 50, + "cell": [ + 0, + 680, + 440, + 680 + ], + "faceRect": [ + 0.052754031528126126, + 0.3972397373003118, + 0.8101360900835558, + 0.5719661348006305 + ], + "landmarks": [ + [ + 0.593233938108791, + 0.6373470544815063 + ], + [ + 0.774149588021365, + 0.6373470544815063 + ], + [ + 0.7076514184474945, + 0.741518354065278 + ], + [ + 0.5755967020988464, + 0.8480571718776927 + ], + [ + 0.7258570811965249, + 0.8471465833046857 + ] + ] + }, + { + "yaw": 60, + "cell": [ + 440, + 680, + 440, + 680 + ], + "faceRect": [ + 0.05318042148243299, + 0.3972397373003118, + 0.8076980016448281, + 0.5719661348006305 + ], + "landmarks": [ + [ + 0.637302761186253, + 0.6373470544815063 + ], + [ + 0.7780302026055076, + 0.6373470544815063 + ], + [ + 0.7341790015047247, + 0.741518354065278 + ], + [ + 0.6118129367178137, + 0.8480571718776927 + ], + [ + 0.72885779467496, + 0.8471465833046857 + ] + ] + }, + { + "yaw": 70, + "cell": [ + 880, + 680, + 440, + 680 + ], + "faceRect": [ + 0.05324053547599101, + 0.3972397373003118, + 0.8038689965551549, + 0.5719661348006305 + ], + "landmarks": [ + [ + 0.6771996614607898, + 0.6373470544815063 + ], + [ + 0.7734629836949435, + 0.6373470544815063 + ], + [ + 0.753591184724461, + 0.741518354065278 + ], + [ + 0.644631740450859, + 0.8480571718776927 + ], + [ + 0.7249047636985779, + 0.8471465833046857 + ] + ] + }, + { + "yaw": 80, + "cell": [ + 1320, + 680, + 440, + 680 + ], + "faceRect": [ + 0.05475428971377288, + 0.3972397373003118, + 0.8099361295049841, + 0.5719661348006305 + ], + "landmarks": [ + [ + 0.7117124449123036, + 0.6373470544815063 + ], + [ + 0.7605867060748014, + 0.6373470544815063 + ], + [ + 0.7652981286699122, + 0.741518354065278 + ], + [ + 0.6730559899048372, + 0.8480571718776927 + ], + [ + 0.7141181230545044, + 0.8471465833046857 + ] + ] + }, + { + "yaw": 90, + "cell": [ + 1760, + 680, + 440, + 680 + ], + "faceRect": [ + 0.05436364195563578, + 0.3972397373003118, + 0.8166640736839987, + 0.5719661348006305 + ], + "landmarks": [ + [ + 0.7397924835031683, + 0.6373470544815063 + ], + [ + 0.7397926699031483, + 0.6373470544815063 + ], + [ + 0.7689441355791958, + 0.741518354065278 + ], + [ + 0.6962220473722978, + 0.8480571718776927 + ], + [ + 0.6968256105076183, + 0.8471465833046857 + ] + ] + } + ], + "fullFace": true, + "scope": "on-device-only" +} diff --git a/apps/ios/InnoLive/InnoLive/Resources/rabbit.png b/apps/ios/InnoLive/InnoLive/Resources/rabbit.png new file mode 100644 index 000000000..a184a08ef Binary files /dev/null and b/apps/ios/InnoLive/InnoLive/Resources/rabbit.png differ diff --git a/apps/ios/InnoLive/InnoLiveTests/PrivacyAnimalSpriteTests.swift b/apps/ios/InnoLive/InnoLiveTests/PrivacyAnimalSpriteTests.swift new file mode 100644 index 000000000..c35142ed5 --- /dev/null +++ b/apps/ios/InnoLive/InnoLiveTests/PrivacyAnimalSpriteTests.swift @@ -0,0 +1,107 @@ +import XCTest +import CoreImage +@testable import InnoLive + +final class PrivacyAnimalSpriteTests: XCTestCase { + private func atlas(_ name: String) throws -> PrivacyAnimalAtlas { + let bundle = Bundle(for: PrivacyModel.self) + let json = try Data(contentsOf: XCTUnwrap(bundle.url(forResource: name, withExtension: "json"))) + let image = try XCTUnwrap(CIImage(contentsOf: XCTUnwrap(bundle.url(forResource: name, withExtension: "png")))) + return try XCTUnwrap(PrivacyAnimalAtlas(json: json, imageSize: image.extent.size)) + } + + /// ArcFace-like landmarks, eyes 100 px apart, rotated by `roll`. + private func face(_ center: CGPoint, roll: Double = 0) -> [CGPoint] { + let template: [(Double, Double)] = [(-0.5, 0), (0.5, 0), (0, 0.57), (-0.41, 1.16), (0.41, 1.16)] + return template.map { (x: Double, y: Double) -> CGPoint in + let dx: Double = 100 * (x * cos(roll) - y * sin(roll)) + let dy: Double = 100 * (x * sin(roll) + y * cos(roll)) + return CGPoint(x: center.x + CGFloat(dx), y: center.y + CGFloat(dy)) + } + } + + func testBlurAndNoFacesKeepProtectedImage() { + let compositor = PrivacyAnimalSprite() + let image = CIImage(color: .black).cropped(to: CGRect(x: 0, y: 0, width: 128, height: 128)) + XCTAssertTrue(compositor.composite(image, faces: [(CGRect(x: 32, y: 32, width: 64, height: 64), 1)], + source: image, mask: image, animal: .blur) === image) + XCTAssertTrue(compositor.composite(image, faces: [], source: image, mask: image, animal: .rabbit) === image) + // Untracked faces keep the blur, as face swap does. + XCTAssertTrue(compositor.composite(image, faces: [(CGRect(x: 32, y: 32, width: 64, height: 64), nil)], + source: image, mask: image, animal: .rabbit) === image) + } + + func testBundledAtlasesCarryYawViewsWithTemplateLandmarks() throws { + for name in ["rabbit", "owl"] { + let atlas = try atlas(name) + XCTAssertTrue(atlas.mirrorNegativeYaw) + XCTAssertEqual(atlas.views.map(\.yaw), stride(from: 0.0, through: 90, by: 10).map { $0 }) + let front = atlas.views[0].landmarks + XCTAssertEqual(front[0].y, front[1].y, accuracy: 1e-3) + XCTAssertGreaterThan(front[3].y, front[2].y) + } + } + + func testLandmarksAlignTheAnimalTemplateWithRoll() throws { + let atlas = try atlas("rabbit") + let landmarks = face(CGPoint(x: 400, y: 300), roll: 0.2) + let place = try XCTUnwrap(atlas.place(landmarks: landmarks, yaw: 0)) + let cell = place.from.cell.size, eye = place.from.landmarks[0] + let x = eye.x * cell.width, y = eye.y * cell.height + XCTAssertEqual(place.a * x - place.b * y + place.tx, landmarks[0].x, accuracy: 6) + XCTAssertEqual(place.b * x + place.a * y + place.ty, landmarks[0].y, accuracy: 6) + XCTAssertEqual(atan2(place.b, place.a), 0.2, accuracy: 0.03) + let mirrored = try XCTUnwrap(atlas.place(landmarks: landmarks, yaw: -50)) + XCTAssertTrue(mirrored.mirrored) + XCTAssertEqual(try XCTUnwrap(atlas.place(landmarks: landmarks, yaw: 25)).blend, 0.5, accuracy: 1e-9) + } + + func testFaceSwapTargetRules() { + let box = CGRect(x: 100, y: 100, width: 100, height: 120) + XCTAssertEqual(PrivacyAnimalTargets.roi(box, frame: CGSize(width: 1280, height: 250)), + CGRect(x: 64, y: 64, width: 172, height: 186)) + XCTAssertNil(PrivacyAnimalTargets.roi(CGRect(x: 0, y: 0, width: 15, height: 40), frame: CGSize(width: 100, height: 100))) + let near = [CGPoint](repeating: CGPoint(x: 1, y: 1), count: 5) + XCTAssertEqual(PrivacyAnimalTargets.match(box, candidates: [(CGRect(x: 300, y: 100, width: 100, height: 120), []), + (CGRect(x: 105, y: 110, width: 100, height: 115), near)]), near) + XCTAssertNil(PrivacyAnimalTargets.match(box, candidates: [(CGRect(x: 170, y: 100, width: 100, height: 120), near)])) + } + + func testLandmarkCacheRefreshesEveryThreeFramesAndFollowsTheBox() { + let cache = PrivacyAnimalLandmarkCache() + var runs = 0 + for frame in 0..<7 { + cache.beginFrame(liveTracks: [1]) + let box = CGRect(x: 5 * CGFloat(frame), y: 0, width: 40, height: 40) + let points = cache.landmarks(track: 1, box: box) { runs += 1; return [CGPoint](repeating: CGPoint(x: 10, y: 20), count: 5) } + if frame < 3 { XCTAssertEqual(points?[0].x ?? 0, 10 + 5 * CGFloat(frame), accuracy: 1e-6) } + } + XCTAssertEqual(runs, 3) + cache.beginFrame(liveTracks: []) + cache.beginFrame(liveTracks: [1]) + _ = cache.landmarks(track: 1, box: CGRect(x: 0, y: 0, width: 40, height: 40)) { runs += 1; return nil } + XCTAssertEqual(runs, 4) + cache.beginFrame(liveTracks: [1]) + XCTAssertNil(cache.landmarks(track: 1, box: CGRect(x: 0, y: 0, width: 40, height: 40)) { runs += 1; return [] }) + XCTAssertEqual(runs, 4, "A failed refresh keeps the blur until the next refresh") + } + + func testPoseRecoversYawSign() throws { + for yaw in [-70.0, -20, 0, 35, 60] { + let r = yaw * .pi / 180 + let template: [(Double, Double, Double)] = [(-0.5, 0, 0), (0.5, 0, 0), (0, 0.55, 0.4), (-0.35, 0.3, 0.8), (0.35, 0.3, 0.8)] + let points = template.map { (x: Double, depth: Double, y: Double) -> CGPoint in + let px: Double = 100 * (x * cos(r) + depth * sin(r)) + return CGPoint(x: px, y: 100 * y) + } + XCTAssertEqual(try XCTUnwrap(PrivacyAnimalPose.yaw(points)), yaw, accuracy: 0.5) + } + } + + func testAnimalsAreLabMethodsForOnDeviceOnly() { + XCTAssertEqual(FaceAnonymizationMode(rawValue: "rabbit"), .rabbit) + XCTAssertEqual(FaceAnonymizationMode.allCases.map(\.rawValue), ["blur", "face_swap", "rabbit", "owl"]) + XCTAssertNil(YouTubeSessionCreateRequest(mode: .server, faceAnonymization: .rabbit).faceAnonymization) + XCTAssertNil(YouTubeSessionCreateRequest(mode: .onDevice, faceAnonymization: .owl).faceAnonymization) + } +} diff --git a/assets-source/animal-filters/.gitignore b/assets-source/animal-filters/.gitignore new file mode 100644 index 000000000..db04dca22 --- /dev/null +++ b/assets-source/animal-filters/.gitignore @@ -0,0 +1,5 @@ +!/build/ +/build/* +!/build/source/ +/build/source/* +!/build/source/*.blend diff --git a/assets-source/animal-filters/README.md b/assets-source/animal-filters/README.md new file mode 100644 index 000000000..58ac9e41a --- /dev/null +++ b/assets-source/animal-filters/README.md @@ -0,0 +1,62 @@ +# InnoLive on-device animal filters + +Original Blender-authored rabbit and owl assets, created on the assistant's cloud computer with Blender 4.3.2. This package contains no downloaded artwork, textures, or models. + +## Runtime integration + +Use only these files in each native client: + +- `build/runtime/rabbit.png` and `build/runtime/rabbit.json` +- `build/runtime/owl.png` and `build/runtime/owl.json` + +Each PNG is a transparent RGBA yaw-view atlas (schema version 2) with sRGB-encoded color and straight alpha. Ten cells hold the same head at yaw 0°–90° in 10° steps, rendered by one fixed orthographic camera at one scale. Positive yaw turns the muzzle toward image right; clients mirror a cell for negative yaw, so the studio lights are left/right symmetric. Composite locally over the client's existing privacy blur. Do not upload camera frames or enable animal effects in server-AI mode. + +| 항목 | 토끼 | 부엉이 | +| --- | --- | --- | +| atlas | 2200 × 1360 | 2240 × 928 | +| cell | 440 × 680 | 448 × 464 | +| yaw views | 0–90°, 10° 간격 10개 | 0–90°, 10° 간격 10개 | + +Sidecar coordinates use a normalized 0–1 top-left origin: + +- `views[].cell`: `[x, y, width, height]` atlas pixels of one yaw view. +- `views[].faceRect`: `[x, y, width, height]` of the head, excluding ears/tufts, normalized inside that cell. +- `views[].landmarks`: the ArcFace 5-point template (InSwapper alignment) ray-cast onto the head and projected for that yaw, YuNet order, normalized inside that cell. The template is scaled by 0.72 around the head centre so the head spans a typical face. +- `faceRect`: the 0° view's face rectangle, kept for front-only readers. + +Clients align a cell like face swap: a least-squares similarity from the view's landmarks to the face's YuNet landmarks, clipped to the protected mask. They never stretch width and height separately, so side faces are not squashed. YuNet's five landmarks also give yaw from the nose offset along the eye line; neighbouring views cross-fade only within ±0.5° of their midpoint. Keep the existing blur underneath because a stylized silhouette is not a privacy guarantee on its own. + +## Editable 3D sources and archival exports + +`build/source/rabbit.blend` and `build/source/owl.blend` (Blender 5.2, zstd-compressed) contain individually named mesh parts, editable PBR materials, a transparent front orthographic camera, and a three-light studio setup. Eyes, muzzles, cheeks, beaks and feather marks are ray-cast onto a deep head shell and oriented to its surface normal, so they stay attached in side views; owl tufts are tapered 3D lofts. `render_pose_views.py` replaces the lights with a symmetric set and fits the camera to every yaw. + +`build/runtime/rabbit.glb` and `build/runtime/owl.glb` are optional self-contained glTF 2.0 archival exports, not required by the current sprite runtime. They have no textures, external buffers, rigs, animations, lights, or cameras. All material alpha modes are OPAQUE. + +- Rabbit: 5,310 authoring vertices; 10,484 triangles; 262,388-byte GLB. +- Owl: 5,607 authoring vertices; 11,022 triangles; 288,892-byte GLB. + +Coordinates: Blender +X right, +Z up, -Y toward camera. The GLB export converts to +X right, +Y up, +Z toward camera. One model unit is the distance between eye centers. The rabbit eye center elevation is 0.11, owl is 0.13. This is a stylized head asset, not a deformable face mesh or ARKit/ARCore rig. + +## Reproduce + +Requires Blender 4.3+; the current assets were generated with Blender 5.2.3 on macOS (Cycles, Metal GPU, denoising). No Blender add-on installation is required. + +```sh +blender -b --factory-startup --python generate_animals.py -- --output ./build --samples 96 +blender -b --factory-startup --python render_pose_views.py -- --output ./build --samples 128 --step 10 +python3 pack_pose_atlas.py ./build +python3 validate_assets.py ./build +``` + +The atlas packer and validator use Pillow. Generation itself uses Blender's built-in Python APIs only. Cycles CPU rendering does not require the optional denoiser. + +`build/asset-manifest.json` documents the package. `build/validation-report.json` records successful asset-level checks. Previews in `build/previews/` are three-quarter renders of the actual geometry, not hand-drawn or AI-generated substitutes. The PNG sprites have been visually inspected along with the 3D previews. + +## Verification limits + +Asset geometry, per-view clipping bounds and opaque face centers, sidecar ranges, opaque materials, file structure, absence of external dependencies, and polygon budget have been checked. Views cover yaw only; pitch keeps the nearest yaw view. Physical-device face tracking, frame rate, orientation/mirroring, filtering lifecycle, and server/on-device mode isolation must be verified by the client implementation. The assets alone do not establish those behaviors. + +The repository retains the editable `.blend` sources and generation scripts. +The PNG/JSON runtime deliverables are also bundled in each native client. +Other `build/` files described above are reproducible local exports and validation +reports; they are not included in the feature commit. diff --git a/assets-source/animal-filters/build/source/owl.blend b/assets-source/animal-filters/build/source/owl.blend new file mode 100644 index 000000000..c8dc0fa7e Binary files /dev/null and b/assets-source/animal-filters/build/source/owl.blend differ diff --git a/assets-source/animal-filters/build/source/rabbit.blend b/assets-source/animal-filters/build/source/rabbit.blend new file mode 100644 index 000000000..f96562c3d Binary files /dev/null and b/assets-source/animal-filters/build/source/rabbit.blend differ diff --git a/assets-source/animal-filters/generate_animals.py b/assets-source/animal-filters/generate_animals.py new file mode 100644 index 000000000..a8e80440d --- /dev/null +++ b/assets-source/animal-filters/generate_animals.py @@ -0,0 +1,264 @@ +"""Create InnoLive on-device animal filters in Blender 4.3+. + +Run: blender -b --factory-startup --python generate_animals.py -- --output ./build +All models are original procedural mesh geometry; no downloaded art or textures. +Facial features are ray-cast onto the head surface so side and diagonal views stay attached. +Blender X = right, Z = up, -Y = camera. glTF is +X right, +Y up, +Z camera. +One unit = interocular distance. Mesh origins are the eye-midpoint plane origin. +""" +import bpy, math, json, argparse, sys, os, struct, zlib +from mathutils import Vector, Quaternion +from pathlib import Path + +args = sys.argv[sys.argv.index('--')+1:] if '--' in sys.argv else [] +p = argparse.ArgumentParser(); p.add_argument('--output', default=str(Path(__file__).parent/'build')); p.add_argument('--samples',type=int,default=96); p.add_argument('--size',type=int,default=1024); p.add_argument('--skip-render',action='store_true'); ns=p.parse_args(args) +OUT=Path(ns.output).resolve(); OUT.mkdir(parents=True,exist_ok=True) +for d in ['source','runtime','previews']: (OUT/d).mkdir(exist_ok=True) + +PALETTE={ + 'ivory':('F7EBD9',.83), 'cream':('FFEED0',.85), 'lavender':('BC9BB5',.9), + 'pink':('DF9BA7',.82), 'rose':('AB5973',.67), 'ink':('362B3E',.24), + 'white':('FFF9ED',.25), 'cocoa':('87554B',.83), 'dark_cocoa':('65434D',.82), + 'caramel':('B97E60',.88), 'gold':('EFB557',.55), 'pale_gold':('FFE0A0',.68) +} +ASSETS=[] +def tag_srgb(path): + # PNG pixels are already display-referred sRGB; add the standard explicit + # color-space marker without decoding or modifying any pixels. + raw=Path(path).read_bytes() + pos=8 + while posI',raw,pos)[0]; kind=raw[pos+4:pos+8] + if kind==b'sRGB': return + pos+=12+n + data=b'sRGB'+b'\x00' + chunk=struct.pack('>I',1)+data+struct.pack('>I',zlib.crc32(data)&0xffffffff) + Path(path).write_bytes(raw[:33]+chunk+raw[33:]) +def srgb(v): return v/12.92 if v<.04045 else ((v+.055)/1.055)**2.4 + +def setup(name): + bpy.ops.wm.read_factory_settings(use_empty=True) + scene=bpy.context.scene + scene.render.engine='CYCLES'; scene.cycles.samples=ns.samples + scene.cycles.use_denoising=False + scene.render.resolution_x=ns.size; scene.render.resolution_y=ns.size; scene.render.resolution_percentage=100 + scene.render.film_transparent=True + scene.render.image_settings.file_format='PNG'; scene.render.image_settings.color_mode='RGBA'; scene.render.image_settings.color_depth='8' + scene.render.image_settings.compression=30 + scene.view_settings.view_transform='AgX'; scene.view_settings.look='AgX - Medium High Contrast' + scene.view_settings.exposure=0; scene.view_settings.gamma=1 + scene.world=bpy.data.worlds.new('Soft studio world'); scene.world.use_nodes=True + scene.world.node_tree.nodes['Background'].inputs[0].default_value=(.65,.72,.86,1) + scene.world.node_tree.nodes['Background'].inputs[1].default_value=.38 + scene.unit_settings.system='NONE' + mats={} + for key,(hx,rough) in PALETTE.items(): + c=tuple(srgb(int(hx[i:i+2],16)/255) for i in (0,2,4))+(1,) + m=bpy.data.materials.new(key); m.diffuse_color=c; m.use_nodes=True + bsdf=m.node_tree.nodes.get('Principled BSDF'); bsdf.inputs['Base Color'].default_value=c + bsdf.inputs['Roughness'].default_value=rough; bsdf.inputs['Metallic'].default_value=0 + mats[key]=m + root=bpy.data.objects.new(name.title()+'Filter',None); scene.collection.objects.link(root) + root['asset_id']=name; root['scope']='on-device-only'; root['interocular_distance']=1.0 + root['version']='1.0.0'; root['axis_contract']='Blender +X right, +Z up, -Y forward; glTF +X right, +Y up, +Z forward' + collection=bpy.data.collections.new('FILTER_GEOMETRY'); scene.collection.children.link(collection) + return scene,mats,root,collection + +def register(o,name,mat,root,coll): + o.name=name; o.data.name=name+'Mesh' + for c in list(o.users_collection): c.objects.unlink(o) + coll.objects.link(o); o.parent=root + if mat: o.data.materials.append(mat) + if o.type=='MESH': + for poly in o.data.polygons: poly.use_smooth=True + return o + +def sphere(name,pos,scale,mat,root,coll,seg=24,rings=14): + bpy.ops.mesh.primitive_uv_sphere_add(segments=seg,ring_count=rings,location=pos) + o=bpy.context.object; o.scale=scale + bpy.ops.object.transform_apply(location=False,rotation=False,scale=True) + return register(o,name,mat,root,coll) + +def ear(name,side,z0,z1,width,depth,y,lean,mat,root,coll,rings=18,segments=16): + verts=[]; faces=[] + for i in range(rings+1): + t=i/rings + r=max(.015,math.sin(math.pi*t)**.65)*(1-.18*t) + cx=side*(.48+lean*t+.04*math.sin(t*math.pi)); z=z0+(z1-z0)*t + for j in range(segments): + a=2*math.pi*j/segments + verts.append((cx+width*r*math.cos(a),y+depth*r*math.sin(a),z)) + for i in range(rings): + for j in range(segments): + a=i*segments+j; b=i*segments+(j+1)%segments + faces.append((a,b,b+segments,a+segments)) + faces.extend([tuple(reversed(range(segments))),tuple(rings*segments+j for j in range(segments))]) + mesh=bpy.data.meshes.new(name+'Mesh'); mesh.from_pydata(verts,[],faces); mesh.update() + o=bpy.data.objects.new(name,mesh); coll.objects.link(o); o.parent=root; mesh.materials.append(mat) + for f in mesh.polygons:f.use_smooth=True + return o + +def stroke(name,points,r,mat,root,coll): + c=bpy.data.curves.new(name,'CURVE'); c.dimensions='3D'; c.resolution_u=6; c.bevel_depth=r; c.bevel_resolution=2 + sp=c.splines.new('BEZIER'); sp.bezier_points.add(len(points)-1) + for bp,co in zip(sp.bezier_points,points): bp.co=co; bp.handle_left_type='AUTO'; bp.handle_right_type='AUTO' + ob=bpy.data.objects.new(name,c); coll.objects.link(ob); ob.parent=root; c.materials.append(mat) + bpy.context.view_layer.objects.active=ob; ob.select_set(True); bpy.ops.object.convert(target='MESH'); ob.select_set(False) + for f in ob.data.polygons:f.use_smooth=True + return ob + +FRONT=Vector((0,-1,0)) +def snap(targets,x,z,lift=0.0): + # Front-most hit of a ray cast toward +Y; features sit on the head instead of a flat front plane. + bpy.context.view_layer.update(); best=None; origin=Vector((x,-6,z)) + for o in targets: + mw=o.matrix_world; inv=mw.inverted() + hit,loc,nor,_=o.ray_cast(inv@origin,(inv.to_3x3()@Vector((0,1,0))).normalized()) + if hit: + w=mw@loc; n=(inv.to_3x3().transposed()@nor).normalized() + if best is None or w.yI', 1)+data+struct.pack('>I', zlib.crc32(data) & 0xffffffff) + Path(path).write_bytes(raw[:33]+chunk+raw[33:]) + +for animal in manifest['animals']: + name = animal['id']; w = animal['cellWidth']; h = animal['cellHeight']; views = animal['views'] + columns = min(a.columns, len(views)); rows = (len(views)+columns-1)//columns + atlas = Image.new('RGBA', (columns*w, rows*h), (0, 0, 0, 0)) + entries = [] + for index, view in enumerate(views): + image = Image.open(root/view['image']).convert('RGBA'); assert image.size == (w, h) + x = index % columns*w; y = index//columns*h + atlas.paste(image, (x, y)) + entries.append({'yaw': view['yaw'], 'cell': [x, y, w, h], 'faceRect': view['faceRect'], + 'landmarks': view['landmarks']}) + out = root/'runtime'/f'{name}.png'; out.parent.mkdir(exist_ok=True) + atlas.save(out, optimize=True); tag_srgb(out) + front = entries[0]['faceRect'] + (root/'runtime'/f'{name}.json').write_text(json.dumps({ + 'id': name, 'version': 2, 'image': f'{name}.png', 'width': atlas.width, 'height': atlas.height, + 'coordinateSystem': 'normalized-top-left', 'cellFaceRect': 'normalized within each cell; excludes ears and tufts', + 'cellLandmarks': 'ArcFace 5-point template on the head, YuNet order, normalized within each cell; align by similarity transform', + 'yawConvention': 'positive yaw turns the muzzle toward image right; mirror the cell for negative yaw', + 'mirrorNegativeYaw': True, 'faceRect': front, 'views': entries, + 'fullFace': True, 'scope': 'on-device-only'}, indent=2)+'\n') + print(name, atlas.size, len(entries), 'views') diff --git a/assets-source/animal-filters/render_pose_views.py b/assets-source/animal-filters/render_pose_views.py new file mode 100644 index 000000000..e69d72c3b --- /dev/null +++ b/assets-source/animal-filters/render_pose_views.py @@ -0,0 +1,123 @@ +"""Render yaw views of the Blender animal heads for side and diagonal faces. + +Run: blender -b --factory-startup --python render_pose_views.py -- --output ./build +Then: python3 pack_pose_atlas.py ./build +Opens build/source/{rabbit,owl}.blend, rotates the model root about Blender +Z, and renders +each yaw with one fixed orthographic camera and the same pixel density. Positive yaw turns +the muzzle toward image right; clients mirror these views for negative yaw, so the studio +lights are left/right symmetric. One cell covers every yaw, so the model scale never changes. +""" +import bpy, math, json, argparse, sys +from mathutils import Vector +from pathlib import Path + +args = sys.argv[sys.argv.index('--')+1:] if '--' in sys.argv else [] +p = argparse.ArgumentParser() +p.add_argument('--output', default=str(Path(__file__).parent/'build')) +p.add_argument('--samples', type=int, default=128) +p.add_argument('--step', type=int, default=10) +p.add_argument('--density', type=float, default=172.0, help='Pixels per interocular distance.') +p.add_argument('--skip-render', action='store_true', help='Recompute pose-views.json only; keeps rendered PNGs.') +ns = p.parse_args(args) +OUT = Path(ns.output).resolve(); VIEWS = OUT/'views'; VIEWS.mkdir(parents=True, exist_ok=True) +YAWS = list(range(0, 91, ns.step)) +# Ears and tufts stay outside the face rectangle, matching the front sprite contract. +NOT_FACE = ('ear', 'tuft') +# ArcFace 5-point template (InSwapper alignment) in interocular units from the eye midpoint, +z up: +# right eye, left eye, nose, right and left mouth corners in YuNet order. Scaled so the animal head +# spans a typical face (forehead to chin about 2.9 interocular distances) around the head centre. +TEMPLATE = [(-.5, 0), (.5, 0), (.009, -.572), (-.413, -1.157), (.415, -1.152)] +FACE_CENTER, FIT = -.35, .72 + +def geometry(scene): + return [o for o in scene.objects if o.type == 'MESH'] + +def bounds(objects, face_only=False): + xs, zs = [], [] + deps = bpy.context.evaluated_depsgraph_get() + for o in objects: + if face_only and any(word in o.name.lower() for word in NOT_FACE): continue + evaluated = o.evaluated_get(deps) + for v in evaluated.data.vertices: + w = evaluated.matrix_world @ v.co + xs.append(w.x); zs.append(w.z) + return min(xs), max(xs), min(zs), max(zs) + +def template_points(scene, shell): + # Ray-cast each template point onto the head shell at rest so it turns with the model. + center = (shell.matrix_world @ Vector((0, 0, 0))).z + points = [] + for x, z in TEMPLATE: + wx, wz = x*FIT, center+(z-FACE_CENTER)*FIT + hit, loc, _, _ = shell.ray_cast(shell.matrix_world.inverted() @ Vector((wx, -6, wz)), Vector((0, 1, 0))) + assert hit, f'Template point misses the head: {x},{z}' + points.append(shell.matrix_world @ loc) + return points + +def symmetric_studio(scene): + for o in [o for o in scene.objects if o.type == 'LIGHT']: bpy.data.objects.remove(o) + def light(name, pos, power, size, color): + data = bpy.data.lights.new(name, 'AREA'); ob = bpy.data.objects.new(name, data); scene.collection.objects.link(ob) + ob.location = pos; data.energy = power; data.shape = 'DISK'; data.size = size; data.color = color + ob.rotation_euler = (Vector((0, 0, .4))-ob.location).to_track_quat('-Z', 'Y').to_euler() + light('Key softbox L', (-3.5, -4.8, 5.5), 300, 4.0, (1, .91, .85)) + light('Key softbox R', (3.5, -4.8, 5.5), 300, 4.0, (1, .91, .85)) + light('Fill softbox', (0, -5, 0), 160, 3.5, (.78, .85, 1)) + light('Rim softbox', (0, 3, 4), 600, 3, (1, .83, .68)) + +def use_gpu(scene): + try: + prefs = bpy.context.preferences.addons['cycles'].preferences + prefs.compute_device_type = 'METAL'; prefs.get_devices() + for device in prefs.devices: device.use = True + scene.cycles.device = 'GPU' + except Exception as error: + print('CPU render:', error) + +manifest = {'schema_version': 2, 'generator': 'Blender '+bpy.app.version_string, 'yawStepDegrees': ns.step, + 'mirrorNegativeYaw': True, 'animals': []} +for name in ('rabbit', 'owl'): + bpy.ops.wm.open_mainfile(filepath=str(OUT/'source'/f'{name}.blend')) + scene = bpy.context.scene + root = next(o for o in scene.objects if o.get('asset_id') == name) + meshes = geometry(scene) + root.rotation_euler = (0, 0, 0); bpy.context.view_layer.update() + shell = next(o for o in meshes if o.name == 'Full face shell') + rest = [root.matrix_world.inverted() @ point for point in template_points(scene, shell)] + use_gpu(scene) + scene.cycles.samples = ns.samples + scene.cycles.use_denoising = True + symmetric_studio(scene) + # Union of all yaw silhouettes plus a margin keeps every view unclipped at one scale. + spans = [] + for yaw in YAWS: + root.rotation_euler = (0, 0, math.radians(yaw)); bpy.context.view_layer.update() + spans.append(bounds(meshes)) + margin = .12 + x0 = min(s[0] for s in spans)-margin; x1 = max(s[1] for s in spans)+margin + # The canvas stays horizontally centered on the eye midpoint so mirroring keeps alignment. + half = max(-x0, x1); x0, x1 = -half, half + z0 = min(s[2] for s in spans)-margin; z1 = max(s[3] for s in spans)+margin + width = int(math.ceil((x1-x0)*ns.density/8))*8; height = int(math.ceil((z1-z0)*ns.density/8))*8 + units_w = width/ns.density; units_h = height/ns.density; cz = (z0+z1)/2 + cam = scene.camera + cam.data.type = 'ORTHO'; cam.data.sensor_fit = 'AUTO'; cam.data.ortho_scale = max(units_w, units_h) + cam.data.shift_x = cam.data.shift_y = 0 + cam.location = (0, -9, cz); cam.rotation_euler = (math.pi/2, 0, 0) + scene.render.resolution_x = width; scene.render.resolution_y = height; scene.render.resolution_percentage = 100 + views = [] + for yaw in YAWS: + root.rotation_euler = (0, 0, math.radians(yaw)); bpy.context.view_layer.update() + fx0, fx1, fz0, fz1 = bounds(meshes, face_only=True) + def u(x): return (x-(-units_w/2))/units_w + def v(z): return ((cz+units_h/2)-z)/units_h + face = [u(fx0), v(fz1), u(fx1)-u(fx0), v(fz0)-v(fz1)] + landmarks = [[u(w.x), v(w.z)] for w in (root.matrix_world @ point for point in rest)] + scene.render.filepath = str(VIEWS/f'{name}-yaw{yaw:02d}.png') + if not ns.skip_render: bpy.ops.render.render(write_still=True) + views.append({'yaw': yaw, 'image': f'views/{name}-yaw{yaw:02d}.png', 'faceRect': face, 'landmarks': landmarks}) + root.rotation_euler = (0, 0, 0) + manifest['animals'].append({'id': name, 'cellWidth': width, 'cellHeight': height, + 'unitsPerCell': [units_w, units_h], 'views': views}) +(OUT/'pose-views.json').write_text(json.dumps(manifest, indent=2)+'\n') +print('POSE_VIEWS', json.dumps(manifest)) diff --git a/assets-source/animal-filters/validate_assets.py b/assets-source/animal-filters/validate_assets.py new file mode 100644 index 000000000..c611f4502 --- /dev/null +++ b/assets-source/animal-filters/validate_assets.py @@ -0,0 +1,36 @@ +"""Validate production PNG, sidecar, and binary glTF exports. Python 3 + Pillow.""" +from pathlib import Path +import argparse, json, struct, hashlib, sys +from PIL import Image +p=argparse.ArgumentParser(); p.add_argument('directory',nargs='?',default=str(Path(__file__).parent/'build'));a=p.parse_args() +root=Path(a.directory); results=[] +for name in ('rabbit','owl'): + im=Image.open(root/'runtime'/f'{name}.png'); assert im.mode=='RGBA' + assert im.info.get('srgb')==0, 'Explicit PNG sRGB marker missing' + sidecar=json.loads((root/'runtime'/f'{name}.json').read_text()) + assert sidecar['version']==2 and sidecar['mirrorNegativeYaw'] is True + assert sidecar['scope']=='on-device-only' and sidecar['coordinateSystem']=='normalized-top-left' + assert [sidecar['width'],sidecar['height']]==list(im.size) and sidecar['image']==f'{name}.png' + views=sidecar['views']; yaws=[v['yaw'] for v in views] + assert yaws[0]==0 and yaws[-1]==90 and yaws==sorted(set(yaws)), 'Yaw views must cover 0..90 ascending' + for v in views: + cx,cy,cw,ch=v['cell']; assert cw==views[0]['cell'][2] and ch==views[0]['cell'][3] and cx+cw<=im.width and cy+ch<=im.height + x,y,w,h=v['faceRect']; assert 00 and bbox[2]