feat: android-on-device-mosaic/#780

apps/android/InnoLive/app/src/androidTest/java/com/framework/innolive/feature/live/privacy/PrivacyGpuFramePipelineDeviceTest.kt

@@ -214,6 +214,57 @@ class PrivacyGpuFramePipelineDeviceTest {
         } finally { source.release() }
     }
 
+    @Test fun strongerOnDeviceStrengthSpreadsTheEdgeFurther() {
+        val source = source(320,320)
+        try {
+            for (y in 0 until 320) for (x in 0 until 320)
+                source.dataY.put(y*source.strideY+x,(if(x<160)16 else 235).toByte())
+            PrivacyGpuFramePipeline(null).use { graph ->
+                val layout = graph.prepare(source,0)
+                fun leftOfEdge(strength: Int): Int {
+                    val texture = graph.finish(ByteArray(160*160) { -1 },layout,320,320,
+                        blurSigma=PrivacyBroadcastBlur.sigma(320,320,strength))
+                    try {
+                        val pixels = texture.toI420()!!
+                        try { return pixels.dataY.get(160*pixels.strideY+100).toInt() and 255 }
+                        finally { pixels.release() }
+                    } finally { texture.release() }
+                }
+                val default = leftOfEdge(PrivacyBroadcastBlur.DEFAULT_STRENGTH)
+                val strongest = leftOfEdge(PrivacyBroadcastBlur.MAX_STRENGTH)
+                // 60px 떨어진 곳까지 밝은 쪽이 번지는 정도는 sigma가 두 배일 때 더 커야 합니다.
+                assertTrue("Stronger blur did not spread: default=$default strongest=$strongest", strongest > default + 5)
+            }
+        } finally { source.release() }
+    }
+
+    @Test fun widerMarginProtectsPixelsOutsideTheDefaultEdge() {
+        val source = source(320, 320)
+        try {
+            for (y in 0 until 320) for (x in 0 until 320)
+                source.dataY.put(y * source.strideY + x, (if (x % 2 == 0) 16 else 235).toByte())
+            PrivacyGpuFramePipeline(null).use { graph ->
+                val layout = graph.prepare(source, 0)
+                val mask = ByteArray(160 * 160)
+                for (y in 60..100) for (x in 60..100) mask[y * 160 + x] = -1
+                fun pixel(mask: ByteArray, edgeCells: Int): Int {
+                    val texture = graph.finish(mask, layout, 320, 320, edgeCells = edgeCells)
+                    try {
+                        val pixels = texture.toI420()!!
+                        try { return pixels.dataY.get(160 * pixels.strideY + 212).toInt() and 255 }
+                        finally { pixels.release() }
+                    } finally { texture.release() }
+                }
+                val unprotected = pixel(ByteArray(160 * 160), PrivacyMaskRenderer.PROTECTED_EDGE_CELLS)
+                // 마스크 101칸(202px)까지가 기본 1칸 팽창입니다. 212px은 기본에서는 그대로이고 8칸이면 가려집니다.
+                assertEquals(unprotected.toDouble(), pixel(mask, PrivacyMaskRenderer.PROTECTED_EDGE_CELLS).toDouble(), 3.0)
+                val widened = pixel(mask, 8)
+                assertTrue("Wider margin left the pixel exposed: before=$unprotected after=$widened",
+                    kotlin.math.abs(widened - unprotected) > 20 && widened in 40..220)
+            }
+        } finally { source.release() }
+    }
+
     @Test fun heldTextureSurvivesNewFramesGeometryChangesAndProcessorClose() {
         val source = source()
         val different = source(48, 32)

apps/android/InnoLive/app/src/main/java/com/framework/innolive/feature/live/CameraFrameAnalyzer.kt

@@ -8,6 +8,7 @@ import com.framework.innolive.BuildConfig
 import com.framework.innolive.feature.live.privacy.PrivacyFrameMode
 import com.framework.innolive.feature.live.privacy.PrivacyFrameProcessor
 import com.framework.innolive.feature.live.privacy.PrivacyFrameRoute
+import com.framework.innolive.feature.live.privacy.PrivacyMosaic
 import com.framework.innolive.feature.live.privacy.PrivacyNativePixels
 import com.framework.innolive.feature.live.privacy.PrivacyFrameAnalysis
 import com.framework.innolive.feature.live.privacy.PrivacyFrameTimings
@@ -46,6 +47,7 @@ class CameraFrameAnalyzer(
 ) : ImageAnalysis.Analyzer, AutoCloseable {
     private val captureLock = Any()
     private val settings = AtomicReference(BroadcastVideoQualitySettings())
+    private val onDeviceMosaic = AtomicReference(PrivacyMosaic())
     private val previewExposure = PreviewExposureFrameGate()
     private val colorProcessor = VideoColorFrameProcessor()
     private val previewRenderer = VideoLookPreviewRenderer()
@@ -157,6 +159,11 @@ class CameraFrameAnalyzer(
         settings.set(value.normalized())
     }
 
+    /** 온디바이스 가림 처리의 블러 강도·영역 너비입니다. 다음 보호 프레임부터 송출·미리보기·가공본 녹화에 함께 적용합니다. */
+    fun setOnDeviceMosaic(value: OnDeviceMosaicSettings) {
+        onDeviceMosaic.set(value.toPrivacyMosaic())
+    }
+
     internal fun beginPreviewExposure() = synchronized(captureLock) {
         previewExposure.begin()
         previewRenderer.invalidatePendingSample()
@@ -479,6 +486,7 @@ class CameraFrameAnalyzer(
                     synchronized(processorLock) {
                         val processor = processor()
                         if (resetPending.getAndSet(false)) processor.resetFaceExceptions()
+                        processor.mosaic = onDeviceMosaic.get()
                         transform(processor)
                     }
                 },

apps/android/InnoLive/app/src/main/java/com/framework/innolive/feature/live/Live.kt

@@ -154,8 +154,10 @@ import com.framework.innolive.feature.live.tutorial.LocalBroadcastTutorialAnchor
 import com.framework.innolive.feature.live.tutorial.broadcastTutorialAnchor
 import com.framework.innolive.feature.live.tutorial.guideFor
 import com.framework.innolive.ui.text.asString
+import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.delay
 import kotlinx.coroutines.launch
+import kotlinx.coroutines.withContext
 
 @Composable
 fun LiveScreen(
@@ -351,6 +353,14 @@ private fun LiveScreenContent(
     }
     val idleFrameAnalyzer = remember { CameraFrameAnalyzer() }
     val frameAnalyzer = webRtcSession.frameAnalyzer ?: idleFrameAnalyzer
+    // 온디바이스 블러 강도·가림 영역 너비는 기기에 저장합니다. 읽는 동안은 하한인 기본값으로 처리하므로 약해지지 않습니다.
+    val onDeviceMosaicPreferences = remember(context) { OnDeviceMosaicPreferences(context) }
+    var onDeviceMosaic by remember(onDeviceMosaicPreferences) { mutableStateOf(OnDeviceMosaicSettings()) }
+    LaunchedEffect(onDeviceMosaicPreferences) {
+        onDeviceMosaic = withContext(Dispatchers.IO) { onDeviceMosaicPreferences.load() }
+    }
+    // 값이 바뀌거나 송출 분석기가 새로 생길 때만 넘깁니다.
+    LaunchedEffect(frameAnalyzer, onDeviceMosaic) { frameAnalyzer.setOnDeviceMosaic(onDeviceMosaic) }
     val lookPreviews by frameAnalyzer.lookPreviews.collectAsState()
     DisposableEffect(idleFrameAnalyzer) {
         onDispose { idleFrameAnalyzer.close() }
@@ -465,19 +475,33 @@ private fun LiveScreenContent(
             onSettingsChanged = props.onVideoQualitySettingsChanged,
             onDismiss = { setVideoControlsOpen(false) },
             mirrorPreviews = props.cameraLensFacing.shouldMirrorPreview,
-            mosaic = MosaicControlsState(
-                config = webRtcSession.mosaicConfig,
-                enabled = webRtcSession.canChangeMosaicConfig,
-                isChanging = webRtcSession.isChangingMosaicConfig,
-                noteRes = mosaicNoteRes(
-                    hasSession = webRtcSession.sessionSnapshot != null,
-                    onDeviceProcessing = webRtcSession.selectedOnDeviceProcessing,
-                    canChange = webRtcSession.canChangeMosaicConfig,
-                    anonymizationEnabled = webRtcSession.anonymizationState == AnonymizationState.ENABLED,
-                ),
-                error = webRtcSession.mosaicConfigError,
-                onChange = { webRtcSession.updateMosaicConfig(it) },
-            ),
+            mosaic = run {
+                // 온디바이스는 기기에 저장한 값을 바로 적용하고, 서버 AI는 세션 설정으로 보냅니다.
+                val onDevice = webRtcSession.selectedOnDeviceProcessing
+                MosaicControlsState(
+                    config = if (onDevice) onDeviceMosaic.toMosaicConfig() else webRtcSession.mosaicConfig,
+                    blurRange = if (onDevice) OnDeviceMosaicSettings.BLUR_STRENGTH_RANGE else MosaicConfig.BLUR_RADIUS_RANGE,
+                    blurStep = if (onDevice) OnDeviceMosaicSettings.BLUR_STRENGTH_STEP else 1,
+                    maskMarginRange = if (onDevice) OnDeviceMosaicSettings.MASK_MARGIN_RANGE else MosaicConfig.MASK_MARGIN_RANGE,
+                    enabled = onDevice || webRtcSession.canChangeMosaicConfig,
+                    isChanging = !onDevice && webRtcSession.isChangingMosaicConfig,
+                    noteRes = mosaicNoteRes(
+                        hasSession = webRtcSession.sessionSnapshot != null,
+                        onDeviceProcessing = onDevice,
+                        canChange = webRtcSession.canChangeMosaicConfig,
+                        anonymization = webRtcSession.anonymizationState,
+                    ),
+                    error = if (onDevice) null else webRtcSession.mosaicConfigError,
+                    onChange = { config ->
+                        if (onDevice) {
+                            onDeviceMosaic = OnDeviceMosaicSettings.from(config)
+                            onDeviceMosaicPreferences.save(onDeviceMosaic)
+                        } else {
+                            webRtcSession.updateMosaicConfig(config)
+                        }
+                    },
+                )
+            },
         )
     }
 

apps/android/InnoLive/app/src/main/java/com/framework/innolive/feature/live/MosaicControls.kt

@@ -29,10 +29,15 @@ import com.framework.innolive.ui.text.UiText
 import com.framework.innolive.ui.text.asString
 import kotlin.math.roundToInt
 
-/** 영상 조절 시트의 모자이크 구역 상태입니다. [config]는 서버가 확정한 값입니다. */
+/** 영상 조절 시트의 모자이크 구역 상태입니다. [config]는 서버가 확정한 값이거나 기기에 저장한 온디바이스 값입니다. */
 @Immutable
 data class MosaicControlsState(
     val config: MosaicConfig,
+    /** 슬라이더 범위입니다. 온디바이스는 지금 강도와 마스크 1칸보다 약하게 내리지 않습니다. */
+    val blurRange: IntRange = MosaicConfig.BLUR_RADIUS_RANGE,
+    /** 온디바이스 강도는 6 단위 다섯 단계입니다. */
+    val blurStep: Int = 1,
+    val maskMarginRange: IntRange = MosaicConfig.MASK_MARGIN_RANGE,
     val enabled: Boolean,
     val isChanging: Boolean,
     /** 조절할 수 없거나 적용되지 않는 이유입니다. */
@@ -42,18 +47,21 @@ data class MosaicControlsState(
 )
 
 /**
- * 조절할 수 없거나 적용되지 않는 이유입니다. iOS와 같은 순서로 방송 준비 전, 온디바이스 처리, 비식별화 꺼짐을 봅니다.
+ * 조절할 수 없거나 적용되지 않는 이유입니다. 온디바이스는 기기에 저장해 방송 준비 전에도 조절하므로, 비식별화를
+ * 실제로 끈 경우가 아니면 하한만 알립니다. 준비 전 비식별화 상태는 [AnonymizationState.UNKNOWN]입니다.
+ * 서버 AI는 iOS와 같은 순서로 방송 준비 전, 비식별화 꺼짐을 봅니다.
  */
 @StringRes
 internal fun mosaicNoteRes(
     hasSession: Boolean,
     onDeviceProcessing: Boolean,
     canChange: Boolean,
-    anonymizationEnabled: Boolean,
+    anonymization: AnonymizationState,
 ): Int? = when {
+    onDeviceProcessing && anonymization == AnonymizationState.DISABLED -> R.string.mosaic_anonymization_off
+    onDeviceProcessing -> R.string.mosaic_on_device_floor
     !hasSession -> R.string.mosaic_prepare_required
-    onDeviceProcessing -> R.string.mosaic_server_only
-    canChange && !anonymizationEnabled -> R.string.mosaic_anonymization_off
+    canChange && anonymization != AnonymizationState.ENABLED -> R.string.mosaic_anonymization_off
     else -> null
 }
 
@@ -95,11 +103,11 @@ internal fun MosaicControls(state: MosaicControlsState) {
             title = stringResource(R.string.mosaic_blur_strength),
             value = shown.blurRadius.toFloat(),
             valueLabel = shown.blurRadius.toString(),
-            range = MosaicConfig.BLUR_RADIUS_RANGE.first.toFloat()..MosaicConfig.BLUR_RADIUS_RANGE.last.toFloat(),
+            range = state.blurRange.first.toFloat()..state.blurRange.last.toFloat(),
             leading = stringResource(R.string.mosaic_weaker),
             trailing = stringResource(R.string.mosaic_stronger),
             enabled = state.enabled,
-            steps = MosaicConfig.BLUR_RADIUS_RANGE.last - MosaicConfig.BLUR_RADIUS_RANGE.first - 1,
+            steps = (state.blurRange.last - state.blurRange.first) / state.blurStep - 1,
             onValueChangeFinished = ::commit,
             onValueChange = {
                 isEditing = true
@@ -110,11 +118,11 @@ internal fun MosaicControls(state: MosaicControlsState) {
             title = stringResource(R.string.mosaic_mask_margin),
             value = shown.maskMargin.toFloat(),
             valueLabel = stringResource(R.string.mosaic_pixels_value, shown.maskMargin),
-            range = MosaicConfig.MASK_MARGIN_RANGE.first.toFloat()..MosaicConfig.MASK_MARGIN_RANGE.last.toFloat(),
+            range = state.maskMarginRange.first.toFloat()..state.maskMarginRange.last.toFloat(),
             leading = stringResource(R.string.mosaic_narrower),
             trailing = stringResource(R.string.mosaic_wider),
             enabled = state.enabled,
-            steps = MosaicConfig.MASK_MARGIN_RANGE.last - MosaicConfig.MASK_MARGIN_RANGE.first - 1,
+            steps = state.maskMarginRange.last - state.maskMarginRange.first - 1,
             onValueChangeFinished = ::commit,
             onValueChange = {
                 isEditing = true

apps/android/InnoLive/app/src/main/java/com/framework/innolive/feature/live/OnDeviceMosaicSettings.kt

@@ -0,0 +1,58 @@
+package com.framework.innolive.feature.live
+
+import android.content.Context
+import com.framework.innolive.feature.live.privacy.PrivacyBroadcastBlur
+import com.framework.innolive.feature.live.privacy.PrivacyMosaic
+
+/**
+ * 온디바이스 AI의 얼굴·번호판 블러 강도와 가림 영역 너비입니다. 서버 AI 슬라이더와 같은 눈금(블러 24, 너비 8px)을
+ * 쓰지만, 블러는 지금 온디바이스 강도보다 약해지지 않고 너비는 마스크 1칸 아래로 줄지 않습니다. 기기에 저장합니다.
+ */
+data class OnDeviceMosaicSettings(
+    val blurStrength: Int = PrivacyBroadcastBlur.DEFAULT_STRENGTH,
+    val maskMargin: Int = MosaicConfig.DEFAULT_MASK_MARGIN,
+) {
+    fun normalized() = copy(
+        blurStrength = PrivacyBroadcastBlur.strengthLevel(blurStrength),
+        maskMargin = maskMargin.coerceIn(MASK_MARGIN_RANGE),
+    )
+
+    /** 영상 조절 시트의 모자이크 구역이 보여 주는 값입니다. */
+    fun toMosaicConfig() = MosaicConfig(blurRadius = blurStrength.toDouble(), maskMargin = maskMargin)
+
+    /** 가림 처리 경로가 프레임마다 읽는 값입니다. */
+    internal fun toPrivacyMosaic() = normalized().let { PrivacyMosaic(it.blurStrength, it.maskMargin) }
+
+    companion object {
+        val BLUR_STRENGTH_RANGE = PrivacyBroadcastBlur.DEFAULT_STRENGTH..PrivacyBroadcastBlur.MAX_STRENGTH
+        const val BLUR_STRENGTH_STEP = PrivacyBroadcastBlur.STRENGTH_STEP
+        /** 8px 아래는 어차피 마스크 1칸이라 슬라이더도 기본값부터 시작합니다. */
+        val MASK_MARGIN_RANGE = MosaicConfig.DEFAULT_MASK_MARGIN..MosaicConfig.MASK_MARGIN_RANGE.last
+
+        /** 시트가 돌려준 값입니다. 기본값으로 되돌리기는 null 필드로 오며, 서버 기본값 24·8px가 곧 지금 강도와 1칸입니다. */
+        fun from(config: MosaicConfig) =
+            OnDeviceMosaicSettings(config.effectiveBlurRadius, config.effectiveMaskMargin).normalized()
+    }
+}
+
+class OnDeviceMosaicPreferences(context: Context) {
+    private val preferences = context.applicationContext
+        .getSharedPreferences("live_on_device_mosaic", Context.MODE_PRIVATE)
+
+    fun load() = OnDeviceMosaicSettings(
+        blurStrength = read("blur_strength", PrivacyBroadcastBlur.DEFAULT_STRENGTH),
+        maskMargin = read("mask_margin", MosaicConfig.DEFAULT_MASK_MARGIN),
+    ).normalized()
+
+    fun save(settings: OnDeviceMosaicSettings) {
+        val value = settings.normalized()
+        preferences.edit()
+            .putInt("blur_strength", value.blurStrength)
+            .putInt("mask_margin", value.maskMargin)
+            .apply()
+    }
+
+    // 저장 형식이 바뀌었거나 깨진 값은 기본값으로 읽어 카메라 시작을 막지 않습니다.
+    private fun read(key: String, fallback: Int) =
+        runCatching { preferences.getInt(key, fallback) }.getOrDefault(fallback)
+}

apps/android/InnoLive/app/src/main/java/com/framework/innolive/feature/live/privacy/PrivacyBitmapBlur.kt

@@ -20,7 +20,7 @@ internal class PrivacyBitmapBlur : AutoCloseable {
     var lastUsedGpu = false
         private set
 
-    fun apply(source: Bitmap): Bitmap {
+    fun apply(source: Bitmap, sigma: Int = PrivacyBroadcastBlur.sigma(source.width, source.height)): Bitmap {
         lastUsedGpu = false
         if (Build.VERSION.SDK_INT >= 31 && !gpuUnavailable) {
             try {
@@ -29,6 +29,8 @@ internal class PrivacyBitmapBlur : AutoCloseable {
                     gpu = null
                     gpu = Gpu(source.width, source.height)
                 }
+                // 강도만 바뀌면 렌더러와 버퍼는 그대로 두고 블러 효과만 바꿉니다.
+                checkNotNull(gpu).sigma = sigma
                 val output = checkNotNull(gpu).apply(source)
                 lastUsedGpu = true
                 return output
@@ -38,7 +40,7 @@ internal class PrivacyBitmapBlur : AutoCloseable {
                 gpuUnavailable = true
             }
         }
-        return PrivacyMaskRenderer.broadcastBlur(source)
+        return PrivacyMaskRenderer.broadcastBlur(source, sigma)
     }
 
     override fun close() { if (Build.VERSION.SDK_INT >= 31) gpu?.close(); gpu = null }
@@ -55,12 +57,18 @@ internal class PrivacyBitmapBlur : AutoCloseable {
                 renderer.setSurface(reader.surface)
                 renderer.setContentRoot(node)
                 node.setPosition(0, 0, width, height)
-                // The same full-resolution sigma as the GPU graph and iOS, via HWUI's radius conversion.
-                val radius = PrivacyBroadcastBlur.renderEffectRadius(width, height)
-                node.setRenderEffect(RenderEffect.createBlurEffect(radius, radius, Shader.TileMode.CLAMP))
             } catch (error: Exception) { close(); throw error }
         }
 
+        /** The same full-resolution sigma as the GPU graph and iOS, via HWUI's radius conversion. */
+        var sigma = 0
+            set(value) {
+                if (field == value) return
+                val radius = PrivacyBroadcastBlur.renderEffectRadius(value)
+                node.setRenderEffect(RenderEffect.createBlurEffect(radius, radius, Shader.TileMode.CLAMP))
+                field = value
+            }
+
         fun apply(source: Bitmap): Bitmap {
             // iOS broadcastStyleBlur: area-average reduction, then one wide Gaussian; no block grid.
             val reduced = PrivacyMaskRenderer.reduce(source)