background blur changes + examples

examples/mobile-client/fishjam-chat/app.json

@@ -56,7 +56,8 @@
           "ios": {
             "enableScreensharing": true,
             "enableVoIPBackgroundMode": true,
-            "supportsPictureInPicture": true
+            "supportsPictureInPicture": true,
+            "iphoneDeploymentTarget": "16.4"
           }
         }
       ],

examples/mobile-client/fishjam-chat/app/room/[roomName].tsx

@@ -13,14 +13,25 @@ import { Platform, StyleSheet, View } from 'react-native';
 import { SafeAreaView } from 'react-native-safe-area-context';
 
 import { InCallButton, VideosGrid } from '../../components';
+import {
+  backgroundBlur,
+  reportBackgroundBlurFailure,
+} from '../../utils/backgroundBlur';
 
 export default function RoomScreen() {
   const { userName } = useLocalSearchParams<{
     roomName: string;
     userName: string;
   }>();
 
-  const { isCameraOn, toggleCamera, stopCamera } = useCamera();
+  const {
+    isCameraOn,
+    toggleCamera,
+    stopCamera,
+    currentCameraMiddleware,
+    setCameraTrackMiddleware,
+  } = useCamera();
+  const isBlurOn = currentCameraMiddleware === backgroundBlur;
   const { isMicrophoneOn, toggleMicrophone, stopMicrophone, startMicrophone } =
     useMicrophone();
   const { leaveRoom } = useConnection();
@@ -141,6 +152,12 @@ export default function RoomScreen() {
           onPress={toggleCamera}
           accessibilityLabel="Toggle Camera"
         />
+        <InCallButton
+          iconName={isBlurOn ? 'blur' : 'blur-off'}
+          onPress={() =>
+            setCameraTrackMiddleware(isBlurOn ? null : backgroundBlur).catch(reportBackgroundBlurFailure)
+          }
+        />
         <InCallButton
           iconName={screenShareStream ? 'monitor-share' : 'monitor-off'}
           onPress={handleToggleScreenShare}

examples/mobile-client/fishjam-chat/app/room/preview.tsx

@@ -12,6 +12,10 @@ import { ActivityIndicator, StyleSheet, Text, View } from 'react-native';
 import { SafeAreaView } from 'react-native-safe-area-context';
 
 import { Button, InCallButton, NoCameraView } from '../../components';
+import {
+  backgroundBlur,
+  reportBackgroundBlurFailure,
+} from '../../utils/backgroundBlur';
 import { useMediaPermissions } from '../../hooks/useMediaPermissions';
 import { BrandColors } from '../../utils/Colors';
 
@@ -26,8 +30,16 @@ export default function PreviewScreen() {
   });
 
   const { initializeDevices } = useInitializeDevices();
-  const { cameraStream, startCamera, stopCamera, isCameraOn, toggleCamera } =
-    useCamera();
+  const {
+    cameraStream,
+    startCamera,
+    stopCamera,
+    isCameraOn,
+    toggleCamera,
+    currentCameraMiddleware,
+    setCameraTrackMiddleware,
+  } = useCamera();
+  const isBlurOn = currentCameraMiddleware === backgroundBlur;
   const { isMicrophoneOn, toggleMicrophone, startMicrophone, stopMicrophone } =
     useMicrophone();
   const { joinRoom, leaveRoom } = useConnection();
@@ -150,6 +162,12 @@ export default function PreviewScreen() {
           onPress={toggleCamera}
           accessibilityLabel="Toggle Camera"
         />
+        <InCallButton
+          iconName={isBlurOn ? 'blur' : 'blur-off'}
+          onPress={() =>
+            setCameraTrackMiddleware(isBlurOn ? null : backgroundBlur).catch(reportBackgroundBlurFailure)
+          }
+        />
       </View>
 
       <View style={styles.joinButton}>

examples/mobile-client/fishjam-chat/babel.config.js

@@ -1,9 +1,9 @@
 module.exports = function (api) {
   api.cache(true);
   return {
-    presets: ['babel-preset-expo'],
+    presets: [['babel-preset-expo', { unstable_transformImportMeta: true }]],
     plugins: [
-      'react-native-reanimated/plugin',
+      '@babel/plugin-transform-class-static-block',
       [
         'module-resolver',
         {
@@ -12,6 +12,8 @@ module.exports = function (api) {
           },
         },
       ],
+      'unplugin-typegpu/babel',
+      'react-native-reanimated/plugin',
     ],
   };
 };

examples/mobile-client/fishjam-chat/metro.config.js

@@ -0,0 +1,7 @@
+const { getDefaultConfig } = require('expo/metro-config');
+
+const config = getDefaultConfig(__dirname);
+
+config.resolver.assetExts = [...config.resolver.assetExts, 'ssgbin'];
+
+module.exports = config;

examples/mobile-client/fishjam-chat/package.json

@@ -14,37 +14,46 @@
   "dependencies": {
     "@expo/vector-icons": "^15.0.3",
     "@fishjam-cloud/react-native-client": "workspace:*",
+    "@fishjam-cloud/react-native-worklets": "0.12.1",
+    "@fishjam-cloud/video-effects": "0.1.3",
     "@react-navigation/bottom-tabs": "^7.4.0",
     "@react-navigation/elements": "^2.6.3",
     "@react-navigation/native": "^7.1.8",
-    "expo": "~54.0.30",
-    "expo-constants": "~18.0.12",
-    "expo-font": "~14.0.10",
-    "expo-haptics": "~15.0.8",
-    "expo-image": "~3.0.11",
-    "expo-linking": "~8.0.11",
-    "expo-router": "~6.0.21",
-    "expo-splash-screen": "~31.0.13",
-    "expo-status-bar": "~3.0.9",
-    "expo-symbols": "~1.0.8",
-    "expo-system-ui": "~6.0.9",
-    "expo-web-browser": "~15.0.10",
-    "react": "19.1.0",
-    "react-dom": "19.1.0",
-    "react-native": "0.81.5",
-    "react-native-gesture-handler": "~2.28.0",
-    "react-native-reanimated": "~4.1.1",
-    "react-native-safe-area-context": "~5.6.0",
-    "react-native-screens": "~4.16.0",
+    "expo": "~57.0.20",
+    "expo-asset": "~57.0.16",
+    "expo-build-properties": "~57.0.17",
+    "expo-constants": "~57.0.17",
+    "expo-file-system": "~57.0.6",
+    "expo-font": "~57.0.3",
+    "expo-haptics": "~57.0.2",
+    "expo-image": "~57.0.4",
+    "expo-linking": "~57.0.9",
+    "expo-router": "~57.0.19",
+    "expo-splash-screen": "~57.0.8",
+    "expo-status-bar": "~57.0.1",
+    "expo-symbols": "~57.0.2",
+    "expo-system-ui": "~57.0.3",
+    "expo-web-browser": "~57.0.2",
+    "react": "19.2.3",
+    "react-dom": "19.2.3",
+    "react-native": "0.86.3",
+    "react-native-gesture-handler": "~2.32.0",
+    "react-native-reanimated": "4.6.0",
+    "react-native-safe-area-context": "~5.7.0",
+    "react-native-screens": "~4.26.0",
     "react-native-web": "~0.21.0",
-    "react-native-worklets": "0.5.1"
+    "react-native-webgpu": "0.5.15",
+    "react-native-worklets": "0.12.1"
   },
   "devDependencies": {
-    "@types/react": "~19.1.0",
+    "@babel/plugin-transform-class-static-block": "^7.28.3",
+    "@types/react": "~19.2.0",
     "babel-plugin-module-resolver": "^5.0.2",
-    "eslint-config-expo": "~8.0.1",
+    "babel-preset-expo": "~57.0.10",
+    "eslint-config-expo": "~57.0.2",
     "eslint-import-resolver-typescript": "^4.4.4",
-    "typescript": "~5.9.2"
+    "typescript": "~5.9.2",
+    "unplugin-typegpu": "^0.12.0"
   },
   "private": true
 }

examples/mobile-client/fishjam-chat/utils/backgroundBlur.ts

@@ -0,0 +1,35 @@
+import { createBackgroundBlurEffect } from '@fishjam-cloud/video-effects/background-blur';
+import { createCameraEffectMiddleware } from '@fishjam-cloud/video-effects/fishjam-react-native';
+import { typeGpuPersonSegmentation } from '@fishjam-cloud/video-effects/segmentation/typegpu';
+import { Asset } from 'expo-asset';
+import { File } from 'expo-file-system';
+
+const segmentationModel = Asset.fromModule(
+  // eslint-disable-next-line @typescript-eslint/no-require-imports
+  require('@fishjam-cloud/video-effects/assets/selfie_segmenter.ssgbin'),
+);
+
+// Android release builds can't fetch a bundled asset by URL, so read it from a local copy.
+async function loadSegmentationModel(): Promise<ArrayBuffer> {
+  await segmentationModel.downloadAsync();
+  const localUri = segmentationModel.localUri;
+  if (!localUri) throw new Error('The segmentation model is not available.');
+  return new File(localUri).arrayBuffer();
+}
+
+const segmentation = typeGpuPersonSegmentation({
+  loadModel: loadSegmentationModel,
+});
+
+export function reportBackgroundBlurFailure(error: unknown) {
+  console.warn('Background blur failed', error);
+}
+
+export const backgroundBlur = createCameraEffectMiddleware(
+  createBackgroundBlurEffect(() => ({ segmentation, radius: 24 })),
+  {
+    onStatus: (status, error) => {
+      if (status === 'error') reportBackgroundBlurFailure(error);
+    },
+  },
+);

package.json

@@ -41,8 +41,8 @@
   "resolutions": {
     "jest-snapshot-prettier": "npm:prettier@^3",
     "@types/react": "19.2.14",
-    "react": "19.1.0",
-    "react-dom": "19.1.0"
+    "react": "19.2.3",
+    "react-dom": "19.2.3"
   },
   "devDependencies": {
     "@fishjam-cloud/protobufs": "workspace:^",

packages/react-client/src/hooks/internal/devices/useDeviceManager.ts

@@ -5,7 +5,7 @@ import { useCallback, useMemo, useRef, useState } from "react";
 import type { DeviceError, DeviceItem, TrackMiddleware } from "../../../types/public";
 import { parseUserMediaError } from "../../../utils/errors";
 import { getTrackFromStream, stopStream } from "../../../utils/track";
-import { useTrackMiddleware } from "../useTrackMiddleware";
+import { type PublishTrack, useTrackMiddleware } from "../useTrackMiddleware";
 import { useHandleTrackEnd } from "./useHandleTrackEnd";
 
 type DeviceManagerProps = {
@@ -28,12 +28,17 @@ export type DeviceManager = {
   selectDevice: (deviceId: string) => Promise<[MediaStreamTrack, null] | [null, DeviceError]> | undefined;
   activeDevice: DeviceItem | null;
   deviceTrack: MediaStreamTrack | null;
+  rawDeviceTrack: MediaStreamTrack | null;
   deviceList: DeviceItem[];
   deviceEnabled: boolean;
   enableDevice: () => void;
   disableDevice: () => void;
   currentMiddleware: TrackMiddleware;
-  applyMiddleware: (middleware: TrackMiddleware) => Promise<MediaStreamTrack | null>;
+  applyMiddleware: (
+    middleware: TrackMiddleware,
+    track: MediaStreamTrack | null,
+    publish?: PublishTrack,
+  ) => Promise<void>;
   deviceError: DeviceError | null;
   selectedDevice: MediaDeviceInfo | null;
 };
@@ -77,7 +82,7 @@ export const useDeviceManager = ({
 
   useHandleTrackEnd(rawTrack, clearStream);
 
-  const { processedTrack, applyMiddleware, currentMiddleware } = useTrackMiddleware(rawTrack);
+  const { processedTrack, applyMiddleware, currentMiddleware } = useTrackMiddleware(rawTrack, logger);
 
   const currentTrack = processedTrack ?? rawTrack;
 
@@ -194,6 +199,7 @@ export const useDeviceManager = ({
     selectDevice,
     activeDevice,
     deviceTrack: processedTrack ?? rawTrack,
+    rawDeviceTrack: rawTrack,
     deviceList,
     enableDevice,
     disableDevice,

packages/react-client/src/hooks/internal/useTrackManager.ts

@@ -36,6 +36,7 @@ export const useTrackManager = ({
     disableDevice,
     deviceTrack,
     applyMiddleware,
+    rawDeviceTrack,
     currentMiddleware,
     selectDevice: _selectDevice,
   } = deviceManager;
@@ -54,27 +55,24 @@ export const useTrackManager = ({
     return currentTrack?.trackId ?? null;
   };
 
+  const replacePublishedTrack = async (track: MediaStreamTrack | null) => {
+    const currentTrackId = await getCurrentTrackId();
+    if (currentTrackId) await tsClient.replaceTrack(currentTrackId, track);
+  };
+
   const selectDevice = useCurrentCallback(async (deviceId: string) => {
     const result = await _selectDevice(deviceId);
     if (!result) return;
 
     const [newTrack, error] = result;
     if (error) return error;
 
-    const currentTrackId = await getCurrentTrackId();
-    if (!currentTrackId) return;
-
-    await tsClient.replaceTrack(currentTrackId, newTrack);
+    await applyMiddleware(currentMiddleware, newTrack, replacePublishedTrack);
   });
 
-  const setTrackMiddleware = useCurrentCallback(async (middleware: TrackMiddleware) => {
-    const processedTrack = await applyMiddleware(middleware);
-
-    const currentTrackId = await getCurrentTrackId();
-    if (!currentTrackId) return;
-
-    await tsClient.replaceTrack(currentTrackId, processedTrack);
-  });
+  const setTrackMiddleware = useCurrentCallback((middleware: TrackMiddleware) =>
+    applyMiddleware(middleware, rawDeviceTrack, replacePublishedTrack),
+  );
 
   const startStreaming = useCurrentCallback(
     async (
@@ -154,11 +152,15 @@ export const useTrackManager = ({
       const [newTrack, error] = await startDevice();
       if (error) return error;
 
-      if (currentTrackId) {
-        await resumeStreaming(currentTrackId, newTrack);
-      } else if (peerStatus === "connected") {
-        await startStreaming(newTrack, streamConfig);
-      }
+      const publishStartedTrack = async (track: MediaStreamTrack | null) => {
+        if (!track) return;
+        if (currentTrackId) {
+          await resumeStreaming(currentTrackId, track);
+        } else if (peerStatus === "connected") {
+          await startStreaming(track, streamConfig);
+        }
+      };
+      await applyMiddleware(currentMiddleware, newTrack, publishStartedTrack);
     }
   });