feat(app): camera for iOS simulator apps (webcam, image, video)

.github/workflows/release-app.yaml

@@ -67,6 +67,11 @@ jobs:
           security import "$RUNNER_TEMP/cert.p12" -P "$MACOS_CERTIFICATE_PASSWORD" -A -t cert -f pkcs12 -k "$KEYCHAIN"
           security set-key-partition-list -S apple-tool:,apple: -k ci "$KEYCHAIN"
           security list-keychains -d user -s "$KEYCHAIN"
+          # --deep skips Mach-O files in Resources: sign the simulator
+          # camera injector first.
+          codesign --force --timestamp --options runtime \
+            --sign "$MACOS_SIGNING_IDENTITY" \
+            "$APP/Contents/Resources/libSimCameraInjector.dylib"
           codesign --force --deep --timestamp --options runtime \
             --entitlements app/macos/Runner/Release.entitlements \
             --sign "$MACOS_SIGNING_IDENTITY" "$APP"

app/CHANGELOG.md

@@ -14,6 +14,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   used from the next start) or the camera button on a running emulator's
   stream (restarts it). SimUtil asks for Camera access before an emulator
   uses your webcam. Agents can set it over MCP too.
+- Give iOS simulator apps a camera (macOS): pick your Mac's webcam, an
+  image or a looping video from the device menu (Camera) or the camera
+  button on its stream, then open an app with it; switching the source
+  updates running apps live, and closing its stream or shutting it down
+  turns the camera off. Agents can pass
+  a camera to launch_app over MCP. Uses serve-sim's camera injector.
 
 ### Fixed
 

app/lib/main.dart

@@ -20,6 +20,7 @@ import 'package:simutil_app/src/mcp/tools/tool_context.dart';
 import 'package:simutil_app/src/recording/grid_recorder.dart';
 import 'package:simutil_app/src/scenarios/scenario_runs_cubit.dart';
 import 'package:simutil_app/src/settings/view_settings_cubit.dart';
+import 'package:simutil_app/src/stream/ios/sim_camera.dart';
 import 'package:simutil_app/src/stream/streams_cubit.dart';
 import 'package:simutil_app/src/stream/touch_marks.dart';
 import 'package:simutil_app/src/update/app_update_cubit.dart';
@@ -41,7 +42,10 @@ Future<void> main() async {
   getIt<AppUpdateCubit>().check();
   final devices = getIt<DevicesCubit>()..start();
   final streams = getIt<StreamsCubit>();
-  devices.state.subscribe((state) => streams.onDevices(state.all));
+  devices.state.subscribe((state) {
+    streams.onDevices(state.all);
+    SimCamera.onDevices(state.all);
+  });
   // Closing the window quits, after streams and recordings are released.
   windowManager.addListener(_QuitOnClose(quitApp));
   await windowManager.setPreventClose(true);

app/lib/src/mcp/tools/input_tools.dart

@@ -6,6 +6,7 @@ import 'package:simutil_app/src/mcp/mcp_server.dart';
 import 'package:simutil_app/src/mcp/tools/tool_context.dart';
 import 'package:simutil_app/src/scenarios/scenario_runs_cubit.dart';
 import 'package:simutil_app/src/scenarios/scenario_yaml.dart';
+import 'package:simutil_app/src/stream/ios/sim_camera.dart';
 import 'package:simutil_apple/simutil_apple.dart';
 import 'package:simutil_core/simutil_core.dart';
 
@@ -69,20 +70,49 @@ List<McpTool> inputTools(ToolContext c) => [
     description:
         'Launch an app by Android package name or iOS bundle id '
         '(e.g. com.example.app), stopping it first if it runs; clear_state '
-        'also wipes its data (simulators and Android).',
+        'also wipes its data (simulators and Android). camera (iOS '
+        'simulators, macOS app) feeds its camera from webcam, '
+        'webcam:<name>, image:<path> or video:<path>.',
     properties: {
       ...deviceArg,
       'app': {'type': 'string'},
       'clear_state': {'type': 'boolean', 'default': false},
+      'camera': {'type': 'string'},
     },
     required: ['device', 'app'],
     handler: (args) async {
       final app = args['app'];
       if (app is! String || app.isEmpty) throw ArgumentError('app required');
       final clear = args['clear_state'] == true;
-      await c.automation.launch(c.find(args), app, clearState: clear);
+      final camera = switch (args['camera']) {
+        final String value when value.isNotEmpty => SimCamera.parse(value),
+        _ => null,
+      };
+      final device = c.find(args);
+      if (camera == null) {
+        await c.automation.launch(device, app, clearState: clear);
+      } else {
+        if (device.os != DeviceOs.ios || device.type.isPhysical) {
+          throw ArgumentError(
+            'camera needs an iOS simulator; pick Android emulator cameras '
+            'with start_device cameraBack / cameraFront',
+          );
+        }
+        if (clear) await c.automation.launch(device, app, clearState: true);
+        await SimCamera.launch(
+          getIt<IosAutomation>(),
+          device,
+          app,
+          source: camera.source,
+          arg: camera.arg,
+        );
+      }
       return c.text(
-        clear ? 'Launched $app with a clean state' : 'Launched $app',
+        [
+          'Launched $app',
+          if (clear) 'with a clean state',
+          if (camera != null) 'with its camera on ${args['camera']}',
+        ].join(' '),
       );
     },
   ),

app/lib/src/stream/ios/ios_sim_session.dart

@@ -7,6 +7,7 @@ import 'package:simutil_core/simutil_core.dart';
 
 import 'package:simutil_app/src/stream/stream_fps.dart';
 import 'package:simutil_app/src/stream/ios/apple_chrome.dart';
+import 'package:simutil_app/src/stream/ios/sim_camera.dart';
 
 /// [DeviceSession] for a booted iOS simulator, backed by the native
 /// `SimStreamPlugin` (macOS): the framebuffer is a Flutter texture and input
@@ -248,6 +249,8 @@ class IosSimSession implements DeviceSession {
     if (_active[udid] == this) _active.remove(udid);
     removeFrameCounter(this, _framesShown);
     await _channel.invokeMethod<void>('stop', {'udid': udid});
+    // Closing the stream turns the simulator's camera (and webcam) off too.
+    await SimCamera.stop(udid);
     await _statusController.close();
   }
 }

app/lib/src/stream/ios/sim_camera.dart

@@ -0,0 +1,107 @@
+import 'package:flutter/services.dart';
+import 'package:simutil_apple/simutil_apple.dart';
+import 'package:simutil_core/simutil_core.dart';
+
+/// Camera of an iOS simulator app: the native `SimCameraPlugin` (macOS)
+/// renders a webcam, image or looping video into a shared feed per
+/// simulator, and apps are launched with the serve-sim camera injector
+/// reading it.
+abstract final class SimCamera {
+  static const _channel = MethodChannel('simutil/sim_camera');
+
+  /// Simulators with a running feed; the host webcam stays on until stopped.
+  static final _running = <String>{};
+
+  /// Host cameras as (unique id, name).
+  static Future<List<({String id, String name})>> webcams() async => [
+    for (final cam in await _channel.invokeListMethod<Map>('webcams') ?? [])
+      (id: cam['id'] as String, name: cam['name'] as String),
+  ];
+
+  /// The source feeding simulator [udid]: `webcam`, `image` or `video` and
+  /// its webcam id / file path; null when no feed runs.
+  static Future<({String source, String? arg})?> status(String udid) async {
+    final reply = await _channel.invokeMapMethod<String, Object?>('status', {
+      'udid': udid,
+    });
+    final source = reply?['source'];
+    if (source is! String) return null;
+    return (source: source, arg: reply!['arg'] as String?);
+  }
+
+  /// Feeds simulator [udid] from [source] (see [status]); a running feed
+  /// switches without relaunching its apps.
+  static Future<({String shm, String dylib})> setSource(
+    String udid,
+    String source, [
+    String? arg,
+  ]) async {
+    try {
+      final reply = (await _channel.invokeMapMethod<String, String>('start', {
+        'udid': udid,
+        'source': source,
+        'arg': ?arg,
+      }))!;
+      _running.add(udid);
+      return (shm: reply['shm']!, dylib: reply['dylib']!);
+    } on PlatformException catch (e) {
+      throw StateError(e.message ?? e.code);
+    }
+  }
+
+  /// (Re)launches [appId] on simulator [device] reading its camera from
+  /// [source], the running source when null (a webcam when none runs).
+  static Future<void> launch(
+    IosAutomation automation,
+    Device device,
+    String appId, {
+    String? source,
+    String? arg,
+  }) async {
+    final current = source == null ? await status(device.id) : null;
+    final feed = await setSource(
+      device.id,
+      source ?? current?.source ?? 'webcam',
+      source != null ? arg : current?.arg,
+    );
+    await automation.launchWithCamera(
+      device,
+      appId,
+      dylib: feed.dylib,
+      shmName: feed.shm,
+    );
+  }
+
+  /// Stops simulator [udid]'s feed (a no-op without one); apps keep their
+  /// last frame.
+  static Future<void> stop(String udid) async {
+    if (!_running.remove(udid)) return;
+    await _channel.invokeMethod('stop', {'udid': udid});
+  }
+
+  /// Stops the feed of every simulator in [devices]' update that is no
+  /// longer booted, so its webcam turns off with it.
+  static void onDevices(List<Device> devices) {
+    final booted = {
+      for (final d in devices)
+        if (d.state == DeviceState.booted) d.id,
+    };
+    _running.where((udid) => !booted.contains(udid)).toList().forEach(stop);
+  }
+
+  /// Parses an MCP / CLI camera value: `webcam`, `webcam:<id or name>`,
+  /// `image:<path>` or `video:<path>`.
+  static ({String source, String? arg}) parse(String value) {
+    final colon = value.indexOf(':');
+    final source = colon < 0 ? value : value.substring(0, colon);
+    final arg = colon < 0 ? null : value.substring(colon + 1);
+    if (!const {'webcam', 'image', 'video'}.contains(source) ||
+        (source != 'webcam' && (arg == null || arg.isEmpty))) {
+      throw FormatException(
+        'camera must be webcam, webcam:<name>, image:<path> or video:<path>',
+        value,
+      );
+    }
+    return (source: source, arg: arg);
+  }
+}

app/lib/src/ui/shared/device_context_menu.dart

@@ -5,12 +5,14 @@ import 'package:file_selector/file_selector.dart';
 import 'package:flutter/services.dart';
 import 'package:flutter/widgets.dart';
 import 'package:simutil_adb/simutil_adb.dart';
+import 'package:simutil_apple/simutil_apple.dart';
 import 'package:simutil_app/src/di.dart';
 import 'package:simutil_app/src/devices/slim_mode_cubit.dart';
 import 'package:simutil_app/src/recording/grid_recorder.dart';
 import 'package:simutil_app/src/devices/devices_cubit.dart';
 import 'package:simutil_app/src/settings/device_settings_cubit.dart';
 import 'package:simutil_app/src/settings/recordings_dir.dart';
+import 'package:simutil_app/src/stream/ios/sim_camera.dart';
 import 'package:simutil_app/src/stream/streams_cubit.dart';
 import 'package:simutil_app/src/ui/shared/device_actions.dart';
 import 'package:simutil_app/src/ui/shared/native_menu.dart';
@@ -20,13 +22,40 @@ import 'package:tray_manager/tray_manager.dart' as native;
 // ponytail: listed once per run, so a camera plugged in later needs an app
 // restart; re-list on every menu open if that matters.
 Future<List<({String id, String name})>>? _webcams;
+Future<List<({String id, String name})>>? _hostWebcams;
+
+/// What an iOS simulator's Camera submenu shows: the host webcams, the
+/// running feed and the apps that can be launched with it.
+typedef _SimCameraState = ({
+  List<({String id, String name})> webcams,
+  ({String source, String? arg})? current,
+  List<({String id, String name})> apps,
+});
+
+Future<_SimCameraState> _simCameraState(Device device) async {
+  final apps = getIt<IosAutomation>()
+      .installedApps(device)
+      .catchError((Object _) => <({String id, String name})>[]);
+  return (
+    webcams: await (_hostWebcams ??= SimCamera.webcams()),
+    current: await SimCamera.status(device.id),
+    apps: await apps,
+  );
+}
 
 Future<List<({String id, String name})>> _listWebcams() =>
     _webcams ??= getIt<AndroidDeviceService>().listWebcams();
 
-/// The camera sources of a running Android emulator as their own menu (from
-/// its stream header); a pick restarts it so the camera changes right away.
+/// The camera sources of a running device as their own menu (from its
+/// stream header). An Android emulator restarts so the camera changes right
+/// away; an iOS simulator's feed switches live.
 Future<void> showCameraMenu(BuildContext context, Device device) async {
+  if (device.os == DeviceOs.ios) {
+    final state = await _simCameraState(device);
+    if (!context.mounted) return;
+    showNativeMenu(() => [_simCameraItems(context, device, state)]);
+    return;
+  }
   final webcams = await _listWebcams();
   if (!context.mounted) return;
   final settingsCubit = context.read<DeviceSettingsCubit>();
@@ -47,6 +76,13 @@ Future<void> showDeviceContextMenu(BuildContext context, Device device) async {
   final webcams = emulator
       ? await _listWebcams()
       : const <({String id, String name})>[];
+  final simCamera =
+      Platform.isMacOS &&
+          device.os == DeviceOs.ios &&
+          !device.type.isPhysical &&
+          device.state == DeviceState.booted
+      ? await _simCameraState(device)
+      : null;
   if (!context.mounted) return;
   final settingsCubit = context.read<DeviceSettingsCubit>();
   void set(DeviceSettings Function(DeviceSettings) change) =>
@@ -65,6 +101,8 @@ Future<void> showDeviceContextMenu(BuildContext context, Device device) async {
             _cameraSides(settingsCubit.of(device), webcams, set),
           ),
         ],
+      if (simCamera != null)
+        [menuSubmenu('Camera', _simCameraItems(context, device, simCamera))],
       _streamItems(context, device, set),
       if (!device.type.isPhysical && !busy) _deleteItems(context, device),
     ],
@@ -221,6 +259,77 @@ List<native.MenuItem> _cameraSides(
   ];
 }
 
+/// Camera of an iOS simulator: pick a host webcam, image or video to feed
+/// it, then launch an app reading that feed (serve-sim's camera injector).
+List<native.MenuItem> _simCameraItems(
+  BuildContext context,
+  Device device,
+  _SimCameraState state,
+) {
+  final devices = context.read<DevicesCubit>();
+  final current = state.current;
+  Future<void> guard(String done, Future<void> Function() action) async {
+    try {
+      await action();
+      devices.notify(done);
+    } on Object catch (e) {
+      final message = e is StateError ? e.message : '$e';
+      devices.notify('Camera: $message', error: true);
+    }
+  }
+
+  void feed(String source, String? arg, String label) => guard(
+    'Simulator camera: $label',
+    () => SimCamera.setSource(device.id, source, arg),
+  );
+
+  native.MenuItem file(String kind, String title, List<String> extensions) {
+    final picked = current?.source == kind;
+    return menuCheck(
+      picked
+          ? '$title: ${current!.arg!.split(Platform.pathSeparator).last}'
+          : '$title…',
+      picked,
+      () async {
+        final file = await openFile(
+          acceptedTypeGroups: [
+            XTypeGroup(label: title, extensions: extensions),
+          ],
+        );
+        if (file != null) feed(kind, file.path, file.name);
+      },
+    );
+  }
+
+  return [
+    for (final cam in state.webcams)
+      menuCheck(
+        cam.name,
+        current?.source == 'webcam' && current?.arg == cam.id,
+        () => feed('webcam', cam.id, cam.name),
+      ),
+    file('image', 'Image', const ['png', 'jpg', 'jpeg', 'heic']),
+    file('video', 'Video', const ['mp4', 'mov', 'm4v']),
+    if (current != null) ...[
+      menuSubmenu('Open app with camera', [
+        for (final app in state.apps)
+          menuItem(
+            app.name,
+            () => guard(
+              'Launched ${app.name} with the camera',
+              () => SimCamera.launch(getIt<IosAutomation>(), device, app.id),
+            ),
+          ),
+        if (state.apps.isEmpty) menuItem('No apps installed', null),
+      ]),
+      menuItem(
+        'Turn off camera',
+        () => guard('Simulator camera off', () => SimCamera.stop(device.id)),
+      ),
+    ],
+  ];
+}
+
 /// Device frame, and the scrcpy options of an Android stream; they apply
 /// the next time the stream opens.
 List<native.MenuItem> _streamItems(