diff --git a/apps/fabric-example/ios/FabricExampleTests/AudioEngineTests.mm b/apps/fabric-example/ios/FabricExampleTests/AudioEngineTests.mm index d98823630..3403c486c 100644 --- a/apps/fabric-example/ios/FabricExampleTests/AudioEngineTests.mm +++ b/apps/fabric-example/ios/FabricExampleTests/AudioEngineTests.mm @@ -12,9 +12,14 @@ - (void)createAudioEngineIfNeeded; @interface FakeAudioInputNode : NSObject @property(nonatomic, strong) AVAudioFormat *outputFormat; +@property(nonatomic, assign) BOOL voiceProcessingEnabled; +@property(nonatomic, assign) NSInteger setVoiceProcessingCallCount; +@property(nonatomic, strong) NSError *voiceProcessingError; - (instancetype)initWithOutputFormat:(AVAudioFormat *)outputFormat; - (AVAudioFormat *)outputFormatForBus:(AVAudioNodeBus)bus; +- (BOOL)isVoiceProcessingEnabled; +- (BOOL)setVoiceProcessingEnabled:(BOOL)enabled error:(NSError **)error; @end @@ -32,6 +37,22 @@ - (AVAudioFormat *)outputFormatForBus:(AVAudioNodeBus)bus { return self.outputFormat; } +- (BOOL)isVoiceProcessingEnabled { + return self.voiceProcessingEnabled; +} + +- (BOOL)setVoiceProcessingEnabled:(BOOL)enabled error:(NSError **)error { + self.setVoiceProcessingCallCount += 1; + if (error != nil) { + *error = self.voiceProcessingError; + } + if (self.voiceProcessingError != nil) { + return NO; + } + self.voiceProcessingEnabled = enabled; + return YES; +} + @end @interface FakeAudioEngine : AVAudioEngine @@ -41,6 +62,7 @@ @interface FakeAudioEngine : AVAudioEngine @property(nonatomic, assign) NSInteger attachNodeCallCount; @property(nonatomic, assign) NSInteger detachNodeCallCount; @property(nonatomic, assign) NSInteger connectCallCount; +@property(nonatomic, assign) BOOL voiceProcessingEnabledWhenInputConnected; @property(nonatomic, assign) NSInteger prepareCallCount; @property(nonatomic, assign) NSInteger startCallCount; @property(nonatomic, assign) NSInteger stopCallCount; @@ -103,6 +125,10 @@ - (void)connect:(AVAudioNode *)node1 to:(AVAudioNode *)node2 format:(AVAudioFormat *)format { self.connectCallCount += 1; + if (node1 == (AVAudioNode *)self.fakeInputNode) { + self.voiceProcessingEnabledWhenInputConnected = + self.fakeInputNode.voiceProcessingEnabled; + } [self.connections addObject:@{ @"from" : node1, @"to" : node2, @@ -264,8 +290,6 @@ - (void)setUp { AVAudioFormat *inputFormat = [self testInputFormat]; self.audioEngine.defaultCreatedEngineInputFormat = inputFormat; [self.audioEngine createAudioEngineIfNeeded]; - self.audioEngine.currentFakeAudioEngine.fakeInputNode.outputFormat = - inputFormat; self.sessionManager = [[FakeAudioSessionManager alloc] init]; self.audioEngine.sessionManager = self.sessionManager; } @@ -480,6 +504,60 @@ - (void)testAttachInputNodeDefersConnectionUntilLiveInputFormatIsAvailable { liveInputFormat); } +- (void)testVoiceProcessingActivatesSessionBeforeMaterializingInput { + FakeAudioEngine *fakeEngine = self.audioEngine.currentFakeAudioEngine; + [self.audioEngine + attachInputNodeWithReceiverBlock:[self testInputReceiverBlock] + voiceProcessingEnabled:YES + onInputConfigurationChange:nil]; + + XCTAssertNil(self.audioEngine.inputNode); + XCTAssertEqual(fakeEngine.connectCallCount, 0); + + XCTAssertTrue([self.audioEngine startIfNecessary]); + XCTAssertGreaterThan(self.sessionManager.ensureActiveCallCount, 0); + XCTAssertEqual(fakeEngine.fakeInputNode.setVoiceProcessingCallCount, 1); + XCTAssertTrue(fakeEngine.fakeInputNode.voiceProcessingEnabled); + XCTAssertNotNil(self.audioEngine.inputNode); + XCTAssertEqual(fakeEngine.connectCallCount, 1); +} + +- (void)testVoiceProcessingFailurePreventsInputMaterialization { + FakeAudioEngine *fakeEngine = self.audioEngine.currentFakeAudioEngine; + fakeEngine.fakeInputNode.voiceProcessingError = + [NSError errorWithDomain:@"AudioEngineTests" code:1 userInfo:nil]; + [self.audioEngine + attachInputNodeWithReceiverBlock:[self testInputReceiverBlock] + voiceProcessingEnabled:YES + onInputConfigurationChange:nil]; + + XCTAssertFalse([self.audioEngine startIfNecessary]); + XCTAssertNil(self.audioEngine.inputNode); + XCTAssertEqual(fakeEngine.connectCallCount, 0); +} + +- (void)testVoiceProcessingIsReappliedBeforeInputConnectionAfterRebuild { + [self.audioEngine + attachInputNodeWithReceiverBlock:[self testInputReceiverBlock] + voiceProcessingEnabled:YES + onInputConfigurationChange:nil]; + + XCTAssertTrue([self.audioEngine startIfNecessary]); + FakeAudioEngine *oldEngine = self.audioEngine.currentFakeAudioEngine; + oldEngine.fakeRunning = YES; + self.audioEngine.state = AudioEngineStateRunning; + + [self.audioEngine restartAudioEngine]; + + FakeAudioEngine *replacementEngine = self.audioEngine.currentFakeAudioEngine; + XCTAssertNotEqual(replacementEngine, oldEngine); + XCTAssertEqual(replacementEngine.fakeInputNode.setVoiceProcessingCallCount, + 1); + XCTAssertTrue(replacementEngine.fakeInputNode.voiceProcessingEnabled); + XCTAssertEqual(replacementEngine.connectCallCount, 1); + XCTAssertTrue(replacementEngine.voiceProcessingEnabledWhenInputConnected); +} + - (void)testDetachInputNodeWithoutInputDoesNothing { FakeAudioEngine *fakeEngine = self.audioEngine.currentFakeAudioEngine; @@ -956,6 +1034,62 @@ - (void)testRestartAudioEngineStopsAndRestartsWhenStateRunning { XCTAssertFalse(self.audioEngine.graphNeedsRebuild); } +- (void)testOnRouteChangedKeepsEngineWhenInputFormatIsUnchanged { + __block NSInteger configurationChangeCount = 0; + [self.audioEngine + attachInputNodeWithReceiverBlock:[self testInputReceiverBlock] + voiceProcessingEnabled:YES + onInputConfigurationChange:^{ + configurationChangeCount += 1; + }]; + XCTAssertTrue([self.audioEngine startIfNecessary]); + FakeAudioEngine *engine = self.audioEngine.currentFakeAudioEngine; + + [self.audioEngine onRouteChanged]; + + XCTAssertEqual(self.audioEngine.currentFakeAudioEngine, engine); + XCTAssertEqual(engine.fakeInputNode.setVoiceProcessingCallCount, 1); + XCTAssertEqual(engine.stopCallCount, 0); + XCTAssertEqual(configurationChangeCount, 1); + XCTAssertEqual(self.audioEngine.state, AudioEngineStateRunning); +} + +- (void)testOnRouteChangedRebuildsWhenInputFormatChanged { + [self.audioEngine + attachInputNodeWithReceiverBlock:[self testInputReceiverBlock] + voiceProcessingEnabled:NO + onInputConfigurationChange:nil]; + XCTAssertTrue([self.audioEngine startIfNecessary]); + FakeAudioEngine *oldEngine = self.audioEngine.currentFakeAudioEngine; + AVAudioFormat *handsFreeFormat = [self testInputFormatWithSampleRate:16000 + channelCount:1]; + oldEngine.fakeInputNode.outputFormat = handsFreeFormat; + self.audioEngine.nextCreatedEngineInputFormat = handsFreeFormat; + + [self.audioEngine onRouteChanged]; + + FakeAudioEngine *newEngine = self.audioEngine.currentFakeAudioEngine; + XCTAssertNotEqual(newEngine, oldEngine); + XCTAssertEqual(oldEngine.stopCallCount, 1); + XCTAssertEqual(newEngine.connectCallCount, 1); + AVAudioFormat *connectionFormat = newEngine.connections.firstObject[@"format"]; + XCTAssertEqualWithAccuracy(connectionFormat.sampleRate, 16000.0, 0.1); + XCTAssertEqual(newEngine.startCallCount, 1); + XCTAssertEqual(self.audioEngine.state, AudioEngineStateRunning); +} + +- (void)testOnRouteChangedLeavesPlaybackOnlyGraphAlone { + [self attachSourceNodeToAudioEngine]; + XCTAssertTrue([self.audioEngine startIfNecessary]); + FakeAudioEngine *engine = self.audioEngine.currentFakeAudioEngine; + + [self.audioEngine onRouteChanged]; + + XCTAssertEqual(self.audioEngine.currentFakeAudioEngine, engine); + XCTAssertEqual(engine.stopCallCount, 0); + XCTAssertEqual(self.audioEngine.state, AudioEngineStateRunning); +} + - (void) testConfigurationChangeCallbackCanReadLiveInputFormatWhileRestartHoldsLock { __block BOOL callbackRan = NO; diff --git a/apps/fabric-example/ios/FabricExampleTests/SystemNotificationManagerTests.mm b/apps/fabric-example/ios/FabricExampleTests/SystemNotificationManagerTests.mm index 9c2161429..eea51e979 100644 --- a/apps/fabric-example/ios/FabricExampleTests/SystemNotificationManagerTests.mm +++ b/apps/fabric-example/ios/FabricExampleTests/SystemNotificationManagerTests.mm @@ -97,11 +97,23 @@ @interface SNMFakeAudioEngine : AudioEngine @property(nonatomic, assign) NSInteger interruptionEndCallCount; @property(nonatomic, assign) NSInteger restartAudioEngineCallCount; @property(nonatomic, assign) BOOL lastShouldResume; +@property(nonatomic, assign) BOOL fakeInUse; +@property(nonatomic, assign) BOOL fakeOwnsNotifyingEngine; @end @implementation SNMFakeAudioEngine +- (bool)isInUse +{ + return self.fakeInUse; +} + +- (bool)ownsAudioEngine:(AVAudioEngine *)engine +{ + return self.fakeOwnsNotifyingEngine; +} + - (void)onInterruptionBegin { self.interruptionBeginCallCount += 1; @@ -118,13 +130,6 @@ - (void)restartAudioEngine self.restartAudioEngineCallCount += 1; } -/// The engine is created lazily, so a fresh fake reads as not in use and the manager would -/// ignore every notification meant for it. -- (bool)isInUse -{ - return true; -} - @end @interface SNMFakeAudioSessionManager : AudioSessionManager @@ -307,8 +312,12 @@ - (void)tearDown [super tearDown]; } -- (void)flushMainQueue +- (void)flushPendingWork { + // Engine reactions run on the manager's own queue; the secondary-audio polling + // timer is started on the main queue. + dispatch_sync(self.manager.engineLifecycleQueue, ^{}); + XCTestExpectation *expectation = [self expectationWithDescription:@"Flush main queue"]; dispatch_async(dispatch_get_main_queue(), ^{ [expectation fulfill]; }); [self waitForExpectations:@[ expectation ] timeout:1.0]; @@ -390,7 +399,7 @@ - (void)testActivelyReclaimSessionRegistersObserverAndStartsPolling self.manager.notificationCenter = (NSNotificationCenter *)fakeNotificationCenter; [self.manager activelyReclaimSession:YES]; - [self flushMainQueue]; + [self flushPendingWork]; XCTAssertEqual(fakeNotificationCenter.addObserverCallCount, 1); XCTAssertEqualObjects(fakeNotificationCenter.addedNotificationNames.firstObject, @@ -411,7 +420,7 @@ - (void)testHandleInterruptionBeganMarksInactiveAndEmitsEventWhenObserved [self.manager handleInterruption:[self interruptionNotificationWithType:AVAudioSessionInterruptionTypeBegan option:0]]; - [self flushMainQueue]; + [self flushPendingWork]; XCTAssertEqual(self.fakeSessionManager.markInactiveCallCount, 1); XCTAssertEqual(self.fakeAudioEngine.interruptionBeginCallCount, 1); @@ -441,7 +450,7 @@ - (void)testHandleInterruptionEndedResumesEngineWhenNotObserved { [self.manager handleInterruption:[self interruptionNotificationWithType:AVAudioSessionInterruptionTypeEnded option:AVAudioSessionInterruptionOptionShouldResume]]; - [self flushMainQueue]; + [self flushPendingWork]; XCTAssertEqual(self.module.eventInvocationCount, 0); XCTAssertEqual(self.fakeAudioEngine.interruptionEndCallCount, 1); @@ -455,7 +464,7 @@ - (void)testHandleSecondaryAudioBeginMarksInactiveAndEmitsEventWhenObserved [self.manager handleSecondaryAudio:[self secondaryAudioNotificationWithType: AVAudioSessionSilenceSecondaryAudioHintTypeBegin]]; - [self flushMainQueue]; + [self flushPendingWork]; XCTAssertEqual(self.fakeSessionManager.markInactiveCallCount, 1); XCTAssertEqual(self.fakeAudioEngine.interruptionBeginCallCount, 1); @@ -471,7 +480,7 @@ - (void)testHandleSecondaryAudioEndResumesEngineWhenNotObserved [self.manager handleSecondaryAudio:[self secondaryAudioNotificationWithType: AVAudioSessionSilenceSecondaryAudioHintTypeEnd]]; - [self flushMainQueue]; + [self flushPendingWork]; XCTAssertEqual(self.module.eventInvocationCount, 0); XCTAssertEqual(self.fakeAudioEngine.interruptionEndCallCount, 1); @@ -516,9 +525,11 @@ - (void)testHandleRouteChangeMapsReasonsAndFallsBackToUnknown - (void)testHandleMediaServicesResetReactivatesSessionAndRestartsEngine { // Only a session that was active gets re-activated after the reset. + self.fakeAudioEngine.fakeInUse = YES; self.fakeSessionManager.isActive = true; + [self.manager handleMediaServicesReset:nil]; - [self flushMainQueue]; + [self flushPendingWork]; XCTAssertEqual(self.fakeSessionManager.markInactiveCallCount, 1); XCTAssertEqual(self.fakeSessionManager.ensureActiveCallCount, 1); @@ -528,8 +539,11 @@ - (void)testHandleMediaServicesResetReactivatesSessionAndRestartsEngine - (void)testHandleEngineConfigurationChangeMarksInactiveAndRestartsEngine { + self.fakeAudioEngine.fakeInUse = YES; + self.fakeAudioEngine.fakeOwnsNotifyingEngine = YES; + [self.manager handleEngineConfigurationChange:nil]; - [self flushMainQueue]; + [self flushPendingWork]; XCTAssertEqual(self.fakeSessionManager.markInactiveCallCount, 1); XCTAssertEqual(self.fakeAudioEngine.restartAudioEngineCallCount, 1); @@ -541,7 +555,7 @@ - (void)testCheckSecondaryAudioHintDoesNothingWhenStateIsUnchanged self.manager.wasOtherAudioPlaying = NO; [self.manager checkSecondaryAudioHint]; - [self flushMainQueue]; + [self flushPendingWork]; XCTAssertEqual(self.module.eventInvocationCount, 0); XCTAssertEqual(self.fakeSessionManager.markInactiveCallCount, 0); @@ -556,7 +570,7 @@ - (void)testCheckSecondaryAudioHintSilencedTransitionMarksInactiveAndEmitsEventW self.manager.wasOtherAudioPlaying = NO; [self.manager checkSecondaryAudioHint]; - [self flushMainQueue]; + [self flushPendingWork]; XCTAssertTrue(self.manager.wasOtherAudioPlaying); XCTAssertEqual(self.fakeSessionManager.markInactiveCallCount, 1); @@ -591,7 +605,7 @@ - (void)testCheckSecondaryAudioHintResumeTransitionResumesEngineWhenNotObserved self.manager.wasOtherAudioPlaying = YES; [self.manager checkSecondaryAudioHint]; - [self flushMainQueue]; + [self flushPendingWork]; XCTAssertFalse(self.manager.wasOtherAudioPlaying); XCTAssertEqual(self.module.eventInvocationCount, 0); diff --git a/packages/audiodocs/docs/inputs/audio-recorder.mdx b/packages/audiodocs/docs/inputs/audio-recorder.mdx index 5b96e2c64..0b45a992b 100644 --- a/packages/audiodocs/docs/inputs/audio-recorder.mdx +++ b/packages/audiodocs/docs/inputs/audio-recorder.mdx @@ -422,7 +422,7 @@ export default MyRecorder; const audioRecorder = new AudioRecorder(); ```
- Full-duplex voice apps (playing audio while recording) need platform echo cancellation on both platforms, otherwise the speaker output leaks back into the microphone: + Full-duplex voice apps can request the platforms' communication capture paths:
```tsx const audioRecorder = new AudioRecorder({ @@ -430,6 +430,9 @@ export default MyRecorder; iosVoiceProcessing: true, }); ``` +
+ These options request platform processing; echo cancellation, noise suppression, and automatic gain control remain device- and route-dependent. +
diff --git a/packages/react-native-audio-api/ios/audioapi/ios/system/AudioEngine.h b/packages/react-native-audio-api/ios/audioapi/ios/system/AudioEngine.h index e87e58aa8..45c13d6b6 100644 --- a/packages/react-native-audio-api/ios/audioapi/ios/system/AudioEngine.h +++ b/packages/react-native-audio-api/ios/audioapi/ios/system/AudioEngine.h @@ -51,6 +51,8 @@ typedef void (^OnOutputRecoveryFailedBlock)(void); - (AudioEngineState)getState; - (bool)isEngineRunning; - (bool)isInUse; +/// Whether `engine` is the live AVAudioEngine instance, as opposed to one a rebuild has replaced. +- (bool)ownsAudioEngine:(AVAudioEngine *)engine; - (bool)startIfNecessary; - (void)pauseIfNecessary; @@ -59,6 +61,9 @@ typedef void (^OnOutputRecoveryFailedBlock)(void); - (void)stopIfPossible; - (void)restartAudioEngine; +/// Rebuilds the graph only when the live input format no longer matches the sink connection; +/// otherwise lets the recorder re-check its format without touching the engine. +- (void)onRouteChanged; - (void)logAudioEngineState; diff --git a/packages/react-native-audio-api/ios/audioapi/ios/system/AudioEngine.mm b/packages/react-native-audio-api/ios/audioapi/ios/system/AudioEngine.mm index fc535c842..ef89c46d6 100644 --- a/packages/react-native-audio-api/ios/audioapi/ios/system/AudioEngine.mm +++ b/packages/react-native-audio-api/ios/audioapi/ios/system/AudioEngine.mm @@ -37,6 +37,9 @@ @interface AudioEngine () { @property (nonatomic, strong) NSMutableDictionary *sourceRegistrations; @property (nonatomic, strong) AudioEngineInputRegistration *inputRegistration; +/// Format the sink node was connected with; compared against the live input +/// format to tell a route change that altered the input from one that did not. +@property (nonatomic, strong) AVAudioFormat *inputConnectionFormat; - (void)createAudioEngineIfNeeded; - (void)destroyAudioEnginePreservingSessionDeactivationState:(BOOL)preserveSessionDeactivationState; @@ -44,12 +47,15 @@ - (BOOL)hasTrackedGraph; - (AVAudioFormat *)currentInputConnectionFormat; - (void)materializeSourceNodeWithId:(NSString *)sourceNodeId; - (BOOL)materializeInputNodeIfNeeded; -- (void)applyVoiceProcessing; -- (void)materializeTrackedNodesIfNeeded; +- (BOOL)hasInputConnectionFormatChanged; +- (BOOL)prepareInputForMaterialization; +- (BOOL)applyVoiceProcessing; +- (BOOL)materializeTrackedNodesIfNeeded; - (AVAudioFormat *)liveInputFormat; - (void)resetInputNode; -- (void)rebuildAudioEngineAndResumeIfNeeded; +- (BOOL)rebuildAudioEngine; +- (BOOL)rebuildAudioEngineAndResumeIfNeeded; - (void)notifyConfigurationChanges; @end @@ -93,6 +99,7 @@ - (void)destroyAudioEnginePreservingSessionDeactivationState:(BOOL)preserveSessi self.sourceNodes = [[NSMutableDictionary alloc] init]; self.sourceFormats = [[NSMutableDictionary alloc] init]; self.inputNode = nil; + self.inputConnectionFormat = nil; self.graphNeedsRebuild = hadGraph; _voiceProcessingApplied = NO; @@ -112,6 +119,7 @@ - (instancetype)init self.state = AudioEngineState::AudioEngineStateIdle; self.audioEngine = nil; self.inputNode = nil; + self.inputConnectionFormat = nil; self.graphNeedsRebuild = false; self.sessionDeactivationInvalidatedGraph = false; @@ -137,6 +145,7 @@ - (void)cleanup self.sourceFormats = nil; self.inputRegistration = nil; self.inputNode = nil; + self.inputConnectionFormat = nil; self.graphNeedsRebuild = false; self.sessionDeactivationInvalidatedGraph = false; @@ -237,15 +246,6 @@ - (BOOL)materializeInputNodeIfNeeded return YES; } - [self applyVoiceProcessing]; - NSError *sessionError = nil; - if (![self.sessionManager ensureActive:true error:&sessionError]) { - NSLog( - @"Error while activating audio session before input materialization: %@", - [sessionError debugDescription]); - return NO; - } - AVAudioFormat *inputFormat = [self currentInputConnectionFormat]; if (inputFormat == nil) { @@ -254,17 +254,56 @@ - (BOOL)materializeInputNodeIfNeeded self.inputNode = [[AVAudioSinkNode alloc] initWithReceiverBlock:self.inputRegistration.receiverBlock]; + self.inputConnectionFormat = inputFormat; [self.audioEngine attachNode:self.inputNode]; [self.audioEngine connect:self.audioEngine.inputNode to:self.inputNode format:inputFormat]; return YES; } +- (BOOL)hasInputConnectionFormatChanged +{ + if (self.inputNode == nil || self.inputConnectionFormat == nil) { + return NO; + } + + AVAudioFormat *liveFormat = [self liveInputFormat]; + + if (liveFormat == nil) { + return NO; + } + + return liveFormat.sampleRate != self.inputConnectionFormat.sampleRate || + liveFormat.channelCount != self.inputConnectionFormat.channelCount || + liveFormat.isInterleaved != self.inputConnectionFormat.isInterleaved; +} + // Apple's voice-processing I/O (echo cancellation, noise suppression, AGC) is // opt-in per recorder. Without it, full-duplex apps (VoIP, voice agents) hear // their own speaker output looped back into the microphone. Toggling it is only // allowed while the engine is stopped, and it changes the hardware input format, // so this has to run before the input connection format is read. -- (void)applyVoiceProcessing +- (BOOL)prepareInputForMaterialization +{ + if (self.inputRegistration == nil) { + return YES; + } + + NSError *sessionError = nil; + if (![self.sessionManager ensureActive:true error:&sessionError]) { + NSLog( + @"Error while activating audio session before input materialization: %@", + [sessionError debugDescription]); + return NO; + } + + if ([self.audioEngine isRunning]) { + [self.audioEngine stop]; + } + + return [self applyVoiceProcessing]; +} + +- (BOOL)applyVoiceProcessing { BOOL wantsVoiceProcessing = self.inputRegistration.voiceProcessingEnabled; @@ -272,22 +311,23 @@ - (void)applyVoiceProcessing // graphs there is nothing to undo - and reading `inputNode` would needlessly // pull the microphone into the engine. if (!wantsVoiceProcessing && !_voiceProcessingApplied) { - return; + return YES; } if (self.audioEngine == nil) { - return; + return NO; } AVAudioInputNode *systemInputNode = self.audioEngine.inputNode; if (systemInputNode.isVoiceProcessingEnabled == wantsVoiceProcessing) { _voiceProcessingApplied = wantsVoiceProcessing; - return; + return YES; } if ([self.audioEngine isRunning]) { - [self.audioEngine stop]; + NSLog(@"[AudioEngine] Voice processing must be configured before the engine starts"); + return NO; } NSError *error = nil; @@ -297,15 +337,18 @@ - (void)applyVoiceProcessing @"[AudioEngine] Error while setting voice processing to %@: %@", wantsVoiceProcessing ? @"true" : @"false", [error debugDescription]); - return; + return NO; } _voiceProcessingApplied = wantsVoiceProcessing; + return YES; } -- (void)materializeTrackedNodesIfNeeded +- (BOOL)materializeTrackedNodesIfNeeded { - [self applyVoiceProcessing]; + if (![self prepareInputForMaterialization]) { + return NO; + } NSArray *sourceNodeIds = [[self.sourceRegistrations allKeys] sortedArrayUsingSelector:@selector(compare:)]; @@ -313,7 +356,7 @@ - (void)materializeTrackedNodesIfNeeded [self materializeSourceNodeWithId:sourceNodeId]; } - [self materializeInputNodeIfNeeded]; + return [self materializeInputNodeIfNeeded]; } - (NSString *)attachSourceNodeWithRenderBlock:(AVAudioSourceNodeRenderBlock)renderBlock @@ -377,7 +420,13 @@ - (void)attachInputNodeWithReceiverBlock:(AVAudioSinkNodeReceiverBlock)receiverB registration.onInputConfigurationChange = onInputConfigurationChange; self.inputRegistration = registration; - [self materializeInputNodeIfNeeded]; + // Voice processing changes the live I/O format, so defer its input-node + // connection until startEngine has activated the configured session. + if (!voiceProcessingEnabled) { + if (![self prepareInputForMaterialization] || ![self materializeInputNodeIfNeeded]) { + NSLog(@"[AudioEngine] Could not materialize input node after attaching recorder"); + } + } } - (void)resetInputNode @@ -390,8 +439,14 @@ - (void)resetInputNode [self.audioEngine detachNode:self.inputNode]; } + // Voice processing left its own mode and options on the session; restore ours. + if (self.inputRegistration.voiceProcessingEnabled) { + [self.sessionManager invalidateAppliedConfiguration]; + } + self.inputRegistration = nil; self.inputNode = nil; + self.inputConnectionFormat = nil; if (![self hasTrackedGraph]) { self.graphNeedsRebuild = false; @@ -463,7 +518,11 @@ - (void)onInterruptionEnd:(bool)shouldResume } [self stopEngine]; - [self rebuildAudioEngine]; + if (![self rebuildAudioEngine]) { + self.state = AudioEngineState::AudioEngineStateIdle; + [self notifyConfigurationChanges]; + return; + } if (!shouldResume) { self.state = AudioEngineState::AudioEngineStatePaused; @@ -516,10 +575,16 @@ - (bool)isInUse return [self hasTrackedGraph] || self.audioEngine != nil; } -- (void)rebuildAudioEngineAndResumeIfNeeded +- (bool)ownsAudioEngine:(AVAudioEngine *)engine +{ + std::scoped_lock lock(_engineLock); + return engine != nil && engine == self.audioEngine; +} + +- (BOOL)rebuildAudioEngineAndResumeIfNeeded { if (_isRebuildingAudioEngine) { - return; + return YES; } _isRebuildingAudioEngine = YES; @@ -528,7 +593,10 @@ - (void)rebuildAudioEngineAndResumeIfNeeded [self.audioEngine stop]; } - [self rebuildAudioEngine]; + if (![self rebuildAudioEngine]) { + _isRebuildingAudioEngine = NO; + return NO; + } self.sessionDeactivationInvalidatedGraph = false; if (self.state == AudioEngineState::AudioEngineStateRunning) { @@ -541,15 +609,19 @@ - (void)rebuildAudioEngineAndResumeIfNeeded [self notifyConfigurationChanges]; _isRebuildingAudioEngine = NO; + return YES; } -- (void)rebuildAudioEngine +- (BOOL)rebuildAudioEngine { [self destroyAudioEnginePreservingSessionDeactivationState:YES]; [self createAudioEngineIfNeeded]; - [self materializeTrackedNodesIfNeeded]; + if (![self materializeTrackedNodesIfNeeded]) { + return NO; + } self.graphNeedsRebuild = false; + return YES; } - (bool)startEngine @@ -570,9 +642,13 @@ - (bool)startEngine if (self.state == AudioEngineState::AudioEngineStateInterrupted || self.graphNeedsRebuild || self.sessionDeactivationInvalidatedGraph) { - [self rebuildAudioEngineAndResumeIfNeeded]; + if (![self rebuildAudioEngineAndResumeIfNeeded]) { + return false; + } } else { - [self materializeTrackedNodesIfNeeded]; + if (![self materializeTrackedNodesIfNeeded]) { + return false; + } } if (self.inputRegistration != nil && self.inputNode == nil) { @@ -611,8 +687,12 @@ - (bool)startIfNecessary std::scoped_lock lock(_engineLock); if (self.state == AudioEngineState::AudioEngineStateRunning && self.audioEngine != nil && [self.audioEngine isRunning]) { - [self materializeTrackedNodesIfNeeded]; - return true; + if (self.inputRegistration == nil || self.inputNode != nil) { + return true; + } + + [self.audioEngine stop]; + self.state = AudioEngineState::AudioEngineStateIdle; } if ([self hasTrackedGraph]) { @@ -668,7 +748,26 @@ - (void)restartAudioEngine return; } - [self rebuildAudioEngineAndResumeIfNeeded]; + if (![self rebuildAudioEngineAndResumeIfNeeded]) { + self.state = AudioEngineState::AudioEngineStatePaused; + [self notifyConfigurationChanges]; + } +} + +- (void)onRouteChanged +{ + std::scoped_lock lock(_engineLock); + + if (![self hasTrackedGraph] && self.audioEngine == nil) { + return; + } + + if (![self hasInputConnectionFormatChanged]) { + [self notifyConfigurationChanges]; + return; + } + + [self restartAudioEngine]; } - (void)logAudioEngineState diff --git a/packages/react-native-audio-api/ios/audioapi/ios/system/AudioSessionManager.h b/packages/react-native-audio-api/ios/audioapi/ios/system/AudioSessionManager.h index 8f584e76a..e132ca059 100644 --- a/packages/react-native-audio-api/ios/audioapi/ios/system/AudioSessionManager.h +++ b/packages/react-native-audio-api/ios/audioapi/ios/system/AudioSessionManager.h @@ -34,6 +34,8 @@ /// Ensures the library-managed session is active. In external owner mode this is a no-op. - (bool)ensureActive:(bool)force error:(NSError **)error; - (bool)setActive:(bool)active error:(NSError **)error; +/// Forces the next activation to re-apply the desired configuration, e.g. after media services reset. +- (void)invalidateAppliedConfiguration; /// Drops the cached active-state flag without reclaiming external ownership. - (void)markInactive; /// Switches the manager into external owner mode and stops all session mutations. diff --git a/packages/react-native-audio-api/ios/audioapi/ios/system/AudioSessionManager.mm b/packages/react-native-audio-api/ios/audioapi/ios/system/AudioSessionManager.mm index fca4ed10d..5b4e0d16d 100644 --- a/packages/react-native-audio-api/ios/audioapi/ios/system/AudioSessionManager.mm +++ b/packages/react-native-audio-api/ios/audioapi/ios/system/AudioSessionManager.mm @@ -12,6 +12,9 @@ - (void)requestSystemRecordPermission:(void (^)(BOOL granted))completion; - (NSInteger)currentRecordPermissionStatus; - (NSString *)recordPermissionStatusString:(NSInteger)status; - (NSString *)formatPorts:(NSArray *)ports; +- (bool)isDesiredConfigurationApplied; + +@property (nonatomic, assign) bool desiredConfigurationApplied; @end @@ -40,6 +43,7 @@ - (instancetype)init self.desiredOptions = 0; self.allowHapticsAndSounds = false; self.notifyOthersOnDeactivation = true; + self.desiredConfigurationApplied = false; } _sharedInstance = self; @@ -90,9 +94,22 @@ - (void)resetPreferredIOBufferFrames [self setPreferredIOBufferFrames:audioapi::RENDER_QUANTUM_SIZE]; } +// Voice processing rewrites the mode and options of the session it runs in. +// Re-applying ours would toggle the route. Once our +// configuration has been applied, only a foreign category change is drift. +- (bool)isDesiredConfigurationApplied +{ + if (self.desiredConfigurationApplied) { + return self.audioSession.category == self.desiredCategory; + } + + return [self areDesiredOptionsSet]; +} + - (bool)configureAudioSession:(NSError **)outError { - if (!self.shouldManageSession || [self areDesiredOptionsSet]) { + if (!self.shouldManageSession || [self isDesiredConfigurationApplied]) { + self.desiredConfigurationApplied = self.shouldManageSession; return true; } @@ -134,6 +151,7 @@ - (bool)configureAudioSession:(NSError **)outError } } + self.desiredConfigurationApplied = true; return true; } @@ -154,6 +172,7 @@ - (void)setAudioSessionOptions:(NSString *)categoryStr } if (configChanged) { + self.desiredConfigurationApplied = false; AudioEngine *audioEngine = [AudioEngine sharedInstance]; [audioEngine markSessionDeactivationInvalidatedGraph]; } @@ -191,6 +210,7 @@ - (bool)setActive:(bool)active error:(NSError **)error if (success) { self.isActive = active; + self.desiredConfigurationApplied = false; } return success; @@ -224,6 +244,11 @@ - (bool)activateSessionIfNeeded:(bool)force error:(NSError **)error return success; } +- (void)invalidateAppliedConfiguration +{ + self.desiredConfigurationApplied = false; +} + - (void)markInactive { // AVAudioSession does not expose a reliable active-state query, so drop our cached flag and diff --git a/packages/react-native-audio-api/ios/audioapi/ios/system/SystemNotificationManager.h b/packages/react-native-audio-api/ios/audioapi/ios/system/SystemNotificationManager.h index 5ddf0d4e2..dd082d842 100644 --- a/packages/react-native-audio-api/ios/audioapi/ios/system/SystemNotificationManager.h +++ b/packages/react-native-audio-api/ios/audioapi/ios/system/SystemNotificationManager.h @@ -17,6 +17,10 @@ @property (nonatomic, assign) bool volumeChangesObserved; @property (nonatomic, assign) bool wasOtherAudioPlaying; +/// Serial queue on which reactions to system notifications reach the audio engine +/// and session: off the thread that delivered the notification, and in arrival order. +@property (nonatomic, strong, readonly) dispatch_queue_t engineLifecycleQueue; + - (instancetype)initWithAudioAPIModule:(AudioAPIModule *)audioAPIModule; - (void)cleanup; diff --git a/packages/react-native-audio-api/ios/audioapi/ios/system/SystemNotificationManager.mm b/packages/react-native-audio-api/ios/audioapi/ios/system/SystemNotificationManager.mm index e3c27a524..1e51c99c6 100644 --- a/packages/react-native-audio-api/ios/audioapi/ios/system/SystemNotificationManager.mm +++ b/packages/react-native-audio-api/ios/audioapi/ios/system/SystemNotificationManager.mm @@ -14,6 +14,8 @@ - (instancetype)initWithAudioAPIModule:(AudioAPIModule *)audioAPIModule self.audioAPIModule = audioAPIModule; self.notificationCenter = [NSNotificationCenter defaultCenter]; self.audioInterruptionsObserved = false; + _engineLifecycleQueue = + dispatch_queue_create("com.swmansion.audioapi.EngineLifecycleQueue", DISPATCH_QUEUE_SERIAL); [self configureNotifications]; } @@ -122,7 +124,7 @@ - (void)handleInterruption:(NSNotification *)notification [notification.userInfo[AVAudioSessionInterruptionOptionKey] integerValue]; if (interruptionType == AVAudioSessionInterruptionTypeBegan) { - dispatch_async(dispatch_get_main_queue(), ^{ + dispatch_async(self.engineLifecycleQueue, ^{ [audioEngine onInterruptionBegin]; [sessionManager markInactive]; }); @@ -144,7 +146,7 @@ - (void)handleInterruption:(NSNotification *)notification payload:audioapi::InterruptionPayload{ .type = "ended", .shouldResume = shouldResume}]; } else { - dispatch_async(dispatch_get_main_queue(), ^{ [audioEngine onInterruptionEnd:shouldResume]; }); + dispatch_async(self.engineLifecycleQueue, ^{ [audioEngine onInterruptionEnd:shouldResume]; }); } } @@ -156,7 +158,7 @@ - (void)handleSecondaryAudio:(NSNotification *)notification [notification.userInfo[AVAudioSessionSilenceSecondaryAudioHintTypeKey] integerValue]; if (secondaryAudioType == AVAudioSessionSilenceSecondaryAudioHintTypeBegin) { - dispatch_async(dispatch_get_main_queue(), ^{ + dispatch_async(self.engineLifecycleQueue, ^{ [sessionManager markInactive]; [audioEngine onInterruptionBegin]; }); @@ -177,7 +179,7 @@ - (void)handleSecondaryAudio:(NSNotification *)notification payload:audioapi::InterruptionPayload{ .type = "ended", .shouldResume = shouldResume}]; } else { - dispatch_async(dispatch_get_main_queue(), ^{ [audioEngine onInterruptionEnd:shouldResume]; }); + dispatch_async(self.engineLifecycleQueue, ^{ [audioEngine onInterruptionEnd:shouldResume]; }); } } @@ -225,9 +227,11 @@ - (void)handleRouteChange:(NSNotification *)notification switch (routeChangeReason) { case AVAudioSessionRouteChangeReasonNewDeviceAvailable: case AVAudioSessionRouteChangeReasonOldDeviceUnavailable: - case AVAudioSessionRouteChangeReasonRouteConfigurationChange: - [self handleEngineConfigurationChange:nil]; + case AVAudioSessionRouteChangeReasonRouteConfigurationChange: { + AudioEngine *audioEngine = self.audioAPIModule.audioEngine; + dispatch_async(self.engineLifecycleQueue, ^{ [audioEngine onRouteChanged]; }); break; + } default: break; } @@ -245,9 +249,10 @@ - (void)handleMediaServicesReset:(NSNotification *)notification NSLog( @"[NotificationManager] Media services have been reset, tearing down and rebuilding everything."); - dispatch_async(dispatch_get_main_queue(), ^{ + dispatch_async(self.engineLifecycleQueue, ^{ bool wasSessionActive = sessionManager.isActive; [sessionManager markInactive]; + [sessionManager invalidateAppliedConfiguration]; if (wasSessionActive) { [sessionManager ensureActive:true error:nil]; @@ -262,12 +267,13 @@ - (void)handleEngineConfigurationChange:(NSNotification *)notification AudioEngine *audioEngine = self.audioAPIModule.audioEngine; AudioSessionManager *sessionManager = self.audioAPIModule.audioSessionManager; - dispatch_async(dispatch_get_main_queue(), ^{ + dispatch_async(self.engineLifecycleQueue, ^{ // This notification is registered with object:nil, so it also fires for - // AVAudioEngine instances owned by other libraries in the host app. Without - // an engine of our own there is nothing to restart, and marking the session + // AVAudioEngine instances owned by other libraries in the host app, and for + // an engine of ours that a rebuild has already replaced. Without a live + // engine of our own there is nothing to restart, and marking the session // inactive would corrupt bookkeeping for apps that only manage the session. - if (![audioEngine isInUse]) { + if (![audioEngine isInUse] || ![audioEngine ownsAudioEngine:notification.object]) { return; } @@ -313,7 +319,7 @@ - (void)checkSecondaryAudioHint self.wasOtherAudioPlaying = shouldSilence; if (shouldSilence) { - dispatch_async(dispatch_get_main_queue(), ^{ + dispatch_async(self.engineLifecycleQueue, ^{ [sessionManager markInactive]; [audioEngine onInterruptionBegin]; }); @@ -331,7 +337,7 @@ - (void)checkSecondaryAudioHint payload:audioapi::InterruptionPayload{ .type = "ended", .shouldResume = true}]; } else { - dispatch_async(dispatch_get_main_queue(), ^{ [audioEngine onInterruptionEnd:true]; }); + dispatch_async(self.engineLifecycleQueue, ^{ [audioEngine onInterruptionEnd:true]; }); } }