From 5a4d90643ec1d507713bda1f11bdefc34fa61ea6 Mon Sep 17 00:00:00 2001 From: Rares Date: Thu, 25 Jun 2026 16:07:09 -0400 Subject: [PATCH 1/9] Add audio session management and playback functionality to Deepgram module - Implemented audio session lifecycle management in Deepgram+AudioSession, including activation, deactivation, and configuration for recording and playback. - Added handling for audio route changes, media services resets, and audio interruptions to ensure seamless audio experience. - Introduced Deepgram+Playback for TTS and voice agent audio output using AVAudioPlayerNode, with support for echo cancellation when voice processing is enabled. - Created Deepgram+Recording to manage microphone capture, supporting both AudioQueue and AVAudioEngine paths for different use cases. - Enhanced private interface in Deepgram+Private.h to manage shared state across audio session, recording, and playback functionalities. - Added error handling and logging throughout the audio management process to aid in debugging and ensure reliability. --- .../java/com/deepgram/AudioFocusManager.kt | 125 ++ .../src/main/java/com/deepgram/AudioPlayer.kt | 412 ++++++ .../main/java/com/deepgram/AudioRecorder.kt | 299 +++++ .../java/com/deepgram/DeepgramAudioService.kt | 37 +- .../main/java/com/deepgram/DeepgramModule.kt | 780 ++--------- ios/DGSupport.h | 77 ++ ios/Deepgram+AudioSession.h | 28 + ios/Deepgram+AudioSession.mm | 438 +++++++ ios/Deepgram+Playback.h | 21 + ios/Deepgram+Playback.mm | 211 +++ ios/Deepgram+Private.h | 65 + ios/Deepgram+Recording.h | 31 + ios/Deepgram+Recording.mm | 411 ++++++ ios/Deepgram.mm | 1142 +---------------- 14 files changed, 2276 insertions(+), 1801 deletions(-) create mode 100644 android/src/main/java/com/deepgram/AudioFocusManager.kt create mode 100644 android/src/main/java/com/deepgram/AudioPlayer.kt create mode 100644 android/src/main/java/com/deepgram/AudioRecorder.kt create mode 100644 ios/DGSupport.h create mode 100644 ios/Deepgram+AudioSession.h create mode 100644 ios/Deepgram+AudioSession.mm create mode 100644 ios/Deepgram+Playback.h create mode 100644 ios/Deepgram+Playback.mm create mode 100644 ios/Deepgram+Private.h create mode 100644 ios/Deepgram+Recording.h create mode 100644 ios/Deepgram+Recording.mm diff --git a/android/src/main/java/com/deepgram/AudioFocusManager.kt b/android/src/main/java/com/deepgram/AudioFocusManager.kt new file mode 100644 index 0000000..57931d8 --- /dev/null +++ b/android/src/main/java/com/deepgram/AudioFocusManager.kt @@ -0,0 +1,125 @@ +package com.deepgram + +import android.content.Context +import android.media.AudioAttributes +import android.media.AudioFocusRequest +import android.media.AudioManager +import android.os.Build +import android.os.Handler +import android.util.Log + +/** + * Owns system audio-focus arbitration for the Deepgram module. + * + * Focus is requested with `AUDIOFOCUS_GAIN_TRANSIENT` so other audio apps + * (Spotify, podcasts, navigation, etc.) pause/duck and then automatically + * resume when we abandon focus. Focus-change events are forwarded to the owner + * through [Listener] so it can stop/pause the right audio components. + */ +internal class AudioFocusManager( + private val context: Context, + private val handler: Handler, + private val listener: Listener, +) { + /** Routes system focus transitions back to the owning module. */ + interface Listener { + /** Permanent loss (another app took focus): tear everything down. */ + fun onFocusLostPermanently() + + /** Transient loss (e.g. incoming call): pause playback but keep state. */ + fun onFocusLostTransiently() + + /** Focus regained: resume playback if it was active. */ + fun onFocusGained() + } + + @Volatile + var hasFocus: Boolean = false + private set + + private var audioManager: AudioManager? = null + private var audioFocusRequest: AudioFocusRequest? = null + + private val focusChangeListener = AudioManager.OnAudioFocusChangeListener { focusChange -> + when (focusChange) { + AudioManager.AUDIOFOCUS_LOSS -> { + Log.i(TAG, "AUDIOFOCUS_LOSS — stopping Deepgram audio") + hasFocus = false + listener.onFocusLostPermanently() + } + AudioManager.AUDIOFOCUS_LOSS_TRANSIENT, + AudioManager.AUDIOFOCUS_LOSS_TRANSIENT_CAN_DUCK -> { + Log.i(TAG, "AUDIOFOCUS_LOSS_TRANSIENT — pausing playback") + listener.onFocusLostTransiently() + } + AudioManager.AUDIOFOCUS_GAIN -> { + Log.i(TAG, "AUDIOFOCUS_GAIN — resuming playback if active") + hasFocus = true + listener.onFocusGained() + } + } + } + + fun requestFocus(): Boolean { + if (hasFocus) return true + try { + val manager = context.getSystemService(Context.AUDIO_SERVICE) as? AudioManager + ?: return false + audioManager = manager + + val result = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + val request = AudioFocusRequest.Builder(AudioManager.AUDIOFOCUS_GAIN_TRANSIENT) + .setAudioAttributes( + AudioAttributes.Builder() + .setUsage(AudioAttributes.USAGE_VOICE_COMMUNICATION) + .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH) + .build() + ) + .setOnAudioFocusChangeListener(focusChangeListener, handler) + .setWillPauseWhenDucked(true) + .build() + audioFocusRequest = request + manager.requestAudioFocus(request) + } else { + @Suppress("DEPRECATION") + manager.requestAudioFocus( + focusChangeListener, + AudioManager.STREAM_VOICE_CALL, + AudioManager.AUDIOFOCUS_GAIN_TRANSIENT + ) + } + + hasFocus = result == AudioManager.AUDIOFOCUS_REQUEST_GRANTED + if (!hasFocus) { + Log.w(TAG, "Audio focus request not granted (result=$result)") + } + return hasFocus + } catch (e: Exception) { + Log.w(TAG, "requestAudioFocus error", e) + hasFocus = false + return false + } + } + + fun abandonFocus() { + if (!hasFocus) return + try { + val manager = audioManager ?: return + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + audioFocusRequest?.let { manager.abandonAudioFocusRequest(it) } + } else { + @Suppress("DEPRECATION") + manager.abandonAudioFocus(focusChangeListener) + } + } catch (e: Exception) { + Log.w(TAG, "abandonAudioFocus error", e) + } finally { + hasFocus = false + audioFocusRequest = null + } + } + + companion object { + private const val TAG = "DeepgramFocus" + } +} diff --git a/android/src/main/java/com/deepgram/AudioPlayer.kt b/android/src/main/java/com/deepgram/AudioPlayer.kt new file mode 100644 index 0000000..0b27cf2 --- /dev/null +++ b/android/src/main/java/com/deepgram/AudioPlayer.kt @@ -0,0 +1,412 @@ +package com.deepgram + +import android.content.BroadcastReceiver +import android.content.Context +import android.content.Intent +import android.content.IntentFilter +import android.media.AudioAttributes +import android.media.AudioFormat +import android.media.AudioManager +import android.media.AudioRecord +import android.media.AudioTrack +import android.os.Build +import android.os.Handler +import android.util.Base64 +import android.util.Log +import java.util.concurrent.LinkedBlockingQueue +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicInteger + +/** + * TTS audio output. Supports two modes: + * - Streaming ([startPlayer] / [feedAudio]): base64 PCM chunks are queued and + * drained on a dedicated thread through a `MODE_STREAM` AudioTrack. + * - One-shot ([playAudioChunk]): a single buffer is played through a + * `MODE_STATIC` AudioTrack and the caller is notified on completion. + * + * Audio focus is delegated to [focusManager]; foreground-service coordination + * is delegated to the owner via the two callbacks. + */ +internal class AudioPlayer( + private val context: Context, + private val mainHandler: Handler, + private val focusManager: AudioFocusManager, + private val onForegroundServiceRequest: () -> Unit, + private val onForegroundServiceRelease: () -> Unit, +) { + @Volatile + var isActive: Boolean = false + private set + + private var playbackSampleRate: Int = DEFAULT_PLAYBACK_SAMPLE_RATE + private var currentOutputChannels: Int = 1 + + private var audioTrack: AudioTrack? = null + private val audioQueue = LinkedBlockingQueue() + private var playbackThread: Thread? = null + private val playbackThreshold = 1024 + private val queuedBytes = AtomicInteger(0) + + // ACTION_AUDIO_BECOMING_NOISY — fires when wired headset is unplugged or A2DP + // disconnects. Without handling this, our AudioTrack would silently re-route + // to the loudspeaker and surprise the user. Standard Android best practice. + private var noisyReceiverRegistered = false + private val becomingNoisyReceiver = object : BroadcastReceiver() { + override fun onReceive(context: Context?, intent: Intent?) { + if (intent?.action == AudioManager.ACTION_AUDIO_BECOMING_NOISY) { + Log.i(TAG, "AUDIO_BECOMING_NOISY — pausing playback") + try { audioTrack?.pause() } catch (_: Exception) {} + } + } + } + + // The streaming AudioTrack is sized to at least 4x the microphone min-buffer. + // Preserved verbatim from the original single-class implementation to keep + // latency / under-run characteristics identical across releases. + private val streamingBufferFloor: Int + get() { + val recMin = AudioRecord.getMinBufferSize( + RECORD_SAMPLE_RATE, AudioFormat.CHANNEL_IN_MONO, AUDIO_FORMAT + ) + val base = if (recMin > 0) recMin else maxOf((RECORD_SAMPLE_RATE / 5) * 2, 4096) + return base * 4 + } + + private fun outputChannelMask(): Int { + return if (currentOutputChannels >= 2) { + AudioFormat.CHANNEL_OUT_STEREO + } else { + AudioFormat.CHANNEL_OUT_MONO + } + } + + private fun minTrackBufferSize(channelMask: Int): Int { + val min = AudioTrack.getMinBufferSize(playbackSampleRate, channelMask, AUDIO_FORMAT) + if (min > 0) return min + + val bytesPerFrame = if (currentOutputChannels >= 2) 4 else 2 + val fallback = maxOf((playbackSampleRate / 5) * bytesPerFrame, 4096) + Log.w(TAG, "Invalid AudioTrack min buffer ($min), using fallback=$fallback") + return fallback + } + + private fun ensureAudioTrack() { + if (audioTrack != null) return + + val channelMask = outputChannelMask() + val minBuf = minTrackBufferSize(channelMask) + + audioTrack = AudioTrack.Builder() + .setAudioAttributes( + AudioAttributes.Builder() + .setUsage(AudioAttributes.USAGE_VOICE_COMMUNICATION) + .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH) + .build() + ) + .setAudioFormat( + AudioFormat.Builder() + .setEncoding(AUDIO_FORMAT) + .setSampleRate(playbackSampleRate) + .setChannelMask(channelMask) + .build() + ) + .setBufferSizeInBytes(maxOf(minBuf, streamingBufferFloor)) + .setTransferMode(AudioTrack.MODE_STREAM) + .build() + } + + // ------------------------------------------------------------------- + // Simple play/stop (kept for API compatibility) + // ------------------------------------------------------------------- + + fun startAudio() { + focusManager.requestFocus() + ensureAudioTrack() + audioTrack?.play() + isActive = true + onForegroundServiceRequest() + } + + fun stopAudio() { + stopAudioInternal(throwOnError = true) + } + + private fun stopAudioInternal(throwOnError: Boolean) { + try { + audioTrack?.stop() + } catch (e: Exception) { + Log.w(TAG, "stopAudioInternal: error stopping AudioTrack", e) + if (throwOnError) throw e + } + + try { + audioTrack?.release() + } catch (e: Exception) { + Log.w(TAG, "stopAudioInternal: error releasing AudioTrack", e) + if (throwOnError) throw e + } + + audioTrack = null + isActive = false + focusManager.abandonFocus() + onForegroundServiceRelease() + } + + // ------------------------------------------------------------------- + // Streaming player. Uses a proper queue instead of O(n²) concatenation. + // ------------------------------------------------------------------- + + fun startPlayer(sampleRate: Int, channels: Int) { + stopStreamingPlayback() + playbackSampleRate = if (sampleRate > 0) sampleRate else DEFAULT_PLAYBACK_SAMPLE_RATE + currentOutputChannels = if (channels >= 2) 2 else 1 + audioQueue.clear() + queuedBytes.set(0) + isActive = false + } + + fun setAudioConfig(sampleRate: Int, channels: Int) { + startPlayer(sampleRate, channels) + } + + fun feedAudio(base64Audio: String) { + try { + val audioData = Base64.decode(base64Audio, Base64.DEFAULT) + if (audioData.isEmpty()) return + + // Bound the queue to prevent unbounded memory growth if JS is producing + // audio faster than the device can play it. Drop the oldest chunk first + // (so we always favor newer audio) when at the limit. + while (queuedBytes.get() + audioData.size > MAX_QUEUED_PLAYBACK_BYTES) { + val dropped = audioQueue.poll() ?: break + queuedBytes.addAndGet(-dropped.size) + Log.w(TAG, "feedAudio: dropping ${dropped.size} bytes — playback queue full") + } + + audioQueue.offer(audioData) + queuedBytes.addAndGet(audioData.size) + + // Start playback when we have enough buffered data + if (!isActive && queuedBytes.get() >= playbackThreshold) { + startStreamingPlayback() + } + } catch (e: Exception) { + Log.e(TAG, "feedAudio error", e) + } + } + + fun interruptAudio() { + try { + stopStreamingPlayback(throwOnError = false) + } catch (e: Exception) { + Log.w(TAG, "interruptAudio error", e) + } + } + + private fun startStreamingPlayback() { + if (isActive) return + + isActive = true + registerNoisyReceiver() + playbackThread = Thread({ + try { + android.os.Process.setThreadPriority(android.os.Process.THREAD_PRIORITY_URGENT_AUDIO) + focusManager.requestFocus() + onForegroundServiceRequest() + ensureAudioTrack() + audioTrack?.play() + + while (isActive) { + // Block until data is available (up to 100ms), avoids busy-wait + val chunk = audioQueue.poll(100, TimeUnit.MILLISECONDS) + ?: continue + + queuedBytes.addAndGet(-chunk.size) + val written = audioTrack?.write(chunk, 0, chunk.size) ?: 0 + if (written < 0) { + Log.e(TAG, "AudioTrack write error: $written") + break + } + } + } catch (e: InterruptedException) { + // Normal shutdown + } catch (e: Exception) { + Log.e(TAG, "Streaming playback error", e) + } finally { + isActive = false + onForegroundServiceRelease() + } + }, "Deepgram-Playback") + + playbackThread?.start() + } + + fun stopStreamingPlayback(throwOnError: Boolean = true) { + isActive = false + playbackThread?.let { thread -> + thread.interrupt() + try { + thread.join(500) + } catch (_: InterruptedException) { + Thread.currentThread().interrupt() + } + } + playbackThread = null + + try { + stopAudioInternal(throwOnError) + } catch (e: Exception) { + if (throwOnError) throw e else Log.w(TAG, "stopStreamingPlayback: error stopping audio", e) + } finally { + audioQueue.clear() + queuedBytes.set(0) + unregisterNoisyReceiver() + onForegroundServiceRelease() + } + } + + // ------------------------------------------------------------------- + // Focus transitions (driven by the owning module) + // ------------------------------------------------------------------- + + /** Pause output on a transient focus loss without discarding state. */ + fun pauseForFocusLoss() { + try { audioTrack?.pause() } catch (_: Exception) {} + } + + /** Resume output after regaining focus, if playback is still active. */ + fun resumeForFocusGain() { + if (isActive) { + try { audioTrack?.play() } catch (_: Exception) {} + } + } + + // ------------------------------------------------------------------- + // One-shot playback (for HTTP synthesis) + // ------------------------------------------------------------------- + + fun playAudioChunk( + base64: String, + onSuccess: () -> Unit, + onError: (code: String, message: String?) -> Unit, + ) { + try { + val audioData = Base64.decode(base64, Base64.DEFAULT) + if (audioData.isEmpty()) { + onError("invalid_data", "Failed to decode audio data") + return + } + + focusManager.requestFocus() + onForegroundServiceRequest() + + val channelMask = outputChannelMask() + val minBuf = minTrackBufferSize(channelMask) + + val tempAudioTrack = AudioTrack.Builder() + .setAudioAttributes( + AudioAttributes.Builder() + .setUsage(AudioAttributes.USAGE_VOICE_COMMUNICATION) + .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH) + .build() + ) + .setAudioFormat( + AudioFormat.Builder() + .setEncoding(AUDIO_FORMAT) + .setSampleRate(playbackSampleRate) + .setChannelMask(channelMask) + .build() + ) + .setBufferSizeInBytes(maxOf(minBuf, audioData.size)) + .setTransferMode(AudioTrack.MODE_STATIC) + .build() + + tempAudioTrack.write(audioData, 0, audioData.size) + + // Use notification marker to know when playback finishes, + // then notify the caller (instead of polling with Thread.sleep). + val bytesPerFrame = if (currentOutputChannels >= 2) 4 else 2 // PCM16 mono/stereo + val frameCount = audioData.size / bytesPerFrame + tempAudioTrack.notificationMarkerPosition = frameCount + val listener = object : AudioTrack.OnPlaybackPositionUpdateListener { + override fun onMarkerReached(track: AudioTrack?) { + try { + track?.stop() + track?.release() + } catch (e: Exception) { + Log.w(TAG, "playAudioChunk cleanup error", e) + } + focusManager.abandonFocus() + onForegroundServiceRelease() + onSuccess() + } + + override fun onPeriodicNotification(track: AudioTrack?) {} + } + + // minSdk is 24, the Handler-aware overload is always available. + tempAudioTrack.setPlaybackPositionUpdateListener(listener, mainHandler) + + tempAudioTrack.play() + } catch (e: Exception) { + Log.e(TAG, "playAudioChunk error", e) + focusManager.abandonFocus() + onForegroundServiceRelease() + onError("playback_error", e.message) + } + } + + /** Tear down all playback resources (called from the module's invalidate). */ + fun release() { + try { + stopStreamingPlayback(throwOnError = false) + } catch (e: Exception) { + Log.w(TAG, "Error stopping playback on release", e) + } + unregisterNoisyReceiver() + } + + private fun registerNoisyReceiver() { + if (noisyReceiverRegistered) return + try { + val filter = IntentFilter(AudioManager.ACTION_AUDIO_BECOMING_NOISY) + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { + context.registerReceiver( + becomingNoisyReceiver, + filter, + Context.RECEIVER_NOT_EXPORTED + ) + } else { + @Suppress("UnspecifiedRegisterReceiverFlag") + context.registerReceiver(becomingNoisyReceiver, filter) + } + noisyReceiverRegistered = true + } catch (e: Exception) { + Log.w(TAG, "registerNoisyReceiver failed", e) + } + } + + private fun unregisterNoisyReceiver() { + if (!noisyReceiverRegistered) return + try { + context.unregisterReceiver(becomingNoisyReceiver) + } catch (e: Exception) { + Log.w(TAG, "unregisterNoisyReceiver failed", e) + } finally { + noisyReceiverRegistered = false + } + } + + companion object { + private const val TAG = "DeepgramPlayer" + + private const val DEFAULT_PLAYBACK_SAMPLE_RATE = 16000 + private const val RECORD_SAMPLE_RATE = 16000 + private const val AUDIO_FORMAT = AudioFormat.ENCODING_PCM_16BIT + + // Cap the streaming playback queue to prevent unbounded memory growth when + // JS feeds audio faster than the device can play it. ~5 s of 24 kHz PCM16 + // mono is roughly 240 KB — we allow a little more headroom. + private const val MAX_QUEUED_PLAYBACK_BYTES = 1_500_000 + } +} diff --git a/android/src/main/java/com/deepgram/AudioRecorder.kt b/android/src/main/java/com/deepgram/AudioRecorder.kt new file mode 100644 index 0000000..5beafbe --- /dev/null +++ b/android/src/main/java/com/deepgram/AudioRecorder.kt @@ -0,0 +1,299 @@ +package com.deepgram + +import android.content.Context +import android.media.AudioFormat +import android.media.AudioManager +import android.media.AudioRecord +import android.media.MediaRecorder +import android.media.audiofx.AcousticEchoCanceler +import android.media.audiofx.AutomaticGainControl +import android.media.audiofx.NoiseSuppressor +import android.util.Log + +/** + * Microphone capture. Records 16 kHz PCM16 mono and streams the raw bytes back + * to the owner via [onAudioChunk]; the JS side downsamples to whatever Deepgram + * is configured for. + * + * Two capture profiles are supported: + * - STT-only (`enableVoiceProcessing = false`): `VOICE_RECOGNITION`, which + * gives the rawest possible signal (platform AEC/NS/AGC disabled). + * - Voice Agent / duplex (`enableVoiceProcessing = true`): `VOICE_COMMUNICATION` + * plus `MODE_IN_COMMUNICATION` so the platform's telephony-grade AEC engages + * against the active playback signal as its reference. + */ +internal class AudioRecorder( + private val context: Context, + private val onAudioChunk: (ByteArray, Int) -> Unit, + private val onForegroundServiceRequest: () -> Unit, + private val onForegroundServiceRelease: () -> Unit, +) { + /** Thrown when AudioRecord fails to initialize (mapped to "init_failed"). */ + class InitializationException(message: String) : Exception(message) + + @Volatile + var isActive: Boolean = false + private set + + @Volatile + private var audioRecord: AudioRecord? = null + private var recordingThread: Thread? = null + + private var acousticEchoCanceler: AcousticEchoCanceler? = null + private var noiseSuppressor: NoiseSuppressor? = null + private var automaticGainControl: AutomaticGainControl? = null + + // When `enableVoiceProcessing = true`, we set the system into + // MODE_IN_COMMUNICATION so the platform engages telephony-grade AEC. We + // remember the prior mode so we can restore it on stop and not strand the + // device in voice-call routing. + private var voiceProcessingActive = false + private var savedAudioMode: Int = AudioManager.MODE_NORMAL + private var audioManager: AudioManager? = null + + private val minRecordBufferSize: Int + get() { + val min = AudioRecord.getMinBufferSize(RECORD_SAMPLE_RATE, CHANNEL_CONFIG, AUDIO_FORMAT) + if (min > 0) return min + + val fallback = maxOf((RECORD_SAMPLE_RATE / 5) * 2, 4096) + Log.w(TAG, "Invalid AudioRecord min buffer ($min), using fallback=$fallback") + return fallback + } + + private val bufferSize: Int + get() = minRecordBufferSize * 4 + + /** + * Start capturing. On failure the recorder is left fully cleaned up and the + * exception is propagated: [InitializationException] for a failed + * `AudioRecord` init, `SecurityException` for a missing permission, or the + * original exception otherwise. + */ + fun start(enableVoiceProcessing: Boolean) { + // Voice Agent / duplex usage: route capture through the telephony stack so + // the platform's hardware AEC engages with the active playback signal as + // its reference. VOICE_RECOGNITION explicitly disables AEC/NS/AGC and is + // therefore the *wrong* source for full-duplex — it's only correct for + // one-way ASR where you want the rawest possible signal. + val audioSource = if (enableVoiceProcessing) { + MediaRecorder.AudioSource.VOICE_COMMUNICATION + } else { + MediaRecorder.AudioSource.VOICE_RECOGNITION + } + + if (enableVoiceProcessing) { + applyVoiceCommunicationMode() + } + + try { + val recorder = AudioRecord.Builder() + .setAudioSource(audioSource) + .setAudioFormat( + AudioFormat.Builder() + .setEncoding(AUDIO_FORMAT) + .setSampleRate(RECORD_SAMPLE_RATE) + .setChannelMask(CHANNEL_CONFIG) + .build() + ) + .setBufferSizeInBytes(bufferSize) + .build() + audioRecord = recorder + + if (recorder.state != AudioRecord.STATE_INITIALIZED) { + recorder.release() + audioRecord = null + releaseAudioEffects() + restoreAudioModeIfNeeded() + throw InitializationException("AudioRecord initialization failed") + } + + enableAudioEffects(recorder) + recorder.startRecording() + isActive = true + onForegroundServiceRequest() + + startRecordingThread() + } catch (e: SecurityException) { + Log.e(TAG, "startRecording security error", e) + audioRecord?.release() + audioRecord = null + releaseAudioEffects() + restoreAudioModeIfNeeded() + throw e + } catch (e: InitializationException) { + throw e + } catch (e: Exception) { + Log.e(TAG, "startRecording error", e) + audioRecord?.release() + audioRecord = null + releaseAudioEffects() + restoreAudioModeIfNeeded() + throw e + } + } + + fun stop(throwOnError: Boolean = true) { + isActive = false + recordingThread?.let { thread -> + thread.interrupt() + try { + thread.join(500) + } catch (_: InterruptedException) { + Thread.currentThread().interrupt() + } + } + recordingThread = null + + try { + audioRecord?.stop() + } catch (e: Exception) { + Log.w(TAG, "stop: error stopping AudioRecord", e) + if (throwOnError) throw e + } + + try { + audioRecord?.release() + } catch (e: Exception) { + Log.w(TAG, "stop: error releasing AudioRecord", e) + if (throwOnError) throw e + } + + audioRecord = null + releaseAudioEffects() + restoreAudioModeIfNeeded() + onForegroundServiceRelease() + } + + private fun applyVoiceCommunicationMode() { + if (voiceProcessingActive) return + try { + val manager = audioManager + ?: (context.getSystemService(Context.AUDIO_SERVICE) as? AudioManager) + if (manager == null) { + Log.w(TAG, "applyVoiceCommunicationMode: AudioManager unavailable") + return + } + audioManager = manager + savedAudioMode = manager.mode + manager.mode = AudioManager.MODE_IN_COMMUNICATION + voiceProcessingActive = true + Log.i(TAG, "Switched audio mode to IN_COMMUNICATION (was $savedAudioMode) for AEC") + } catch (e: Exception) { + Log.w(TAG, "applyVoiceCommunicationMode failed", e) + } + } + + private fun restoreAudioModeIfNeeded() { + if (!voiceProcessingActive) return + try { + val manager = audioManager + ?: (context.getSystemService(Context.AUDIO_SERVICE) as? AudioManager) + manager?.mode = savedAudioMode + Log.i(TAG, "Restored audio mode to $savedAudioMode") + } catch (e: Exception) { + Log.w(TAG, "restoreAudioModeIfNeeded failed", e) + } finally { + voiceProcessingActive = false + } + } + + private fun enableAudioEffects(recorder: AudioRecord) { + val sessionId = recorder.audioSessionId + if (sessionId == AudioRecord.ERROR || sessionId == AudioRecord.ERROR_BAD_VALUE) { + return + } + + releaseAudioEffects() + + if (AcousticEchoCanceler.isAvailable()) { + try { + acousticEchoCanceler = AcousticEchoCanceler.create(sessionId)?.apply { + enabled = true + } + } catch (e: Exception) { + Log.w(TAG, "Unable to enable AcousticEchoCanceler", e) + } + } + + if (NoiseSuppressor.isAvailable()) { + try { + noiseSuppressor = NoiseSuppressor.create(sessionId)?.apply { + enabled = true + } + } catch (e: Exception) { + Log.w(TAG, "Unable to enable NoiseSuppressor", e) + } + } + + if (AutomaticGainControl.isAvailable()) { + try { + automaticGainControl = AutomaticGainControl.create(sessionId)?.apply { + enabled = true + } + } catch (e: Exception) { + Log.w(TAG, "Unable to enable AutomaticGainControl", e) + } + } + } + + private fun releaseAudioEffects() { + try { + acousticEchoCanceler?.release() + } catch (e: Exception) { + Log.w(TAG, "Error releasing AcousticEchoCanceler", e) + } finally { + acousticEchoCanceler = null + } + + try { + noiseSuppressor?.release() + } catch (e: Exception) { + Log.w(TAG, "Error releasing NoiseSuppressor", e) + } finally { + noiseSuppressor = null + } + + try { + automaticGainControl?.release() + } catch (e: Exception) { + Log.w(TAG, "Error releasing AutomaticGainControl", e) + } finally { + automaticGainControl = null + } + } + + private fun startRecordingThread() { + recordingThread = Thread({ + try { + android.os.Process.setThreadPriority(android.os.Process.THREAD_PRIORITY_URGENT_AUDIO) + val buffer = ByteArray(bufferSize) + while (isActive) { + val recorder = audioRecord ?: break + val read = recorder.read(buffer, 0, buffer.size) + if (read > 0) { + onAudioChunk(buffer, read) + } else if (read < 0) { + // ERROR_INVALID_OPERATION / ERROR_BAD_VALUE / ERROR_DEAD_OBJECT — + // bail rather than spin in a tight loop. + Log.w(TAG, "AudioRecord.read returned $read — stopping thread") + break + } + } + } catch (e: Exception) { + Log.e(TAG, "recording thread error", e) + } + }, "Deepgram-Recording") + recordingThread?.start() + } + + companion object { + private const val TAG = "DeepgramRecorder" + + // Recording is locked to 16 kHz PCM16 mono — exposed so the module can tag + // the `AudioChunk` event it emits to JS with the matching sample rate. + const val RECORD_SAMPLE_RATE = 16000 + private const val CHANNEL_CONFIG = AudioFormat.CHANNEL_IN_MONO + private const val AUDIO_FORMAT = AudioFormat.ENCODING_PCM_16BIT + } +} diff --git a/android/src/main/java/com/deepgram/DeepgramAudioService.kt b/android/src/main/java/com/deepgram/DeepgramAudioService.kt index c2d0170..ffeeec7 100644 --- a/android/src/main/java/com/deepgram/DeepgramAudioService.kt +++ b/android/src/main/java/com/deepgram/DeepgramAudioService.kt @@ -6,17 +6,22 @@ import android.app.NotificationManager import android.app.Service import android.content.Context import android.content.Intent +import android.content.pm.ServiceInfo import android.os.Build import android.os.IBinder +import android.util.Log class DeepgramAudioService : Service() { companion object { + private const val TAG = "DeepgramAudioService" private const val CHANNEL_ID = "deepgram_audio_channel" private const val CHANNEL_NAME = "Deepgram Audio" private const val NOTIFICATION_ID = 43123 + private const val EXTRA_WITH_MICROPHONE = "with_microphone" - fun start(context: Context) { + fun start(context: Context, withMicrophone: Boolean) { val intent = Intent(context, DeepgramAudioService::class.java) + intent.putExtra(EXTRA_WITH_MICROPHONE, withMicrophone) if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { context.startForegroundService(intent) } else { @@ -38,16 +43,28 @@ class DeepgramAudioService : Service() { } override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int { + val withMicrophone = intent?.getBooleanExtra(EXTRA_WITH_MICROPHONE, false) ?: false val notification = buildNotification() - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { - startForeground( - NOTIFICATION_ID, - notification, - android.content.pm.ServiceInfo.FOREGROUND_SERVICE_TYPE_MICROPHONE or - android.content.pm.ServiceInfo.FOREGROUND_SERVICE_TYPE_MEDIA_PLAYBACK - ) - } else { - startForeground(NOTIFICATION_ID, notification) + try { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + // Always allow media playback; only advertise the microphone type when + // we are actually capturing. Declaring FOREGROUND_SERVICE_TYPE_MICROPHONE + // for a playback-only session throws a SecurityException on Android 14+ + // when the RECORD_AUDIO permission has not been granted. + var serviceType = ServiceInfo.FOREGROUND_SERVICE_TYPE_MEDIA_PLAYBACK + if (withMicrophone) { + serviceType = serviceType or ServiceInfo.FOREGROUND_SERVICE_TYPE_MICROPHONE + } + startForeground(NOTIFICATION_ID, notification, serviceType) + } else { + startForeground(NOTIFICATION_ID, notification) + } + } catch (e: Exception) { + // e.g. a missing-permission or background-start restriction. Don't crash + // the host app over the keep-alive notification — just bail out cleanly. + Log.w(TAG, "startForeground failed", e) + stopSelf() + return START_NOT_STICKY } return START_STICKY diff --git a/android/src/main/java/com/deepgram/DeepgramModule.kt b/android/src/main/java/com/deepgram/DeepgramModule.kt index 6f8ab25..4cad9ec 100644 --- a/android/src/main/java/com/deepgram/DeepgramModule.kt +++ b/android/src/main/java/com/deepgram/DeepgramModule.kt @@ -1,22 +1,7 @@ package com.deepgram import android.Manifest -import android.content.BroadcastReceiver -import android.content.Context -import android.content.Intent -import android.content.IntentFilter import android.content.pm.PackageManager -import android.media.AudioAttributes -import android.media.AudioFormat -import android.media.AudioManager -import android.media.AudioFocusRequest -import android.media.AudioRecord -import android.media.AudioTrack -import android.media.MediaRecorder -import android.media.audiofx.AcousticEchoCanceler -import android.media.audiofx.AutomaticGainControl -import android.media.audiofx.NoiseSuppressor -import android.os.Build import android.os.Handler import android.os.Looper import android.util.Base64 @@ -29,150 +14,84 @@ import com.facebook.react.bridge.ReactMethod import com.facebook.react.bridge.ReadableMap import com.facebook.react.bridge.WritableNativeMap import com.facebook.react.modules.core.DeviceEventManagerModule -import java.util.concurrent.LinkedBlockingQueue -import java.util.concurrent.atomic.AtomicInteger /** - * Native module that records raw PCM audio and plays back base64 encoded - * PCM chunks. Audio data recorded from the microphone is emitted to JS as the - * `AudioChunk` event with base64-encoded PCM. + * React Native bridge for Deepgram audio. + * + * This class is intentionally thin: it owns the JS-facing `@ReactMethod` + * surface, the foreground-service lifecycle, and audio-focus arbitration, and + * delegates the actual audio work to three focused collaborators: + * + * - [AudioRecorder] microphone capture (emitted to JS as the `AudioChunk` event) + * - [AudioPlayer] TTS streaming + one-shot playback + * - [AudioFocusManager] system audio-focus arbitration + * + * Microphone audio is emitted to JS as the `AudioChunk` event with + * base64-encoded 16 kHz PCM16. */ class DeepgramModule(private val reactContext: ReactApplicationContext) : - ReactContextBaseJavaModule(reactContext) { - - companion object { - const val NAME = "Deepgram" - private const val TAG = "DeepgramModule" - // Recording is locked to 16 kHz PCM16 mono — the JS side downsamples from - // here to whatever Deepgram is configured for. Do NOT mix this with the - // playback rate (which can vary, e.g. 24 kHz for Aura TTS). - private const val RECORD_SAMPLE_RATE = 16000 - private const val DEFAULT_PLAYBACK_SAMPLE_RATE = 16000 - private const val CHANNEL_CONFIG = AudioFormat.CHANNEL_IN_MONO - private const val AUDIO_FORMAT = AudioFormat.ENCODING_PCM_16BIT - // Cap the streaming playback queue to prevent unbounded memory growth - // when JS feeds audio faster than the device can play it. ~5 s of 24 kHz - // PCM16 mono is roughly 240 KB — we allow a little more headroom. - private const val MAX_QUEUED_PLAYBACK_BYTES = 1_500_000 - } - - private var playbackSampleRate: Int = DEFAULT_PLAYBACK_SAMPLE_RATE - private var currentOutputChannels: Int = 1 + ReactContextBaseJavaModule(reactContext), AudioFocusManager.Listener { private val mainHandler by lazy { Handler(Looper.getMainLooper()) } - private val minRecordBufferSize: Int - get() { - val min = AudioRecord.getMinBufferSize(RECORD_SAMPLE_RATE, CHANNEL_CONFIG, AUDIO_FORMAT) - if (min > 0) return min + private val focusManager = AudioFocusManager(reactContext, mainHandler, this) - val fallback = maxOf((RECORD_SAMPLE_RATE / 5) * 2, 4096) - Log.w(TAG, "Invalid AudioRecord min buffer ($min), using fallback=$fallback") - return fallback - } + private val recorder = AudioRecorder( + context = reactContext, + onAudioChunk = { data, length -> sendAudioChunk(data, length) }, + onForegroundServiceRequest = { startForegroundAudioService() }, + onForegroundServiceRelease = { stopForegroundAudioServiceIfInactive() }, + ) - private val bufferSize: Int - get() = minRecordBufferSize * 4 - - @Volatile - private var audioRecord: AudioRecord? = null - private var recordingThread: Thread? = null - @Volatile - private var isRecording = false - private var acousticEchoCanceler: AcousticEchoCanceler? = null - private var noiseSuppressor: NoiseSuppressor? = null - private var automaticGainControl: AutomaticGainControl? = null - - // When `enableVoiceProcessing=true`, we set the system into - // MODE_IN_COMMUNICATION so the platform engages telephony-grade AEC. - // We remember the prior mode so we can restore it on stop and not - // strand the device in voice-call routing. - private var voiceProcessingActive = false - private var savedAudioMode: Int = AudioManager.MODE_NORMAL - - // TTS Streaming Playback - private var audioTrack: AudioTrack? = null - private val audioQueue = LinkedBlockingQueue() - @Volatile - private var isPlaying = false - private var playbackThread: Thread? = null - private val playbackThreshold = 1024 - private val queuedBytes = AtomicInteger(0) - - // Audio focus - private var audioManager: AudioManager? = null - private var audioFocusRequest: AudioFocusRequest? = null - private var hasAudioFocus = false - - // ACTION_AUDIO_BECOMING_NOISY — fires when wired headset is unplugged or A2DP - // disconnects. Without handling this, our AudioTrack would silently re-route - // to the loudspeaker and surprise the user. Standard Android best practice. - private var noisyReceiverRegistered = false - private val becomingNoisyReceiver = object : BroadcastReceiver() { - override fun onReceive(context: Context?, intent: Intent?) { - if (intent?.action == AudioManager.ACTION_AUDIO_BECOMING_NOISY) { - Log.i(TAG, "AUDIO_BECOMING_NOISY — pausing playback") - try { audioTrack?.pause() } catch (_: Exception) {} - } - } - } - - private val audioFocusListener = AudioManager.OnAudioFocusChangeListener { focusChange -> - when (focusChange) { - AudioManager.AUDIOFOCUS_LOSS -> { - // Permanent loss (e.g. another app took focus). Tear down so we don't - // fight other audio packages and so we release the microphone. - Log.i(TAG, "AUDIOFOCUS_LOSS — stopping Deepgram audio") - hasAudioFocus = false - try { stopStreamingPlayback(throwOnError = false) } catch (_: Exception) {} - try { stopRecordingInternal(throwOnError = false) } catch (_: Exception) {} - } - AudioManager.AUDIOFOCUS_LOSS_TRANSIENT, - AudioManager.AUDIOFOCUS_LOSS_TRANSIENT_CAN_DUCK -> { - // Transient loss (e.g. incoming call) — pause playback but keep state. - Log.i(TAG, "AUDIOFOCUS_LOSS_TRANSIENT — pausing playback") - try { audioTrack?.pause() } catch (_: Exception) {} - } - AudioManager.AUDIOFOCUS_GAIN -> { - Log.i(TAG, "AUDIOFOCUS_GAIN — resuming playback if active") - hasAudioFocus = true - if (isPlaying) { - try { audioTrack?.play() } catch (_: Exception) {} - } - } - } - } + private val player = AudioPlayer( + context = reactContext, + mainHandler = mainHandler, + focusManager = focusManager, + onForegroundServiceRequest = { startForegroundAudioService() }, + onForegroundServiceRelease = { stopForegroundAudioServiceIfInactive() }, + ) override fun getName() = NAME - private fun outputChannelMask(): Int { - return if (currentOutputChannels >= 2) { - AudioFormat.CHANNEL_OUT_STEREO - } else { - AudioFormat.CHANNEL_OUT_MONO - } + // ------------------------------------------------------------------- + // Audio focus (AudioFocusManager.Listener) + // ------------------------------------------------------------------- + + override fun onFocusLostPermanently() { + // Permanent loss (e.g. another app took focus). Tear down so we don't fight + // other audio packages and so we release the microphone. Order matches the + // original: stop playback first, then recording. + try { player.stopStreamingPlayback(throwOnError = false) } catch (_: Exception) {} + try { recorder.stop(throwOnError = false) } catch (_: Exception) {} } - private fun minTrackBufferSize(channelMask: Int): Int { - val min = AudioTrack.getMinBufferSize(playbackSampleRate, channelMask, AUDIO_FORMAT) - if (min > 0) return min + override fun onFocusLostTransiently() { + // Transient loss (e.g. incoming call) — pause playback but keep state. + player.pauseForFocusLoss() + } - val bytesPerFrame = if (currentOutputChannels >= 2) 4 else 2 - val fallback = maxOf((playbackSampleRate / 5) * bytesPerFrame, 4096) - Log.w(TAG, "Invalid AudioTrack min buffer ($min), using fallback=$fallback") - return fallback + override fun onFocusGained() { + player.resumeForFocusGain() } + // ------------------------------------------------------------------- + // Foreground service coordination + // + // The keep-alive foreground service is shared by recording and playback: it + // is (re)started whenever either becomes active and stopped only once BOTH + // are inactive. It advertises the microphone type only while capturing. + // ------------------------------------------------------------------- + private fun startForegroundAudioService() { try { - DeepgramAudioService.start(reactContext) + DeepgramAudioService.start(reactContext, withMicrophone = recorder.isActive) } catch (e: Exception) { Log.w(TAG, "Unable to start foreground audio service", e) } } private fun stopForegroundAudioServiceIfInactive() { - if (isRecording || isPlaying) return + if (recorder.isActive || player.isActive) return try { DeepgramAudioService.stop(reactContext) } catch (e: Exception) { @@ -186,7 +105,7 @@ class DeepgramModule(private val reactContext: ReactApplicationContext) : @ReactMethod fun startRecording(options: ReadableMap?, promise: Promise) { - if (isRecording) { + if (recorder.isActive) { promise.resolve(null) return } @@ -206,137 +125,22 @@ class DeepgramModule(private val reactContext: ReactApplicationContext) : it.getBoolean("enableVoiceProcessing") } ?: false - // Voice Agent / duplex usage: route capture through the telephony stack so - // the platform's hardware AEC engages with the active playback signal as - // its reference. VOICE_RECOGNITION explicitly disables AEC/NS/AGC and is - // therefore the *wrong* source for full-duplex — it's only correct for - // one-way ASR where you want the rawest possible signal. - val audioSource = if (enableVoiceProcessing) { - MediaRecorder.AudioSource.VOICE_COMMUNICATION - } else { - MediaRecorder.AudioSource.VOICE_RECOGNITION - } - - if (enableVoiceProcessing) { - applyVoiceCommunicationMode() - } - try { - val recorder = AudioRecord.Builder() - .setAudioSource(audioSource) - .setAudioFormat( - AudioFormat.Builder() - .setEncoding(AUDIO_FORMAT) - .setSampleRate(RECORD_SAMPLE_RATE) - .setChannelMask(CHANNEL_CONFIG) - .build() - ) - .setBufferSizeInBytes(bufferSize) - .build() - audioRecord = recorder - - if (recorder.state != AudioRecord.STATE_INITIALIZED) { - recorder.release() - audioRecord = null - releaseAudioEffects() - restoreAudioModeIfNeeded() - promise.reject("init_failed", "AudioRecord initialization failed") - return - } - - enableAudioEffects(recorder) - recorder.startRecording() - isRecording = true - startForegroundAudioService() - - startRecordingThread() + recorder.start(enableVoiceProcessing) promise.resolve(null) } catch (e: SecurityException) { - Log.e(TAG, "startRecording security error", e) - audioRecord?.release() - audioRecord = null - releaseAudioEffects() - restoreAudioModeIfNeeded() promise.reject("permission_denied", e) + } catch (e: AudioRecorder.InitializationException) { + promise.reject("init_failed", e.message) } catch (e: Exception) { - Log.e(TAG, "startRecording error", e) - audioRecord?.release() - audioRecord = null - releaseAudioEffects() - restoreAudioModeIfNeeded() promise.reject("start_error", e) } } - private fun applyVoiceCommunicationMode() { - if (voiceProcessingActive) return - try { - val manager = audioManager - ?: (reactContext.getSystemService(Context.AUDIO_SERVICE) as? AudioManager) - if (manager == null) { - Log.w(TAG, "applyVoiceCommunicationMode: AudioManager unavailable") - return - } - audioManager = manager - savedAudioMode = manager.mode - manager.mode = AudioManager.MODE_IN_COMMUNICATION - voiceProcessingActive = true - Log.i(TAG, "Switched audio mode to IN_COMMUNICATION (was $savedAudioMode) for AEC") - } catch (e: Exception) { - Log.w(TAG, "applyVoiceCommunicationMode failed", e) - } - } - - private fun restoreAudioModeIfNeeded() { - if (!voiceProcessingActive) return - try { - val manager = audioManager - ?: (reactContext.getSystemService(Context.AUDIO_SERVICE) as? AudioManager) - manager?.mode = savedAudioMode - Log.i(TAG, "Restored audio mode to $savedAudioMode") - } catch (e: Exception) { - Log.w(TAG, "restoreAudioModeIfNeeded failed", e) - } finally { - voiceProcessingActive = false - } - } - - private fun stopRecordingInternal(throwOnError: Boolean = true) { - isRecording = false - recordingThread?.let { thread -> - thread.interrupt() - try { - thread.join(500) - } catch (_: InterruptedException) { - Thread.currentThread().interrupt() - } - } - recordingThread = null - - try { - audioRecord?.stop() - } catch (e: Exception) { - Log.w(TAG, "stopRecordingInternal: error stopping AudioRecord", e) - if (throwOnError) throw e - } - - try { - audioRecord?.release() - } catch (e: Exception) { - Log.w(TAG, "stopRecordingInternal: error releasing AudioRecord", e) - if (throwOnError) throw e - } - - audioRecord = null - releaseAudioEffects() - restoreAudioModeIfNeeded() - stopForegroundAudioServiceIfInactive() - } - @ReactMethod fun stopRecording(promise: Promise) { try { - stopRecordingInternal() + recorder.stop() promise.resolve(null) } catch (e: Exception) { Log.e(TAG, "stopRecording error", e) @@ -344,214 +148,14 @@ class DeepgramModule(private val reactContext: ReactApplicationContext) : } } - private fun enableAudioEffects(recorder: AudioRecord) { - val sessionId = recorder.audioSessionId - if (sessionId == AudioRecord.ERROR || sessionId == AudioRecord.ERROR_BAD_VALUE) { - return - } - - releaseAudioEffects() - - if (AcousticEchoCanceler.isAvailable()) { - try { - acousticEchoCanceler = AcousticEchoCanceler.create(sessionId)?.apply { - enabled = true - } - } catch (e: Exception) { - Log.w(TAG, "Unable to enable AcousticEchoCanceler", e) - } - } - - if (NoiseSuppressor.isAvailable()) { - try { - noiseSuppressor = NoiseSuppressor.create(sessionId)?.apply { - enabled = true - } - } catch (e: Exception) { - Log.w(TAG, "Unable to enable NoiseSuppressor", e) - } - } - - if (AutomaticGainControl.isAvailable()) { - try { - automaticGainControl = AutomaticGainControl.create(sessionId)?.apply { - enabled = true - } - } catch (e: Exception) { - Log.w(TAG, "Unable to enable AutomaticGainControl", e) - } - } - } - - private fun releaseAudioEffects() { - try { - acousticEchoCanceler?.release() - } catch (e: Exception) { - Log.w(TAG, "Error releasing AcousticEchoCanceler", e) - } finally { - acousticEchoCanceler = null - } - - try { - noiseSuppressor?.release() - } catch (e: Exception) { - Log.w(TAG, "Error releasing NoiseSuppressor", e) - } finally { - noiseSuppressor = null - } - - try { - automaticGainControl?.release() - } catch (e: Exception) { - Log.w(TAG, "Error releasing AutomaticGainControl", e) - } finally { - automaticGainControl = null - } - } - - private fun startRecordingThread() { - recordingThread = Thread({ - try { - android.os.Process.setThreadPriority(android.os.Process.THREAD_PRIORITY_URGENT_AUDIO) - val buffer = ByteArray(bufferSize) - while (isRecording) { - val recorder = audioRecord ?: break - val read = recorder.read(buffer, 0, buffer.size) - if (read > 0) { - sendAudioChunk(buffer, read) - } else if (read < 0) { - // ERROR_INVALID_OPERATION / ERROR_BAD_VALUE / ERROR_DEAD_OBJECT — - // bail rather than spin in a tight loop. - Log.w(TAG, "AudioRecord.read returned $read — stopping thread") - break - } - } - } catch (e: Exception) { - Log.e(TAG, "recording thread error", e) - } - }, "Deepgram-Recording") - recordingThread?.start() - } - - /** - * Emit audio data to JS as base64-encoded PCM (matches iOS format). - * This is far more efficient than sending individual byte values. - */ - private fun sendAudioChunk(data: ByteArray, length: Int) { - if (!reactContext.hasActiveReactInstance()) return - val b64 = Base64.encodeToString(data, 0, length, Base64.NO_WRAP) - val map = WritableNativeMap() - map.putString("b64", b64) - map.putInt("sampleRate", RECORD_SAMPLE_RATE) - try { - reactContext - .getJSModule(DeviceEventManagerModule.RCTDeviceEventEmitter::class.java) - .emit("AudioChunk", map) - } catch (e: Exception) { - // Catalyst tearing down or JS bundle not ready — drop silently. - Log.w(TAG, "sendAudioChunk emit failed", e) - } - } - // ------------------------------------------------------------------- // Playback // ------------------------------------------------------------------- - private fun ensureAudioTrack() { - if (audioTrack != null) return - - val channelMask = outputChannelMask() - val minBuf = minTrackBufferSize(channelMask) - - audioTrack = AudioTrack.Builder() - .setAudioAttributes( - AudioAttributes.Builder() - .setUsage(AudioAttributes.USAGE_VOICE_COMMUNICATION) - .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH) - .build() - ) - .setAudioFormat( - AudioFormat.Builder() - .setEncoding(AUDIO_FORMAT) - .setSampleRate(playbackSampleRate) - .setChannelMask(channelMask) - .build() - ) - .setBufferSizeInBytes(maxOf(minBuf, bufferSize)) - .setTransferMode(AudioTrack.MODE_STREAM) - .build() - } - - private fun requestAudioFocus(): Boolean { - if (hasAudioFocus) return true - try { - val manager = reactContext.getSystemService(android.content.Context.AUDIO_SERVICE) as? AudioManager - ?: return false - audioManager = manager - - // Use AUDIOFOCUS_GAIN_TRANSIENT so other audio apps (Spotify, podcasts, - // navigation, etc.) pause/duck and then automatically resume when we - // abandon focus. AUDIOFOCUS_GAIN would permanently stop them. - val result = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { - val request = AudioFocusRequest.Builder(AudioManager.AUDIOFOCUS_GAIN_TRANSIENT) - .setAudioAttributes( - AudioAttributes.Builder() - .setUsage(AudioAttributes.USAGE_VOICE_COMMUNICATION) - .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH) - .build() - ) - .setOnAudioFocusChangeListener(audioFocusListener, mainHandler) - .setWillPauseWhenDucked(true) - .build() - audioFocusRequest = request - manager.requestAudioFocus(request) - } else { - @Suppress("DEPRECATION") - manager.requestAudioFocus( - audioFocusListener, - AudioManager.STREAM_VOICE_CALL, - AudioManager.AUDIOFOCUS_GAIN_TRANSIENT - ) - } - - hasAudioFocus = result == AudioManager.AUDIOFOCUS_REQUEST_GRANTED - if (!hasAudioFocus) { - Log.w(TAG, "Audio focus request not granted (result=$result)") - } - return hasAudioFocus - } catch (e: Exception) { - Log.w(TAG, "requestAudioFocus error", e) - hasAudioFocus = false - return false - } - } - - private fun abandonAudioFocus() { - if (!hasAudioFocus) return - try { - val manager = audioManager ?: return - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { - audioFocusRequest?.let { manager.abandonAudioFocusRequest(it) } - } else { - @Suppress("DEPRECATION") - manager.abandonAudioFocus(audioFocusListener) - } - } catch (e: Exception) { - Log.w(TAG, "abandonAudioFocus error", e) - } finally { - hasAudioFocus = false - audioFocusRequest = null - } - } - @ReactMethod fun startAudio(promise: Promise) { try { - requestAudioFocus() - ensureAudioTrack() - audioTrack?.play() - isPlaying = true - startForegroundAudioService() + player.startAudio() promise.resolve(null) } catch (e: Exception) { Log.e(TAG, "startAudio error", e) @@ -559,31 +163,10 @@ class DeepgramModule(private val reactContext: ReactApplicationContext) : } } - private fun stopAudioInternal(throwOnError: Boolean = true) { - try { - audioTrack?.stop() - } catch (e: Exception) { - Log.w(TAG, "stopAudioInternal: error stopping AudioTrack", e) - if (throwOnError) throw e - } - - try { - audioTrack?.release() - } catch (e: Exception) { - Log.w(TAG, "stopAudioInternal: error releasing AudioTrack", e) - if (throwOnError) throw e - } - - audioTrack = null - isPlaying = false - abandonAudioFocus() - stopForegroundAudioServiceIfInactive() - } - @ReactMethod fun stopAudio(promise: Promise) { try { - stopAudioInternal() + player.stopAudio() promise.resolve(null) } catch (e: Exception) { Log.e(TAG, "stopAudio error", e) @@ -591,128 +174,30 @@ class DeepgramModule(private val reactContext: ReactApplicationContext) : } } - // ------------------------------------------------------------------- - // TTS Streaming Audio Player - // Uses a proper queue instead of O(n²) array concatenation. - // ------------------------------------------------------------------- - @ReactMethod fun startPlayer(sampleRate: Int, channels: Int) { - stopStreamingPlayback() - playbackSampleRate = if (sampleRate > 0) sampleRate else DEFAULT_PLAYBACK_SAMPLE_RATE - currentOutputChannels = if (channels >= 2) 2 else 1 - audioQueue.clear() - queuedBytes.set(0) - isPlaying = false + player.startPlayer(sampleRate, channels) } @ReactMethod fun setAudioConfig(sampleRate: Int, channels: Int) { - startPlayer(sampleRate, channels) + player.setAudioConfig(sampleRate, channels) } @ReactMethod fun feedAudio(base64Audio: String) { - try { - val audioData = Base64.decode(base64Audio, Base64.DEFAULT) - if (audioData.isEmpty()) return - - // Bound the queue to prevent unbounded memory growth if JS is producing - // audio faster than the device can play it. Drop the oldest chunk first - // (so we always favor newer audio) when at the limit. - while (queuedBytes.get() + audioData.size > MAX_QUEUED_PLAYBACK_BYTES) { - val dropped = audioQueue.poll() ?: break - queuedBytes.addAndGet(-dropped.size) - Log.w(TAG, "feedAudio: dropping ${dropped.size} bytes — playback queue full") - } - - audioQueue.offer(audioData) - queuedBytes.addAndGet(audioData.size) - - // Start playback when we have enough buffered data - if (!isPlaying && queuedBytes.get() >= playbackThreshold) { - startStreamingPlayback() - } - } catch (e: Exception) { - Log.e(TAG, "feedAudio error", e) - } + player.feedAudio(base64Audio) } @ReactMethod fun interruptAudio() { - try { - stopStreamingPlayback(throwOnError = false) - } catch (e: Exception) { - Log.w(TAG, "interruptAudio error", e) - } - } - - private fun startStreamingPlayback() { - if (isPlaying) return - - isPlaying = true - registerNoisyReceiver() - playbackThread = Thread({ - try { - android.os.Process.setThreadPriority(android.os.Process.THREAD_PRIORITY_URGENT_AUDIO) - requestAudioFocus() - startForegroundAudioService() - ensureAudioTrack() - audioTrack?.play() - - while (isPlaying) { - // Block until data is available (up to 100ms), avoids busy-wait - val chunk = audioQueue.poll(100, java.util.concurrent.TimeUnit.MILLISECONDS) - ?: continue - - queuedBytes.addAndGet(-chunk.size) - val written = audioTrack?.write(chunk, 0, chunk.size) ?: 0 - if (written < 0) { - Log.e(TAG, "AudioTrack write error: $written") - break - } - } - } catch (e: InterruptedException) { - // Normal shutdown - } catch (e: Exception) { - Log.e(TAG, "Streaming playback error", e) - } finally { - isPlaying = false - stopForegroundAudioServiceIfInactive() - } - }, "Deepgram-Playback") - - playbackThread?.start() - } - - private fun stopStreamingPlayback(throwOnError: Boolean = true) { - isPlaying = false - playbackThread?.let { thread -> - thread.interrupt() - try { - thread.join(500) - } catch (_: InterruptedException) { - Thread.currentThread().interrupt() - } - } - playbackThread = null - - try { - stopAudioInternal(throwOnError) - } catch (e: Exception) { - if (throwOnError) throw e else Log.w(TAG, "stopStreamingPlayback: error stopping audio", e) - } finally { - audioQueue.clear() - queuedBytes.set(0) - unregisterNoisyReceiver() - stopForegroundAudioServiceIfInactive() - } + player.interruptAudio() } @ReactMethod fun stopPlayer(promise: Promise?) { try { - stopStreamingPlayback() + player.stopStreamingPlayback() promise?.resolve(null) } catch (e: Exception) { Log.e(TAG, "stopPlayer error", e) @@ -720,124 +205,59 @@ class DeepgramModule(private val reactContext: ReactApplicationContext) : } } + // ------------------------------------------------------------------- + // One-shot playback (for HTTP synthesis) + // ------------------------------------------------------------------- + + @ReactMethod + fun playAudioChunk(chunk: String, promise: Promise) { + player.playAudioChunk( + chunk, + onSuccess = { promise.resolve(null) }, + onError = { code, message -> promise.reject(code, message) }, + ) + } + override fun invalidate() { try { - stopRecordingInternal(false) + recorder.stop(false) } catch (e: Exception) { Log.w(TAG, "Error stopping recording on invalidate", e) } try { - stopStreamingPlayback(false) + player.release() } catch (e: Exception) { Log.w(TAG, "Error stopping playback on invalidate", e) } - abandonAudioFocus() - unregisterNoisyReceiver() + focusManager.abandonFocus() super.invalidate() } - private fun registerNoisyReceiver() { - if (noisyReceiverRegistered) return - try { - val filter = IntentFilter(AudioManager.ACTION_AUDIO_BECOMING_NOISY) - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { - reactContext.registerReceiver( - becomingNoisyReceiver, - filter, - Context.RECEIVER_NOT_EXPORTED - ) - } else { - @Suppress("UnspecifiedRegisterReceiverFlag") - reactContext.registerReceiver(becomingNoisyReceiver, filter) - } - noisyReceiverRegistered = true - } catch (e: Exception) { - Log.w(TAG, "registerNoisyReceiver failed", e) - } - } - - private fun unregisterNoisyReceiver() { - if (!noisyReceiverRegistered) return + /** + * Emit microphone audio to JS as base64-encoded PCM (matches the iOS + * `DeepgramAudioPCM` payload shape). Far more efficient than sending + * individual byte values. + */ + private fun sendAudioChunk(data: ByteArray, length: Int) { + if (!reactContext.hasActiveReactInstance()) return + val b64 = Base64.encodeToString(data, 0, length, Base64.NO_WRAP) + val map = WritableNativeMap() + map.putString("b64", b64) + map.putInt("sampleRate", AudioRecorder.RECORD_SAMPLE_RATE) try { - reactContext.unregisterReceiver(becomingNoisyReceiver) + reactContext + .getJSModule(DeviceEventManagerModule.RCTDeviceEventEmitter::class.java) + .emit("AudioChunk", map) } catch (e: Exception) { - Log.w(TAG, "unregisterNoisyReceiver failed", e) - } finally { - noisyReceiverRegistered = false + // Catalyst tearing down or JS bundle not ready — drop silently. + Log.w(TAG, "sendAudioChunk emit failed", e) } } - // ------------------------------------------------------------------- - // One-shot playback (for HTTP synthesis) - // ------------------------------------------------------------------- - - @ReactMethod - fun playAudioChunk(chunk: String, promise: Promise) { - try { - val audioData = Base64.decode(chunk, Base64.DEFAULT) - if (audioData.isEmpty()) { - promise.reject("invalid_data", "Failed to decode audio data") - return - } - - requestAudioFocus() - startForegroundAudioService() - - val channelMask = outputChannelMask() - val minBuf = minTrackBufferSize(channelMask) - - val tempAudioTrack = AudioTrack.Builder() - .setAudioAttributes( - AudioAttributes.Builder() - .setUsage(AudioAttributes.USAGE_VOICE_COMMUNICATION) - .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH) - .build() - ) - .setAudioFormat( - AudioFormat.Builder() - .setEncoding(AUDIO_FORMAT) - .setSampleRate(playbackSampleRate) - .setChannelMask(channelMask) - .build() - ) - .setBufferSizeInBytes(maxOf(minBuf, audioData.size)) - .setTransferMode(AudioTrack.MODE_STATIC) - .build() - - tempAudioTrack.write(audioData, 0, audioData.size) - - // Use notification marker to know when playback finishes, - // then resolve the promise (instead of polling with Thread.sleep). - val bytesPerFrame = if (currentOutputChannels >= 2) 4 else 2 // PCM16 mono/stereo - val frameCount = audioData.size / bytesPerFrame - tempAudioTrack.notificationMarkerPosition = frameCount - val listener = object : AudioTrack.OnPlaybackPositionUpdateListener { - override fun onMarkerReached(track: AudioTrack?) { - try { - track?.stop() - track?.release() - } catch (e: Exception) { - Log.w(TAG, "playAudioChunk cleanup error", e) - } - abandonAudioFocus() - stopForegroundAudioServiceIfInactive() - promise.resolve(null) - } - - override fun onPeriodicNotification(track: AudioTrack?) {} - } - - // minSdk is 24, the Handler-aware overload is always available. - tempAudioTrack.setPlaybackPositionUpdateListener(listener, mainHandler) - - tempAudioTrack.play() - } catch (e: Exception) { - Log.e(TAG, "playAudioChunk error", e) - abandonAudioFocus() - stopForegroundAudioServiceIfInactive() - promise.reject("playback_error", e.message) - } + companion object { + const val NAME = "Deepgram" + private const val TAG = "DeepgramModule" } } diff --git a/ios/DGSupport.h b/ios/DGSupport.h new file mode 100644 index 0000000..fac2ae2 --- /dev/null +++ b/ios/DGSupport.h @@ -0,0 +1,77 @@ +#import +#import +#import +#import + +/** + * Shared, state-free support for the Deepgram native module: logging macros, + * error helpers and the recording-state struct. Kept in one header so the main + * implementation and its category files (`Deepgram+AudioSession.mm`, + * `Deepgram+Recording.mm`, `Deepgram+Playback.mm`) all share a single copy. + */ + +#define DGNumberBuffers 3 + +#ifndef DG_ENABLE_DEBUG_LOGS +#define DG_ENABLE_DEBUG_LOGS 0 +#endif + +#if DG_ENABLE_DEBUG_LOGS +#define DGLogDebug(...) NSLog(__VA_ARGS__) +#else +#define DGLogDebug(...) +#endif + +#define DGLogWarn(...) RCTLogWarn(__VA_ARGS__) +#define DGLogError(...) RCTLogError(__VA_ARGS__) + +@class Deepgram; + +/** Backing state for the AudioQueue-based microphone capture path. */ +typedef struct { + __unsafe_unretained Deepgram *mSelf; + AudioStreamBasicDescription dataFormat; + AudioQueueRef queue; + AudioQueueBufferRef buffers[DGNumberBuffers]; + UInt32 bufferByteSize; + SInt64 currentPacket; + bool isRunning; +} DGRecordState; + +/** Build a generic native error with a localized description. */ +static inline NSError *DGNativeError(NSString *domain, NSInteger code, + NSString *message) { + return [NSError errorWithDomain:domain + code:code + userInfo:@{ + NSLocalizedDescriptionKey : message + ?: @"Unknown Deepgram native error" + }]; +} + +/** Wrap an OSStatus failure from a named CoreAudio operation. */ +static inline NSError *DGOSStatusError(NSString *operation, OSStatus status) { + NSString *message = [NSString + stringWithFormat:@"%@ failed with OSStatus %d", operation, (int)status]; + return DGNativeError(NSOSStatusErrorDomain, (NSInteger)status, message); +} + +/** + * Reject a promise with a guaranteed-concrete `NSError`. Passing `nil` for the + * error synthesizes one so JS never sees a generic "unknown error from native + * module" that hides the real cause. + */ +static inline void DGRejectPromise(RCTPromiseRejectBlock reject, NSString *code, + NSString *message, NSError *error) { + if (!reject) { + return; + } + + NSString *safeCode = code ?: @"deepgram_native_error"; + NSString *safeMessage = message ?: error.localizedDescription + ?: @"Deepgram native error"; + NSError *safeError = + error ?: DGNativeError(@"DeepgramNativeError", 0, safeMessage); + DGLogError(@"[Deepgram] reject: code=%@ message=%@", safeCode, safeMessage); + reject(safeCode, safeMessage, safeError); +} diff --git a/ios/Deepgram+AudioSession.h b/ios/Deepgram+AudioSession.h new file mode 100644 index 0000000..ad07c1f --- /dev/null +++ b/ios/Deepgram+AudioSession.h @@ -0,0 +1,28 @@ +#import "Deepgram.h" + +#import + +/** + * Audio-session lifecycle: activation/deactivation, the category fallback + * ladder (sensitive to OSStatus -50) and the route-change / interruption / + * media-services-reset notification handlers. + */ +@interface Deepgram (AudioSession) + +- (BOOL)activateAudioSession:(NSError **)outError; +- (void)deactivateAudioSession; +- (BOOL)configureAudioSessionIfNeeded:(NSError **)outError; +- (BOOL)configureAudioSession:(NSError **)outError; +- (BOOL)configureAudioSessionForRecording:(BOOL)needsMicrophone + error:(NSError **)outError; +- (AVAudioSessionCategoryOptions)bluetoothHFPOption; +- (BOOL)applyAudioSessionCategory:(AVAudioSessionCategory)category + mode:(NSString *)mode + options:(AVAudioSessionCategoryOptions)options + error:(NSError **)outError; +- (void)maybeDeactivateAudioSession; +- (void)handleAudioRouteChange:(NSNotification *)note; +- (void)handleMediaServicesReset:(NSNotification *)note; +- (void)handleAudioInterruption:(NSNotification *)note; + +@end diff --git a/ios/Deepgram+AudioSession.mm b/ios/Deepgram+AudioSession.mm new file mode 100644 index 0000000..26b3098 --- /dev/null +++ b/ios/Deepgram+AudioSession.mm @@ -0,0 +1,438 @@ +#import "Deepgram+AudioSession.h" +#import "Deepgram+Private.h" +#import "Deepgram+Recording.h" + +#import +#import + +/** + * Audio-session lifecycle for the Deepgram module: activation/deactivation, the + * category fallback ladder (which is highly sensitive to OSStatus -50), and the + * route-change / interruption / media-services-reset notification handlers. + */ +@implementation Deepgram (AudioSession) + +- (BOOL)activateAudioSession:(NSError **)outError { + DGLogDebug(@"[Deepgram] activateAudioSession: begin (configured=%@)", + self.audioSessionConfigured ? @"YES" : @"NO"); + __block BOOL success = YES; + __block NSError *activationError = nil; + + DGLogDebug(@"[Deepgram] activateAudioSession: configuring on current queue"); + success = [self configureAudioSessionIfNeeded:&activationError]; + + if (!success && activationError) { + DGLogError(@"[Deepgram] Failed to activate audio session: %@", + activationError.localizedDescription ?: activationError); + } else { + DGLogDebug(@"[Deepgram] activateAudioSession: success=%@", + success ? @"YES" : @"NO"); + } + + if (outError) { + *outError = activationError; + } + + return success; +} + +- (void)deactivateAudioSession { + DGLogDebug(@"[Deepgram] deactivateAudioSession: begin"); + if (!self.audioSessionConfigured) { + DGLogDebug(@"[Deepgram] deactivateAudioSession: skipped (not configured)"); + return; + } + + NSError *error = nil; + // Use NotifyOthersOnDeactivation so other audio sessions (expo-av, etc.) + // know they can resume. + BOOL success = NO; + @try { + success = [[AVAudioSession sharedInstance] + setActive:NO + withOptions:AVAudioSessionSetActiveOptionNotifyOthersOnDeactivation + error:&error]; + } + @catch (NSException *e) { + DGLogWarn(@"[Deepgram] deactivateAudioSession threw exception: %@", + e.reason ?: e); + self.audioSessionConfigured = NO; + return; + } + + if (!success && error) { + // Deactivation can legitimately fail during competing audio-route + // transitions; treat as a warning rather than an error. + DGLogWarn(@"[Deepgram] Failed to deactivate audio session: %@", + error.localizedDescription ?: error); + } + self.audioSessionConfigured = NO; +} + +- (BOOL)configureAudioSessionIfNeeded:(NSError **)outError +{ + BOOL needsMic = _recordState.isRunning || self.audioQueueCaptureRequested || + self.engineCaptureActive || self.voiceProcessingRequested; + AVAudioSession *session = [AVAudioSession sharedInstance]; + + if (self.audioSessionConfigured) { + BOOL isCurrentlyPlayAndRecord = [session.category isEqualToString:AVAudioSessionCategoryPlayAndRecord]; + if (isCurrentlyPlayAndRecord == needsMic) { + DGLogDebug(@"[Deepgram] configureAudioSessionIfNeeded: already configured correctly, ensuring active"); + NSError *activeError = nil; + BOOL success = [[AVAudioSession sharedInstance] setActive:YES error:&activeError]; + if (success && !activeError) { + return YES; + } + } + } + + DGLogDebug(@"[Deepgram] configureAudioSessionIfNeeded: configuring (needsMic=%@)", needsMic ? @"YES" : @"NO"); + return [self configureAudioSession:outError]; +} + +- (BOOL)configureAudioSession:(NSError **)outError { + BOOL needsMic = _recordState.isRunning || self.audioQueueCaptureRequested || + self.engineCaptureActive || self.voiceProcessingRequested; + return [self configureAudioSessionForRecording:needsMic error:outError]; +} + +/** + * Build the Bluetooth input option appropriate for the running OS. + * iOS 17 deprecated `AllowBluetooth` (HFP route) in favor of + * `AllowBluetoothHFP`. Prefer the new symbol when building against the iOS + * 17+ SDK and fall back to the legacy spelling on older toolchains. + */ +- (AVAudioSessionCategoryOptions)bluetoothHFPOption { +#if defined(__IPHONE_17_0) && (__IPHONE_OS_VERSION_MAX_ALLOWED >= __IPHONE_17_0) + if (@available(iOS 17.0, *)) { + return AVAudioSessionCategoryOptionAllowBluetoothHFP; + } +#endif +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wdeprecated-declarations" + return AVAudioSessionCategoryOptionAllowBluetooth; +#pragma clang diagnostic pop +} + +/** + * Attempt to apply a single category/mode/options tuple and activate the + * session. Returns YES on success. On failure, populates `outError` so the + * caller can decide whether to try a more conservative configuration. + */ +- (BOOL)applyAudioSessionCategory:(AVAudioSessionCategory)category + mode:(NSString *)mode + options:(AVAudioSessionCategoryOptions)options + error:(NSError **)outError { + AVAudioSession *session = [AVAudioSession sharedInstance]; + NSError *error = nil; + + DGLogDebug(@"[Deepgram] configureAudioSession: setCategory %@ mode %@ " + @"options 0x%lx", + category, mode, (unsigned long)options); + + if (![session setCategory:category + mode:mode + options:options + error:&error] || + error) { + DGLogError(@"[Deepgram] configureAudioSession: category failed error=%@", + error.localizedDescription ?: error); + if (outError) { + *outError = error; + } + return NO; + } + + NSError *activeError = nil; + if (![session setActive:YES error:&activeError]) { + DGLogError(@"[Deepgram] configureAudioSession: setActive failed error=%@", + activeError.localizedDescription ?: activeError); + if (outError) { + *outError = activeError; + } + return NO; + } + + return YES; +} + +/** + * Configure audio session with mode appropriate to current usage. + * + * The configuration is attempted from most- to least-capable. If a richer + * setup is rejected we automatically fall back to a simpler one so audio still + * works instead of hard-failing. + * + * Two subtle rules drive the ordering below; getting them wrong yields + * OSStatus -50 (kAudioSession/paramErr) on `setCategory`, which then leaves the + * session in a half-configured state and makes a later AudioQueueStart fail + * with 'what' (kAudioSessionUnspecifiedError): + * + * 1. `DefaultToSpeaker` forces output to the built-in speaker, while + * `AllowBluetoothA2DP` / `AllowAirPlay` route output to external devices. + * Combining them is contradictory and is rejected with -50, so those + * output-routing options are NEVER mixed with a recording configuration. + * A2DP/AirPlay devices are output-only and can't supply a mic input + * anyway — Bluetooth input uses the HFP route instead. + * 2. `MixWithOthers` lets us coexist with other audio packages (expo-av, + * etc.) but can prevent us from acquiring the input on some routes. We try + * it first to be a good citizen, then degrade to exclusive access so + * recording still starts reliably. + * + * VoiceChat mode (which engages Apple's hardware AEC / VPIO) is used only when + * echo cancellation is required and only on a device — VPIO is unavailable on + * the Simulator and requesting it there guarantees a -50. + */ +- (BOOL)configureAudioSessionForRecording:(BOOL)needsMicrophone + error:(NSError **)outError { + DGLogDebug(@"[Deepgram] configureAudioSession: begin (mic=%@)", + needsMicrophone ? @"YES" : @"NO"); + + const AVAudioSessionCategoryOptions mixOption = + AVAudioSessionCategoryOptionMixWithOthers; + + // Build an ordered list of attempts, best first. + NSMutableArray *attempts = [NSMutableArray array]; + + if (needsMicrophone) { + BOOL needsAEC = self.engineCaptureActive || self.voiceProcessingRequested || + self.isPlaying; + + // Input-safe options for PlayAndRecord. DefaultToSpeaker + HFP are valid + // here; A2DP/AirPlay are intentionally excluded (see rule 1 above). + AVAudioSessionCategoryOptions recordOptions = + AVAudioSessionCategoryOptionDefaultToSpeaker | [self bluetoothHFPOption]; + +#if !TARGET_OS_SIMULATOR + // Preferred: hardware AEC via VoiceChat (device only). VoiceChat mode + // manages the route itself, so DefaultToSpeaker / HFP must NOT be combined + // with it — doing so is rejected with -50 on some iOS versions. Only + // MixWithOthers is safe to layer on top. + if (needsAEC) { + [attempts addObject:@{ + @"category" : AVAudioSessionCategoryPlayAndRecord, + @"mode" : AVAudioSessionModeVoiceChat, + @"options" : @(mixOption), + }]; + [attempts addObject:@{ + @"category" : AVAudioSessionCategoryPlayAndRecord, + @"mode" : AVAudioSessionModeVoiceChat, + @"options" : @(0), + }]; + } +#endif + + // PlayAndRecord without VoiceChat (no hardware AEC). Try with MixWithOthers + // first to coexist with other packages, then exclusively. + [attempts addObject:@{ + @"category" : AVAudioSessionCategoryPlayAndRecord, + @"mode" : AVAudioSessionModeDefault, + @"options" : @(recordOptions | mixOption), + }]; + [attempts addObject:@{ + @"category" : AVAudioSessionCategoryPlayAndRecord, + @"mode" : AVAudioSessionModeDefault, + @"options" : @(recordOptions), + }]; + + // Last resort: bare PlayAndRecord with no options. Always valid. + [attempts addObject:@{ + @"category" : AVAudioSessionCategoryPlayAndRecord, + @"mode" : AVAudioSessionModeDefault, + @"options" : @(0), + }]; + } else { + // Playback already supports AirPlay and A2DP routes; the explicit route + // options are for PlayAndRecord and can be rejected with OSStatus -50. + AVAudioSessionCategoryOptions playbackOptions = mixOption; + [attempts addObject:@{ + @"category" : AVAudioSessionCategoryPlayback, + @"mode" : AVAudioSessionModeDefault, + @"options" : @(playbackOptions), + }]; + + // Last resort: bare Playback with no options. + [attempts addObject:@{ + @"category" : AVAudioSessionCategoryPlayback, + @"mode" : AVAudioSessionModeDefault, + @"options" : @(0), + }]; + } + + NSError *lastError = nil; + NSUInteger attemptIndex = 0; + for (NSDictionary *attempt in attempts) { + attemptIndex++; + NSError *attemptError = nil; + BOOL success = [self + applyAudioSessionCategory:attempt[@"category"] + mode:attempt[@"mode"] + options:(AVAudioSessionCategoryOptions) + [attempt[@"options"] unsignedIntegerValue] + error:&attemptError]; + if (success) { + if (attemptIndex > 1) { + DGLogDebug(@"[Deepgram] configureAudioSession: succeeded on fallback " + @"attempt %lu/%lu", + (unsigned long)attemptIndex, (unsigned long)attempts.count); + } + self.audioSessionConfigured = YES; + DGLogDebug(@"[Deepgram] configureAudioSession: success"); + return YES; + } + lastError = attemptError; + DGLogDebug(@"[Deepgram] configureAudioSession: attempt %lu/%lu failed, " + @"trying next configuration", + (unsigned long)attemptIndex, (unsigned long)attempts.count); + } + + DGLogError(@"[Deepgram] configureAudioSession: all %lu configurations failed; " + @"last error=%@", + (unsigned long)attempts.count, + lastError.localizedDescription ?: lastError); + if (outError) { + *outError = lastError; + } + self.audioSessionConfigured = NO; + return NO; +} + +- (void)maybeDeactivateAudioSession { + DGLogDebug(@"[Deepgram] maybeDeactivateAudioSession: running=%@ playing=%@ " + @"engineCapture=%@", + _recordState.isRunning ? @"YES" : @"NO", + self.isPlaying ? @"YES" : @"NO", + self.engineCaptureActive ? @"YES" : @"NO"); + if (!_recordState.isRunning && !self.isPlaying && !self.engineCaptureActive) { + DGLogDebug(@"[Deepgram] maybeDeactivateAudioSession: deactivating"); + [self deactivateAudioSession]; + } +} + +- (void)handleAudioRouteChange:(NSNotification *)note { + DGLogDebug(@"[Deepgram] handleAudioRouteChange: %@", note.userInfo); + NSNumber *reasonValue = note.userInfo[AVAudioSessionRouteChangeReasonKey]; + AVAudioSessionRouteChangeReason reason = + reasonValue + ? (AVAudioSessionRouteChangeReason)reasonValue.unsignedIntegerValue + : AVAudioSessionRouteChangeReasonUnknown; + + // Headphones / Bluetooth headset unplugged — pause playback so we don't + // surprise the user by suddenly blasting through the loud speaker. + // (System would route audio to the speaker automatically; that's the point + // of this notification.) Matches expo-audio behavior and Apple's guidance. + if (reason == AVAudioSessionRouteChangeReasonOldDeviceUnavailable) { + DGLogDebug(@"[Deepgram] handleAudioRouteChange: old device unavailable — " + @"pausing playback"); + if (self.isPlaying && self.playerNode) { + [self.playerNode pause]; + } + } + + BOOL causedByCategory = + reason == AVAudioSessionRouteChangeReasonCategoryChange || + reason == AVAudioSessionRouteChangeReasonNoSuitableRouteForCategory; + + if (!causedByCategory) { + self.audioSessionConfigured = NO; + } + + if (!_recordState.isRunning && !self.isPlaying) { + DGLogDebug(@"[Deepgram] handleAudioRouteChange: ignoring (inactive)"); + return; + } + + if (!self.appIsActive) { + DGLogDebug(@"[Deepgram] handleAudioRouteChange: app inactive, skipping"); + return; + } + + if (causedByCategory) { + DGLogDebug(@"[Deepgram] handleAudioRouteChange: category change detected, " + @"keeping session"); + return; + } + + DGLogDebug(@"[Deepgram] handleAudioRouteChange: reactivating session"); + [self activateAudioSession:NULL]; +} + +// AVAudioSession can post this when mediaserverd restarts (e.g. after a +// rare system audio HAL hiccup). Once it fires, every AudioQueue / +// AVAudioEngine instance becomes a zombie. Apple's documented fix is to throw +// everything away and rebuild on demand. Match expo-audio's approach. +- (void)handleMediaServicesReset:(NSNotification *)note { + DGLogDebug(@"[Deepgram] handleMediaServicesReset: tearing down audio stack"); + self.audioSessionConfigured = NO; + + // Recording queue must be torn down — it's a zombie after a media services + // reset. + if (_recordState.isRunning || _recordState.queue) { + [self cleanupRecordingQueue]; + } + + // Engine-based capture path also dies with mediaserverd; force a full + // teardown so the next startRecording rebuilds VPIO from scratch. + self.engineCaptureActive = NO; + self.captureConverter = nil; + self.captureOutputFormat = nil; + + // AVAudioEngine + player node are no longer valid. Stop and discard so the + // next setup call rebuilds from scratch. + @try { + if (self.playerNode) { + [self.playerNode stop]; + } + if (self.audioEngine) { + [self.audioEngine stop]; + } + } @catch (NSException *e) { + DGLogError(@"[Deepgram] handleMediaServicesReset: teardown exception %@", + e); + } + self.playerNode = nil; + self.audioEngine = nil; + self.isPlaying = NO; +} + +- (void)handleAudioInterruption:(NSNotification *)note { + DGLogDebug(@"[Deepgram] handleAudioInterruption: %@", note.userInfo); + NSNumber *typeValue = note.userInfo[AVAudioSessionInterruptionTypeKey]; + AVAudioSessionInterruptionType type = + (AVAudioSessionInterruptionType)typeValue.unsignedIntegerValue; + + if (type == AVAudioSessionInterruptionTypeBegan) { + DGLogDebug(@"[Deepgram] handleAudioInterruption: interruption began"); + if (_recordState.isRunning && _recordState.queue) { + DGLogDebug( + @"[Deepgram] handleAudioInterruption: pausing recording queue"); + AudioQueuePause(_recordState.queue); + } + if (self.isPlaying && self.playerNode) { + DGLogDebug(@"[Deepgram] handleAudioInterruption: pausing player node"); + [self.playerNode pause]; + } + } else if (type == AVAudioSessionInterruptionTypeEnded) { + DGLogDebug(@"[Deepgram] handleAudioInterruption: interruption ended"); + NSNumber *optionValue = note.userInfo[AVAudioSessionInterruptionOptionKey]; + AVAudioSessionInterruptionOptions options = + (AVAudioSessionInterruptionOptions)optionValue.unsignedIntegerValue; + if (options & AVAudioSessionInterruptionOptionShouldResume) { + DGLogDebug(@"[Deepgram] handleAudioInterruption: should resume"); + [self activateAudioSession:nil]; + + if (_recordState.isRunning && _recordState.queue) { + DGLogDebug( + @"[Deepgram] handleAudioInterruption: resuming recording queue"); + AudioQueueStart(_recordState.queue, NULL); + } + if (self.isPlaying && self.playerNode) { + DGLogDebug(@"[Deepgram] handleAudioInterruption: resuming player node"); + [self.playerNode play]; + } + } + } +} + +@end diff --git a/ios/Deepgram+Playback.h b/ios/Deepgram+Playback.h new file mode 100644 index 0000000..850bf57 --- /dev/null +++ b/ios/Deepgram+Playback.h @@ -0,0 +1,21 @@ +#import "Deepgram.h" + +#import + +/** + * TTS / Voice Agent audio output through an `AVAudioPlayerNode` on the shared + * `AVAudioEngine`. When voice processing is requested the same engine drives + * both output and capture so the VPIO Audio Unit can cancel our own output out + * of the captured input (hardware echo cancellation). + */ +@interface Deepgram (Playback) + +- (void)stopAndDetachPlayerNode; +- (void)interruptPlayerPlayback; +- (BOOL)setupAudioEngineWithSampleRate:(int)sampleRate + channels:(int)channels + enableVoiceProcessing:(BOOL)enableVoiceProcessing + error:(NSError **)outError; +- (AVAudioPCMBuffer *)createPCMBufferFromData:(NSData *)data; + +@end diff --git a/ios/Deepgram+Playback.mm b/ios/Deepgram+Playback.mm new file mode 100644 index 0000000..30e9aa3 --- /dev/null +++ b/ios/Deepgram+Playback.mm @@ -0,0 +1,211 @@ +#import "Deepgram+Playback.h" +#import "Deepgram+Private.h" + +#import + +#include + +/** + * TTS / Voice Agent audio output for the Deepgram module. Playback runs through + * an `AVAudioPlayerNode` on a shared `AVAudioEngine`. When voice processing is + * requested the same engine drives both output and microphone capture so the + * VPIO Audio Unit can cancel our own output out of the captured input + * (hardware echo cancellation). + */ +@implementation Deepgram (Playback) + +/* ================================================================== */ +/* 2. AVAUDIOENGINE PLAYBACK WITH VOICE PROCESSING (ECHO CANCEL) */ +/* ================================================================== */ + +/** + * Setup AVAudioEngine for output. When `enableVoiceProcessing` is YES we also + * configure the input node for hardware echo cancellation (Voice Agent / + * duplex use case). When NO (pure TTS playback), we deliberately avoid + * touching `inputNode` so we don't request the microphone or interfere with + * other audio libraries. + */ +- (void)stopAndDetachPlayerNode { + if (self.playerNode) { + _playbackGeneration.fetch_add(1); + + @try { + [self.playerNode stop]; + } @catch (NSException *e) { + DGLogWarn(@"[Deepgram] playerNode stop exception: %@", e); + } + + if (self.audioEngine) { + @try { + [self.audioEngine detachNode:self.playerNode]; + } @catch (NSException *e) { + DGLogWarn(@"[Deepgram] playerNode detach exception: %@", e); + } + } + } + + self.playerNode = nil; + self.playbackFormat = nil; + self.isPlaying = NO; + _scheduledBufferCount = 0; +} + +- (void)interruptPlayerPlayback { + if (self.playerNode) { + _playbackGeneration.fetch_add(1); + + @try { + [self.playerNode stop]; + } @catch (NSException *e) { + DGLogWarn(@"[Deepgram] playerNode interrupt exception: %@", e); + } + } + + self.isPlaying = NO; + _scheduledBufferCount = 0; +} + +- (BOOL)setupAudioEngineWithSampleRate:(int)sampleRate + channels:(int)channels + enableVoiceProcessing:(BOOL)enableVoiceProcessing + error:(NSError **)outError { + BOOL reuseCaptureEngine = self.audioEngine && self.engineCaptureActive; + + if (self.audioEngine && self.audioEngine.isRunning) { + [self.audioEngine stop]; + } + + [self stopAndDetachPlayerNode]; + + if (!reuseCaptureEngine) { + if (self.audioEngine) { + [self.audioEngine reset]; + } + self.audioEngine = [[AVAudioEngine alloc] init]; + } + + if (!self.audioEngine) { + self.audioEngine = [[AVAudioEngine alloc] init]; + } + + self.playerNode = [[AVAudioPlayerNode alloc] init]; + + self.playbackFormat = + [[AVAudioFormat alloc] initWithCommonFormat:AVAudioPCMFormatInt16 + sampleRate:sampleRate + channels:channels + interleaved:YES]; + + if (!self.playbackFormat) { + if (outError) { + *outError = [NSError + errorWithDomain:@"DeepgramAudioEngine" + code:-1 + userInfo:@{ + NSLocalizedDescriptionKey : @"Failed to create audio format" + }]; + } + return NO; + } + + [self.audioEngine attachNode:self.playerNode]; + [self.audioEngine connect:self.playerNode + to:self.audioEngine.mainMixerNode + format:self.playbackFormat]; + + if (enableVoiceProcessing) { +#if TARGET_IPHONE_SIMULATOR + // VPIO is not implemented on the simulator. Calling + // setVoiceProcessingEnabled:YES there raises an internal AVAEInternal + // exception about IsFormatSampleRateAndChannelCountValid because the + // simulator's input node cannot satisfy the VPIO format constraints. + // Skip VP entirely; AEC has to be tested on a physical device. + DGLogWarn(@"[Deepgram] NOTE: Voice Processing I/O (Echo Cancellation) is " + @"NOT supported on the iOS Simulator. Audio output may be picked " + @"up by the microphone. Please test on a physical device for " + @"proper AEC behavior."); +#else + @try { + AVAudioInputNode *inputNode = self.audioEngine.inputNode; + AVAudioOutputNode *outputNode = self.audioEngine.outputNode; + // Apple's hardware AEC (VPIO Audio Unit) only engages when *both* + // input and output flow through a voice-processing-enabled audio + // unit. Enabling VP only on the input node is a no-op for capture + // unless rendering goes through the same unit, so we enable both. + if (inputNode) { + NSError *voiceProcessingError = nil; + if (![inputNode setVoiceProcessingEnabled:YES + error:&voiceProcessingError]) { + DGLogWarn(@"[Deepgram] inputNode VP enable failed: %@", + voiceProcessingError); + } + } + if (outputNode) { + NSError *voiceProcessingError = nil; + if (![outputNode setVoiceProcessingEnabled:YES + error:&voiceProcessingError]) { + DGLogWarn(@"[Deepgram] outputNode VP enable failed: %@", + voiceProcessingError); + } + } + } @catch (NSException *exception) { + // Continue - voice processing not critical for basic playback + DGLogWarn(@"[Deepgram] VP enable threw: %@", exception); + } +#endif + } + + [self.audioEngine prepare]; + NSError *startError = nil; + BOOL started = [self.audioEngine startAndReturnError:&startError]; + + if (!started) { + DGLogError(@"[Deepgram] Failed to start audio engine: %@", + startError.localizedDescription); + if (outError) { + *outError = startError; + } + return NO; + } + + return YES; +} + +/** + * Create AVAudioPCMBuffer from raw PCM data. + */ +- (AVAudioPCMBuffer *)createPCMBufferFromData:(NSData *)data { + if (!data || data.length == 0 || !self.playbackFormat) { + return nil; + } + + // Calculate frame count (data length / bytes per frame) + const AudioStreamBasicDescription *asbd = + self.playbackFormat.streamDescription; + int bytesPerFrame = asbd ? (int)asbd->mBytesPerFrame : 0; + if (bytesPerFrame <= 0) { + return nil; + } + AVAudioFrameCount frameCount = + (AVAudioFrameCount)(data.length / bytesPerFrame); + + if (frameCount == 0) { + return nil; + } + + AVAudioPCMBuffer *buffer = + [[AVAudioPCMBuffer alloc] initWithPCMFormat:self.playbackFormat + frameCapacity:frameCount]; + + buffer.frameLength = frameCount; + + // Copy only whole frames. `data.length` may not be an exact multiple of + // bytesPerFrame (e.g. a truncated network chunk); copying the raw length + // would write past the buffer, which is sized for `frameCount` frames. + NSUInteger usableBytes = (NSUInteger)frameCount * (NSUInteger)bytesPerFrame; + memcpy(buffer.int16ChannelData[0], data.bytes, usableBytes); + + return buffer; +} + +@end diff --git a/ios/Deepgram+Private.h b/ios/Deepgram+Private.h new file mode 100644 index 0000000..59ccf5c --- /dev/null +++ b/ios/Deepgram+Private.h @@ -0,0 +1,65 @@ +#import "DGSupport.h" +#import "Deepgram.h" + +#import +#import +#import + +#include + +/** + * Shared instance state for the `Deepgram` module. The class is deliberately + * kept as a single unit (rather than split into collaborator objects) because + * the audio session, the `AVAudioEngine` and the record/playback state are + * mutually dependent — the session configuration, for example, has to reason + * about recording and playback at the same time. + * + * Behaviour is split across category files for readability; each declares its + * own methods in a matching category header: + * - Deepgram+AudioSession.h / .mm : audio session lifecycle + * - Deepgram+Recording.h / .mm : microphone capture + * - Deepgram+Playback.h / .mm : TTS / Voice Agent playback + * + * Only the shared state lives here, in a class extension, so it auto-synthesizes + * in the primary `@implementation` while remaining visible to every category. + */ +@interface Deepgram () { +@protected + DGRecordState _recordState; + std::atomic _scheduledBufferCount; + std::atomic _playbackGeneration; +} + +// Recording +@property(nonatomic, strong) NSMutableData *pendingPCMBuffer; +@property(nonatomic, strong) dispatch_queue_t emitterQueue; +@property(nonatomic, assign) NSUInteger chunkSizeBytes; +@property(atomic, assign) BOOL hasListeners; +@property(atomic, assign) BOOL appIsActive; + +// Playback / TTS (AVAudioEngine-based with echo cancellation) +@property(nonatomic, strong) AVAudioEngine *audioEngine; +@property(nonatomic, strong) AVAudioPlayerNode *playerNode; +@property(nonatomic, strong) AVAudioFormat *playbackFormat; +@property(atomic, assign) BOOL isPlaying; +@property(atomic, assign) int currentSampleRate; +@property(nonatomic, assign) BOOL audioSessionConfigured; +// YES while we're capturing the microphone through `audioEngine.inputNode` +// (Voice Agent / duplex). When YES, the AudioQueue path is bypassed and the +// session must use VoiceChat mode so Apple's Voice-Processing I/O Audio Unit +// engages and performs hardware echo cancellation. +@property(atomic, assign) BOOL engineCaptureActive; +@property(atomic, assign) BOOL voiceProcessingRequested; +@property(atomic, assign) BOOL audioQueueCaptureRequested; +@property(nonatomic, strong) AVAudioConverter *captureConverter; +@property(nonatomic, strong) AVAudioFormat *captureOutputFormat; + +// Exported workers implemented in Deepgram.mm (via RCT_EXPORT_METHOD) but +// invoked from other methods before their textual definition. Declared here so +// those call sites don't trip -Wundeclared-selector. +- (void)startPlayer:(nonnull NSNumber *)sampleRate + channels:(nonnull NSNumber *)channels; +- (void)stopPlayer:(nullable RCTPromiseResolveBlock)resolve + rejecter:(nullable RCTPromiseRejectBlock)reject; + +@end diff --git a/ios/Deepgram+Recording.h b/ios/Deepgram+Recording.h new file mode 100644 index 0000000..dfaf8f1 --- /dev/null +++ b/ios/Deepgram+Recording.h @@ -0,0 +1,31 @@ +#import "DGSupport.h" +#import "Deepgram.h" + +#import +#import + +/** + * Microphone capture. Two capture paths live in the implementation: the + * AudioQueue path (STT-only, no echo cancellation) and the `AVAudioEngine` + * path (Voice Agent / duplex, with VPIO hardware AEC). + */ +@interface Deepgram (Recording) + +- (void)emitPCMChunk:(NSData *)chunk sampleRate:(int)sampleRate; +- (void)appendPCMDataAndEmitIfNeeded:(NSData *)pcmData; +- (void)flushPendingPCM; +- (void)cleanupRecordingQueue; +- (BOOL)startEngineCaptureAndReturnError:(NSError **)outError; +- (void)stopEngineCapture; + +@end + +/** + * AudioQueue input callback. Defined in `Deepgram+Recording.mm`; referenced by + * `startRecording` in `Deepgram.mm`, so it must have external linkage. + */ +extern void DGHandleInputBuffer(void *inUserData, AudioQueueRef inAQ, + AudioQueueBufferRef inBuffer, + const AudioTimeStamp *inStartTime, + UInt32 inNumPackets, + const AudioStreamPacketDescription *inPacketDesc); diff --git a/ios/Deepgram+Recording.mm b/ios/Deepgram+Recording.mm new file mode 100644 index 0000000..3ae36a2 --- /dev/null +++ b/ios/Deepgram+Recording.mm @@ -0,0 +1,411 @@ +#import "Deepgram+Recording.h" +#import "Deepgram+AudioSession.h" +#import "Deepgram+Playback.h" +#import "Deepgram+Private.h" + +#import +#import + +#include +#include + +/** + * Microphone capture for the Deepgram module. Two capture paths share this + * file: + * + * - The AudioQueue path (`DGHandleInputBuffer` + `cleanupRecordingQueue`), + * used for STT-only recording where hardware echo cancellation is not + * wanted. + * - The `AVAudioEngine` path (`startEngineCaptureAndReturnError:`), used for + * Voice Agent / duplex where VPIO hardware AEC is required. It shares the + * same `AVAudioEngine` instance as playback so the Voice-Processing I/O + * unit can cancel our own output from the captured input. + */ + +void DGHandleInputBuffer( + void *inUserData, __unused AudioQueueRef inAQ, AudioQueueBufferRef inBuffer, + __unused const AudioTimeStamp *inStartTime, __unused UInt32 inNumPackets, + __unused const AudioStreamPacketDescription *inPacketDesc) { + @autoreleasepool { + DGRecordState *state = (DGRecordState *)inUserData; + if (!state || !state->isRunning) { + DGLogDebug(@"[Deepgram] DGHandleInputBuffer: inactive state"); + return; + } + + Deepgram *strongSelf = state->mSelf; + if (!strongSelf) { + DGLogDebug(@"[Deepgram] DGHandleInputBuffer: missing self"); + return; + } + + if (!inBuffer || inBuffer->mAudioDataByteSize == 0) { + DGLogDebug(@"[Deepgram] DGHandleInputBuffer: empty buffer"); + return; + } + + DGLogDebug(@"[Deepgram] DGHandleInputBuffer: received %u bytes", + (unsigned int)inBuffer->mAudioDataByteSize); + NSData *data = [NSData dataWithBytes:inBuffer->mAudioData + length:inBuffer->mAudioDataByteSize]; + [strongSelf appendPCMDataAndEmitIfNeeded:data]; + + if (state->queue) { + DGLogDebug(@"[Deepgram] DGHandleInputBuffer: re-enqueue buffer"); + AudioQueueEnqueueBuffer(state->queue, inBuffer, 0, NULL); + } + } +} + +@implementation Deepgram (Recording) + +- (void)emitPCMChunk:(NSData *)chunk sampleRate:(int)sampleRate { + if (!chunk || chunk.length == 0) { + DGLogDebug(@"[Deepgram] emitPCMChunk: empty chunk, skipping"); + return; + } + + __weak __typeof(self) weakSelf = self; + NSData *chunkCopy = [chunk copy]; + dispatch_queue_t queue = self.emitterQueue ?: dispatch_get_main_queue(); + dispatch_async(queue, ^{ + if (!weakSelf) { + DGLogDebug(@"[Deepgram] emitPCMChunk: self released, aborting"); + return; + } + + if (!weakSelf.hasListeners) { + DGLogDebug(@"[Deepgram] emitPCMChunk: no listeners, dropping %lu bytes", + (unsigned long)chunkCopy.length); + return; + } + + if (!weakSelf.bridge || !weakSelf.callableJSModules) { + DGLogDebug( + @"[Deepgram] Skipping DeepgramAudioPCM event (bridge not ready)"); + return; + } + + DGLogDebug(@"[Deepgram] emitPCMChunk: sending %lu bytes sampleRate=%d", + (unsigned long)chunkCopy.length, sampleRate); + NSString *b64 = [chunkCopy base64EncodedStringWithOptions:0]; + [weakSelf sendEventWithName:@"DeepgramAudioPCM" + body:@{ + @"b64" : b64, + @"sampleRate" : @(sampleRate) + }]; + }); +} + +- (void)appendPCMDataAndEmitIfNeeded:(NSData *)pcmData { + if (!pcmData || pcmData.length == 0) { + DGLogDebug(@"[Deepgram] appendPCMDataAndEmitIfNeeded: empty PCM, skipping"); + return; + } + + if (!self.pendingPCMBuffer) { + DGLogDebug( + @"[Deepgram] appendPCMDataAndEmitIfNeeded: allocate pending buffer"); + self.pendingPCMBuffer = [[NSMutableData alloc] init]; + } + + DGLogDebug(@"[Deepgram] appendPCMDataAndEmitIfNeeded: append %lu bytes " + @"(pending=%lu)", + (unsigned long)pcmData.length, + (unsigned long)self.pendingPCMBuffer.length); + [self.pendingPCMBuffer appendData:pcmData]; + + NSUInteger chunkSize = + self.chunkSizeBytes > 0 ? self.chunkSizeBytes : pcmData.length; + + while (self.pendingPCMBuffer.length >= chunkSize) { + NSData *chunk = + [self.pendingPCMBuffer subdataWithRange:NSMakeRange(0, chunkSize)]; + [self.pendingPCMBuffer replaceBytesInRange:NSMakeRange(0, chunkSize) + withBytes:NULL + length:0]; + DGLogDebug(@"[Deepgram] appendPCMDataAndEmitIfNeeded: emitting chunk %lu " + @"bytes remaining=%lu", + (unsigned long)chunk.length, + (unsigned long)self.pendingPCMBuffer.length); + [self emitPCMChunk:chunk sampleRate:self.currentSampleRate]; + } +} + +- (void)flushPendingPCM { + if (self.pendingPCMBuffer.length == 0) { + DGLogDebug(@"[Deepgram] flushPendingPCM: nothing to flush"); + return; + } + + NSData *remaining = [self.pendingPCMBuffer copy]; + [self.pendingPCMBuffer setLength:0]; + DGLogDebug(@"[Deepgram] flushPendingPCM: flushing %lu bytes", + (unsigned long)remaining.length); + [self emitPCMChunk:remaining sampleRate:self.currentSampleRate]; +} + +- (void)cleanupRecordingQueue { + DGLogDebug(@"[Deepgram] cleanupRecordingQueue: begin"); + self.audioQueueCaptureRequested = NO; + _recordState.isRunning = false; + + if (_recordState.queue) { + DGLogDebug(@"[Deepgram] cleanupRecordingQueue: stopping queue"); + AudioQueueStop(_recordState.queue, true); + + for (int i = 0; i < DGNumberBuffers; i++) { + if (_recordState.buffers[i]) { + DGLogDebug(@"[Deepgram] cleanupRecordingQueue: freeing buffer %d", i); + AudioQueueFreeBuffer(_recordState.queue, _recordState.buffers[i]); + _recordState.buffers[i] = NULL; + } + } + + DGLogDebug(@"[Deepgram] cleanupRecordingQueue: disposing queue"); + AudioQueueDispose(_recordState.queue, true); + } + + memset(&_recordState, 0, sizeof(DGRecordState)); + DGLogDebug(@"[Deepgram] cleanupRecordingQueue: state cleared"); + [self maybeDeactivateAudioSession]; +} + +/* ================================================================== */ +/* 1. MICROPHONE CAPTURE (16 kHz PCM16 emission) */ +/* ================================================================== */ + +/** + * Engine-based microphone capture path used when the JS side opts in to + * hardware voice processing (Voice Agent / duplex). This routes through + * `AVAudioEngine.inputNode` with `setVoiceProcessingEnabled:YES` on both + * input and output nodes — the only Apple-supported way to actually engage + * the VPIO Audio Unit's hardware echo cancellation on iOS. The legacy + * AudioQueue path is preserved for STT-only usage where AEC is undesirable. + */ +- (BOOL)startEngineCaptureAndReturnError:(NSError **)outError { + self.currentSampleRate = 16000; + + if (!self.audioEngine) { + NSError *engineError = nil; + if (![self setupAudioEngineWithSampleRate:self.currentSampleRate + channels:1 + enableVoiceProcessing:YES + error:&engineError]) { + if (outError) + *outError = engineError; + return NO; + } + } else { +#if TARGET_IPHONE_SIMULATOR + // VPIO is unsupported on the simulator and toggling VP corrupts the + // input node's format (sampleRate becomes 0), which would later fail + // the IsFormatSampleRateAndChannelCountValid check. Skip entirely. + DGLogWarn(@"[Deepgram] NOTE: Voice Processing I/O (Echo Cancellation) is " + @"NOT supported on the iOS Simulator. Audio output may be picked " + @"up by the microphone. Please test on a physical device for " + @"proper AEC behavior."); +#else + @try { + NSError *vpError = nil; + if (self.audioEngine.inputNode && + ![self.audioEngine.inputNode setVoiceProcessingEnabled:YES + error:&vpError]) { + DGLogWarn(@"[Deepgram] inputNode VP enable failed: %@", vpError); + } + if (self.audioEngine.outputNode && + ![self.audioEngine.outputNode setVoiceProcessingEnabled:YES + error:&vpError]) { + DGLogWarn(@"[Deepgram] outputNode VP enable failed: %@", vpError); + } + } @catch (NSException *e) { + DGLogWarn(@"[Deepgram] VP enable threw: %@", e); + } +#endif + } + + AVAudioInputNode *inputNode = self.audioEngine.inputNode; + if (!inputNode) { + if (outError) { + *outError = [NSError + errorWithDomain:@"DeepgramAudioEngine" + code:-2 + userInfo:@{ + NSLocalizedDescriptionKey : @"No input node available" + }]; + } + return NO; + } + + AVAudioFormat *hwFormat = [inputNode inputFormatForBus:0]; + if (!hwFormat || hwFormat.sampleRate <= 0) { + if (outError) { + *outError = + [NSError errorWithDomain:@"DeepgramAudioEngine" + code:-3 + userInfo:@{ + NSLocalizedDescriptionKey : @"Invalid input format" + }]; + } + return NO; + } + + AVAudioFormat *outputFormat = + [[AVAudioFormat alloc] initWithCommonFormat:AVAudioPCMFormatInt16 + sampleRate:self.currentSampleRate + channels:1 + interleaved:YES]; + self.captureOutputFormat = outputFormat; + self.captureConverter = + [[AVAudioConverter alloc] initFromFormat:hwFormat toFormat:outputFormat]; + if (!self.captureConverter) { + if (outError) { + *outError = [NSError errorWithDomain:@"DeepgramAudioEngine" + code:-4 + userInfo:@{ + NSLocalizedDescriptionKey : + @"Failed to build AVAudioConverter" + }]; + } + return NO; + } + + AVAudioFrameCount tapFrames = (AVAudioFrameCount)MAX( + 1024, (NSUInteger)round(hwFormat.sampleRate * 0.2)); + + __weak __typeof(self) weakSelf = self; + @try { + [inputNode removeTapOnBus:0]; + } @catch (__unused NSException *e) { + // No prior tap — fine. + } + + @try { + [inputNode + installTapOnBus:0 + bufferSize:tapFrames + format:hwFormat + block:^(AVAudioPCMBuffer *_Nonnull inBuf, + __unused AVAudioTime *_Nonnull when) { + __strong __typeof(weakSelf) strongSelf = weakSelf; + if (!strongSelf || !strongSelf.engineCaptureActive) + return; + AVAudioConverter *converter = strongSelf.captureConverter; + AVAudioFormat *outFmt = strongSelf.captureOutputFormat; + if (!converter || !outFmt || inBuf.frameLength == 0) + return; + + AVAudioFrameCount outCapacity = + (AVAudioFrameCount)ceil((double)inBuf.frameLength * + outFmt.sampleRate / + inBuf.format.sampleRate) + + 16; + AVAudioPCMBuffer *outBuf = [[AVAudioPCMBuffer alloc] + initWithPCMFormat:outFmt + frameCapacity:outCapacity]; + if (!outBuf) + return; + + __block BOOL provided = NO; + NSError *cvtError = nil; + AVAudioConverterInputBlock inputBlock = + ^AVAudioBuffer *_Nullable( + __unused AVAudioPacketCount inNumPackets, + AVAudioConverterInputStatus *_Nonnull outStatus) { + if (provided) { + *outStatus = AVAudioConverterInputStatus_NoDataNow; + return nil; + } + provided = YES; + *outStatus = AVAudioConverterInputStatus_HaveData; + return inBuf; + }; + + AVAudioConverterOutputStatus status = + [converter convertToBuffer:outBuf + error:&cvtError + withInputFromBlock:inputBlock]; + if (status == AVAudioConverterOutputStatus_Error || + cvtError) { + DGLogWarn(@"[Deepgram] capture convert failed: %@", + cvtError); + return; + } + if (outBuf.frameLength == 0 || !outBuf.int16ChannelData) + return; + + NSUInteger byteCount = (NSUInteger)outBuf.frameLength * 2; + NSData *pcm = + [NSData dataWithBytes:outBuf.int16ChannelData[0] + length:byteCount]; + [strongSelf appendPCMDataAndEmitIfNeeded:pcm]; + }]; + } @catch (NSException *e) { + DGLogError(@"[Deepgram] installTapOnBus exception: %@", e); + if (outError) { + *outError = [NSError + errorWithDomain:@"DeepgramAudioEngine" + code:-5 + userInfo:@{ + NSLocalizedDescriptionKey : e.reason ?: @"Tap install failed" + }]; + } + return NO; + } + + self.chunkSizeBytes = + (NSUInteger)MAX(1, (int)round(self.currentSampleRate * 2 * 0.2)); + self.engineCaptureActive = YES; + + if (!self.audioEngine.isRunning) { + NSError *startError = nil; + if (![self.audioEngine startAndReturnError:&startError]) { + DGLogError(@"[Deepgram] engine start failed: %@", startError); + @try { + [inputNode removeTapOnBus:0]; + } @catch (__unused NSException *e) { + } + self.engineCaptureActive = NO; + if (outError) + *outError = startError; + return NO; + } + } + return YES; +} + +- (void)stopEngineCapture { + if (!self.engineCaptureActive && !self.captureConverter) + return; + DGLogDebug(@"[Deepgram] stopEngineCapture"); + self.engineCaptureActive = NO; + + @try { + [self.audioEngine.inputNode removeTapOnBus:0]; + } @catch (NSException *e) { + DGLogWarn(@"[Deepgram] removeTapOnBus exception: %@", e); + } + +#if !TARGET_IPHONE_SIMULATOR + @try { + NSError *vpError = nil; + if (self.audioEngine.inputNode) { + [self.audioEngine.inputNode setVoiceProcessingEnabled:NO error:&vpError]; + } + } @catch (__unused NSException *e) { + // best-effort + } +#endif + + self.captureConverter = nil; + self.captureOutputFormat = nil; + + if (!self.isPlaying && self.audioEngine.isRunning) { + @try { + [self.audioEngine stop]; + } @catch (__unused NSException *e) { + } + } +} + +@end diff --git a/ios/Deepgram.mm b/ios/Deepgram.mm index 42650c3..07fab4a 100644 --- a/ios/Deepgram.mm +++ b/ios/Deepgram.mm @@ -1,155 +1,31 @@ +#import "Deepgram.h" +#import "Deepgram+Private.h" +#import "Deepgram+AudioSession.h" +#import "Deepgram+Recording.h" +#import "Deepgram+Playback.h" + #import #import -#import -#import #import #import #if TARGET_OS_IOS #import #endif -#include + #include #include -#define DGNumberBuffers 3 - -#ifndef DG_ENABLE_DEBUG_LOGS -#define DG_ENABLE_DEBUG_LOGS 0 -#endif - -#if DG_ENABLE_DEBUG_LOGS -#define DGLogDebug(...) NSLog(__VA_ARGS__) -#else -#define DGLogDebug(...) -#endif - -#define DGLogWarn(...) RCTLogWarn(__VA_ARGS__) -#define DGLogError(...) RCTLogError(__VA_ARGS__) - -static NSError *DGNativeError(NSString *domain, NSInteger code, - NSString *message) { - return [NSError errorWithDomain:domain - code:code - userInfo:@{ - NSLocalizedDescriptionKey : message - ?: @"Unknown Deepgram native error" - }]; -} - -static NSError *DGOSStatusError(NSString *operation, OSStatus status) { - NSString *message = [NSString - stringWithFormat:@"%@ failed with OSStatus %d", operation, (int)status]; - return DGNativeError(NSOSStatusErrorDomain, (NSInteger)status, message); -} - -static void DGRejectPromise(RCTPromiseRejectBlock reject, NSString *code, - NSString *message, NSError *error) { - if (!reject) { - return; - } - - NSString *safeCode = code ?: @"deepgram_native_error"; - NSString *safeMessage = message ?: error.localizedDescription - ?: @"Deepgram native error"; - NSError *safeError = error ?: DGNativeError(@"DeepgramNativeError", 0, - safeMessage); - DGLogError(@"[Deepgram] reject: code=%@ message=%@", safeCode, - safeMessage); - reject(safeCode, safeMessage, safeError); -} - -@class Deepgram; - -typedef struct { - __unsafe_unretained Deepgram *mSelf; - AudioStreamBasicDescription dataFormat; - AudioQueueRef queue; - AudioQueueBufferRef buffers[DGNumberBuffers]; - UInt32 bufferByteSize; - SInt64 currentPacket; - bool isRunning; -} DGRecordState; - -@interface Deepgram : RCTEventEmitter { - DGRecordState _recordState; - std::atomic _scheduledBufferCount; - std::atomic _playbackGeneration; -} - -// Recording -@property(nonatomic, strong) NSMutableData *pendingPCMBuffer; -@property(nonatomic, strong) dispatch_queue_t emitterQueue; -@property(nonatomic, assign) NSUInteger chunkSizeBytes; -@property(atomic, assign) BOOL hasListeners; -@property(atomic, assign) BOOL appIsActive; - -// Playback / TTS (AVAudioEngine-based with echo cancellation) -@property(nonatomic, strong) AVAudioEngine *audioEngine; -@property(nonatomic, strong) AVAudioPlayerNode *playerNode; -@property(nonatomic, strong) AVAudioFormat *playbackFormat; -@property(atomic, assign) BOOL isPlaying; -@property(atomic, assign) int currentSampleRate; -@property(nonatomic, assign) BOOL audioSessionConfigured; -// YES while we're capturing the microphone through `audioEngine.inputNode` -// (Voice Agent / duplex). When YES, the AudioQueue path is bypassed and the -// session must use VoiceChat mode so Apple's Voice-Processing I/O Audio Unit -// engages and performs hardware echo cancellation. -@property(atomic, assign) BOOL engineCaptureActive; -@property(atomic, assign) BOOL voiceProcessingRequested; -@property(atomic, assign) BOOL audioQueueCaptureRequested; -@property(nonatomic, strong) AVAudioConverter *captureConverter; -@property(nonatomic, strong) AVAudioFormat *captureOutputFormat; -@end - -@interface Deepgram (RecordingPrivate) -- (void)appendPCMDataAndEmitIfNeeded:(NSData *)pcmData; -- (BOOL)activateAudioSession:(NSError **)outError; -- (BOOL)configureAudioSessionIfNeeded:(NSError **)outError; -- (BOOL)configureAudioSession:(NSError **)outError; -- (AVAudioSessionCategoryOptions)bluetoothHFPOption; -- (BOOL)applyAudioSessionCategory:(AVAudioSessionCategory)category - mode:(NSString *)mode - options:(AVAudioSessionCategoryOptions)options - error:(NSError **)outError; -- (void)stopAndDetachPlayerNode; -- (void)interruptPlayerPlayback; -- (void)maybeDeactivateAudioSession; -@end - -static void DGHandleInputBuffer( - void *inUserData, __unused AudioQueueRef inAQ, AudioQueueBufferRef inBuffer, - __unused const AudioTimeStamp *inStartTime, __unused UInt32 inNumPackets, - __unused const AudioStreamPacketDescription *inPacketDesc) { - @autoreleasepool { - DGRecordState *state = (DGRecordState *)inUserData; - if (!state || !state->isRunning) { - DGLogDebug(@"[Deepgram] DGHandleInputBuffer: inactive state"); - return; - } - - Deepgram *strongSelf = state->mSelf; - if (!strongSelf) { - DGLogDebug(@"[Deepgram] DGHandleInputBuffer: missing self"); - return; - } - - if (!inBuffer || inBuffer->mAudioDataByteSize == 0) { - DGLogDebug(@"[Deepgram] DGHandleInputBuffer: empty buffer"); - return; - } - - DGLogDebug(@"[Deepgram] DGHandleInputBuffer: received %u bytes", - (unsigned int)inBuffer->mAudioDataByteSize); - NSData *data = [NSData dataWithBytes:inBuffer->mAudioData - length:inBuffer->mAudioDataByteSize]; - [strongSelf appendPCMDataAndEmitIfNeeded:data]; - - if (state->queue) { - DGLogDebug(@"[Deepgram] DGHandleInputBuffer: re-enqueue buffer"); - AudioQueueEnqueueBuffer(state->queue, inBuffer, 0, NULL); - } - } -} +// The class state (ivars + properties), the shared C helpers, the +// `DGRecordState` struct and the `DGHandleInputBuffer` AudioQueue callback now +// live in the shared headers and category files: +// - DGSupport.h : logging macros, error helpers, DGRecordState +// - Deepgram+Private.h : class extension (state) + cross-file decls +// - Deepgram+AudioSession.mm : audio session lifecycle +// - Deepgram+Recording.mm : microphone capture (+ DGHandleInputBuffer) +// - Deepgram+Playback.mm : TTS / Voice Agent playback +// This file keeps the module registration, lifecycle and the exported +// RCT_EXPORT_METHOD entry points (kept here so the macros register on the +// concrete class rather than a category). @implementation Deepgram RCT_EXPORT_MODULE(); @@ -233,428 +109,7 @@ - (void)dealloc { [[NSNotificationCenter defaultCenter] removeObserver:self]; } -- (BOOL)activateAudioSession:(NSError **)outError { - DGLogDebug(@"[Deepgram] activateAudioSession: begin (configured=%@)", - self.audioSessionConfigured ? @"YES" : @"NO"); - __block BOOL success = YES; - __block NSError *activationError = nil; - - DGLogDebug(@"[Deepgram] activateAudioSession: configuring on current queue"); - success = [self configureAudioSessionIfNeeded:&activationError]; - - if (!success && activationError) { - DGLogError(@"[Deepgram] Failed to activate audio session: %@", - activationError.localizedDescription ?: activationError); - } else { - DGLogDebug(@"[Deepgram] activateAudioSession: success=%@", - success ? @"YES" : @"NO"); - } - - if (outError) { - *outError = activationError; - } - - return success; -} - -- (void)deactivateAudioSession { - DGLogDebug(@"[Deepgram] deactivateAudioSession: begin"); - if (!self.audioSessionConfigured) { - DGLogDebug(@"[Deepgram] deactivateAudioSession: skipped (not configured)"); - return; - } - - NSError *error = nil; - // Use NotifyOthersOnDeactivation so other audio sessions (expo-av, etc.) - // know they can resume. - BOOL success = NO; - @try { - success = [[AVAudioSession sharedInstance] - setActive:NO - withOptions:AVAudioSessionSetActiveOptionNotifyOthersOnDeactivation - error:&error]; - } - @catch (NSException *e) { - DGLogWarn(@"[Deepgram] deactivateAudioSession threw exception: %@", - e.reason ?: e); - self.audioSessionConfigured = NO; - return; - } - - if (!success && error) { - // Deactivation can legitimately fail during competing audio-route - // transitions; treat as a warning rather than an error. - DGLogWarn(@"[Deepgram] Failed to deactivate audio session: %@", - error.localizedDescription ?: error); - } - self.audioSessionConfigured = NO; -} - -- (BOOL)configureAudioSessionIfNeeded:(NSError **)outError -{ - BOOL needsMic = _recordState.isRunning || self.audioQueueCaptureRequested || - self.engineCaptureActive || self.voiceProcessingRequested; - AVAudioSession *session = [AVAudioSession sharedInstance]; - - if (self.audioSessionConfigured) { - BOOL isCurrentlyPlayAndRecord = [session.category isEqualToString:AVAudioSessionCategoryPlayAndRecord]; - if (isCurrentlyPlayAndRecord == needsMic) { - DGLogDebug(@"[Deepgram] configureAudioSessionIfNeeded: already configured correctly, ensuring active"); - NSError *activeError = nil; - BOOL success = [[AVAudioSession sharedInstance] setActive:YES error:&activeError]; - if (success && !activeError) { - return YES; - } - } - } - - DGLogDebug(@"[Deepgram] configureAudioSessionIfNeeded: configuring (needsMic=%@)", needsMic ? @"YES" : @"NO"); - return [self configureAudioSession:outError]; -} - -- (BOOL)configureAudioSession:(NSError **)outError { - BOOL needsMic = _recordState.isRunning || self.audioQueueCaptureRequested || - self.engineCaptureActive || self.voiceProcessingRequested; - return [self configureAudioSessionForRecording:needsMic error:outError]; -} - -/** - * Build the Bluetooth input option appropriate for the running OS. - * iOS 17 deprecated `AllowBluetooth` (HFP route) in favor of - * `AllowBluetoothHFP`. Prefer the new symbol when building against the iOS - * 17+ SDK and fall back to the legacy spelling on older toolchains. - */ -- (AVAudioSessionCategoryOptions)bluetoothHFPOption { -#if defined(__IPHONE_17_0) && (__IPHONE_OS_VERSION_MAX_ALLOWED >= __IPHONE_17_0) - if (@available(iOS 17.0, *)) { - return AVAudioSessionCategoryOptionAllowBluetoothHFP; - } -#endif -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wdeprecated-declarations" - return AVAudioSessionCategoryOptionAllowBluetooth; -#pragma clang diagnostic pop -} - -/** - * Attempt to apply a single category/mode/options tuple and activate the - * session. Returns YES on success. On failure, populates `outError` so the - * caller can decide whether to try a more conservative configuration. - */ -- (BOOL)applyAudioSessionCategory:(AVAudioSessionCategory)category - mode:(NSString *)mode - options:(AVAudioSessionCategoryOptions)options - error:(NSError **)outError { - AVAudioSession *session = [AVAudioSession sharedInstance]; - NSError *error = nil; - - DGLogDebug(@"[Deepgram] configureAudioSession: setCategory %@ mode %@ " - @"options 0x%lx", - category, mode, (unsigned long)options); - - if (![session setCategory:category - mode:mode - options:options - error:&error] || - error) { - DGLogError(@"[Deepgram] configureAudioSession: category failed error=%@", - error.localizedDescription ?: error); - if (outError) { - *outError = error; - } - return NO; - } - - NSError *activeError = nil; - if (![session setActive:YES error:&activeError]) { - DGLogError(@"[Deepgram] configureAudioSession: setActive failed error=%@", - activeError.localizedDescription ?: activeError); - if (outError) { - *outError = activeError; - } - return NO; - } - - return YES; -} - -/** - * Configure audio session with mode appropriate to current usage. - * - * The configuration is attempted from most- to least-capable. If a richer - * setup is rejected we automatically fall back to a simpler one so audio still - * works instead of hard-failing. - * - * Two subtle rules drive the ordering below; getting them wrong yields - * OSStatus -50 (kAudioSession/paramErr) on `setCategory`, which then leaves the - * session in a half-configured state and makes a later AudioQueueStart fail - * with 'what' (kAudioSessionUnspecifiedError): - * - * 1. `DefaultToSpeaker` forces output to the built-in speaker, while - * `AllowBluetoothA2DP` / `AllowAirPlay` route output to external devices. - * Combining them is contradictory and is rejected with -50, so those - * output-routing options are NEVER mixed with a recording configuration. - * A2DP/AirPlay devices are output-only and can't supply a mic input - * anyway — Bluetooth input uses the HFP route instead. - * 2. `MixWithOthers` lets us coexist with other audio packages (expo-av, - * etc.) but can prevent us from acquiring the input on some routes. We try - * it first to be a good citizen, then degrade to exclusive access so - * recording still starts reliably. - * - * VoiceChat mode (which engages Apple's hardware AEC / VPIO) is used only when - * echo cancellation is required and only on a device — VPIO is unavailable on - * the Simulator and requesting it there guarantees a -50. - */ -- (BOOL)configureAudioSessionForRecording:(BOOL)needsMicrophone - error:(NSError **)outError { - DGLogDebug(@"[Deepgram] configureAudioSession: begin (mic=%@)", - needsMicrophone ? @"YES" : @"NO"); - - const AVAudioSessionCategoryOptions mixOption = - AVAudioSessionCategoryOptionMixWithOthers; - - // Build an ordered list of attempts, best first. - NSMutableArray *attempts = [NSMutableArray array]; - - if (needsMicrophone) { - BOOL needsAEC = self.engineCaptureActive || self.voiceProcessingRequested || - self.isPlaying; - - // Input-safe options for PlayAndRecord. DefaultToSpeaker + HFP are valid - // here; A2DP/AirPlay are intentionally excluded (see rule 1 above). - AVAudioSessionCategoryOptions recordOptions = - AVAudioSessionCategoryOptionDefaultToSpeaker | [self bluetoothHFPOption]; - -#if !TARGET_OS_SIMULATOR - // Preferred: hardware AEC via VoiceChat (device only). VoiceChat mode - // manages the route itself, so DefaultToSpeaker / HFP must NOT be combined - // with it — doing so is rejected with -50 on some iOS versions. Only - // MixWithOthers is safe to layer on top. - if (needsAEC) { - [attempts addObject:@{ - @"category" : AVAudioSessionCategoryPlayAndRecord, - @"mode" : AVAudioSessionModeVoiceChat, - @"options" : @(mixOption), - }]; - [attempts addObject:@{ - @"category" : AVAudioSessionCategoryPlayAndRecord, - @"mode" : AVAudioSessionModeVoiceChat, - @"options" : @(0), - }]; - } -#endif - - // PlayAndRecord without VoiceChat (no hardware AEC). Try with MixWithOthers - // first to coexist with other packages, then exclusively. - [attempts addObject:@{ - @"category" : AVAudioSessionCategoryPlayAndRecord, - @"mode" : AVAudioSessionModeDefault, - @"options" : @(recordOptions | mixOption), - }]; - [attempts addObject:@{ - @"category" : AVAudioSessionCategoryPlayAndRecord, - @"mode" : AVAudioSessionModeDefault, - @"options" : @(recordOptions), - }]; - - // Last resort: bare PlayAndRecord with no options. Always valid. - [attempts addObject:@{ - @"category" : AVAudioSessionCategoryPlayAndRecord, - @"mode" : AVAudioSessionModeDefault, - @"options" : @(0), - }]; - } else { - // Playback already supports AirPlay and A2DP routes; the explicit route - // options are for PlayAndRecord and can be rejected with OSStatus -50. - AVAudioSessionCategoryOptions playbackOptions = mixOption; - [attempts addObject:@{ - @"category" : AVAudioSessionCategoryPlayback, - @"mode" : AVAudioSessionModeDefault, - @"options" : @(playbackOptions), - }]; - - // Last resort: bare Playback with no options. - [attempts addObject:@{ - @"category" : AVAudioSessionCategoryPlayback, - @"mode" : AVAudioSessionModeDefault, - @"options" : @(0), - }]; - } - - NSError *lastError = nil; - NSUInteger attemptIndex = 0; - for (NSDictionary *attempt in attempts) { - attemptIndex++; - NSError *attemptError = nil; - BOOL success = [self - applyAudioSessionCategory:attempt[@"category"] - mode:attempt[@"mode"] - options:(AVAudioSessionCategoryOptions) - [attempt[@"options"] unsignedIntegerValue] - error:&attemptError]; - if (success) { - if (attemptIndex > 1) { - DGLogDebug(@"[Deepgram] configureAudioSession: succeeded on fallback " - @"attempt %lu/%lu", - (unsigned long)attemptIndex, (unsigned long)attempts.count); - } - self.audioSessionConfigured = YES; - DGLogDebug(@"[Deepgram] configureAudioSession: success"); - return YES; - } - lastError = attemptError; - DGLogDebug(@"[Deepgram] configureAudioSession: attempt %lu/%lu failed, " - @"trying next configuration", - (unsigned long)attemptIndex, (unsigned long)attempts.count); - } - - DGLogError(@"[Deepgram] configureAudioSession: all %lu configurations failed; " - @"last error=%@", - (unsigned long)attempts.count, - lastError.localizedDescription ?: lastError); - if (outError) { - *outError = lastError; - } - self.audioSessionConfigured = NO; - return NO; -} - -- (void)maybeDeactivateAudioSession { - DGLogDebug(@"[Deepgram] maybeDeactivateAudioSession: running=%@ playing=%@ " - @"engineCapture=%@", - _recordState.isRunning ? @"YES" : @"NO", - self.isPlaying ? @"YES" : @"NO", - self.engineCaptureActive ? @"YES" : @"NO"); - if (!_recordState.isRunning && !self.isPlaying && !self.engineCaptureActive) { - DGLogDebug(@"[Deepgram] maybeDeactivateAudioSession: deactivating"); - [self deactivateAudioSession]; - } -} - -- (void)handleAudioRouteChange:(NSNotification *)note { - DGLogDebug(@"[Deepgram] handleAudioRouteChange: %@", note.userInfo); - NSNumber *reasonValue = note.userInfo[AVAudioSessionRouteChangeReasonKey]; - AVAudioSessionRouteChangeReason reason = - reasonValue - ? (AVAudioSessionRouteChangeReason)reasonValue.unsignedIntegerValue - : AVAudioSessionRouteChangeReasonUnknown; - - // Headphones / Bluetooth headset unplugged — pause playback so we don't - // surprise the user by suddenly blasting through the loud speaker. - // (System would route audio to the speaker automatically; that's the point - // of this notification.) Matches expo-audio behavior and Apple's guidance. - if (reason == AVAudioSessionRouteChangeReasonOldDeviceUnavailable) { - DGLogDebug(@"[Deepgram] handleAudioRouteChange: old device unavailable — " - @"pausing playback"); - if (self.isPlaying && self.playerNode) { - [self.playerNode pause]; - } - } - - BOOL causedByCategory = - reason == AVAudioSessionRouteChangeReasonCategoryChange || - reason == AVAudioSessionRouteChangeReasonNoSuitableRouteForCategory; - - if (!causedByCategory) { - self.audioSessionConfigured = NO; - } - - if (!_recordState.isRunning && !self.isPlaying) { - DGLogDebug(@"[Deepgram] handleAudioRouteChange: ignoring (inactive)"); - return; - } - - if (!self.appIsActive) { - DGLogDebug(@"[Deepgram] handleAudioRouteChange: app inactive, skipping"); - return; - } - - if (causedByCategory) { - DGLogDebug(@"[Deepgram] handleAudioRouteChange: category change detected, " - @"keeping session"); - return; - } - - DGLogDebug(@"[Deepgram] handleAudioRouteChange: reactivating session"); - [self activateAudioSession:NULL]; -} - -// AVAudioSession can post this when mediaserverd restarts (e.g. after a -// rare system audio HAL hiccup). Once it fires, every AudioQueue / -// AVAudioEngine instance becomes a zombie. Apple's documented fix is to throw -// everything away and rebuild on demand. Match expo-audio's approach. -- (void)handleMediaServicesReset:(NSNotification *)note { - DGLogDebug(@"[Deepgram] handleMediaServicesReset: tearing down audio stack"); - self.audioSessionConfigured = NO; - - // Recording queue must be torn down — it's a zombie after a media services - // reset. - if (_recordState.isRunning || _recordState.queue) { - [self cleanupRecordingQueue]; - } - - // Engine-based capture path also dies with mediaserverd; force a full - // teardown so the next startRecording rebuilds VPIO from scratch. - self.engineCaptureActive = NO; - self.captureConverter = nil; - self.captureOutputFormat = nil; - - // AVAudioEngine + player node are no longer valid. Stop and discard so the - // next setup call rebuilds from scratch. - @try { - if (self.playerNode) { - [self.playerNode stop]; - } - if (self.audioEngine) { - [self.audioEngine stop]; - } - } @catch (NSException *e) { - DGLogError(@"[Deepgram] handleMediaServicesReset: teardown exception %@", - e); - } - self.playerNode = nil; - self.audioEngine = nil; - self.isPlaying = NO; -} - -- (void)handleAudioInterruption:(NSNotification *)note { - DGLogDebug(@"[Deepgram] handleAudioInterruption: %@", note.userInfo); - NSNumber *typeValue = note.userInfo[AVAudioSessionInterruptionTypeKey]; - AVAudioSessionInterruptionType type = - (AVAudioSessionInterruptionType)typeValue.unsignedIntegerValue; - - if (type == AVAudioSessionInterruptionTypeBegan) { - DGLogDebug(@"[Deepgram] handleAudioInterruption: interruption began"); - if (_recordState.isRunning && _recordState.queue) { - DGLogDebug( - @"[Deepgram] handleAudioInterruption: pausing recording queue"); - AudioQueuePause(_recordState.queue); - } - if (self.isPlaying && self.playerNode) { - DGLogDebug(@"[Deepgram] handleAudioInterruption: pausing player node"); - [self.playerNode pause]; - } - } else if (type == AVAudioSessionInterruptionTypeEnded) { - DGLogDebug(@"[Deepgram] handleAudioInterruption: interruption ended"); - NSNumber *optionValue = note.userInfo[AVAudioSessionInterruptionOptionKey]; - AVAudioSessionInterruptionOptions options = - (AVAudioSessionInterruptionOptions)optionValue.unsignedIntegerValue; - if (options & AVAudioSessionInterruptionOptionShouldResume) { - DGLogDebug(@"[Deepgram] handleAudioInterruption: should resume"); - [self activateAudioSession:nil]; - - if (_recordState.isRunning && _recordState.queue) { - DGLogDebug( - @"[Deepgram] handleAudioInterruption: resuming recording queue"); - AudioQueueStart(_recordState.queue, NULL); - } - if (self.isPlaying && self.playerNode) { - DGLogDebug(@"[Deepgram] handleAudioInterruption: resuming player node"); - [self.playerNode play]; - } - } - } -} +// MARK: - Audio session lifecycle lives in Deepgram+AudioSession.mm #if TARGET_OS_IOS - (void)handleAppDidBecomeActive:(NSNotification *)note { @@ -673,356 +128,10 @@ - (void)handleAppDidEnterBackground:(NSNotification *)note { } #endif -- (void)emitPCMChunk:(NSData *)chunk sampleRate:(int)sampleRate { - if (!chunk || chunk.length == 0) { - DGLogDebug(@"[Deepgram] emitPCMChunk: empty chunk, skipping"); - return; - } - - __weak __typeof(self) weakSelf = self; - NSData *chunkCopy = [chunk copy]; - dispatch_queue_t queue = self.emitterQueue ?: dispatch_get_main_queue(); - dispatch_async(queue, ^{ - if (!weakSelf) { - DGLogDebug(@"[Deepgram] emitPCMChunk: self released, aborting"); - return; - } - - if (!weakSelf.hasListeners) { - DGLogDebug(@"[Deepgram] emitPCMChunk: no listeners, dropping %lu bytes", - (unsigned long)chunkCopy.length); - return; - } - - if (!weakSelf.bridge || !weakSelf.callableJSModules) { - DGLogDebug( - @"[Deepgram] Skipping DeepgramAudioPCM event (bridge not ready)"); - return; - } - - DGLogDebug(@"[Deepgram] emitPCMChunk: sending %lu bytes sampleRate=%d", - (unsigned long)chunkCopy.length, sampleRate); - NSString *b64 = [chunkCopy base64EncodedStringWithOptions:0]; - [weakSelf sendEventWithName:@"DeepgramAudioPCM" - body:@{ - @"b64" : b64, - @"sampleRate" : @(sampleRate) - }]; - }); -} - -- (void)appendPCMDataAndEmitIfNeeded:(NSData *)pcmData { - if (!pcmData || pcmData.length == 0) { - DGLogDebug(@"[Deepgram] appendPCMDataAndEmitIfNeeded: empty PCM, skipping"); - return; - } - - if (!self.pendingPCMBuffer) { - DGLogDebug( - @"[Deepgram] appendPCMDataAndEmitIfNeeded: allocate pending buffer"); - self.pendingPCMBuffer = [[NSMutableData alloc] init]; - } - - DGLogDebug(@"[Deepgram] appendPCMDataAndEmitIfNeeded: append %lu bytes " - @"(pending=%lu)", - (unsigned long)pcmData.length, - (unsigned long)self.pendingPCMBuffer.length); - [self.pendingPCMBuffer appendData:pcmData]; - - NSUInteger chunkSize = - self.chunkSizeBytes > 0 ? self.chunkSizeBytes : pcmData.length; - - while (self.pendingPCMBuffer.length >= chunkSize) { - NSData *chunk = - [self.pendingPCMBuffer subdataWithRange:NSMakeRange(0, chunkSize)]; - [self.pendingPCMBuffer replaceBytesInRange:NSMakeRange(0, chunkSize) - withBytes:NULL - length:0]; - DGLogDebug(@"[Deepgram] appendPCMDataAndEmitIfNeeded: emitting chunk %lu " - @"bytes remaining=%lu", - (unsigned long)chunk.length, - (unsigned long)self.pendingPCMBuffer.length); - [self emitPCMChunk:chunk sampleRate:self.currentSampleRate]; - } -} - -- (void)flushPendingPCM { - if (self.pendingPCMBuffer.length == 0) { - DGLogDebug(@"[Deepgram] flushPendingPCM: nothing to flush"); - return; - } - - NSData *remaining = [self.pendingPCMBuffer copy]; - [self.pendingPCMBuffer setLength:0]; - DGLogDebug(@"[Deepgram] flushPendingPCM: flushing %lu bytes", - (unsigned long)remaining.length); - [self emitPCMChunk:remaining sampleRate:self.currentSampleRate]; -} - -- (void)cleanupRecordingQueue { - DGLogDebug(@"[Deepgram] cleanupRecordingQueue: begin"); - self.audioQueueCaptureRequested = NO; - _recordState.isRunning = false; - - if (_recordState.queue) { - DGLogDebug(@"[Deepgram] cleanupRecordingQueue: stopping queue"); - AudioQueueStop(_recordState.queue, true); - - for (int i = 0; i < DGNumberBuffers; i++) { - if (_recordState.buffers[i]) { - DGLogDebug(@"[Deepgram] cleanupRecordingQueue: freeing buffer %d", i); - AudioQueueFreeBuffer(_recordState.queue, _recordState.buffers[i]); - _recordState.buffers[i] = NULL; - } - } - - DGLogDebug(@"[Deepgram] cleanupRecordingQueue: disposing queue"); - AudioQueueDispose(_recordState.queue, true); - } - - memset(&_recordState, 0, sizeof(DGRecordState)); - DGLogDebug(@"[Deepgram] cleanupRecordingQueue: state cleared"); - [self maybeDeactivateAudioSession]; -} - -/* ================================================================== */ -/* 1. MICROPHONE CAPTURE (16 kHz PCM16 emission) */ -/* ================================================================== */ - -/** - * Engine-based microphone capture path used when the JS side opts in to - * hardware voice processing (Voice Agent / duplex). This routes through - * `AVAudioEngine.inputNode` with `setVoiceProcessingEnabled:YES` on both - * input and output nodes — the only Apple-supported way to actually engage - * the VPIO Audio Unit's hardware echo cancellation on iOS. The legacy - * AudioQueue path is preserved for STT-only usage where AEC is undesirable. - */ -- (BOOL)startEngineCaptureAndReturnError:(NSError **)outError { - self.currentSampleRate = 16000; - - if (!self.audioEngine) { - NSError *engineError = nil; - if (![self setupAudioEngineWithSampleRate:self.currentSampleRate - channels:1 - enableVoiceProcessing:YES - error:&engineError]) { - if (outError) - *outError = engineError; - return NO; - } - } else { -#if TARGET_IPHONE_SIMULATOR - // VPIO is unsupported on the simulator and toggling VP corrupts the - // input node's format (sampleRate becomes 0), which would later fail - // the IsFormatSampleRateAndChannelCountValid check. Skip entirely. - DGLogWarn(@"[Deepgram] NOTE: Voice Processing I/O (Echo Cancellation) is " - @"NOT supported on the iOS Simulator. Audio output may be picked " - @"up by the microphone. Please test on a physical device for " - @"proper AEC behavior."); -#else - @try { - NSError *vpError = nil; - if (self.audioEngine.inputNode && - ![self.audioEngine.inputNode setVoiceProcessingEnabled:YES - error:&vpError]) { - DGLogWarn(@"[Deepgram] inputNode VP enable failed: %@", vpError); - } - if (self.audioEngine.outputNode && - ![self.audioEngine.outputNode setVoiceProcessingEnabled:YES - error:&vpError]) { - DGLogWarn(@"[Deepgram] outputNode VP enable failed: %@", vpError); - } - } @catch (NSException *e) { - DGLogWarn(@"[Deepgram] VP enable threw: %@", e); - } -#endif - } - - AVAudioInputNode *inputNode = self.audioEngine.inputNode; - if (!inputNode) { - if (outError) { - *outError = [NSError - errorWithDomain:@"DeepgramAudioEngine" - code:-2 - userInfo:@{ - NSLocalizedDescriptionKey : @"No input node available" - }]; - } - return NO; - } - - AVAudioFormat *hwFormat = [inputNode inputFormatForBus:0]; - if (!hwFormat || hwFormat.sampleRate <= 0) { - if (outError) { - *outError = - [NSError errorWithDomain:@"DeepgramAudioEngine" - code:-3 - userInfo:@{ - NSLocalizedDescriptionKey : @"Invalid input format" - }]; - } - return NO; - } - - AVAudioFormat *outputFormat = - [[AVAudioFormat alloc] initWithCommonFormat:AVAudioPCMFormatInt16 - sampleRate:self.currentSampleRate - channels:1 - interleaved:YES]; - self.captureOutputFormat = outputFormat; - self.captureConverter = - [[AVAudioConverter alloc] initFromFormat:hwFormat toFormat:outputFormat]; - if (!self.captureConverter) { - if (outError) { - *outError = [NSError errorWithDomain:@"DeepgramAudioEngine" - code:-4 - userInfo:@{ - NSLocalizedDescriptionKey : - @"Failed to build AVAudioConverter" - }]; - } - return NO; - } - - AVAudioFrameCount tapFrames = (AVAudioFrameCount)MAX( - 1024, (NSUInteger)round(hwFormat.sampleRate * 0.2)); - - __weak __typeof(self) weakSelf = self; - @try { - [inputNode removeTapOnBus:0]; - } @catch (__unused NSException *e) { - // No prior tap — fine. - } - - @ - try { - [inputNode - installTapOnBus:0 - bufferSize:tapFrames - format:hwFormat - block:^(AVAudioPCMBuffer *_Nonnull inBuf, - __unused AVAudioTime *_Nonnull when) { - __strong __typeof(weakSelf) strongSelf = weakSelf; - if (!strongSelf || !strongSelf.engineCaptureActive) - return; - AVAudioConverter *converter = strongSelf.captureConverter; - AVAudioFormat *outFmt = strongSelf.captureOutputFormat; - if (!converter || !outFmt || inBuf.frameLength == 0) - return; - - AVAudioFrameCount outCapacity = - (AVAudioFrameCount)ceil((double)inBuf.frameLength * - outFmt.sampleRate / - inBuf.format.sampleRate) + - 16; - AVAudioPCMBuffer *outBuf = [[AVAudioPCMBuffer alloc] - initWithPCMFormat:outFmt - frameCapacity:outCapacity]; - if (!outBuf) - return; - - __block BOOL provided = NO; - NSError *cvtError = nil; - AVAudioConverterInputBlock inputBlock = - ^AVAudioBuffer *_Nullable( - __unused AVAudioPacketCount inNumPackets, - AVAudioConverterInputStatus *_Nonnull outStatus) { - if (provided) { - *outStatus = AVAudioConverterInputStatus_NoDataNow; - return nil; - } - provided = YES; - *outStatus = AVAudioConverterInputStatus_HaveData; - return inBuf; - }; - - AVAudioConverterOutputStatus status = - [converter convertToBuffer:outBuf - error:&cvtError - withInputFromBlock:inputBlock]; - if (status == AVAudioConverterOutputStatus_Error || - cvtError) { - DGLogWarn(@"[Deepgram] capture convert failed: %@", - cvtError); - return; - } - if (outBuf.frameLength == 0 || !outBuf.int16ChannelData) - return; - - NSUInteger byteCount = (NSUInteger)outBuf.frameLength * 2; - NSData *pcm = - [NSData dataWithBytes:outBuf.int16ChannelData[0] - length:byteCount]; - [strongSelf appendPCMDataAndEmitIfNeeded:pcm]; - }]; - } @catch (NSException *e) { - DGLogError(@"[Deepgram] installTapOnBus exception: %@", e); - if (outError) { - *outError = [NSError - errorWithDomain:@"DeepgramAudioEngine" - code:-5 - userInfo:@{ - NSLocalizedDescriptionKey : e.reason ?: @"Tap install failed" - }]; - } - return NO; - } - - self.chunkSizeBytes = - (NSUInteger)MAX(1, (int)round(self.currentSampleRate * 2 * 0.2)); - self.engineCaptureActive = YES; - - if (!self.audioEngine.isRunning) { - NSError *startError = nil; - if (![self.audioEngine startAndReturnError:&startError]) { - DGLogError(@"[Deepgram] engine start failed: %@", startError); - @try { - [inputNode removeTapOnBus:0]; - } @catch (__unused NSException *e) { - } - self.engineCaptureActive = NO; - if (outError) - *outError = startError; - return NO; - } - } - return YES; -} - -- (void)stopEngineCapture { - if (!self.engineCaptureActive && !self.captureConverter) - return; - DGLogDebug(@"[Deepgram] stopEngineCapture"); - self.engineCaptureActive = NO; - - @try { - [self.audioEngine.inputNode removeTapOnBus:0]; - } @catch (NSException *e) { - DGLogWarn(@"[Deepgram] removeTapOnBus exception: %@", e); - } - -#if !TARGET_IPHONE_SIMULATOR - @ - try { - NSError *vpError = nil; - if (self.audioEngine.inputNode) { - [self.audioEngine.inputNode setVoiceProcessingEnabled:NO error:&vpError]; - } - } @catch (__unused NSException *e) { - // best-effort - } -#endif - - self.captureConverter = nil; - self.captureOutputFormat = nil; - - if (!self.isPlaying && self.audioEngine.isRunning) { - @try { - [self.audioEngine stop]; - } @catch (__unused NSException *e) { - } - } -} +// MARK: - Microphone capture lives in Deepgram+Recording.mm +// (emitPCMChunk / appendPCMDataAndEmitIfNeeded / flushPendingPCM / +// cleanupRecordingQueue / startEngineCaptureAndReturnError / +// stopEngineCapture, plus the DGHandleInputBuffer callback) RCT_EXPORT_METHOD(startRecording : (NSDictionary *)options resolver : ( RCTPromiseResolveBlock)resolve rejecter : (RCTPromiseRejectBlock)reject) { @@ -1222,8 +331,7 @@ - (void)stopEngineCapture { resolve(nil); } @catch (NSException *e) { DGLogError(@"[Deepgram] stopRecording: exception %@", e); - if (reject) - reject(@"record_stop_error", e.reason, nil); + DGRejectPromise(reject, @"record_stop_error", e.reason, nil); } } @@ -1250,8 +358,7 @@ - (void)stopEngineCapture { resolve(nil); } @catch (NSException *e) { DGLogError(@"[Deepgram] startAudio: exception %@", e); - if (reject) - reject(@"audio_start_error", e.reason, nil); + DGRejectPromise(reject, @"audio_start_error", e.reason, nil); } } @@ -1265,198 +372,13 @@ - (void)stopEngineCapture { resolve(nil); } @catch (NSException *e) { DGLogError(@"[Deepgram] stopAudio: exception %@", e); - if (reject) - reject(@"audio_stop_error", e.reason, nil); - } -} - -/* ================================================================== */ -/* 2. AVAUDIOENGINE PLAYBACK WITH VOICE PROCESSING (ECHO CANCEL) */ -/* ================================================================== */ - -/** - * Setup AVAudioEngine for output. When `enableVoiceProcessing` is YES we also - * configure the input node for hardware echo cancellation (Voice Agent / - * duplex use case). When NO (pure TTS playback), we deliberately avoid - * touching `inputNode` so we don't request the microphone or interfere with - * other audio libraries. - */ -- (void)stopAndDetachPlayerNode { - if (self.playerNode) { - _playbackGeneration.fetch_add(1); - - @try { - [self.playerNode stop]; - } @catch (NSException *e) { - DGLogWarn(@"[Deepgram] playerNode stop exception: %@", e); - } - - if (self.audioEngine) { - @try { - [self.audioEngine detachNode:self.playerNode]; - } @catch (NSException *e) { - DGLogWarn(@"[Deepgram] playerNode detach exception: %@", e); - } - } - } - - self.playerNode = nil; - self.playbackFormat = nil; - self.isPlaying = NO; - _scheduledBufferCount = 0; -} - -- (void)interruptPlayerPlayback { - if (self.playerNode) { - _playbackGeneration.fetch_add(1); - - @try { - [self.playerNode stop]; - } @catch (NSException *e) { - DGLogWarn(@"[Deepgram] playerNode interrupt exception: %@", e); - } + DGRejectPromise(reject, @"audio_stop_error", e.reason, nil); } - - self.isPlaying = NO; - _scheduledBufferCount = 0; } -- (BOOL)setupAudioEngineWithSampleRate:(int)sampleRate - channels:(int)channels - enableVoiceProcessing:(BOOL)enableVoiceProcessing - error:(NSError **)outError { - BOOL reuseCaptureEngine = self.audioEngine && self.engineCaptureActive; - - if (self.audioEngine && self.audioEngine.isRunning) { - [self.audioEngine stop]; - } - - [self stopAndDetachPlayerNode]; - - if (!reuseCaptureEngine) { - if (self.audioEngine) { - [self.audioEngine reset]; - } - self.audioEngine = [[AVAudioEngine alloc] init]; - } - - if (!self.audioEngine) { - self.audioEngine = [[AVAudioEngine alloc] init]; - } - - self.playerNode = [[AVAudioPlayerNode alloc] init]; - - self.playbackFormat = - [[AVAudioFormat alloc] initWithCommonFormat:AVAudioPCMFormatInt16 - sampleRate:sampleRate - channels:channels - interleaved:YES]; - - if (!self.playbackFormat) { - if (outError) { - *outError = [NSError - errorWithDomain:@"DeepgramAudioEngine" - code:-1 - userInfo:@{ - NSLocalizedDescriptionKey : @"Failed to create audio format" - }]; - } - return NO; - } - - [self.audioEngine attachNode:self.playerNode]; - [self.audioEngine connect:self.playerNode - to:self.audioEngine.mainMixerNode - format:self.playbackFormat]; - - if (enableVoiceProcessing) { -#if TARGET_IPHONE_SIMULATOR - // VPIO is not implemented on the simulator. Calling - // setVoiceProcessingEnabled:YES there raises an internal AVAEInternal - // exception about IsFormatSampleRateAndChannelCountValid because the - // simulator's input node cannot satisfy the VPIO format constraints. - // Skip VP entirely; AEC has to be tested on a physical device. - DGLogWarn(@"[Deepgram] NOTE: Voice Processing I/O (Echo Cancellation) is " - @"NOT supported on the iOS Simulator. Audio output may be picked " - @"up by the microphone. Please test on a physical device for " - @"proper AEC behavior."); -#else - @try { - AVAudioInputNode *inputNode = self.audioEngine.inputNode; - AVAudioOutputNode *outputNode = self.audioEngine.outputNode; - // Apple's hardware AEC (VPIO Audio Unit) only engages when *both* - // input and output flow through a voice-processing-enabled audio - // unit. Enabling VP only on the input node is a no-op for capture - // unless rendering goes through the same unit, so we enable both. - if (inputNode) { - NSError *voiceProcessingError = nil; - if (![inputNode setVoiceProcessingEnabled:YES - error:&voiceProcessingError]) { - DGLogWarn(@"[Deepgram] inputNode VP enable failed: %@", - voiceProcessingError); - } - } - if (outputNode) { - NSError *voiceProcessingError = nil; - if (![outputNode setVoiceProcessingEnabled:YES - error:&voiceProcessingError]) { - DGLogWarn(@"[Deepgram] outputNode VP enable failed: %@", - voiceProcessingError); - } - } - } @catch (NSException *exception) { - // Continue - voice processing not critical for basic playback - DGLogWarn(@"[Deepgram] VP enable threw: %@", exception); - } -#endif - } - - [self.audioEngine prepare]; - NSError *startError = nil; - BOOL started = [self.audioEngine startAndReturnError:&startError]; - - if (!started) { - DGLogError(@"[Deepgram] Failed to start audio engine: %@", - startError.localizedDescription); - if (outError) { - *outError = startError; - } - return NO; - } - - return YES; -} - -/** - * Create AVAudioPCMBuffer from raw PCM data. - */ -- (AVAudioPCMBuffer *)createPCMBufferFromData:(NSData *)data { - if (!data || data.length == 0 || !self.playbackFormat) { - return nil; - } - - // Calculate frame count (data length / bytes per frame) - const AudioStreamBasicDescription *asbd = - self.playbackFormat.streamDescription; - int bytesPerFrame = asbd->mBytesPerFrame; - AVAudioFrameCount frameCount = - (AVAudioFrameCount)(data.length / bytesPerFrame); - - if (frameCount == 0) { - return nil; - } - - AVAudioPCMBuffer *buffer = - [[AVAudioPCMBuffer alloc] initWithPCMFormat:self.playbackFormat - frameCapacity:frameCount]; - - buffer.frameLength = frameCount; - - // Copy audio data to buffer - memcpy(buffer.int16ChannelData[0], data.bytes, data.length); - - return buffer; -} +// MARK: - TTS / Voice Agent playback lives in Deepgram+Playback.mm +// (stopAndDetachPlayerNode / interruptPlayerPlayback / +// setupAudioEngineWithSampleRate / createPCMBufferFromData) RCT_EXPORT_METHOD(startPlayer : (nonnull NSNumber *) sampleRate channels : (nonnull NSNumber *)channels) { @@ -1600,8 +522,7 @@ - (AVAudioPCMBuffer *)createPCMBufferFromData:(NSData *)data { resolve(nil); } @catch (NSException *e) { DGLogError(@"[Deepgram] stopPlayer: exception %@", e); - if (reject) - reject(@"player_stop_error", e.reason, nil); + DGRejectPromise(reject, @"player_stop_error", e.reason, nil); } } @@ -1704,8 +625,7 @@ - (AVAudioPCMBuffer *)createPCMBufferFromData:(NSData *)data { (unsigned long)pcmData.length); } @catch (NSException *e) { DGLogError(@"[Deepgram] playAudioChunk: exception %@", e); - if (reject) - reject(@"audio_chunk_error", e.reason, nil); + DGRejectPromise(reject, @"audio_chunk_error", e.reason, nil); } } From bec6d2fbe10d65382864a373dee2e48e33f57d9f Mon Sep 17 00:00:00 2001 From: Rares Date: Fri, 26 Jun 2026 10:53:23 -0400 Subject: [PATCH 2/9] feat: enhance audio playback management with one-shot playback support and permission handling --- .../src/main/java/com/deepgram/AudioPlayer.kt | 200 +++++++++++++----- .../main/java/com/deepgram/DeepgramModule.kt | 1 + example/app.json | 13 +- example/package.json | 1 + ios/Deepgram.mm | 74 +++++-- yarn.lock | 13 ++ 6 files changed, 229 insertions(+), 73 deletions(-) diff --git a/android/src/main/java/com/deepgram/AudioPlayer.kt b/android/src/main/java/com/deepgram/AudioPlayer.kt index 0b27cf2..4c489c6 100644 --- a/android/src/main/java/com/deepgram/AudioPlayer.kt +++ b/android/src/main/java/com/deepgram/AudioPlayer.kt @@ -13,8 +13,10 @@ import android.os.Build import android.os.Handler import android.util.Base64 import android.util.Log +import java.util.Collections import java.util.concurrent.LinkedBlockingQueue import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicBoolean import java.util.concurrent.atomic.AtomicInteger /** @@ -34,9 +36,25 @@ internal class AudioPlayer( private val onForegroundServiceRequest: () -> Unit, private val onForegroundServiceRelease: () -> Unit, ) { + // `streamingActive` drives the streaming playback loop and its teardown. + // The simple play/stop path is tracked independently via `simpleActive`, and + // one-shot playbacks via `oneShotPlaybacks`, so none of the three paths can + // tear down another's AudioTrack or shared focus/foreground-service state. @Volatile - var isActive: Boolean = false - private set + private var streamingActive: Boolean = false + + // The simple "prepare for playback" path (mirrors iOS `startAudio`, which only + // activates the audio session). It owns no AudioTrack — it just keeps audio + // focus + the foreground service alive until `stopAudio` is called. + @Volatile + private var simpleActive: Boolean = false + + private val oneShotPlaybacks = + Collections.synchronizedSet(mutableSetOf()) + + /** True while streaming, simple, or any one-shot playback is running. */ + val isActive: Boolean + get() = streamingActive || simpleActive || oneShotPlaybacks.isNotEmpty() private var playbackSampleRate: Int = DEFAULT_PLAYBACK_SAMPLE_RATE private var currentOutputChannels: Int = 1 @@ -120,34 +138,50 @@ internal class AudioPlayer( // ------------------------------------------------------------------- fun startAudio() { + // Mirror iOS `startAudio`: only prepare the session for playback (request + // focus + keep-alive service). Do NOT create/play the streaming AudioTrack + // here — the streaming player owns that track, and sharing it would let a + // later `stopAudio`/`startPlayer` tear down the other path's playback. focusManager.requestFocus() - ensureAudioTrack() - audioTrack?.play() - isActive = true + simpleActive = true onForegroundServiceRequest() } fun stopAudio() { - stopAudioInternal(throwOnError = true) + // Mirror iOS `stopAudio` -> `stopPlayer`: end the simple session and stop + // any streaming playback, then release shared resources once nothing is + // left active. + simpleActive = false + stopStreamingPlayback(throwOnError = true) } - private fun stopAudioInternal(throwOnError: Boolean) { + private fun releaseStreamingTrack(throwOnError: Boolean) { try { audioTrack?.stop() } catch (e: Exception) { - Log.w(TAG, "stopAudioInternal: error stopping AudioTrack", e) + Log.w(TAG, "releaseStreamingTrack: error stopping AudioTrack", e) if (throwOnError) throw e } try { audioTrack?.release() } catch (e: Exception) { - Log.w(TAG, "stopAudioInternal: error releasing AudioTrack", e) + Log.w(TAG, "releaseStreamingTrack: error releasing AudioTrack", e) if (throwOnError) throw e } audioTrack = null - isActive = false + streamingActive = false + releaseSharedResourcesIfIdle() + } + + /** + * Abandon audio focus and release the foreground service only when none of + * the streaming, simple, or one-shot playback paths remain active, so a + * finishing path cannot tear down shared resources still in use by another. + */ + private fun releaseSharedResourcesIfIdle() { + if (isActive) return focusManager.abandonFocus() onForegroundServiceRelease() } @@ -162,7 +196,7 @@ internal class AudioPlayer( currentOutputChannels = if (channels >= 2) 2 else 1 audioQueue.clear() queuedBytes.set(0) - isActive = false + streamingActive = false } fun setAudioConfig(sampleRate: Int, channels: Int) { @@ -187,7 +221,7 @@ internal class AudioPlayer( queuedBytes.addAndGet(audioData.size) // Start playback when we have enough buffered data - if (!isActive && queuedBytes.get() >= playbackThreshold) { + if (!streamingActive && queuedBytes.get() >= playbackThreshold) { startStreamingPlayback() } } catch (e: Exception) { @@ -204,9 +238,9 @@ internal class AudioPlayer( } private fun startStreamingPlayback() { - if (isActive) return + if (streamingActive) return - isActive = true + streamingActive = true registerNoisyReceiver() playbackThread = Thread({ try { @@ -216,7 +250,7 @@ internal class AudioPlayer( ensureAudioTrack() audioTrack?.play() - while (isActive) { + while (streamingActive) { // Block until data is available (up to 100ms), avoids busy-wait val chunk = audioQueue.poll(100, TimeUnit.MILLISECONDS) ?: continue @@ -233,8 +267,8 @@ internal class AudioPlayer( } catch (e: Exception) { Log.e(TAG, "Streaming playback error", e) } finally { - isActive = false - onForegroundServiceRelease() + streamingActive = false + releaseSharedResourcesIfIdle() } }, "Deepgram-Playback") @@ -242,7 +276,7 @@ internal class AudioPlayer( } fun stopStreamingPlayback(throwOnError: Boolean = true) { - isActive = false + streamingActive = false playbackThread?.let { thread -> thread.interrupt() try { @@ -254,14 +288,13 @@ internal class AudioPlayer( playbackThread = null try { - stopAudioInternal(throwOnError) + releaseStreamingTrack(throwOnError) } catch (e: Exception) { if (throwOnError) throw e else Log.w(TAG, "stopStreamingPlayback: error stopping audio", e) } finally { audioQueue.clear() queuedBytes.set(0) unregisterNoisyReceiver() - onForegroundServiceRelease() } } @@ -276,7 +309,7 @@ internal class AudioPlayer( /** Resume output after regaining focus, if playback is still active. */ fun resumeForFocusGain() { - if (isActive) { + if (streamingActive) { try { audioTrack?.play() } catch (_: Exception) {} } } @@ -290,16 +323,39 @@ internal class AudioPlayer( onSuccess: () -> Unit, onError: (code: String, message: String?) -> Unit, ) { - try { - val audioData = Base64.decode(base64, Base64.DEFAULT) - if (audioData.isEmpty()) { - onError("invalid_data", "Failed to decode audio data") - return - } + val audioData = try { + Base64.decode(base64, Base64.DEFAULT) + } catch (e: Exception) { + Log.e(TAG, "playAudioChunk decode error", e) + onError("invalid_data", "Failed to decode audio data") + return + } + + if (audioData.isEmpty()) { + onError("invalid_data", "Failed to decode audio data") + return + } + + // Reject malformed / oversized PCM before building the AudioTrack. A + // sub-frame buffer would make frameCount (and notificationMarkerPosition) + // 0, so onMarkerReached would never fire — the promise would hang forever + // while leaking audio focus + the foreground service. The upper bound + // mirrors the streaming queue cap. + val bytesPerFrame = if (currentOutputChannels >= 2) 4 else 2 // PCM16 mono/stereo + if (audioData.size < bytesPerFrame || audioData.size % bytesPerFrame != 0) { + onError("invalid_data", "Audio data must contain whole PCM16 frames") + return + } + if (audioData.size > MAX_QUEUED_PLAYBACK_BYTES) { + onError("invalid_data", "Audio data exceeds the maximum one-shot size") + return + } - focusManager.requestFocus() - onForegroundServiceRequest() + focusManager.requestFocus() + onForegroundServiceRequest() + var playback: OneShotPlayback? = null + try { val channelMask = outputChannelMask() val minBuf = minTrackBufferSize(channelMask) @@ -323,46 +379,90 @@ internal class AudioPlayer( tempAudioTrack.write(audioData, 0, audioData.size) - // Use notification marker to know when playback finishes, - // then notify the caller (instead of polling with Thread.sleep). - val bytesPerFrame = if (currentOutputChannels >= 2) 4 else 2 // PCM16 mono/stereo + // Use a notification marker to know when playback finishes (instead of + // polling with Thread.sleep). Track the one-shot so it contributes to + // isActive and can be torn down on focus loss / invalidate. val frameCount = audioData.size / bytesPerFrame tempAudioTrack.notificationMarkerPosition = frameCount - val listener = object : AudioTrack.OnPlaybackPositionUpdateListener { - override fun onMarkerReached(track: AudioTrack?) { - try { - track?.stop() - track?.release() - } catch (e: Exception) { - Log.w(TAG, "playAudioChunk cleanup error", e) - } - focusManager.abandonFocus() - onForegroundServiceRelease() - onSuccess() - } - override fun onPeriodicNotification(track: AudioTrack?) {} - } + val created = OneShotPlayback(tempAudioTrack, onSuccess, onError) + playback = created + oneShotPlaybacks.add(created) // minSdk is 24, the Handler-aware overload is always available. - tempAudioTrack.setPlaybackPositionUpdateListener(listener, mainHandler) + tempAudioTrack.setPlaybackPositionUpdateListener( + object : AudioTrack.OnPlaybackPositionUpdateListener { + override fun onMarkerReached(track: AudioTrack?) = created.complete() + override fun onPeriodicNotification(track: AudioTrack?) {} + }, + mainHandler, + ) tempAudioTrack.play() } catch (e: Exception) { Log.e(TAG, "playAudioChunk error", e) - focusManager.abandonFocus() - onForegroundServiceRelease() - onError("playback_error", e.message) + val created = playback + if (created != null) { + created.fail("playback_error", e.message) + } else { + releaseSharedResourcesIfIdle() + onError("playback_error", e.message) + } + } + } + + /** + * A single MODE_STATIC one-shot playback. Guarantees the JS promise settles + * exactly once (via [complete] or [fail]), tears the AudioTrack down, and + * releases the shared focus / foreground service only when nothing else is + * still playing. + */ + private inner class OneShotPlayback( + private val track: AudioTrack, + private val onSuccess: () -> Unit, + private val onError: (code: String, message: String?) -> Unit, + ) { + private val settled = AtomicBoolean(false) + + fun complete() { + if (!settled.compareAndSet(false, true)) return + teardown() + onSuccess() + } + + fun fail(code: String, message: String?) { + if (!settled.compareAndSet(false, true)) return + teardown() + onError(code, message) + } + + private fun teardown() { + try { + track.stop() + track.release() + } catch (e: Exception) { + Log.w(TAG, "playAudioChunk cleanup error", e) + } + oneShotPlaybacks.remove(this) + releaseSharedResourcesIfIdle() } } + /** Stop every in-flight one-shot playback, rejecting their pending promises. */ + fun stopOneShotPlayback() { + val pending = synchronized(oneShotPlaybacks) { oneShotPlaybacks.toList() } + pending.forEach { it.fail("playback_error", "Playback interrupted") } + } + /** Tear down all playback resources (called from the module's invalidate). */ fun release() { + simpleActive = false try { stopStreamingPlayback(throwOnError = false) } catch (e: Exception) { Log.w(TAG, "Error stopping playback on release", e) } + stopOneShotPlayback() unregisterNoisyReceiver() } diff --git a/android/src/main/java/com/deepgram/DeepgramModule.kt b/android/src/main/java/com/deepgram/DeepgramModule.kt index 4cad9ec..4c7b8e0 100644 --- a/android/src/main/java/com/deepgram/DeepgramModule.kt +++ b/android/src/main/java/com/deepgram/DeepgramModule.kt @@ -62,6 +62,7 @@ class DeepgramModule(private val reactContext: ReactApplicationContext) : // other audio packages and so we release the microphone. Order matches the // original: stop playback first, then recording. try { player.stopStreamingPlayback(throwOnError = false) } catch (_: Exception) {} + try { player.stopOneShotPlayback() } catch (_: Exception) {} try { recorder.stop(throwOnError = false) } catch (_: Exception) {} } diff --git a/example/app.json b/example/app.json index a3223c7..3fecd6d 100644 --- a/example/app.json +++ b/example/app.json @@ -4,10 +4,7 @@ "slug": "react-native-deepgram-example", "version": "1.0.0", "orientation": "portrait", - "platforms": [ - "ios", - "android" - ], + "platforms": ["ios", "android"], "jsEngine": "hermes", "newArchEnabled": true, "ios": { @@ -19,6 +16,14 @@ { "microphonePermission": "Allow $(PRODUCT_NAME) to access your microphone" } + ], + [ + "expo-build-properties", + { + "android": { + "kotlinVersion": "1.9.24" + } + } ] ], "android": { diff --git a/example/package.json b/example/package.json index f730d96..2d814f3 100644 --- a/example/package.json +++ b/example/package.json @@ -16,6 +16,7 @@ "dependencies": { "@expo/metro-runtime": "~4.0.1", "expo": "~52.0.27", + "expo-build-properties": "~0.13.3", "expo-document-picker": "^12.0.2", "react": "18.3.1", "react-dom": "18.3.1", diff --git a/ios/Deepgram.mm b/ios/Deepgram.mm index 07fab4a..7aca3ee 100644 --- a/ios/Deepgram.mm +++ b/ios/Deepgram.mm @@ -27,6 +27,21 @@ // RCT_EXPORT_METHOD entry points (kept here so the macros register on the // concrete class rather than a category). +/** + * Whether the user has explicitly granted microphone record permission. Read + * via the deprecated `AVAudioSession` accessor (still functional on iOS 17+) + * with the deprecation warning suppressed locally — mirroring MicPermission.m. + * Lets `startRecording` fail fast with the contract `permission_denied` code, + * matching the Android module. + */ +static BOOL DGHasRecordPermission(void) { +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wdeprecated-declarations" + return [AVAudioSession sharedInstance].recordPermission == + AVAudioSessionRecordPermissionGranted; +#pragma clang diagnostic pop +} + @implementation Deepgram RCT_EXPORT_MODULE(); @@ -138,6 +153,17 @@ - (void)handleAppDidEnterBackground:(NSNotification *)note { @try { DGLogDebug(@"[Deepgram] startRecording: begin"); + if (!DGHasRecordPermission()) { + DGLogError( + @"[Deepgram] startRecording: microphone permission not granted"); + DGRejectPromise( + reject, @"permission_denied", + @"Microphone permission has not been granted. Request it from JS " + @"(MicPermission.request) before calling startRecording.", + nil); + return; + } + BOOL enableVoiceProcessing = NO; if ([options isKindOfClass:[NSDictionary class]]) { id raw = options[@"enableVoiceProcessing"]; @@ -158,7 +184,7 @@ - (void)handleAppDidEnterBackground:(NSNotification *)note { DGLogError(@"[Deepgram] startRecording: activation failed %@", message); self.voiceProcessingRequested = NO; self.audioQueueCaptureRequested = NO; - DGRejectPromise(reject, @"record_start_error", message, sessionError); + DGRejectPromise(reject, @"start_error", message, sessionError); return; } @@ -176,7 +202,7 @@ - (void)handleAppDidEnterBackground:(NSNotification *)note { DGLogError(@"[Deepgram] startRecording: %@", message); self.voiceProcessingRequested = NO; [self maybeDeactivateAudioSession]; - DGRejectPromise(reject, @"record_start_error", message, engineError); + DGRejectPromise(reject, @"start_error", message, engineError); return; } DGLogDebug(@"[Deepgram] startRecording: engine capture started"); @@ -233,7 +259,7 @@ - (void)handleAppDidEnterBackground:(NSNotification *)note { NSString *message = error.localizedDescription; DGLogError(@"[Deepgram] startRecording: %@", message); [self cleanupRecordingQueue]; - DGRejectPromise(reject, @"record_start_error", message, error); + DGRejectPromise(reject, @"start_error", message, error); return; } @@ -277,7 +303,7 @@ - (void)handleAppDidEnterBackground:(NSNotification *)note { NSString *message = error.localizedDescription; DGLogError(@"[Deepgram] startRecording: %@", message); [self cleanupRecordingQueue]; - DGRejectPromise(reject, @"record_start_error", message, error); + DGRejectPromise(reject, @"start_error", message, error); return; } @@ -288,7 +314,7 @@ - (void)handleAppDidEnterBackground:(NSNotification *)note { NSString *message = error.localizedDescription; DGLogError(@"[Deepgram] startRecording: %@", message); [self cleanupRecordingQueue]; - DGRejectPromise(reject, @"record_start_error", message, error); + DGRejectPromise(reject, @"start_error", message, error); return; } } @@ -299,7 +325,7 @@ - (void)handleAppDidEnterBackground:(NSNotification *)note { NSString *message = error.localizedDescription; DGLogError(@"[Deepgram] startRecording: %@", message); [self cleanupRecordingQueue]; - DGRejectPromise(reject, @"record_start_error", message, error); + DGRejectPromise(reject, @"start_error", message, error); return; } @@ -312,7 +338,7 @@ - (void)handleAppDidEnterBackground:(NSNotification *)note { [self cleanupRecordingQueue]; NSString *message = e.reason ?: @"Deepgram native exception"; NSError *error = DGNativeError(@"DeepgramNativeException", 0, message); - DGRejectPromise(reject, @"record_start_error", message, error); + DGRejectPromise(reject, @"start_error", message, error); } } @@ -331,7 +357,7 @@ - (void)handleAppDidEnterBackground:(NSNotification *)note { resolve(nil); } @catch (NSException *e) { DGLogError(@"[Deepgram] stopRecording: exception %@", e); - DGRejectPromise(reject, @"record_stop_error", e.reason, nil); + DGRejectPromise(reject, @"stop_error", e.reason, nil); } } @@ -522,7 +548,7 @@ - (void)handleAppDidEnterBackground:(NSNotification *)note { resolve(nil); } @catch (NSException *e) { DGLogError(@"[Deepgram] stopPlayer: exception %@", e); - DGRejectPromise(reject, @"player_stop_error", e.reason, nil); + DGRejectPromise(reject, @"stop_player_error", e.reason, nil); } } @@ -545,7 +571,7 @@ - (void)handleAppDidEnterBackground:(NSNotification *)note { @try { if (!b64 || b64.length == 0) { if (reject) - reject(@"audio_chunk_error", @"Empty audio chunk", nil); + reject(@"invalid_data", @"Empty audio chunk", nil); return; } @@ -553,7 +579,7 @@ - (void)handleAppDidEnterBackground:(NSNotification *)note { options:0]; if (!pcmData || pcmData.length == 0) { if (reject) - reject(@"audio_chunk_error", @"Failed to decode audio chunk", nil); + reject(@"invalid_data", @"Failed to decode audio chunk", nil); return; } @@ -564,7 +590,7 @@ - (void)handleAppDidEnterBackground:(NSNotification *)note { ?: @"Failed to activate audio session"; DGLogError(@"[Deepgram] playAudioChunk: activation failed %@", message); if (reject) - reject(@"audio_chunk_error", message, sessionError); + reject(@"playback_error", message, sessionError); return; } @@ -586,7 +612,7 @@ - (void)handleAppDidEnterBackground:(NSNotification *)note { @"[Deepgram] playAudioChunk: failed to setup audio engine: %@", engineError); if (reject) - reject(@"audio_chunk_error", @"Failed to setup audio engine", + reject(@"playback_error", @"Failed to setup audio engine", engineError); return; } @@ -597,20 +623,30 @@ - (void)handleAppDidEnterBackground:(NSNotification *)note { if (!buffer) { DGLogError(@"[Deepgram] playAudioChunk: failed to create PCM buffer"); if (reject) - reject(@"audio_chunk_error", @"Failed to create PCM buffer", nil); + reject(@"invalid_data", @"Failed to create PCM buffer", nil); return; } self.isPlaying = YES; + _scheduledBufferCount++; + int playbackGeneration = _playbackGeneration.load(); - // Schedule buffer with completion handler to resolve promise + // Schedule buffer with completion handler to resolve promise. The session + // teardown is guarded by the playback generation and remaining buffer count + // (mirroring feedAudio) so a one-shot finishing mid-stream cannot deactivate + // the audio session while streaming playback is still active. __weak Deepgram *weakSelf = self; [self.playerNode scheduleBuffer:buffer completionHandler:^{ Deepgram *strongSelf = weakSelf; - if (strongSelf) { - strongSelf.isPlaying = NO; - [strongSelf maybeDeactivateAudioSession]; + if (strongSelf && + strongSelf->_playbackGeneration.load() == + playbackGeneration) { + int remaining = --strongSelf->_scheduledBufferCount; + if (remaining <= 0) { + strongSelf.isPlaying = NO; + [strongSelf maybeDeactivateAudioSession]; + } } if (resolve) resolve(nil); @@ -625,7 +661,7 @@ - (void)handleAppDidEnterBackground:(NSNotification *)note { (unsigned long)pcmData.length); } @catch (NSException *e) { DGLogError(@"[Deepgram] playAudioChunk: exception %@", e); - DGRejectPromise(reject, @"audio_chunk_error", e.reason, nil); + DGRejectPromise(reject, @"playback_error", e.reason, nil); } } diff --git a/yarn.lock b/yarn.lock index bea4983..52dfa0d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8232,6 +8232,18 @@ __metadata: languageName: node linkType: hard +"expo-build-properties@npm:~0.13.3": + version: 0.13.3 + resolution: "expo-build-properties@npm:0.13.3" + dependencies: + ajv: ^8.11.0 + semver: ^7.6.0 + peerDependencies: + expo: "*" + checksum: 65b97a82dc46b5d92f7ed7e1d4e85bb441ac719a6dacea54ba0ae0c9080ae5c465b5ccab6e41cc8bdca63a26128a72d73ed6901e5c38793fd95fac0fa6dc2588 + languageName: node + linkType: hard + "expo-constants@npm:~17.0.8": version: 17.0.8 resolution: "expo-constants@npm:17.0.8" @@ -13649,6 +13661,7 @@ __metadata: "@react-native/typescript-config": ^0.81.1 babel-plugin-module-resolver: ^5.0.2 expo: ~52.0.27 + expo-build-properties: ~0.13.3 expo-document-picker: ^12.0.2 react: 18.3.1 react-dom: 18.3.1 From 6fcb3d3c9727b871843fe628ccf6df280c3b3a92 Mon Sep 17 00:00:00 2001 From: Rares Date: Fri, 26 Jun 2026 12:15:52 -0400 Subject: [PATCH 3/9] feat: update version to 2.2.0 and enhance configuration options - Bump version in package.json to 2.2.0. - Add new constants for agent URL and improve base URL handling. - Enhance `configure` function to accept base URL, WebSocket URL, and agent URL overrides. - Introduce reconnect options for live streaming sockets in shared types. - Update speech-to-text hook to support reconnect options and manage socket state. - Implement pause and resume functionality for streaming in speech-to-text hook. - Refactor text intelligence and text-to-speech hooks to use dynamic base URLs. - Add reconnect logic to voice agent hook for improved resilience. - Introduce mute/unmute functionality in voice agent hook with keep-alive support. --- README.md | 95 ++++++- assets/example.webp | Bin 0 -> 263762 bytes example/src/App.tsx | 13 + example/src/SpeechToText.tsx | 179 +++++++++++-- example/src/VoiceAgent.tsx | 166 ++++++++++-- package.json | 2 +- src/constants/deepgram.ts | 40 +++ src/index.ts | 29 +- src/types/deepgram/shared.ts | 16 ++ src/types/deepgram/speech-to-text.ts | 28 +- src/useDeepgramManagement.ts | 4 +- src/useDeepgramSpeechToText.ts | 350 ++++++++++++++++++------ src/useDeepgramTextIntelligence.ts | 4 +- src/useDeepgramTextToSpeech.ts | 12 +- src/useDeepgramVoiceAgent.ts | 382 ++++++++++++++++++++++----- 15 files changed, 1118 insertions(+), 202 deletions(-) create mode 100644 assets/example.webp diff --git a/README.md b/README.md index a0a7938..d4ad523 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ yarn add react-native-deepgram | 🧠 **Text Intelligence** | Summaries, topics, intents, sentiment over text or URLs. | | 🛠️ **Management API** | Typed wrapper for projects, models, keys, usage, balances, members, invitations, scopes, purchases, and temporary tokens. | | ⚙️ **Expo plugin** | One-line install — handles permissions, background-audio modes, and Android foreground service automatically. | -| 🧰 **Resilient by default** | Audio interruptions, route changes, headphone unplugging, mediaserverd resets, audio-focus loss, and queue overflow are all handled. | +| 🧰 **Resilient by default** | Audio interruptions, route changes, headphone unplugging, mediaserverd resets, audio-focus loss, and queue overflow are all handled — plus opt-in WebSocket auto-reconnect with exponential backoff. | | 🆕 **Modern iOS APIs** | `AVAudioApplication.requestRecordPermission` and `AllowBluetoothHFP` on iOS 17+, with safe fallbacks. | --- @@ -232,6 +232,27 @@ configure({ apiKey: process.env.EXPO_PUBLIC_DEEPGRAM_API_KEY! }); > **Never ship raw API keys.** Use `EXPO_PUBLIC_*` env vars, secrets storage, or a backend proxy that mints scoped keys. +#### Custom endpoints (regional, Dedicated & self-hosted) + +Point the SDK at a non-default Deepgram deployment by passing base URLs to `configure`. Every override is optional and falls back to the public Deepgram endpoints. + +```ts +configure({ + apiKey: process.env.EXPO_PUBLIC_DEEPGRAM_API_KEY!, + baseUrl: 'https://api.beta.deepgram.com/v1', // REST (must include the version segment) + baseWss: 'wss://api.beta.deepgram.com/v1', // live STT + streaming TTS sockets + agentUrl: 'wss://agent.deepgram.com/v1/agent/converse', // Voice Agent socket +}); +``` + +| Option | Applies to | Default | +| ------ | ---------- | ------- | +| `baseUrl` | REST calls (file STT, TTS, Text Intelligence, Management) | `https://api.deepgram.com/v1` | +| `baseWss` | Live STT + streaming TTS sockets | `wss://api.deepgram.com/v1` | +| `agentUrl` | Voice Agent socket | `wss://agent.deepgram.com/v1/agent/converse` | + +`baseUrl`/`baseWss` **must include the version segment** (`/v1`). Flux v2 endpoints are derived automatically by swapping the trailing `/v1` for `/v2`. See Deepgram's [custom endpoints reference](https://developers.deepgram.com/reference/custom-endpoints). + ### 2. Hooks return reactive state — opt in Every hook supports `track*` flags that turn on a reactive return value. Without them, the hook stays event-driven (callbacks only) so it never causes unnecessary re-renders. @@ -411,6 +432,8 @@ return ( const { // Connection connect, disconnect, isConnected, + // Mic control + mute, unmute, isMuted, // Messaging sendMessage, sendSettings, sendMedia, sendKeepAlive, injectUserMessage, injectAgentMessage, updatePrompt, @@ -432,6 +455,7 @@ const { | `autoStartMicrophone` | `boolean` | `true` | Automatically requests mic access and starts streaming PCM. | | `autoPlayAudio` | `boolean` | `true` | Plays received audio using the native player. | | `downsampleFactor` | `number` | heuristic | Manually override the downsample ratio applied to captured audio. | +| `reconnect` | `DeepgramReconnectOptions` | `{ enabled: false }` | Auto-reconnect config for the agent socket. The stored `Settings` payload is re-sent on every successful reconnect. | **Reactive tracking flags** (opt-in — extra renders only when set) @@ -452,6 +476,8 @@ const { | `onConnect` | `() => void` | Socket opens and the initial settings payload is delivered. | | `onClose` | `(event?: any) => void` | Socket closes (manual disconnect or remote). | | `onError` | `(error: unknown) => void` | Unexpected client-side error (mic, playback, socket send). | +| `onReconnecting` | `(attempt: number) => void` | A reconnect attempt begins (1-based attempt number). Requires `reconnect.enabled`. | +| `onReconnected` | `() => void` | The socket reconnects and the stored settings are re-sent. | | `onServerError` | `(message: DeepgramVoiceAgentErrorMessage) => void` | API reports a structured error (`description` + `code`). | | `onWarning` | `(message: DeepgramVoiceAgentWarningMessage) => void` | Non-fatal warning (e.g. degraded audio quality). | @@ -499,6 +525,8 @@ const { | `connect` | `(settings?: DeepgramVoiceAgentSettings) => Promise` | Opens the socket, optionally merges settings, starts mic streaming. | | `disconnect` | `() => void` | Tears down the socket, stops recording, removes listeners. | | `isConnected` | `() => boolean` | Returns `true` when the socket is open. | +| `mute` | `() => void` | Stops forwarding mic audio while keeping the socket alive with periodic `KeepAlive` frames. | +| `unmute` | `() => void` | Resumes forwarding microphone audio. | **Messaging** @@ -528,6 +556,7 @@ Each value is `undefined` unless its corresponding `track*` flag is `true`. | Return | Type | Requires | | ------ | ---- | -------- | | `state` | `{ connectionState: 'idle' \| 'connecting' \| 'connected' \| 'disconnected'; error: string \| null; warning: string \| null }` | `trackState: true` | +| `isMuted` | `boolean` | `trackState: true` | | `conversation` | `Array<{ role: string; content: string }>` | `trackConversation: true` | | `agentStatus` | `{ thinking: string \| null; latency: { total?: number; tts?: number; ttt?: number } \| null }` | `trackAgentStatus: true` | | `clearConversation` | `() => void` | `trackConversation: true` | @@ -619,11 +648,11 @@ const pickFile = async () => { ```ts const { // Live streaming - startListening, stopListening, + startListening, stopListening, pause, resume, // File transcription transcribeFile, // Reactive state (opt-in) - state, transcript, interimTranscript, + state, transcript, interimTranscript, isPaused, } = useDeepgramSpeechToText(props); ``` @@ -635,6 +664,7 @@ const { | ---- | ---- | ----------- | | `live` | `DeepgramLiveListenOptions` | Default options merged into every live stream. | | `prerecorded` | `DeepgramPrerecordedOptions` | Default options merged into every file transcription. | +| `reconnect` | `DeepgramReconnectOptions` | Auto-reconnect config for the live socket. Disabled unless `reconnect.enabled` is `true`. | **Live streaming callbacks** @@ -644,7 +674,9 @@ const { | `onStart` | `() => void` | The WebSocket opens. | | `onTranscript` | `(transcript: string, event?: DeepgramTranscriptEvent) => void` | Every transcript update (partial and final). | | `onError` | `(error: unknown) => void` | A streaming error occurs. | -| `onEnd` | `() => void` | The socket closes. | +| `onReconnecting` | `(attempt: number) => void` | A reconnect attempt begins (1-based attempt number). Requires `reconnect.enabled`. | +| `onReconnected` | `() => void` | The live socket successfully reconnects. | +| `onEnd` | `() => void` | The socket closes (manual stop, or after reconnect attempts are exhausted). | **File transcription callbacks** @@ -667,6 +699,8 @@ const { | ------ | --------- | ----------- | | `startListening` | `(options?: DeepgramLiveListenOptions) => Promise` | Requests mic access, starts recording, streams audio to Deepgram. | | `stopListening` | `() => void` | Stops recording and closes the active WebSocket. | +| `pause` | `() => void` | Pauses mic streaming while keeping the socket alive via `KeepAlive`. On v1 a `Finalize` is sent first to flush buffered audio. | +| `resume` | `() => void` | Resumes mic streaming after a `pause`. | | `transcribeFile` | `(file: DeepgramPrerecordedSource, options?: DeepgramPrerecordedOptions) => Promise` | Uploads a file/URI/URL and resolves via the success/error callbacks. | #### Reactive state @@ -674,6 +708,7 @@ const { | Return | Type | Requires | | ------ | ---- | -------- | | `state` | `{ status: 'idle' \| 'loading' \| 'listening' \| 'transcribing' \| 'error'; error: Error \| null }` | `trackState: true` | +| `isPaused` | `boolean` | `trackState: true` | | `transcript` | `string` | `trackTranscript: true` | | `interimTranscript` | `string` | `trackTranscript: true` | @@ -1277,6 +1312,54 @@ const stt = useDeepgramSpeechToText({ }); ``` +### Pause and resume a live stream + +```tsx +const { startListening, pause, resume, isPaused } = useDeepgramSpeechToText({ + trackState: true, + live: { punctuate: true, interimResults: true }, +}); + +// Stop forwarding mic audio without dropping the socket. Deepgram is kept warm +// with periodic KeepAlive frames, so resume() continues the same session. +