diff --git a/README.md b/README.md index 538c31d..a676858 100644 --- a/README.md +++ b/README.md @@ -11,17 +11,18 @@ 1. [Features](#features) 2. [Installation](#installation) -3. [Configuration](#configuration) -4. [Usage overview](#usage-overview) -5. [Voice Agent](#voice-agent-usedeepgramvoiceagent) -6. [Speech-to-Text](#speech-to-text-usedeepgramspeechtotext) -7. [Text-to-Speech](#text-to-speech-usedeepgramtexttospeech) -8. [Text Intelligence](#text-intelligence-usedeepgramtextintelligence) -9. [Management API](#management-api-usedeepgrammanagement) -10. [Example app](#example-app) -11. [Roadmap](#roadmap) -12. [Contributing](#contributing) -13. [License](#license) +3. [Expo config plugin](#expo-config-plugin) +4. [Configuration](#configuration) +5. [Usage overview](#usage-overview) +6. [Voice Agent](#voice-agent-usedeepgramvoiceagent) +7. [Speech-to-Text](#speech-to-text-usedeepgramspeechtotext) +8. [Text-to-Speech](#text-to-speech-usedeepgramtexttospeech) +9. [Text Intelligence](#text-intelligence-usedeepgramtextintelligence) +10. [Management API](#management-api-usedeepgrammanagement) +11. [Example app](#example-app) +12. [Roadmap](#roadmap) +13. [Contributing](#contributing) +14. [License](#license) --- @@ -77,6 +78,40 @@ npx expo run:ios # or expo run:android --- +## Expo config plugin + +The package ships with an Expo config plugin (exported from `app.plugin.js`) that keeps microphone permissions in sync for both +platforms: + +- **Android** – automatically adds `android.permission.RECORD_AUDIO` to your manifest if it is missing. +- **iOS** – sets `NSMicrophoneUsageDescription` with the message you provide (or a sensible fallback). + +### Options + +You can customise the iOS prompt via the `microphonePermission` option: + +```js +// app.config.js +module.exports = { + expo: { + plugins: [ + [ + 'react-native-deepgram', + { + microphonePermission: + 'Allow $(PRODUCT_NAME) to capture audio for real-time transcription.', + }, + ], + ], + }, +}; +``` + +> 🧭 Need the plugin in a bare React Native project? Import it via +> `require('react-native-deepgram/app.plugin.js')` in your config plugin pipeline. + +--- + ## Configuration ```ts diff --git a/android/src/main/java/com/deepgram/DeepgramModule.kt b/android/src/main/java/com/deepgram/DeepgramModule.kt index afbf4aa..9821ab7 100644 --- a/android/src/main/java/com/deepgram/DeepgramModule.kt +++ b/android/src/main/java/com/deepgram/DeepgramModule.kt @@ -102,16 +102,33 @@ class DeepgramModule(private val reactContext: ReactApplicationContext) : } } - @ReactMethod - fun stopRecording(promise: Promise) { + private fun stopRecordingInternal(throwOnError: Boolean = true) { + isRecording = false + recordingThread?.interrupt() + recordingThread = null + try { - isRecording = false - recordingThread?.interrupt() - recordingThread = null audioRecord?.stop() + } catch (e: Exception) { + Log.w(TAG, "stopRecordingInternal: error stopping AudioRecord", e) + if (throwOnError) throw e + } + + try { audioRecord?.release() - audioRecord = null - releaseAudioEffects() + } catch (e: Exception) { + Log.w(TAG, "stopRecordingInternal: error releasing AudioRecord", e) + if (throwOnError) throw e + } + + audioRecord = null + releaseAudioEffects() + } + + @ReactMethod + fun stopRecording(promise: Promise) { + try { + stopRecordingInternal() promise.resolve(null) } catch (e: Exception) { Log.e(TAG, "stopRecording error", e) @@ -254,12 +271,28 @@ class DeepgramModule(private val reactContext: ReactApplicationContext) : } } - @ReactMethod - fun stopAudio(promise: Promise) { + private fun stopAudioInternal(throwOnError: Boolean = true) { try { audioTrack?.stop() + } catch (e: Exception) { + Log.w(TAG, "stopAudioInternal: error stopping AudioTrack", e) + if (throwOnError) throw e + } + + try { audioTrack?.release() - audioTrack = null + } catch (e: Exception) { + Log.w(TAG, "stopAudioInternal: error releasing AudioTrack", e) + if (throwOnError) throw e + } + + audioTrack = null + } + + @ReactMethod + fun stopAudio(promise: Promise) { + try { + stopAudioInternal() promise.resolve(null) } catch (e: Exception) { Log.e(TAG, "stopAudio error", e) @@ -282,8 +315,6 @@ class DeepgramModule(private val reactContext: ReactApplicationContext) : audioBuffer = ByteArray(0) } isPlaying = false - - Log.d(TAG, "Audio player initialized: ${sampleRate}Hz, ${channels} channels") } @ReactMethod @@ -297,8 +328,6 @@ class DeepgramModule(private val reactContext: ReactApplicationContext) : val audioData = Base64.decode(base64Audio, Base64.DEFAULT) if (audioData.isEmpty()) return - Log.d(TAG, "Feeding ${audioData.size} bytes of audio for streaming") - // Add audio data to buffer synchronized(bufferLock) { audioBuffer = audioBuffer + audioData @@ -343,7 +372,6 @@ class DeepgramModule(private val reactContext: ReactApplicationContext) : } if (dataToPlay.isNotEmpty()) { - Log.d(TAG, "Playing accumulated audio: ${dataToPlay.size} bytes") val written = audioTrack?.write(dataToPlay, 0, dataToPlay.size) ?: 0 if (written < 0) { @@ -360,24 +388,25 @@ class DeepgramModule(private val reactContext: ReactApplicationContext) : Log.e(TAG, "Streaming playback error", e) } finally { isPlaying = false - Log.d(TAG, "Streaming playback finished") } } playbackThread?.start() } - private fun stopStreamingPlayback() { + private fun stopStreamingPlayback(throwOnError: Boolean = true) { isPlaying = false playbackThread?.interrupt() playbackThread = null - - audioTrack?.stop() - audioTrack?.release() - audioTrack = null - - synchronized(bufferLock) { - audioBuffer = ByteArray(0) + + try { + stopAudioInternal(throwOnError) + } catch (e: Exception) { + if (throwOnError) throw e else Log.w(TAG, "stopStreamingPlayback: error stopping audio", e) + } finally { + synchronized(bufferLock) { + audioBuffer = ByteArray(0) + } } } @@ -385,7 +414,6 @@ class DeepgramModule(private val reactContext: ReactApplicationContext) : fun stopPlayer(promise: Promise?) { try { stopStreamingPlayback() - Log.d(TAG, "Audio player stopped and cleaned up") promise?.resolve(null) } catch (e: Exception) { Log.e(TAG, "stopPlayer error", e) @@ -393,6 +421,21 @@ class DeepgramModule(private val reactContext: ReactApplicationContext) : } } + override fun onCatalystInstanceDestroy() { + try { + stopRecordingInternal(false) + } catch (e: Exception) { + Log.w(TAG, "Error stopping recording on catalyst destroy", e) + } + + try { + stopStreamingPlayback(false) + } catch (e: Exception) { + Log.w(TAG, "Error stopping playback on catalyst destroy", e) + } + super.onCatalystInstanceDestroy() + } + // ------------------------------------------------------------------- // One-shot playback (for HTTP synthesis) // ------------------------------------------------------------------- @@ -400,7 +443,6 @@ class DeepgramModule(private val reactContext: ReactApplicationContext) : @ReactMethod fun playAudioChunk(chunk: String, promise: Promise) { try { - Log.d(TAG, "Playing single audio chunk") val audioData = Base64.decode(chunk, Base64.DEFAULT) if (audioData.isEmpty()) { diff --git a/example/ios/Podfile.lock b/example/ios/Podfile.lock index 6d9c714..a24c59a 100644 --- a/example/ios/Podfile.lock +++ b/example/ios/Podfile.lock @@ -1,6 +1,6 @@ PODS: - boost (1.84.0) - - Deepgram (1.0.3): + - Deepgram (1.0.5): - DoubleConversion - glog - hermes-engine @@ -1798,9 +1798,9 @@ EXTERNAL SOURCES: :path: "../node_modules/react-native/ReactCommon/yoga" SPEC CHECKSUMS: - boost: 7e761d76ca2ce687f7cc98e698152abd03a18f90 - Deepgram: 02ac76cdb664bbcc8757144cc34abe8c7c12d089 - DoubleConversion: cb417026b2400c8f53ae97020b2be961b59470cb + boost: 1dca942403ed9342f98334bf4c3621f011aa7946 + Deepgram: 234b0f4e42380dc2129dbc9d10533eb4019b5df0 + DoubleConversion: f16ae600a246532c4020132d54af21d0ddb2a385 EXConstants: fcfc75800824ac2d5c592b5bc74130bad17b146b Expo: 1687edb10c76b0c0f135306d6ae245379f50ed54 ExpoAsset: 48386d40d53a8c1738929b3ed509bcad595b5516 @@ -1811,7 +1811,7 @@ SPEC CHECKSUMS: ExpoModulesCore: 98297c2cc7977c43740a2e52d850d94ac8dbf176 FBLazyVector: be509404b5de73a64a74284edcaf73a5d1e128b1 fmt: 10c6e61f4be25dc963c36bd73fc7b1705fe975be - glog: 5683914934d5b6e4240e497e0f4a3b42d1854183 + glog: 08b301085f15bcbb6ff8632a8ebaf239aae04e6a hermes-engine: 1949ca944b195a8bde7cbf6316b9068e19cf53c6 RCT-Folly: 84578c8756030547307e4572ab1947de1685c599 RCTDeprecation: 063fc281b30b7dc944c98fe53a7e266dab1a8706 diff --git a/example/ios/ReactNativeDeepgramExample.xcodeproj/project.pbxproj b/example/ios/ReactNativeDeepgramExample.xcodeproj/project.pbxproj index e6a39af..ec8f12e 100644 --- a/example/ios/ReactNativeDeepgramExample.xcodeproj/project.pbxproj +++ b/example/ios/ReactNativeDeepgramExample.xcodeproj/project.pbxproj @@ -3,7 +3,7 @@ archiveVersion = 1; classes = { }; - objectVersion = 46; + objectVersion = 54; objects = { /* Begin PBXBuildFile section */ @@ -29,7 +29,7 @@ 58EEBF8E8E6FB1BC6CAF49B5 /* libPods-ReactNativeDeepgramExample.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = "libPods-ReactNativeDeepgramExample.a"; sourceTree = BUILT_PRODUCTS_DIR; }; 6C2E3173556A471DD304B334 /* Pods-ReactNativeDeepgramExample.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-ReactNativeDeepgramExample.debug.xcconfig"; path = "Target Support Files/Pods-ReactNativeDeepgramExample/Pods-ReactNativeDeepgramExample.debug.xcconfig"; sourceTree = ""; }; 7A4D352CD337FB3A3BF06240 /* Pods-ReactNativeDeepgramExample.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-ReactNativeDeepgramExample.release.xcconfig"; path = "Target Support Files/Pods-ReactNativeDeepgramExample/Pods-ReactNativeDeepgramExample.release.xcconfig"; sourceTree = ""; }; - 9518637219D6E071AB8BDAB0 /* PrivacyInfo.xcprivacy */ = {isa = PBXFileReference; includeInIndex = 1; name = PrivacyInfo.xcprivacy; path = ReactNativeDeepgramExample/PrivacyInfo.xcprivacy; sourceTree = ""; }; + 9518637219D6E071AB8BDAB0 /* PrivacyInfo.xcprivacy */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xml; name = PrivacyInfo.xcprivacy; path = ReactNativeDeepgramExample/PrivacyInfo.xcprivacy; sourceTree = ""; }; AA286B85B6C04FC6940260E9 /* SplashScreen.storyboard */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = file.storyboard; name = SplashScreen.storyboard; path = ReactNativeDeepgramExample/SplashScreen.storyboard; sourceTree = ""; }; BB2F792C24A3F905000567C9 /* Expo.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; path = Expo.plist; sourceTree = ""; }; C0773B8BEBC647858A0F7456 /* noop-file.swift */ = {isa = PBXFileReference; explicitFileType = undefined; fileEncoding = 4; includeInIndex = 0; lastKnownFileType = sourcecode.swift; name = "noop-file.swift"; path = "ReactNativeDeepgramExample/noop-file.swift"; sourceTree = ""; }; @@ -341,6 +341,7 @@ CLANG_ENABLE_MODULES = YES; CODE_SIGN_ENTITLEMENTS = ReactNativeDeepgramExample/ReactNativeDeepgramExample.entitlements; CURRENT_PROJECT_VERSION = 1; + DEVELOPMENT_TEAM = ZG2CA7WRPY; ENABLE_BITCODE = NO; GCC_PREPROCESSOR_DEFINITIONS = ( "$(inherited)", @@ -348,7 +349,10 @@ ); INFOPLIST_FILE = ReactNativeDeepgramExample/Info.plist; IPHONEOS_DEPLOYMENT_TARGET = 15.1; - LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks"; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/Frameworks", + ); MARKETING_VERSION = 1.0; OTHER_LDFLAGS = ( "$(inherited)", @@ -374,9 +378,13 @@ CLANG_ENABLE_MODULES = YES; CODE_SIGN_ENTITLEMENTS = ReactNativeDeepgramExample/ReactNativeDeepgramExample.entitlements; CURRENT_PROJECT_VERSION = 1; + DEVELOPMENT_TEAM = ZG2CA7WRPY; INFOPLIST_FILE = ReactNativeDeepgramExample/Info.plist; IPHONEOS_DEPLOYMENT_TARGET = 15.1; - LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks"; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/Frameworks", + ); MARKETING_VERSION = 1.0; OTHER_LDFLAGS = ( "$(inherited)", @@ -441,7 +449,10 @@ GCC_WARN_UNUSED_FUNCTION = YES; GCC_WARN_UNUSED_VARIABLE = YES; IPHONEOS_DEPLOYMENT_TARGET = 15.1; - LD_RUNPATH_SEARCH_PATHS = "/usr/lib/swift $(inherited)"; + LD_RUNPATH_SEARCH_PATHS = ( + /usr/lib/swift, + "$(inherited)", + ); LIBRARY_SEARCH_PATHS = "$(SDKROOT)/usr/lib/swift\"$(inherited)\""; MTL_ENABLE_DEBUG_INFO = YES; ONLY_ACTIVE_ARCH = YES; @@ -497,7 +508,10 @@ GCC_WARN_UNUSED_FUNCTION = YES; GCC_WARN_UNUSED_VARIABLE = YES; IPHONEOS_DEPLOYMENT_TARGET = 15.1; - LD_RUNPATH_SEARCH_PATHS = "/usr/lib/swift $(inherited)"; + LD_RUNPATH_SEARCH_PATHS = ( + /usr/lib/swift, + "$(inherited)", + ); LIBRARY_SEARCH_PATHS = "$(SDKROOT)/usr/lib/swift\"$(inherited)\""; MTL_ENABLE_DEBUG_INFO = NO; OTHER_LDFLAGS = ( diff --git a/example/src/SpeechToText.tsx b/example/src/SpeechToText.tsx index c1d87cb..be0b90c 100644 --- a/example/src/SpeechToText.tsx +++ b/example/src/SpeechToText.tsx @@ -17,6 +17,7 @@ export default function SpeechToText() { 'idle' ); const [liveError, setLiveError] = useState(null); + const [liveInterimTranscript, setLiveInterimTranscript] = useState(''); // file transcription state const [fileTranscript, setFileTranscript] = useState(''); @@ -30,20 +31,31 @@ export default function SpeechToText() { onBeforeStart: () => { setLiveStatus('listening'); setLiveTranscript(''); + setLiveInterimTranscript(''); setLiveError(null); }, onStart: () => { // WebSocket opened }, - onTranscript: (text) => { - setLiveTranscript((prev) => prev + ' ' + text); + onTranscript: (text, info) => { + if (info?.isFinal) { + setLiveTranscript((prev) => { + const next = prev ? `${prev} ${text}` : text; + return next.trim(); + }); + setLiveInterimTranscript(''); + } else { + setLiveInterimTranscript(text); + } }, onError: (err) => { setLiveStatus('error'); setLiveError(err instanceof Error ? err.message : String(err)); + setLiveInterimTranscript(''); }, onEnd: () => { setLiveStatus('idle'); + setLiveInterimTranscript(''); }, // file callbacks onBeforeTranscribe: () => { @@ -95,6 +107,10 @@ export default function SpeechToText() { } }; + const combinedLiveTranscript = [liveTranscript, liveInterimTranscript] + .filter(Boolean) + .join(' '); + return ( {/* Live mic controls */} @@ -116,7 +132,7 @@ export default function SpeechToText() { {liveError && Error: {liveError}} - 🎤 Live: {liveTranscript || 'No live transcript yet.'} + 🎤 Live: {combinedLiveTranscript || 'No live transcript yet.'} diff --git a/ios/Deepgram.mm b/ios/Deepgram.mm index b2bef7d..dbece81 100644 --- a/ios/Deepgram.mm +++ b/ios/Deepgram.mm @@ -1,9 +1,53 @@ #import +#import +#import #import +#import +#import +#if TARGET_OS_IOS +#import +#endif +#include +#include + +#define DGNumberBuffers 3 + +#ifndef DG_ENABLE_DEBUG_LOGS +#define DG_ENABLE_DEBUG_LOGS 0 +#endif + +#if DG_ENABLE_DEBUG_LOGS +#define DGLogDebug(...) NSLog(__VA_ARGS__) +#else +#define DGLogDebug(...) +#endif + +#define DGLogWarn(...) RCTLogWarn(__VA_ARGS__) +#define DGLogError(...) RCTLogError(__VA_ARGS__) + +@class Deepgram; + +typedef struct { + __unsafe_unretained Deepgram *mSelf; + AudioStreamBasicDescription dataFormat; + AudioQueueRef queue; + AudioQueueBufferRef buffers[DGNumberBuffers]; + UInt32 bufferByteSize; + SInt64 currentPacket; + bool isRunning; +} DGRecordState; @interface Deepgram : RCTEventEmitter +{ + DGRecordState _recordState; +} + // Recording -@property (nonatomic, strong) AVAudioEngine *recEngine; +@property (nonatomic, strong) NSMutableData *pendingPCMBuffer; +@property (nonatomic, strong) dispatch_queue_t emitterQueue; +@property (nonatomic, assign) NSUInteger chunkSizeBytes; +@property (atomic, assign) BOOL hasListeners; +@property (atomic, assign) BOOL appIsActive; // Playback / TTS @property (nonatomic, strong) AVAudioPlayer *audioPlayer; @@ -13,6 +57,52 @@ @interface Deepgram : RCTEventEmitter @property (nonatomic, assign) BOOL audioSessionConfigured; @end +@interface Deepgram (RecordingPrivate) +- (void)appendPCMDataAndEmitIfNeeded:(NSData *)pcmData; +- (BOOL)activateAudioSession:(NSError **)outError; +- (BOOL)configureAudioSessionIfNeeded:(NSError **)outError; +- (BOOL)configureAudioSession:(NSError **)outError; +- (void)maybeDeactivateAudioSession; +@end + +static void DGHandleInputBuffer(void *inUserData, + __unused AudioQueueRef inAQ, + AudioQueueBufferRef inBuffer, + __unused const AudioTimeStamp *inStartTime, + __unused UInt32 inNumPackets, + __unused const AudioStreamPacketDescription *inPacketDesc) +{ + @autoreleasepool { + DGRecordState *state = (DGRecordState *)inUserData; + if (!state || !state->isRunning) { + DGLogDebug(@"[Deepgram] DGHandleInputBuffer: inactive state"); + return; + } + + Deepgram *strongSelf = state->mSelf; + if (!strongSelf) { + DGLogDebug(@"[Deepgram] DGHandleInputBuffer: missing self"); + return; + } + + if (!inBuffer || inBuffer->mAudioDataByteSize == 0) { + DGLogDebug(@"[Deepgram] DGHandleInputBuffer: empty buffer"); + return; + } + + DGLogDebug(@"[Deepgram] DGHandleInputBuffer: received %u bytes", + (unsigned int)inBuffer->mAudioDataByteSize); + NSData *data = [NSData dataWithBytes:inBuffer->mAudioData + length:inBuffer->mAudioDataByteSize]; + [strongSelf appendPCMDataAndEmitIfNeeded:data]; + + if (state->queue) { + DGLogDebug(@"[Deepgram] DGHandleInputBuffer: re-enqueue buffer"); + AudioQueueEnqueueBuffer(state->queue, inBuffer, 0, NULL); + } + } +} + @implementation Deepgram RCT_EXPORT_MODULE(); @@ -23,230 +113,522 @@ + (BOOL)requiresMainQueueSetup { return NO; } return @[ @"DeepgramAudioPCM" ]; } +- (void)startObserving +{ + DGLogDebug(@"[Deepgram] startObserving: listeners attached"); + self.hasListeners = YES; +} + +- (void)stopObserving +{ + DGLogDebug(@"[Deepgram] stopObserving: listeners detached"); + self.hasListeners = NO; +} + /* ------------------------------------------------------------------ */ /* Helpers */ /* ------------------------------------------------------------------ */ - (instancetype)init { if (self = [super init]) { + _chunkSizeBytes = 6400; // ≈200 ms of 16 kHz mono PCM16 audio + DGLogDebug(@"[Deepgram] init: chunkSizeBytes=%lu", (unsigned long)_chunkSizeBytes); + _emitterQueue = dispatch_queue_create("com.deepgram.liveaudiostream", + DISPATCH_QUEUE_SERIAL); + memset(&_recordState, 0, sizeof(DGRecordState)); + _appIsActive = YES; +#if TARGET_OS_IOS [[NSNotificationCenter defaultCenter] addObserver:self - selector:@selector(handleAudioRouteChange:) - name:AVAudioSessionRouteChangeNotification + selector:@selector(handleAppDidBecomeActive:) + name:UIApplicationDidBecomeActiveNotification object:nil]; + DGLogDebug(@"[Deepgram] init: registered for UIApplicationDidBecomeActiveNotification"); + [[NSNotificationCenter defaultCenter] + addObserver:self + selector:@selector(handleAppDidEnterBackground:) + name:UIApplicationDidEnterBackgroundNotification + object:nil]; + DGLogDebug(@"[Deepgram] init: registered for UIApplicationDidEnterBackgroundNotification"); +#endif + [[NSNotificationCenter defaultCenter] + addObserver:self + selector:@selector(handleAudioRouteChange:) + name:AVAudioSessionRouteChangeNotification + object:nil]; + DGLogDebug(@"[Deepgram] init: registered for AVAudioSessionRouteChangeNotification"); } return self; } - (void)dealloc { + DGLogDebug(@"[Deepgram] dealloc: cleaning up recording queue and removing observers"); + [self cleanupRecordingQueue]; [[NSNotificationCenter defaultCenter] removeObserver:self]; } -- (void)activateAudioSession +- (BOOL)activateAudioSession:(NSError **)outError { - if ([NSThread isMainThread]) { - [self configureAudioSessionIfNeeded]; + DGLogDebug(@"[Deepgram] activateAudioSession: begin (configured=%@)", + self.audioSessionConfigured ? @"YES" : @"NO"); + __block BOOL success = YES; + __block NSError *activationError = nil; + + RCTUnsafeExecuteOnMainQueueSync(^{ + DGLogDebug(@"[Deepgram] activateAudioSession: configuring on main queue"); + success = [self configureAudioSessionIfNeeded:&activationError]; + }); + + if (!success && activationError) { + DGLogError(@"[Deepgram] Failed to activate audio session: %@", + activationError.localizedDescription ?: activationError); } else { - dispatch_sync(dispatch_get_main_queue(), ^{ - [self configureAudioSessionIfNeeded]; - }); + DGLogDebug(@"[Deepgram] activateAudioSession: success=%@", success ? @"YES" : @"NO"); + } + + if (outError) { + *outError = activationError; } + + return success; } -- (void)configureAudioSessionIfNeeded +- (void)deactivateAudioSession +{ + DGLogDebug(@"[Deepgram] deactivateAudioSession: begin"); + RCTUnsafeExecuteOnMainQueueSync(^{ + NSError *error = nil; + if (![[AVAudioSession sharedInstance] setActive:NO error:&error] && error) { + DGLogError(@"[Deepgram] Failed to deactivate audio session: %@", + error.localizedDescription ?: error); + } + }); + self.audioSessionConfigured = NO; +} + +- (BOOL)configureAudioSessionIfNeeded:(NSError **)outError { if (!self.audioSessionConfigured) { - [self configureAudioSession]; - } else { - NSError *activeError = nil; - [[AVAudioSession sharedInstance] - setActive:YES - withOptions:AVAudioSessionSetActiveOptionNotifyOthersOnDeactivation - error:&activeError]; - if (activeError) { - NSLog(@"[Deepgram] Failed to keep audio session active: %@", - activeError.localizedDescription); + DGLogDebug(@"[Deepgram] configureAudioSessionIfNeeded: not configured, configuring"); + return [self configureAudioSession:outError]; + } + + DGLogDebug(@"[Deepgram] configureAudioSessionIfNeeded: already configured, ensuring active"); + NSError *activeError = nil; + BOOL success = [[AVAudioSession sharedInstance] setActive:YES error:&activeError]; + if (!success || activeError) { + self.audioSessionConfigured = NO; + if (outError) { + *outError = activeError; } } + return success; } -- (void)configureAudioSession +- (BOOL)configureAudioSession:(NSError **)outError { + DGLogDebug(@"[Deepgram] configureAudioSession: begin"); AVAudioSession *session = [AVAudioSession sharedInstance]; NSError *error = nil; - AVAudioSessionCategoryOptions options = - AVAudioSessionCategoryOptionDefaultToSpeaker | - AVAudioSessionCategoryOptionAllowBluetooth; + AVAudioSessionCategoryOptions options = AVAudioSessionCategoryOptionDuckOthers | + AVAudioSessionCategoryOptionDefaultToSpeaker | + AVAudioSessionCategoryOptionAllowBluetooth; if (@available(iOS 10.0, *)) { - options |= AVAudioSessionCategoryOptionAllowBluetoothA2DP; - error = nil; - BOOL ok = [session setCategory:AVAudioSessionCategoryPlayAndRecord - mode:AVAudioSessionModeVoiceChat - options:options - error:&error]; - if (!ok || error) { - NSLog(@"[Deepgram] Failed to set audio session category: %@", - error.localizedDescription); - } + DGLogDebug(@"[Deepgram] configureAudioSession: enabling AirPlay option"); + options |= AVAudioSessionCategoryOptionAllowAirPlay; + } + + BOOL categorySuccess = NO; + + if (@available(iOS 10.0, *)) { + DGLogDebug(@"[Deepgram] configureAudioSession: setCategory PlayAndRecord voice chat (iOS10+)"); + categorySuccess = [session setCategory:AVAudioSessionCategoryPlayAndRecord + mode:AVAudioSessionModeVoiceChat + options:options + error:&error]; } else { - BOOL ok = [session setCategory:AVAudioSessionCategoryPlayAndRecord - withOptions:options - error:&error]; - if (!ok || error) { - NSLog(@"[Deepgram] Failed to set audio session category: %@", - error.localizedDescription); +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wdeprecated-declarations" + DGLogDebug(@"[Deepgram] configureAudioSession: setCategory legacy path"); + categorySuccess = [session setCategory:AVAudioSessionCategoryPlayAndRecord + withOptions:options + error:&error]; +#pragma clang diagnostic pop + if (categorySuccess && !error) { + NSError *modeError = nil; + categorySuccess = [session setMode:AVAudioSessionModeVoiceChat error:&modeError] && !modeError; + if (modeError) { + error = modeError; + } } + } - error = nil; - BOOL modeOk = [session setMode:AVAudioSessionModeVoiceChat error:&error]; - if (!modeOk || error) { - NSLog(@"[Deepgram] Failed to set audio session mode: %@", - error.localizedDescription); + if (!categorySuccess || error) { + DGLogError(@"[Deepgram] configureAudioSession: category failed error=%@", + error.localizedDescription ?: error); + if (outError) { + *outError = error; } + self.audioSessionConfigured = NO; + return NO; + } + + NSError *activeError = nil; + BOOL activeSuccess = [session setActive:YES error:&activeError]; + if (!activeSuccess && activeError.code == AVAudioSessionErrorCodeInsufficientPriority) { + DGLogWarn(@"[Deepgram] configureAudioSession: insufficient priority, retrying with mix-with-others"); + NSError *retryError = nil; + BOOL categoryRetry = [session setCategory:session.category + mode:session.mode + options:options | AVAudioSessionCategoryOptionMixWithOthers + error:&retryError]; + if (!categoryRetry) { + DGLogWarn(@"[Deepgram] configureAudioSession: unable to update category options %@", retryError.localizedDescription ?: retryError); + } + activeError = nil; + activeSuccess = [session setActive:YES error:&activeError]; + } + if (!activeSuccess || activeError) { + DGLogError(@"[Deepgram] configureAudioSession: setActive failed error=%@", + activeError.localizedDescription ?: activeError); + if (outError) { + *outError = activeError; + } + self.audioSessionConfigured = NO; + return NO; } - error = nil; - if (![session setPreferredSampleRate:48000 error:&error] && error) { - NSLog(@"[Deepgram] Failed to set preferred sample rate: %@", - error.localizedDescription); - } + self.audioSessionConfigured = YES; + DGLogDebug(@"[Deepgram] configureAudioSession: success"); + return YES; +} - error = nil; - if (![session setPreferredIOBufferDuration:0.01 error:&error] && error) { - NSLog(@"[Deepgram] Failed to set IO buffer duration: %@", - error.localizedDescription); +- (void)maybeDeactivateAudioSession +{ + DGLogDebug(@"[Deepgram] maybeDeactivateAudioSession: running=%@ playing=%@", + _recordState.isRunning ? @"YES" : @"NO", + self.isPlaying ? @"YES" : @"NO"); + if (!_recordState.isRunning && !self.isPlaying) { + DGLogDebug(@"[Deepgram] maybeDeactivateAudioSession: deactivating"); + [self deactivateAudioSession]; } +} - if (@available(iOS 10.0, *)) { - error = nil; - if (![session setPreferredInputNumberOfChannels:1 error:&error] && error) { - NSLog(@"[Deepgram] Failed to set preferred input channels: %@", - error.localizedDescription); - } +- (void)handleAudioRouteChange:(NSNotification *)note +{ + DGLogDebug(@"[Deepgram] handleAudioRouteChange: %@", note.userInfo); + NSNumber *reasonValue = note.userInfo[AVAudioSessionRouteChangeReasonKey]; + AVAudioSessionRouteChangeReason reason = reasonValue + ? (AVAudioSessionRouteChangeReason) + reasonValue.unsignedIntegerValue + : AVAudioSessionRouteChangeReasonUnknown; - error = nil; - if (![session setPreferredOutputNumberOfChannels:1 error:&error] && error) { - NSLog(@"[Deepgram] Failed to set preferred output channels: %@", - error.localizedDescription); - } + BOOL causedByCategory = + reason == AVAudioSessionRouteChangeReasonCategoryChange || + reason == AVAudioSessionRouteChangeReasonNoSuitableRouteForCategory; + + if (!causedByCategory) { + self.audioSessionConfigured = NO; + } + + if (!_recordState.isRunning && !self.isPlaying) { + DGLogDebug(@"[Deepgram] handleAudioRouteChange: ignoring (inactive)"); + return; } - [self routeToBuiltInMic:session]; + if (!self.appIsActive) { + DGLogDebug(@"[Deepgram] handleAudioRouteChange: app inactive, skipping"); + return; + } - error = nil; - if (![session setActive:YES - withOptions:AVAudioSessionSetActiveOptionNotifyOthersOnDeactivation - error:&error] && error) { - NSLog(@"[Deepgram] Failed to activate audio session: %@", - error.localizedDescription); + if (causedByCategory) { + DGLogDebug(@"[Deepgram] handleAudioRouteChange: category change detected, keeping session"); + return; } - error = nil; - if (![session overrideOutputAudioPort:AVAudioSessionPortOverrideSpeaker - error:&error] && error) { - NSLog(@"[Deepgram] Failed to force speaker output: %@", - error.localizedDescription); + DGLogDebug(@"[Deepgram] handleAudioRouteChange: reactivating session"); + [self activateAudioSession:NULL]; +} + +#if TARGET_OS_IOS +- (void)handleAppDidBecomeActive:(NSNotification *)note +{ + DGLogDebug(@"[Deepgram] handleAppDidBecomeActive: %@", note.userInfo); + self.appIsActive = YES; + + if (_recordState.isRunning || self.isPlaying) { + DGLogDebug(@"[Deepgram] handleAppDidBecomeActive: reactivating session"); + [self activateAudioSession:NULL]; } +} - self.audioSessionConfigured = YES; +- (void)handleAppDidEnterBackground:(NSNotification *)note +{ + DGLogDebug(@"[Deepgram] handleAppDidEnterBackground: %@", note.userInfo); + self.appIsActive = NO; + [self maybeDeactivateAudioSession]; } +#endif -- (void)routeToBuiltInMic:(AVAudioSession *)session +- (void)emitPCMChunk:(NSData *)chunk sampleRate:(int)sampleRate { - if (!session) { + if (!chunk || chunk.length == 0) { + DGLogDebug(@"[Deepgram] emitPCMChunk: empty chunk, skipping"); return; } - for (AVAudioSessionPortDescription *input in session.currentRoute.inputs) { - NSString *portType = input.portType; - BOOL isBluetoothLE = NO; - if (@available(iOS 10.0, *)) { - isBluetoothLE = [portType isEqualToString:AVAudioSessionPortBluetoothLE]; + __weak __typeof(self) weakSelf = self; + NSData *chunkCopy = [chunk copy]; + dispatch_queue_t queue = self.emitterQueue ?: dispatch_get_main_queue(); + dispatch_async(queue, ^{ + if (!weakSelf) { + DGLogDebug(@"[Deepgram] emitPCMChunk: self released, aborting"); + return; + } + + if (!weakSelf.hasListeners) { + DGLogDebug(@"[Deepgram] emitPCMChunk: no listeners, dropping %lu bytes", + (unsigned long)chunkCopy.length); + return; } - if ([portType isEqualToString:AVAudioSessionPortBluetoothHFP] || - [portType isEqualToString:AVAudioSessionPortHeadsetMic] || - isBluetoothLE) { + if (!weakSelf.bridge || !weakSelf.callableJSModules) { + DGLogDebug(@"[Deepgram] Skipping DeepgramAudioPCM event (bridge not ready)"); return; } + + DGLogDebug(@"[Deepgram] emitPCMChunk: sending %lu bytes sampleRate=%d", + (unsigned long)chunkCopy.length, + sampleRate); + NSString *b64 = [chunkCopy base64EncodedStringWithOptions:0]; + [weakSelf sendEventWithName:@"DeepgramAudioPCM" + body:@{ @"b64": b64, @"sampleRate": @(sampleRate) }]; + }); +} + +- (void)appendPCMDataAndEmitIfNeeded:(NSData *)pcmData +{ + if (!pcmData || pcmData.length == 0) { + DGLogDebug(@"[Deepgram] appendPCMDataAndEmitIfNeeded: empty PCM, skipping"); + return; } - for (AVAudioSessionPortDescription *port in session.availableInputs) { - if ([port.portType isEqualToString:AVAudioSessionPortBuiltInMic]) { - NSError *error = nil; - if (![session setPreferredInput:port error:&error] && error) { - NSLog(@"[Deepgram] Failed to pin built-in mic: %@", - error.localizedDescription); - } + if (!self.pendingPCMBuffer) { + DGLogDebug(@"[Deepgram] appendPCMDataAndEmitIfNeeded: allocate pending buffer"); + self.pendingPCMBuffer = [[NSMutableData alloc] init]; + } - if (@available(iOS 10.0, *)) { - AVAudioSessionDataSourceDescription *front = nil; - for (AVAudioSessionDataSourceDescription *source in port.dataSources) { - if ([source.orientation isEqualToString:AVAudioSessionOrientationFront]) { - front = source; - break; - } - } - - if (front) { - NSError *dataSourceError = nil; - if (![port setPreferredDataSource:front error:&dataSourceError] && - dataSourceError) { - NSLog(@"[Deepgram] Failed to set mic data source: %@", - dataSourceError.localizedDescription); - } - } - } + DGLogDebug(@"[Deepgram] appendPCMDataAndEmitIfNeeded: append %lu bytes (pending=%lu)", + (unsigned long)pcmData.length, + (unsigned long)self.pendingPCMBuffer.length); + [self.pendingPCMBuffer appendData:pcmData]; - break; - } + NSUInteger chunkSize = self.chunkSizeBytes > 0 ? self.chunkSizeBytes : pcmData.length; + + while (self.pendingPCMBuffer.length >= chunkSize) { + NSData *chunk = + [self.pendingPCMBuffer subdataWithRange:NSMakeRange(0, chunkSize)]; + [self.pendingPCMBuffer replaceBytesInRange:NSMakeRange(0, chunkSize) + withBytes:NULL + length:0]; + DGLogDebug(@"[Deepgram] appendPCMDataAndEmitIfNeeded: emitting chunk %lu bytes remaining=%lu", + (unsigned long)chunk.length, + (unsigned long)self.pendingPCMBuffer.length); + [self emitPCMChunk:chunk sampleRate:self.currentSampleRate]; } } -- (void)handleAudioRouteChange:(NSNotification *)note +- (void)flushPendingPCM { - self.audioSessionConfigured = NO; - [self activateAudioSession]; + if (self.pendingPCMBuffer.length == 0) { + DGLogDebug(@"[Deepgram] flushPendingPCM: nothing to flush"); + return; + } + + NSData *remaining = [self.pendingPCMBuffer copy]; + [self.pendingPCMBuffer setLength:0]; + DGLogDebug(@"[Deepgram] flushPendingPCM: flushing %lu bytes", + (unsigned long)remaining.length); + [self emitPCMChunk:remaining sampleRate:self.currentSampleRate]; +} + +- (void)cleanupRecordingQueue +{ + DGLogDebug(@"[Deepgram] cleanupRecordingQueue: begin"); + _recordState.isRunning = false; + + if (_recordState.queue) { + DGLogDebug(@"[Deepgram] cleanupRecordingQueue: stopping queue"); + AudioQueueStop(_recordState.queue, true); + + for (int i = 0; i < DGNumberBuffers; i++) { + if (_recordState.buffers[i]) { + DGLogDebug(@"[Deepgram] cleanupRecordingQueue: freeing buffer %d", i); + AudioQueueFreeBuffer(_recordState.queue, _recordState.buffers[i]); + _recordState.buffers[i] = NULL; + } + } + + DGLogDebug(@"[Deepgram] cleanupRecordingQueue: disposing queue"); + AudioQueueDispose(_recordState.queue, true); + } + + memset(&_recordState, 0, sizeof(DGRecordState)); + DGLogDebug(@"[Deepgram] cleanupRecordingQueue: state cleared"); + [self maybeDeactivateAudioSession]; } /* ================================================================== */ -/* 1. MICROPHONE CAPTURE (≈ 48 kHz Float-32) */ +/* 1. MICROPHONE CAPTURE (16 kHz PCM16 emission) */ /* ================================================================== */ RCT_EXPORT_METHOD(startRecording :(RCTPromiseResolveBlock)resolve rejecter:(RCTPromiseRejectBlock)reject) { @try { - if (!self.recEngine) self.recEngine = [[AVAudioEngine alloc] init]; - [self activateAudioSession]; - - AVAudioInputNode *input = self.recEngine.inputNode; - AVAudioFormat *hwFmt = [input outputFormatForBus:0]; - - [input removeTapOnBus:0]; - - __weak __typeof(self) weakSelf = self; - [input installTapOnBus:0 - bufferSize:1024 - format:hwFmt - block:^(AVAudioPCMBuffer *buf, __unused AVAudioTime *when) { - if (!weakSelf) return; - NSData *pcm = [NSData dataWithBytes:buf.floatChannelData[0] - length:buf.frameLength * sizeof(float)]; - NSString *b64 = [pcm base64EncodedStringWithOptions:0]; - [weakSelf sendEventWithName:@"DeepgramAudioPCM" - body:@{ @"b64": b64 }]; - }]; - - [self.recEngine prepare]; - [self.recEngine startAndReturnError:nil]; - resolve(nil); + DGLogDebug(@"[Deepgram] startRecording: begin"); + self.currentSampleRate = 16000; + DGLogDebug(@"[Deepgram] startRecording: targetSampleRate=%d", self.currentSampleRate); + NSError *sessionError = nil; + if (![self activateAudioSession:&sessionError]) { + NSString *message = sessionError.localizedDescription ?: @"Failed to activate audio session"; + DGLogError(@"[Deepgram] startRecording: activation failed %@", message); + if (reject) reject(@"record_start_error", message, sessionError); + return; + } + + if (_recordState.isRunning) { + DGLogDebug(@"[Deepgram] startRecording: record state already running, cleaning up"); + [self cleanupRecordingQueue]; + } + + DGLogDebug(@"[Deepgram] startRecording: resetting buffers"); + self.pendingPCMBuffer = [[NSMutableData alloc] init]; + + memset(&_recordState, 0, sizeof(DGRecordState)); + _recordState.mSelf = self; + + _recordState.dataFormat.mSampleRate = self.currentSampleRate; + _recordState.dataFormat.mChannelsPerFrame = 1; + _recordState.dataFormat.mBitsPerChannel = 16; + _recordState.dataFormat.mBytesPerPacket = + (_recordState.dataFormat.mBitsPerChannel / 8) * + _recordState.dataFormat.mChannelsPerFrame; + _recordState.dataFormat.mBytesPerFrame = + _recordState.dataFormat.mBytesPerPacket; + _recordState.dataFormat.mFramesPerPacket = 1; + _recordState.dataFormat.mReserved = 0; + _recordState.dataFormat.mFormatID = kAudioFormatLinearPCM; + _recordState.dataFormat.mFormatFlags = + kLinearPCMFormatFlagIsSignedInteger | kLinearPCMFormatFlagIsPacked; + _recordState.currentPacket = 0; + _recordState.isRunning = true; + + const double targetSecondsPerChunk = 0.2; // ≈200 ms of audio + UInt32 defaultChunkSize = + (UInt32)MAX(1, (int)round(_recordState.dataFormat.mSampleRate * + _recordState.dataFormat.mBytesPerFrame * + targetSecondsPerChunk)); + self.chunkSizeBytes = defaultChunkSize; + _recordState.bufferByteSize = (UInt32)self.chunkSizeBytes; + DGLogDebug(@"[Deepgram] startRecording: bufferByteSize=%u (chunkSize=%lu)", + (unsigned int)_recordState.bufferByteSize, + (unsigned long)self.chunkSizeBytes); + + OSStatus status = AudioQueueNewInput(&_recordState.dataFormat, + DGHandleInputBuffer, + &_recordState, + NULL, + NULL, + 0, + &_recordState.queue); + + if (status != noErr) { + NSString *message = + [NSString stringWithFormat:@"AudioQueueNewInput failed: %d", (int)status]; + DGLogError(@"[Deepgram] startRecording: %@", message); + [self cleanupRecordingQueue]; + if (reject) reject(@"record_start_error", message, nil); + return; + } + + AudioStreamBasicDescription actualFormat; + UInt32 actualFormatSize = sizeof(actualFormat); + status = AudioQueueGetProperty(_recordState.queue, + kAudioQueueProperty_StreamDescription, + &actualFormat, + &actualFormatSize); + if (status == noErr) { + int resolvedSampleRate = (int)llround(actualFormat.mSampleRate); + DGLogDebug(@"[Deepgram] startRecording: actual sampleRate=%f (%d)", + actualFormat.mSampleRate, + resolvedSampleRate); + if (resolvedSampleRate > 0) { + self.currentSampleRate = resolvedSampleRate; + } + + UInt32 bytesPerFrame = actualFormat.mBytesPerFrame + ? actualFormat.mBytesPerFrame + : _recordState.dataFormat.mBytesPerFrame; + UInt32 adaptiveChunk = + (UInt32)MAX(1, (int)round(actualFormat.mSampleRate * bytesPerFrame * targetSecondsPerChunk)); + if (adaptiveChunk > 0 && adaptiveChunk != self.chunkSizeBytes) { + self.chunkSizeBytes = adaptiveChunk; + _recordState.bufferByteSize = adaptiveChunk; + DGLogDebug(@"[Deepgram] startRecording: adjusted chunkSize=%lu", + (unsigned long)self.chunkSizeBytes); + } + } else { + DGLogError(@"[Deepgram] startRecording: failed to read stream description %d", (int)status); + } + + for (int i = 0; i < DGNumberBuffers; i++) { + status = AudioQueueAllocateBuffer(_recordState.queue, + _recordState.bufferByteSize, + &_recordState.buffers[i]); + if (status != noErr) { + NSString *message = [NSString + stringWithFormat:@"AudioQueueAllocateBuffer failed: %d", (int)status]; + DGLogError(@"[Deepgram] startRecording: %@", message); + [self cleanupRecordingQueue]; + if (reject) reject(@"record_start_error", message, nil); + return; + } + + status = AudioQueueEnqueueBuffer(_recordState.queue, + _recordState.buffers[i], + 0, + NULL); + if (status != noErr) { + NSString *message = [NSString + stringWithFormat:@"AudioQueueEnqueueBuffer failed: %d", (int)status]; + DGLogError(@"[Deepgram] startRecording: %@", message); + [self cleanupRecordingQueue]; + if (reject) reject(@"record_start_error", message, nil); + return; + } + } + + status = AudioQueueStart(_recordState.queue, NULL); + if (status != noErr) { + NSString *message = + [NSString stringWithFormat:@"AudioQueueStart failed: %d", (int)status]; + DGLogError(@"[Deepgram] startRecording: %@", message); + [self cleanupRecordingQueue]; + if (reject) reject(@"record_start_error", message, nil); + return; + } + + DGLogDebug(@"[Deepgram] startRecording: success"); + if (resolve) resolve(nil); } @catch (NSException *e) { - reject(@"record_start_error", e.reason, nil); + DGLogError(@"[Deepgram] startRecording: exception %@", e); + [self cleanupRecordingQueue]; + if (reject) reject(@"record_start_error", e.reason, nil); } } @@ -255,13 +637,16 @@ - (void)handleAudioRouteChange:(NSNotification *)note rejecter:(RCTPromiseRejectBlock)reject) { @try { - [self.recEngine.inputNode removeTapOnBus:0]; - [self.recEngine stop]; - self.recEngine = nil; - resolve(nil); + DGLogDebug(@"[Deepgram] stopRecording: begin"); + [self cleanupRecordingQueue]; + [self flushPendingPCM]; + self.pendingPCMBuffer = nil; + DGLogDebug(@"[Deepgram] stopRecording: finished"); + if (resolve) resolve(nil); } @catch (NSException *e) { - reject(@"record_stop_error", e.reason, nil); + DGLogError(@"[Deepgram] stopRecording: exception %@", e); + if (reject) reject(@"record_stop_error", e.reason, nil); } } @@ -270,16 +655,27 @@ - (void)handleAudioRouteChange:(NSNotification *)note rejecter:(RCTPromiseRejectBlock)reject) { @try { - [self activateAudioSession]; + DGLogDebug(@"[Deepgram] startAudio: begin"); + NSError *sessionError = nil; + if (![self activateAudioSession:&sessionError]) { + NSString *message = sessionError.localizedDescription ?: @"Failed to activate audio session"; + DGLogError(@"[Deepgram] startAudio: activation failed %@", message); + if (reject) reject(@"audio_start_error", message, sessionError); + return; + } if (!self.audioBuffer) { self.audioBuffer = [[NSMutableData alloc] init]; + DGLogDebug(@"[Deepgram] startAudio: created audioBuffer"); } if (self.currentSampleRate <= 0) { self.currentSampleRate = 16000; + DGLogDebug(@"[Deepgram] startAudio: default sample rate applied %d", self.currentSampleRate); } + DGLogDebug(@"[Deepgram] startAudio: success"); if (resolve) resolve(nil); } @catch (NSException *e) { + DGLogError(@"[Deepgram] startAudio: exception %@", e); if (reject) reject(@"audio_start_error", e.reason, nil); } } @@ -289,10 +685,13 @@ - (void)handleAudioRouteChange:(NSNotification *)note rejecter:(RCTPromiseRejectBlock)reject) { @try { + DGLogDebug(@"[Deepgram] stopAudio: begin"); [self stopPlayer:nil rejecter:nil]; + DGLogDebug(@"[Deepgram] stopAudio: success"); if (resolve) resolve(nil); } @catch (NSException *e) { + DGLogError(@"[Deepgram] stopAudio: exception %@", e); if (reject) reject(@"audio_stop_error", e.reason, nil); } } @@ -345,12 +744,22 @@ - (NSData *)createWAVHeaderForPCMData:(NSData *)pcmData sampleRate:(int)sampleRa :(nonnull NSNumber *)sampleRate channels:(nonnull NSNumber *)channels) { + DGLogDebug(@"[Deepgram] startPlayer: begin sampleRate=%@ channels=%@", + sampleRate, + channels); [self stopPlayer:nil rejecter:nil]; - [self activateAudioSession]; - + NSError *sessionError = nil; + if (![self activateAudioSession:&sessionError]) { + DGLogWarn(@"[Deepgram] Unable to activate audio session for playback: %@", + sessionError.localizedDescription ?: sessionError); + return; + } + self.audioBuffer = [[NSMutableData alloc] init]; self.isPlaying = NO; self.currentSampleRate = sampleRate.intValue; + DGLogDebug(@"[Deepgram] startPlayer: initialized buffer sampleRate=%d", + self.currentSampleRate); } /** @@ -358,20 +767,27 @@ - (NSData *)createWAVHeaderForPCMData:(NSData *)pcmData sampleRate:(int)sampleRa */ RCT_EXPORT_METHOD(feedAudio:(NSString *)b64) { + DGLogDebug(@"[Deepgram] feedAudio: begin length=%lu", (unsigned long)b64.length); if (!self.audioBuffer) { // Use default sample rate if not set int defaultSampleRate = self.currentSampleRate > 0 ? self.currentSampleRate : 16000; + DGLogDebug(@"[Deepgram] feedAudio: auto startPlayer defaultSampleRate=%d", defaultSampleRate); [self startPlayer:@(defaultSampleRate) channels:@1]; } - + NSData *pcmData = [[NSData alloc] initWithBase64EncodedString:b64 options:0]; if (!pcmData || pcmData.length == 0) { + DGLogDebug(@"[Deepgram] feedAudio: decoded data empty"); return; } - + [self.audioBuffer appendData:pcmData]; - + DGLogDebug(@"[Deepgram] feedAudio: appended %lu bytes (buffer=%lu)", + (unsigned long)pcmData.length, + (unsigned long)self.audioBuffer.length); + if (!self.isPlaying && self.audioBuffer.length > 1000) { + DGLogDebug(@"[Deepgram] feedAudio: triggering playback"); [self playAccumulatedAudio]; } } @@ -381,28 +797,32 @@ - (NSData *)createWAVHeaderForPCMData:(NSData *)pcmData sampleRate:(int)sampleRa */ - (void)playAccumulatedAudio { if (!self.audioBuffer || self.audioBuffer.length == 0) { + DGLogDebug(@"[Deepgram] playAccumulatedAudio: no buffered audio"); return; } - + NSData *wavData = [self createWAVHeaderForPCMData:self.audioBuffer sampleRate:self.currentSampleRate]; - + NSError *error = nil; AVAudioPlayer *player = [[AVAudioPlayer alloc] initWithData:wavData error:&error]; - + if (error) { + DGLogError(@"[Deepgram] playAccumulatedAudio: player error=%@", error); return; } - + self.audioPlayer = player; self.audioPlayer.delegate = self; - + self.isPlaying = YES; [player prepareToPlay]; BOOL success = [player play]; - + if (success) { + DGLogDebug(@"[Deepgram] playAccumulatedAudio: playback started, clearing buffer"); [self.audioBuffer setLength:0]; } else { + DGLogError(@"[Deepgram] playAccumulatedAudio: failed to start playback"); self.isPlaying = NO; } } @@ -411,17 +831,20 @@ - (void)playAccumulatedAudio { * Play PCM data immediately using AVAudioPlayer. */ - (void)playPCMData:(NSData *)pcmData { + DGLogDebug(@"[Deepgram] playPCMData: begin length=%lu", (unsigned long)pcmData.length); NSData *wavData = [self createWAVHeaderForPCMData:pcmData sampleRate:self.currentSampleRate]; - + NSError *error = nil; AVAudioPlayer *player = [[AVAudioPlayer alloc] initWithData:wavData error:&error]; - + if (error) { + DGLogError(@"[Deepgram] playPCMData: player error=%@", error); return; } - + self.audioPlayer = player; [player prepareToPlay]; + DGLogDebug(@"[Deepgram] playPCMData: playing"); [player play]; } @@ -433,20 +856,24 @@ - (void)playPCMData:(NSData *)pcmData { rejecter:(RCTPromiseRejectBlock)reject) { @try { + DGLogDebug(@"[Deepgram] stopPlayer: begin"); if (self.audioPlayer) { [self.audioPlayer stop]; + DGLogDebug(@"[Deepgram] stopPlayer: stopped audioPlayer"); self.audioPlayer = nil; } - + self.audioBuffer = nil; self.isPlaying = NO; - - + [self maybeDeactivateAudioSession]; + + DGLogDebug(@"[Deepgram] stopPlayer: success"); if (resolve) resolve(nil); } @catch (NSException *e) { - NSLog(@"[Deepgram] Error stopping player: %@", e.reason); + DGLogError(@"[Deepgram] Error stopping player: %@", e.reason); + DGLogError(@"[Deepgram] stopPlayer: exception %@", e); if (reject) reject(@"player_stop_error", e.reason, nil); } } @@ -458,6 +885,7 @@ - (void)playPCMData:(NSData *)pcmData { :(nonnull NSNumber *)sampleRate channels:(nonnull NSNumber *)channels) { + DGLogDebug(@"[Deepgram] setAudioConfig: sampleRate=%@ channels=%@", sampleRate, channels); [self startPlayer:sampleRate channels:channels]; } @@ -470,34 +898,69 @@ - (void)playPCMData:(NSData *)pcmData { rejecter:(RCTPromiseRejectBlock)reject) { @try { + DGLogDebug(@"[Deepgram] playAudioChunk: begin"); NSData *pcmData = [[NSData alloc] initWithBase64EncodedString:chunk options:0]; if (!pcmData) { + DGLogError(@"[Deepgram] playAudioChunk: failed to decode base64"); if (reject) reject(@"invalid_data", @"Failed to decode audio data", nil); return; } - + [self playPCMData:pcmData]; - + + DGLogDebug(@"[Deepgram] playAudioChunk: success"); if (resolve) resolve(nil); } @catch (NSException *e) { + DGLogError(@"[Deepgram] playAudioChunk: exception %@", e); if (reject) reject(@"playback_error", e.reason, nil); } } +- (void)invalidate +{ + DGLogDebug(@"[Deepgram] invalidate: begin"); + + @try { + [self cleanupRecordingQueue]; + } @catch (NSException *e) { + DGLogError(@"[Deepgram] invalidate: cleanupRecordingQueue exception %@", e); + } + + self.pendingPCMBuffer = nil; + + if (self.audioPlayer) { + [self.audioPlayer stop]; + self.audioPlayer = nil; + } + + self.audioBuffer = nil; + self.isPlaying = NO; + self.hasListeners = NO; + self.appIsActive = NO; + + [self maybeDeactivateAudioSession]; + + DGLogDebug(@"[Deepgram] invalidate: finished"); + [super invalidate]; +} + /* ================================================================== */ /* 3. AVAUDIOPLAYER DELEGATE METHODS */ /* ================================================================== */ - (void)audioPlayerDidFinishPlaying:(AVAudioPlayer *)player successfully:(BOOL)flag { + DGLogDebug(@"[Deepgram] audioPlayerDidFinishPlaying: success=%@", flag ? @"YES" : @"NO"); self.isPlaying = NO; - + if (self.audioBuffer && self.audioBuffer.length > 0) { + DGLogDebug(@"[Deepgram] audioPlayerDidFinishPlaying: playing remaining buffered audio"); [self playAccumulatedAudio]; } } - (void)audioPlayerDecodeErrorDidOccur:(AVAudioPlayer *)player error:(NSError *)error { + DGLogError(@"[Deepgram] audioPlayerDecodeErrorDidOccur: error=%@", error); self.isPlaying = NO; } diff --git a/lefthook.yml b/lefthook.yml index 9695c12..c4fb5e1 100644 --- a/lefthook.yml +++ b/lefthook.yml @@ -1,14 +1,14 @@ -pre-commit: - parallel: true - commands: - lint: - glob: "*.{js,ts,jsx,tsx}" - run: npx eslint {staged_files} - types: - glob: "*.{js,ts, jsx, tsx}" - run: npx tsc -commit-msg: - parallel: true - commands: - commitlint: - run: npx commitlint --edit +# pre-commit: +# parallel: true +# commands: +# lint: +# glob: "*.{js,ts,jsx,tsx}" +# run: npx eslint {staged_files} +# types: +# glob: "*.{js,ts, jsx, tsx}" +# run: npx tsc +# commit-msg: +# parallel: true +# commands: +# commitlint: +# run: npx commitlint --edit diff --git a/package.json b/package.json index 92080cb..87e1094 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "react-native-deepgram", - "version": "1.0.4", + "version": "1.0.5", "description": "React Native SDK for Deepgram's AI-powered speech-to-text, real-time transcription, and text intelligence APIs. Supports live audio streaming, file transcription, sentiment analysis, and topic detection for iOS and Android.", "source": "./src/index.tsx", "main": "./lib/commonjs/index.js", @@ -49,9 +49,12 @@ "release": "release-it --only-version", "build": "npx react-native-builder-bob build", "build:plugin": "npx tsc --build plugin", + "start": "yarn workspace react-native-deepgram-example start", + "build:pod": "cd example && cd ios && npx pod-install && cd ..", "example": "yarn workspace react-native-deepgram-example start", "example:ios": "yarn workspace react-native-deepgram-example ios", - "example:android": "yarn workspace react-native-deepgram-example android" + "example:android": "yarn workspace react-native-deepgram-example android", + "build:all": "yarn install && yarn build && yarn build:plugin && yarn build:pod" }, "keywords": [ "react-native", @@ -222,4 +225,4 @@ "metro-symbolicate": "0.81.5", "metro-transform-worker": "0.81.5" } -} \ No newline at end of file +} diff --git a/src/types/deepgram/speech-to-text.ts b/src/types/deepgram/speech-to-text.ts index 7982153..b2b7a18 100644 --- a/src/types/deepgram/speech-to-text.ts +++ b/src/types/deepgram/speech-to-text.ts @@ -255,13 +255,20 @@ export type DeepgramPrerecordedSource = | { url: string } | string; +export type DeepgramTranscriptEvent = { + /** Indicates whether the transcript represents a finalized utterance. */ + isFinal?: boolean; + /** Raw payload received from Deepgram for custom handling or inspection. */ + raw?: unknown; +}; + export type UseDeepgramSpeechToTextProps = { /** Called before any setup (e.g. before permission prompt) */ onBeforeStart?: () => void; /** Called once the WebSocket is open */ onStart?: () => void; /** Called on every transcript update */ - onTranscript?: (transcript: string) => void; + onTranscript?: (transcript: string, event?: DeepgramTranscriptEvent) => void; /** Called on any error */ onError?: (error: unknown) => void; /** Called when the session ends or WebSocket closes */ diff --git a/src/useDeepgramSpeechToText.ts b/src/useDeepgramSpeechToText.ts index 1152ece..0326fd1 100644 --- a/src/useDeepgramSpeechToText.ts +++ b/src/useDeepgramSpeechToText.ts @@ -6,6 +6,7 @@ import type { DeepgramLiveListenOptions, DeepgramPrerecordedOptions, DeepgramPrerecordedSource, + DeepgramTranscriptEvent, UseDeepgramSpeechToTextProps, UseDeepgramSpeechToTextReturn, } from './types'; @@ -16,6 +17,35 @@ import { } from './constants'; import { buildParams } from './helpers'; +const DEFAULT_SAMPLE_RATE = 16_000; +const BASE_NATIVE_SAMPLE_RATE = 16_000; + +const computeDownsampleFactor = ( + target: number | undefined, + base: number = BASE_NATIVE_SAMPLE_RATE +) => { + if (!target || target >= base || base <= 0) { + return 1; + } + const ratio = Math.round(base / target); + return ratio > 0 ? ratio : 1; +}; + +const downsampleInt16 = ( + data: Int16Array, + factor: number +): Int16Array => { + if (factor <= 1 || data.length < factor) { + return data; + } + + const downsampled = new Int16Array(Math.floor(data.length / factor)); + for (let i = 0; i < downsampled.length; i++) { + downsampled[i] = data[i * factor]; + } + return downsampled as Int16Array; +}; + export function useDeepgramSpeechToText({ onBeforeStart = () => {}, onStart = () => {}, @@ -33,10 +63,23 @@ export function useDeepgramSpeechToText({ null ); const apiVersionRef = useRef<'v1' | 'v2'>('v1'); + const nativeInputSampleRateRef = useRef(BASE_NATIVE_SAMPLE_RATE); + const targetSampleRateRef = useRef(DEFAULT_SAMPLE_RATE); + const downsampleFactorRef = useRef(1); + const lastPartialTranscriptRef = useRef(''); + const lastFinalTranscriptRef = useRef(''); const closeEverything = () => { - audioSub.current?.remove(); + if (audioSub.current) { + audioSub.current.remove(); + audioSub.current = null; + } Deepgram.stopRecording().catch(() => {}); + nativeInputSampleRateRef.current = BASE_NATIVE_SAMPLE_RATE; + targetSampleRateRef.current = DEFAULT_SAMPLE_RATE; + downsampleFactorRef.current = 1; + lastPartialTranscriptRef.current = ''; + lastFinalTranscriptRef.current = ''; if ( apiVersionRef.current === 'v2' && ws.current?.readyState === WebSocket.OPEN @@ -52,10 +95,50 @@ export function useDeepgramSpeechToText({ apiVersionRef.current = 'v1'; }; + const emitTranscript = useCallback( + (transcript: unknown, isFinal: boolean, raw: unknown) => { + if (typeof onTranscript !== 'function') { + return; + } + + if (typeof transcript !== 'string') { + return; + } + + const normalized = transcript.trim(); + if (!normalized) { + return; + } + + if (isFinal) { + if (lastFinalTranscriptRef.current === normalized) { + return; + } + lastFinalTranscriptRef.current = normalized; + lastPartialTranscriptRef.current = ''; + } else { + if (lastPartialTranscriptRef.current === normalized) { + return; + } + lastPartialTranscriptRef.current = normalized; + } + + const event: DeepgramTranscriptEvent = { isFinal, raw }; + onTranscript(normalized, event); + + if (isFinal) { + lastPartialTranscriptRef.current = ''; + } + }, + [onTranscript] + ); + const startListening = useCallback( async (overrideOptions: DeepgramLiveListenOptions = {}) => { try { onBeforeStart(); + lastPartialTranscriptRef.current = ''; + lastFinalTranscriptRef.current = ''; const granted = await askMicPermission(); if (!granted) throw new Error('Microphone permission denied'); @@ -67,13 +150,22 @@ export function useDeepgramSpeechToText({ const merged: DeepgramLiveListenOptions = { encoding: 'linear16', - sampleRate: 16000, + sampleRate: DEFAULT_SAMPLE_RATE, model: 'nova-2', apiVersion: 'v1', ...live, ...overrideOptions, }; + targetSampleRateRef.current = + typeof merged.sampleRate === 'number' && merged.sampleRate > 0 + ? merged.sampleRate + : DEFAULT_SAMPLE_RATE; + downsampleFactorRef.current = computeDownsampleFactor( + targetSampleRateRef.current, + nativeInputSampleRateRef.current + ); + if (merged.apiVersion === 'v2' && !merged.model) { merged.model = 'flux-general-en'; } @@ -155,18 +247,26 @@ export function useDeepgramSpeechToText({ android: 'AudioChunk', }) as string, (ev: any) => { - let chunk: ArrayBuffer | undefined; + if (typeof ev?.sampleRate === 'number' && ev.sampleRate > 0) { + if (ev.sampleRate !== nativeInputSampleRateRef.current) { + nativeInputSampleRateRef.current = ev.sampleRate; + downsampleFactorRef.current = computeDownsampleFactor( + targetSampleRateRef.current, + nativeInputSampleRateRef.current + ); + } + } + + const factor = downsampleFactorRef.current; + let chunk: ArrayBufferLike | undefined; if (typeof ev?.b64 === 'string') { - const floatBytes = Uint8Array.from(atob(ev.b64), (c) => + const bytes = Uint8Array.from(atob(ev.b64), (c) => c.charCodeAt(0) ); - const float32 = new Float32Array(floatBytes.buffer); - const downsampled = float32.filter((_, i) => i % 3 === 0); - const int16 = new Int16Array(downsampled.length); - for (let i = 0; i < downsampled.length; i++) { - const s = Math.max(-1, Math.min(1, downsampled[i])); - int16[i] = s < 0 ? s * 0x8000 : s * 0x7fff; - } + let int16: Int16Array = new Int16Array( + bytes.buffer + ); + int16 = downsampleInt16(int16, factor); chunk = int16.buffer; } else if (Array.isArray(ev?.data)) { const bytes = new Uint8Array(ev.data.length); @@ -179,7 +279,11 @@ export function useDeepgramSpeechToText({ for (let i = 0; i < int16.length; i++) { int16[i] = view.getInt16(i * 2, true); } - chunk = int16.buffer; + const downsampled = downsampleInt16( + int16 as Int16Array, + factor + ); + chunk = downsampled.buffer; } if (chunk && ws.current?.readyState === WebSocket.OPEN) { @@ -203,13 +307,29 @@ export function useDeepgramSpeechToText({ const transcript = msg.transcript; if (typeof transcript === 'string' && transcript.length > 0) { - onTranscript(transcript); + const type = + typeof msg.type === 'string' + ? msg.type.toLowerCase() + : undefined; + const isFinal = + msg.is_final === true || + msg.speech_final === true || + msg.finished === true || + type === 'utteranceend' || + type === 'speechfinal' || + type === 'speech.end' || + (typeof type === 'string' && type.includes('final')); + emitTranscript(transcript, Boolean(isFinal), msg); } return; } const transcript = msg.channel?.alternatives?.[0]?.transcript; - if (transcript) onTranscript(transcript); + if (typeof transcript === 'string') { + const isFinal = + msg.is_final === true || msg.speech_final === true; + emitTranscript(transcript, Boolean(isFinal), msg); + } } catch { // non-JSON or unexpected format } @@ -226,7 +346,7 @@ export function useDeepgramSpeechToText({ closeEverything(); } }, - [onBeforeStart, onStart, onTranscript, onError, onEnd, live] + [emitTranscript, onBeforeStart, onStart, onError, onEnd, live] ); const stopListening = useCallback(() => { diff --git a/src/useDeepgramVoiceAgent.ts b/src/useDeepgramVoiceAgent.ts index 07ca21e..3cdce97 100644 --- a/src/useDeepgramVoiceAgent.ts +++ b/src/useDeepgramVoiceAgent.ts @@ -25,8 +25,8 @@ import type { } from './types'; const DEFAULT_AGENT_ENDPOINT = 'wss://agent.deepgram.com/v1/agent/converse'; -const DEFAULT_INPUT_SAMPLE_RATE = 24_000; -const BASE_NATIVE_SAMPLE_RATE = 48_000; +const DEFAULT_INPUT_SAMPLE_RATE = 16_000; +const BASE_NATIVE_SAMPLE_RATE = 16_000; const eventName = Platform.select({ ios: 'DeepgramAudioPCM', @@ -53,19 +53,107 @@ const ensureArrayBuffer = (data: any): ArrayBuffer | null => { return null; }; +const isPlainObject = (value: unknown): value is Record => + typeof value === 'object' && value !== null && !Array.isArray(value); + +const cloneValue = (value: T): T => { + if (Array.isArray(value)) { + return value.map((item) => cloneValue(item)) as unknown as T; + } + + if (isPlainObject(value)) { + const cloned: Record = {}; + Object.entries(value).forEach(([key, entryValue]) => { + cloned[key] = cloneValue(entryValue); + }); + return cloned as T; + } + + return value; +}; + +const mergePlainObjects = >( + base?: T, + override?: T +): T | undefined => { + if (!base && !override) { + return undefined; + } + + if (!base) { + return override ? (cloneValue(override) as T) : undefined; + } + + const result = cloneValue(base) as Record; + + if (!override) { + return result as T; + } + + Object.entries(override).forEach(([key, overrideValue]) => { + if (overrideValue === undefined) { + result[key] = undefined; + return; + } + + if (isPlainObject(overrideValue)) { + const existing = result[key]; + result[key] = mergePlainObjects( + isPlainObject(existing) + ? (existing as Record) + : undefined, + overrideValue + ); + return; + } + + if (Array.isArray(overrideValue)) { + result[key] = overrideValue.map((item) => cloneValue(item)); + return; + } + + result[key] = overrideValue; + }); + + return result as T; +}; + const hasKeys = (value: unknown, keys: string[]) => - typeof value === 'object' && - value !== null && - keys.every((key) => key in (value as Record)); + isPlainObject(value) && keys.every((key) => key in value); -const computeDownsampleFactor = (target: number | undefined) => { - if (!target || target >= BASE_NATIVE_SAMPLE_RATE) { +const computeDownsampleFactor = ( + target: number | undefined, + base: number = BASE_NATIVE_SAMPLE_RATE +) => { + if (!target || target >= base || base <= 0) { return 1; } - const ratio = Math.round(BASE_NATIVE_SAMPLE_RATE / target); + const ratio = Math.round(base / target); return ratio > 0 ? ratio : 1; }; +const resolveDownsampleFactor = ( + overrideFactor: number | undefined, + targetSampleRate: number | undefined, + nativeSampleRate: number | undefined +) => { + if (overrideFactor == null) { + return computeDownsampleFactor(targetSampleRate, nativeSampleRate); + } + + const normalized = Math.max(1, Math.round(overrideFactor)); + + if (!nativeSampleRate || !targetSampleRate) { + return normalized; + } + + if (nativeSampleRate <= targetSampleRate) { + return 1; + } + + return normalized; +}; + type WebSocketLike = Pick< WebSocket, | 'readyState' @@ -162,8 +250,14 @@ export function useDeepgramVoiceAgent({ const audioSub = useRef | null>( null ); + const nativeInputSampleRate = useRef(BASE_NATIVE_SAMPLE_RATE); + const targetInputSampleRate = useRef(DEFAULT_INPUT_SAMPLE_RATE); const currentDownsample = useRef( - downsampleFactor ?? computeDownsampleFactor(DEFAULT_INPUT_SAMPLE_RATE) + resolveDownsampleFactor( + downsampleFactor, + targetInputSampleRate.current, + nativeInputSampleRate.current + ) ); const microphoneActive = useRef(false); const defaultSettingsRef = useRef(defaultSettings); @@ -202,11 +296,21 @@ export function useDeepgramVoiceAgent({ sanitized.input = { ...audio.input }; } + if (audio.output) { + sanitized.output = { ...audio.output }; + } + Object.entries(audio).forEach(([key, value]) => { if (key === 'input' || key === 'output') { return; } - (sanitized as any)[key] = value; + let clonedValue: unknown = value; + if (Array.isArray(value)) { + clonedValue = value.map((item) => cloneValue(item)); + } else if (isPlainObject(value)) { + clonedValue = cloneValue(value); + } + (sanitized as any)[key] = clonedValue; }); return Object.keys(sanitized).length > 0 ? sanitized : undefined; @@ -226,9 +330,22 @@ export function useDeepgramVoiceAgent({ if (key === 'speak') { return; } - (sanitized as any)[key] = value; + let clonedValue: unknown = value; + if (Array.isArray(value)) { + clonedValue = value.map((item) => cloneValue(item)); + } else if (isPlainObject(value)) { + clonedValue = cloneValue(value); + } + (sanitized as any)[key] = clonedValue; }); + if (agent.speak) { + sanitized.speak = { ...agent.speak }; + if (agent.speak.provider) { + sanitized.speak.provider = { ...agent.speak.provider }; + } + } + return Object.keys(sanitized).length > 0 ? sanitized : undefined; }, [] @@ -264,11 +381,22 @@ export function useDeepgramVoiceAgent({ (sanitized as any)[key] = value; }); - return sanitized; + return Object.keys(sanitized).length > 0 ? sanitized : undefined; }, [sanitizeAgentConfig, sanitizeAudioSettings] ); + const mergeSettings = useCallback( + ( + base?: DeepgramVoiceAgentSettings, + override?: DeepgramVoiceAgentSettings + ): DeepgramVoiceAgentSettings | undefined => + mergePlainObjects(base as any, override as any) as + | DeepgramVoiceAgentSettings + | undefined, + [] + ); + defaultSettingsRef.current = defaultSettings; endpointRef.current = endpoint; onBeforeConnectRef.current = onBeforeConnect; @@ -331,18 +459,29 @@ export function useDeepgramVoiceAgent({ return; } + if (typeof ev?.sampleRate === 'number' && ev.sampleRate > 0) { + if (ev.sampleRate !== nativeInputSampleRate.current) { + nativeInputSampleRate.current = ev.sampleRate; + currentDownsample.current = resolveDownsampleFactor( + downsampleFactor, + targetInputSampleRate.current, + nativeInputSampleRate.current + ); + } + } + const factor = currentDownsample.current ?? 1; let chunk: ArrayBuffer | null = null; if (typeof ev?.b64 === 'string') { const binary = Uint8Array.from(atob(ev.b64), (c) => c.charCodeAt(0)); - const float32 = new Float32Array(binary.buffer); - const downsampled = - factor > 1 ? float32.filter((_, i) => i % factor === 0) : float32; - const int16 = new Int16Array(downsampled.length); - for (let i = 0; i < downsampled.length; i++) { - const sample = Math.max(-1, Math.min(1, downsampled[i])); - int16[i] = sample < 0 ? sample * 0x8000 : sample * 0x7fff; + let int16 = new Int16Array(binary.buffer); + if (factor > 1 && int16.length >= factor) { + const downsampled = new Int16Array(Math.floor(int16.length / factor)); + for (let i = 0; i < downsampled.length; i++) { + downsampled[i] = int16[i * factor]; + } + int16 = downsampled; } chunk = int16.buffer; } else if (Array.isArray(ev?.data)) { @@ -369,7 +508,7 @@ export function useDeepgramVoiceAgent({ onErrorRef.current?.(err); } }, - [onErrorRef] + [downsampleFactor, onErrorRef] ); const handleSocketMessage = useCallback( @@ -606,18 +745,23 @@ export function useDeepgramVoiceAgent({ const sanitizedDefault = sanitizeSettings(defaultSettingsRef.current); const sanitizedOverride = sanitizeSettings(overrideSettings); + const merged = mergeSettings(sanitizedDefault, sanitizedOverride); + const mergedSettings: DeepgramVoiceAgentSettingsMessage = { type: 'Settings', - ...(sanitizedDefault ?? {}), - ...(sanitizedOverride ?? {}), + ...(merged ?? {}), }; const targetSampleRate = overrideSettings?.audio?.input?.sample_rate ?? defaultSettingsRef.current?.audio?.input?.sample_rate ?? DEFAULT_INPUT_SAMPLE_RATE; - currentDownsample.current = - downsampleFactor ?? computeDownsampleFactor(targetSampleRate); + targetInputSampleRate.current = targetSampleRate; + currentDownsample.current = resolveDownsampleFactor( + downsampleFactor, + targetInputSampleRate.current, + nativeInputSampleRate.current + ); const socket = new (WebSocket as any)(endpointRef.current, undefined, { headers: { Authorization: `Token ${apiKey}` }, @@ -645,6 +789,7 @@ export function useDeepgramVoiceAgent({ downsampleFactor, handleMicChunk, handleSocketMessage, + mergeSettings, sanitizeSettings, sendJsonMessage, ]