From 4266be2669a523f5a47ad982a6deb2db851e7603 Mon Sep 17 00:00:00 2001 From: Rares Date: Fri, 28 Nov 2025 14:53:12 -0500 Subject: [PATCH 1/5] feat: Add automatic audio playback, internal state tracking for connection, conversation, and agent status, and new audio utilities to the `useDeepgramVoiceAgent` hook. --- example/src/VoiceAgent.tsx | 50 ++-- ios/Deepgram.mm | 370 +++++++++++++++--------------- src/types/deepgram/voice-agent.ts | 16 ++ src/useDeepgramVoiceAgent.ts | 27 ++- 4 files changed, 260 insertions(+), 203 deletions(-) diff --git a/example/src/VoiceAgent.tsx b/example/src/VoiceAgent.tsx index 5e8b40b..3b3a399 100644 --- a/example/src/VoiceAgent.tsx +++ b/example/src/VoiceAgent.tsx @@ -8,9 +8,13 @@ import { TextInput, Switch, } from 'react-native'; -import { useDeepgramVoiceAgent } from 'react-native-deepgram'; +import { useDeepgramVoiceAgent, Deepgram } from 'react-native-deepgram'; import OptionSelect from './components/OptionSelect'; +declare const Buffer: { + from(data: Uint8Array | string, encoding?: string): { toString(encoding: string): string }; +}; + type ConversationEntry = { role: string; content: string; @@ -41,9 +45,8 @@ const THINK_MODEL_OPTIONS = [ ]; const SAMPLE_RATE_OPTIONS = [ - { label: '48 kHz', value: '48000' }, - { label: '24 kHz', value: '24000' }, - { label: '16 kHz', value: '16000' }, + { label: '16 kHz (recommended)', value: '16000' }, + { label: '8 kHz (lower quality)', value: '8000' }, ]; export default function VoiceAgent() { @@ -64,7 +67,7 @@ export default function VoiceAgent() { ); const [tagsInput, setTagsInput] = useState('demo'); const [autoStartMic, setAutoStartMic] = useState(true); - const [inputSampleRate, setInputSampleRate] = useState('24000'); + const [inputSampleRate, setInputSampleRate] = useState('16000'); const [temperature, setTemperature] = useState('0.7'); const [customMessage, setCustomMessage] = useState(''); @@ -75,7 +78,7 @@ export default function VoiceAgent() { .filter(Boolean); const resolvedListenModel = listenModel.trim() || 'nova-3'; const resolvedThinkModel = thinkModel.trim() || 'gpt-4o'; - const inputRate = Number(inputSampleRate) || 24_000; + const inputRate = Number(inputSampleRate) || 16_000; const parsedTemperature = parseFloat(temperature); const boundedTemperature = Number.isFinite(parsedTemperature) ? Math.max(0, Math.min(2, parsedTemperature)) @@ -84,6 +87,7 @@ export default function VoiceAgent() { return { audio: { input: { encoding: 'linear16', sample_rate: inputRate }, + output: { encoding: 'linear16', sample_rate: inputRate, container: 'none' }, }, agent: { language: language.trim() || 'en', @@ -117,11 +121,6 @@ export default function VoiceAgent() { thinkModel, ]); - const downsampleFactor = useMemo(() => { - const rate = Number(inputSampleRate) || 24_000; - return Math.max(1, Math.round(48_000 / rate)); - }, [inputSampleRate]); - const { connect, disconnect, @@ -133,18 +132,21 @@ export default function VoiceAgent() { } = useDeepgramVoiceAgent({ defaultSettings: agentSettings, autoStartMicrophone: autoStartMic, - downsampleFactor, onBeforeConnect: () => { setStatus('connecting'); setError(null); setWarning(null); setConversation([]); }, - onConnect: () => setStatus('connected'), + onConnect: () => { + setStatus('connected'); + Deepgram.startPlayer?.(16000, 1); + }, onClose: () => { setStatus('idle'); setThinking(null); setLatency(null); + Deepgram.stopPlayer?.(); }, onError: (err) => { setStatus('error'); @@ -173,6 +175,20 @@ export default function VoiceAgent() { setStatus('error'); setError(message.description || message.code || 'Voice agent error'); }, + onAudioConfig: (message: { sample_rate?: number; channels?: number }) => { + const sampleRate = message.sample_rate || 16000; + const channels = message.channels || 1; + Deepgram.startPlayer?.(sampleRate, channels); + }, + onAudio: async (audioData: ArrayBuffer) => { + try { + const bytes = new Uint8Array(audioData); + const b64 = Buffer.from(bytes).toString('base64'); + await Deepgram.feedAudio?.(b64); + } catch (err) { + console.error('[VoiceAgentDemo] onAudio error:', err); + } + }, }); const startAgent = async () => { @@ -319,8 +335,7 @@ export default function VoiceAgent() { - 🎧 Agent replies are delivered as text only. Speak into the - microphone and read the responses below. + 🎧 The agent will respond with voice. Make sure your volume is up! @@ -380,11 +395,10 @@ export default function VoiceAgent() { {warning && Warning: {warning}} {error && Error: {error}} - Conversation (Text Only) + Conversation - 💬 The agent responds with text transcripts so playback can't echo - back into the microphone. + 💬 Text transcripts of the conversation appear here alongside audio playback. diff --git a/ios/Deepgram.mm b/ios/Deepgram.mm index dbece81..5c3bd7f 100644 --- a/ios/Deepgram.mm +++ b/ios/Deepgram.mm @@ -37,7 +37,7 @@ bool isRunning; } DGRecordState; -@interface Deepgram : RCTEventEmitter +@interface Deepgram : RCTEventEmitter { DGRecordState _recordState; } @@ -49,8 +49,10 @@ @interface Deepgram : RCTEventEmitter @property (atomic, assign) BOOL hasListeners; @property (atomic, assign) BOOL appIsActive; -// Playback / TTS -@property (nonatomic, strong) AVAudioPlayer *audioPlayer; +// Playback / TTS (AVAudioEngine-based with echo cancellation) +@property (nonatomic, strong) AVAudioEngine *audioEngine; +@property (nonatomic, strong) AVAudioPlayerNode *playerNode; +@property (nonatomic, strong) AVAudioFormat *playbackFormat; @property (nonatomic, strong) NSMutableData *audioBuffer; @property (nonatomic, assign) BOOL isPlaying; @property (nonatomic, assign) int currentSampleRate; @@ -157,6 +159,12 @@ - (instancetype)init name:AVAudioSessionRouteChangeNotification object:nil]; DGLogDebug(@"[Deepgram] init: registered for AVAudioSessionRouteChangeNotification"); + [[NSNotificationCenter defaultCenter] + addObserver:self + selector:@selector(handleAudioInterruption:) + name:AVAudioSessionInterruptionNotification + object:nil]; + DGLogDebug(@"[Deepgram] init: registered for AVAudioSessionInterruptionNotification"); } return self; } @@ -353,6 +361,42 @@ - (void)handleAudioRouteChange:(NSNotification *)note [self activateAudioSession:NULL]; } +- (void)handleAudioInterruption:(NSNotification *)note +{ + DGLogDebug(@"[Deepgram] handleAudioInterruption: %@", note.userInfo); + NSNumber *typeValue = note.userInfo[AVAudioSessionInterruptionTypeKey]; + AVAudioSessionInterruptionType type = (AVAudioSessionInterruptionType)typeValue.unsignedIntegerValue; + + if (type == AVAudioSessionInterruptionTypeBegan) { + DGLogDebug(@"[Deepgram] handleAudioInterruption: interruption began"); + if (_recordState.isRunning && _recordState.queue) { + DGLogDebug(@"[Deepgram] handleAudioInterruption: pausing recording queue"); + AudioQueuePause(_recordState.queue); + } + if (self.isPlaying && self.playerNode) { + DGLogDebug(@"[Deepgram] handleAudioInterruption: pausing player node"); + [self.playerNode pause]; + } + } else if (type == AVAudioSessionInterruptionTypeEnded) { + DGLogDebug(@"[Deepgram] handleAudioInterruption: interruption ended"); + NSNumber *optionValue = note.userInfo[AVAudioSessionInterruptionOptionKey]; + AVAudioSessionInterruptionOptions options = (AVAudioSessionInterruptionOptions)optionValue.unsignedIntegerValue; + if (options & AVAudioSessionInterruptionOptionShouldResume) { + DGLogDebug(@"[Deepgram] handleAudioInterruption: should resume"); + [self activateAudioSession:nil]; + + if (_recordState.isRunning && _recordState.queue) { + DGLogDebug(@"[Deepgram] handleAudioInterruption: resuming recording queue"); + AudioQueueStart(_recordState.queue, NULL); + } + if (self.isPlaying && self.playerNode) { + DGLogDebug(@"[Deepgram] handleAudioInterruption: resuming player node"); + [self.playerNode play]; + } + } + } +} + #if TARGET_OS_IOS - (void)handleAppDidBecomeActive:(NSNotification *)note { @@ -697,182 +741,183 @@ - (void)cleanupRecordingQueue } /* ================================================================== */ -/* 2. SIMPLE AUDIO PLAYBACK (USING AVAUDIOPLAYER) */ +/* 2. AVAUDIOENGINE PLAYBACK WITH VOICE PROCESSING (ECHO CANCEL) */ /* ================================================================== */ /** - * Create a WAV header for PCM data. - * This allows us to use AVAudioPlayer with raw PCM data. + * Setup AVAudioEngine with Voice Processing I/O for echo cancellation. + * This enables hardware-level echo cancellation during simultaneous recording + playback. */ -- (NSData *)createWAVHeaderForPCMData:(NSData *)pcmData sampleRate:(int)sampleRate { - uint32_t dataSize = (uint32_t)pcmData.length; - uint32_t fileSize = 36 + dataSize; - uint16_t channels = 1; - uint16_t bitsPerSample = 16; - uint32_t byteRate = sampleRate * channels * (bitsPerSample / 8); - uint16_t blockAlign = channels * (bitsPerSample / 8); +- (BOOL)setupAudioEngineWithSampleRate:(int)sampleRate + channels:(int)channels + error:(NSError **)outError { + if (self.audioEngine && self.audioEngine.isRunning) { + [self.audioEngine stop]; + } - NSMutableData *wavData = [NSMutableData data]; + self.audioEngine = [[AVAudioEngine alloc] init]; + self.playerNode = [[AVAudioPlayerNode alloc] init]; - // RIFF header - [wavData appendBytes:"RIFF" length:4]; - [wavData appendBytes:&fileSize length:4]; - [wavData appendBytes:"WAVE" length:4]; + self.playbackFormat = [[AVAudioFormat alloc] + initWithCommonFormat:AVAudioPCMFormatInt16 + sampleRate:sampleRate + channels:channels + interleaved:YES]; - // fmt chunk - [wavData appendBytes:"fmt " length:4]; - uint32_t fmtSize = 16; - [wavData appendBytes:&fmtSize length:4]; - uint16_t audioFormat = 1; // PCM - [wavData appendBytes:&audioFormat length:2]; - [wavData appendBytes:&channels length:2]; - uint32_t sampleRateValue = sampleRate; - [wavData appendBytes:&sampleRateValue length:4]; - [wavData appendBytes:&byteRate length:4]; - [wavData appendBytes:&blockAlign length:2]; - [wavData appendBytes:&bitsPerSample length:2]; + if (!self.playbackFormat) { + if (outError) { + *outError = [NSError errorWithDomain:@"DeepgramAudioEngine" + code:-1 + userInfo:@{NSLocalizedDescriptionKey: @"Failed to create audio format"}]; + } + return NO; + } + + [self.audioEngine attachNode:self.playerNode]; + [self.audioEngine connect:self.playerNode + to:self.audioEngine.mainMixerNode + format:self.playbackFormat]; + + @try { + AVAudioInputNode *inputNode = self.audioEngine.inputNode; + if (inputNode) { + NSError *voiceProcessingError = nil; + [inputNode setVoiceProcessingEnabled:YES error:&voiceProcessingError]; + } + } @catch (NSException *exception) { + // Continue - voice processing not critical for basic playback + } - // data chunk - [wavData appendBytes:"data" length:4]; - [wavData appendBytes:&dataSize length:4]; - [wavData appendData:pcmData]; + [self.audioEngine prepare]; + NSError *startError = nil; + BOOL started = [self.audioEngine startAndReturnError:&startError]; - return wavData; + if (!started) { + DGLogError(@"[Deepgram] Failed to start audio engine: %@", startError.localizedDescription); + if (outError) { + *outError = startError; + } + return NO; + } + + return YES; } -RCT_EXPORT_METHOD(startPlayer - :(nonnull NSNumber *)sampleRate +/** + * Create AVAudioPCMBuffer from raw PCM data. + */ +- (AVAudioPCMBuffer *)createPCMBufferFromData:(NSData *)data { + if (!data || data.length == 0 || !self.playbackFormat) { + return nil; + } + + // Calculate frame count (data length / bytes per frame) + const AudioStreamBasicDescription *asbd = self.playbackFormat.streamDescription; + int bytesPerFrame = asbd->mBytesPerFrame; + AVAudioFrameCount frameCount = (AVAudioFrameCount)(data.length / bytesPerFrame); + + if (frameCount == 0) { + return nil; + } + + AVAudioPCMBuffer *buffer = [[AVAudioPCMBuffer alloc] + initWithPCMFormat:self.playbackFormat + frameCapacity:frameCount]; + + buffer.frameLength = frameCount; + + // Copy audio data to buffer + memcpy(buffer.int16ChannelData[0], data.bytes, data.length); + + return buffer; +} + +RCT_EXPORT_METHOD(startPlayer:(nonnull NSNumber *)sampleRate channels:(nonnull NSNumber *)channels) { - DGLogDebug(@"[Deepgram] startPlayer: begin sampleRate=%@ channels=%@", - sampleRate, - channels); [self stopPlayer:nil rejecter:nil]; + NSError *sessionError = nil; if (![self activateAudioSession:&sessionError]) { - DGLogWarn(@"[Deepgram] Unable to activate audio session for playback: %@", - sessionError.localizedDescription ?: sessionError); + DGLogWarn(@"[Deepgram] Unable to activate audio session: %@", sessionError); + return; + } + + NSError *engineError = nil; + if (![self setupAudioEngineWithSampleRate:sampleRate.intValue + channels:channels.intValue + error:&engineError]) { + DGLogError(@"[Deepgram] Failed to setup audio engine: %@", engineError); return; } self.audioBuffer = [[NSMutableData alloc] init]; self.isPlaying = NO; self.currentSampleRate = sampleRate.intValue; - DGLogDebug(@"[Deepgram] startPlayer: initialized buffer sampleRate=%d", - self.currentSampleRate); } /** - * Add audio data to the buffer for streaming playback. + * Feed base64-encoded PCM audio data for playback using AVAudioEngine. + * Uses real-time PCM buffer streaming instead of WAV files. */ RCT_EXPORT_METHOD(feedAudio:(NSString *)b64) { - DGLogDebug(@"[Deepgram] feedAudio: begin length=%lu", (unsigned long)b64.length); - if (!self.audioBuffer) { - // Use default sample rate if not set - int defaultSampleRate = self.currentSampleRate > 0 ? self.currentSampleRate : 16000; - DGLogDebug(@"[Deepgram] feedAudio: auto startPlayer defaultSampleRate=%d", defaultSampleRate); - [self startPlayer:@(defaultSampleRate) channels:@1]; - } - - NSData *pcmData = [[NSData alloc] initWithBase64EncodedString:b64 options:0]; - if (!pcmData || pcmData.length == 0) { - DGLogDebug(@"[Deepgram] feedAudio: decoded data empty"); - return; - } - - [self.audioBuffer appendData:pcmData]; - DGLogDebug(@"[Deepgram] feedAudio: appended %lu bytes (buffer=%lu)", - (unsigned long)pcmData.length, - (unsigned long)self.audioBuffer.length); - - if (!self.isPlaying && self.audioBuffer.length > 1000) { - DGLogDebug(@"[Deepgram] feedAudio: triggering playback"); - [self playAccumulatedAudio]; - } -} - -/** - * Play accumulated audio buffer for streaming. - */ -- (void)playAccumulatedAudio { - if (!self.audioBuffer || self.audioBuffer.length == 0) { - DGLogDebug(@"[Deepgram] playAccumulatedAudio: no buffered audio"); - return; - } - - NSData *wavData = [self createWAVHeaderForPCMData:self.audioBuffer sampleRate:self.currentSampleRate]; - - NSError *error = nil; - AVAudioPlayer *player = [[AVAudioPlayer alloc] initWithData:wavData error:&error]; - - if (error) { - DGLogError(@"[Deepgram] playAccumulatedAudio: player error=%@", error); - return; - } - - self.audioPlayer = player; - self.audioPlayer.delegate = self; - - self.isPlaying = YES; - [player prepareToPlay]; - BOOL success = [player play]; - - if (success) { - DGLogDebug(@"[Deepgram] playAccumulatedAudio: playback started, clearing buffer"); - [self.audioBuffer setLength:0]; - } else { - DGLogError(@"[Deepgram] playAccumulatedAudio: failed to start playback"); - self.isPlaying = NO; + @try { + if (!self.playerNode || !self.audioEngine || !self.audioEngine.isRunning) { + return; + } + + NSData *pcmData = [[NSData alloc] initWithBase64EncodedString:b64 options:0]; + if (!pcmData || pcmData.length == 0) { + return; + } + + AVAudioPCMBuffer *buffer = [self createPCMBufferFromData:pcmData]; + if (!buffer) { + DGLogError(@"[Deepgram] feedAudio: failed to create PCM buffer"); + return; + } + + [self.playerNode scheduleBuffer:buffer completionHandler:nil]; + + if (!self.playerNode.isPlaying) { + [self.playerNode play]; + } } -} - -/** - * Play PCM data immediately using AVAudioPlayer. - */ -- (void)playPCMData:(NSData *)pcmData { - DGLogDebug(@"[Deepgram] playPCMData: begin length=%lu", (unsigned long)pcmData.length); - NSData *wavData = [self createWAVHeaderForPCMData:pcmData sampleRate:self.currentSampleRate]; - - NSError *error = nil; - AVAudioPlayer *player = [[AVAudioPlayer alloc] initWithData:wavData error:&error]; - - if (error) { - DGLogError(@"[Deepgram] playPCMData: player error=%@", error); - return; + @catch (NSException *e) { + DGLogError(@"[Deepgram] feedAudio: exception %@", e); } - - self.audioPlayer = player; - [player prepareToPlay]; - DGLogDebug(@"[Deepgram] playPCMData: playing"); - [player play]; -} - /** - * Stop and cleanup the audio player. + * Stop audio playback and cleanup AVAudioEngine. */ RCT_EXPORT_METHOD(stopPlayer :(RCTPromiseResolveBlock)resolve rejecter:(RCTPromiseRejectBlock)reject) { @try { - DGLogDebug(@"[Deepgram] stopPlayer: begin"); - if (self.audioPlayer) { - [self.audioPlayer stop]; - DGLogDebug(@"[Deepgram] stopPlayer: stopped audioPlayer"); - self.audioPlayer = nil; + + // Stop player node + if (self.playerNode) { + [self.playerNode stop]; } - + + // Stop and cleanup audio engine + if (self.audioEngine && self.audioEngine.isRunning) { + [self.audioEngine stop]; + } + + // Clear properties + self.playerNode = nil; + self.audioEngine = nil; + self.playbackFormat = nil; self.audioBuffer = nil; self.isPlaying = NO; [self maybeDeactivateAudioSession]; - DGLogDebug(@"[Deepgram] stopPlayer: success"); if (resolve) resolve(nil); } @catch (NSException *e) { - DGLogError(@"[Deepgram] Error stopping player: %@", e.reason); DGLogError(@"[Deepgram] stopPlayer: exception %@", e); if (reject) reject(@"player_stop_error", e.reason, nil); } @@ -889,34 +934,6 @@ - (void)playPCMData:(NSData *)pcmData { [self startPlayer:sampleRate channels:channels]; } -/** - * Play a single audio chunk (one-shot playback). - */ -RCT_EXPORT_METHOD(playAudioChunk - :(NSString *)chunk - resolver:(RCTPromiseResolveBlock)resolve - rejecter:(RCTPromiseRejectBlock)reject) -{ - @try { - DGLogDebug(@"[Deepgram] playAudioChunk: begin"); - NSData *pcmData = [[NSData alloc] initWithBase64EncodedString:chunk options:0]; - if (!pcmData) { - DGLogError(@"[Deepgram] playAudioChunk: failed to decode base64"); - if (reject) reject(@"invalid_data", @"Failed to decode audio data", nil); - return; - } - - [self playPCMData:pcmData]; - - DGLogDebug(@"[Deepgram] playAudioChunk: success"); - if (resolve) resolve(nil); - } - @catch (NSException *e) { - DGLogError(@"[Deepgram] playAudioChunk: exception %@", e); - if (reject) reject(@"playback_error", e.reason, nil); - } -} - - (void)invalidate { DGLogDebug(@"[Deepgram] invalidate: begin"); @@ -929,10 +946,16 @@ - (void)invalidate self.pendingPCMBuffer = nil; - if (self.audioPlayer) { - [self.audioPlayer stop]; - self.audioPlayer = nil; + if (self.playerNode) { + [self.playerNode stop]; + self.playerNode = nil; + } + + if (self.audioEngine && self.audioEngine.isRunning) { + [self.audioEngine stop]; } + self.audioEngine = nil; + self.playbackFormat = nil; self.audioBuffer = nil; self.isPlaying = NO; @@ -945,23 +968,4 @@ - (void)invalidate [super invalidate]; } -/* ================================================================== */ -/* 3. AVAUDIOPLAYER DELEGATE METHODS */ -/* ================================================================== */ - -- (void)audioPlayerDidFinishPlaying:(AVAudioPlayer *)player successfully:(BOOL)flag { - DGLogDebug(@"[Deepgram] audioPlayerDidFinishPlaying: success=%@", flag ? @"YES" : @"NO"); - self.isPlaying = NO; - - if (self.audioBuffer && self.audioBuffer.length > 0) { - DGLogDebug(@"[Deepgram] audioPlayerDidFinishPlaying: playing remaining buffered audio"); - [self playAccumulatedAudio]; - } -} - -- (void)audioPlayerDecodeErrorDidOccur:(AVAudioPlayer *)player error:(NSError *)error { - DGLogError(@"[Deepgram] audioPlayerDecodeErrorDidOccur: error=%@", error); - self.isPlaying = NO; -} - @end diff --git a/src/types/deepgram/voice-agent.ts b/src/types/deepgram/voice-agent.ts index fa076e3..e56b4e3 100644 --- a/src/types/deepgram/voice-agent.ts +++ b/src/types/deepgram/voice-agent.ts @@ -243,6 +243,20 @@ export interface DeepgramVoiceAgentWarningMessage { [key: string]: unknown; } +export interface DeepgramVoiceAgentAudioConfigMessage { + type: 'AudioConfig'; + sample_rate?: number; + channels?: number; + encoding?: string; + [key: string]: unknown; +} + +export interface DeepgramVoiceAgentAudioMessage { + type: 'Audio'; + // Binary audio data will be in the WebSocket message payload + [key: string]: unknown; +} + export type DeepgramVoiceAgentServerMessage = | DeepgramVoiceAgentWelcomeMessage | DeepgramVoiceAgentSettingsAppliedMessage @@ -258,6 +272,8 @@ export type DeepgramVoiceAgentServerMessage = | DeepgramVoiceAgentInjectionRefusedMessage | DeepgramVoiceAgentErrorMessage | DeepgramVoiceAgentWarningMessage + | DeepgramVoiceAgentAudioConfigMessage + | DeepgramVoiceAgentAudioMessage | DeepgramVoiceAgentSettingsMessage | DeepgramVoiceAgentUpdateSpeakMessage | DeepgramVoiceAgentInjectUserMessage diff --git a/src/useDeepgramVoiceAgent.ts b/src/useDeepgramVoiceAgent.ts index 3cdce97..20f5747 100644 --- a/src/useDeepgramVoiceAgent.ts +++ b/src/useDeepgramVoiceAgent.ts @@ -22,6 +22,7 @@ import type { DeepgramVoiceAgentInjectionRefusedMessage, DeepgramVoiceAgentWarningMessage, DeepgramVoiceAgentErrorMessage, + DeepgramVoiceAgentAudioConfigMessage, } from './types'; const DEFAULT_AGENT_ENDPOINT = 'wss://agent.deepgram.com/v1/agent/converse'; @@ -203,6 +204,8 @@ export interface UseDeepgramVoiceAgentProps { ) => void; onWarning?: (message: DeepgramVoiceAgentWarningMessage) => void; onServerError?: (message: DeepgramVoiceAgentErrorMessage) => void; + onAudioConfig?: (message: DeepgramVoiceAgentAudioConfigMessage) => void; + onAudio?: (audioData: ArrayBuffer) => void; } export interface UseDeepgramVoiceAgentReturn { @@ -245,6 +248,8 @@ export function useDeepgramVoiceAgent({ onInjectionRefused, onWarning, onServerError, + onAudioConfig, + onAudio, }: UseDeepgramVoiceAgentProps = {}): UseDeepgramVoiceAgentReturn { const ws = useRef(null); const audioSub = useRef | null>( @@ -282,6 +287,8 @@ export function useDeepgramVoiceAgent({ const onInjectionRefusedRef = useRef(onInjectionRefused); const onWarningRef = useRef(onWarning); const onServerErrorRef = useRef(onServerError); + const onAudioConfigRef = useRef(onAudioConfig); + const onAudioRef = useRef(onAudio); const autoStartMicRef = useRef(autoStartMicrophone); const sanitizeAudioSettings = useCallback( @@ -418,6 +425,8 @@ export function useDeepgramVoiceAgent({ onInjectionRefusedRef.current = onInjectionRefused; onWarningRef.current = onWarning; onServerErrorRef.current = onServerError; + onAudioConfigRef.current = onAudioConfig; + onAudioRef.current = onAudio; autoStartMicRef.current = autoStartMicrophone; if (downsampleFactor != null) { currentDownsample.current = downsampleFactor; @@ -432,6 +441,9 @@ export function useDeepgramVoiceAgent({ microphoneActive.current = false; } + // Cleanup audio session for playback + Deepgram.stopAudio().catch(() => {}); + const socket = ws.current; if (socket) { ws.current = null; @@ -587,8 +599,14 @@ export function useDeepgramVoiceAgent({ ); break; case 'Audio': + // Audio binary data will be handled by onmessage binary path + break; case 'AudioConfig': - // Audio responses are ignored in text-only mode. + if (hasKeys(message, ['sample_rate'])) { + onAudioConfigRef.current?.( + message as DeepgramVoiceAgentAudioConfigMessage + ); + } break; case 'InjectionRefused': if (hasKeys(message, ['message'])) { @@ -637,7 +655,8 @@ export function useDeepgramVoiceAgent({ const buffer = ensureArrayBuffer(ev.data); if (buffer) { - // Binary audio responses are ignored in text-only mode. + // Binary audio from server + onAudioRef.current?.(buffer); } }, [ @@ -740,6 +759,10 @@ export function useDeepgramVoiceAgent({ if (eventName) { audioSub.current = emitter.addListener(eventName, handleMicChunk); } + } else { + // Only initialize audio session for playback if not recording + // (startRecording already activates the audio session) + await Deepgram.startAudio(); } const sanitizedDefault = sanitizeSettings(defaultSettingsRef.current); From 73ff7af6bc56e58e0c35b785fba5c5d25f0e2d9c Mon Sep 17 00:00:00 2001 From: Rares Date: Fri, 28 Nov 2025 16:43:52 -0500 Subject: [PATCH 2/5] feat: add useAsyncCall hook and enhance Deepgram API hooks with state tracking - Introduced `useAsyncCall` hook for managing asynchronous calls with loading and error states. - Updated `index.ts` to export new helper functions. - Enhanced `useDeepgramSpeechToText`, `useDeepgramTextIntelligence`, and `useDeepgramTextToSpeech` hooks to support internal state tracking. - Added `trackState` and `trackTranscript` options to `useDeepgramSpeechToText` for managing transcription state. - Implemented conversation tracking in `useDeepgramVoiceAgent` with `trackConversation` option. - Improved type definitions in Deepgram management and speech-to-text interfaces with detailed comments. - Added support for agent status tracking in `useDeepgramVoiceAgent`. --- README.md | 45 +++++- example/src/SpeechToText.tsx | 120 ++++++-------- example/src/TextIntelligence.tsx | 28 ++-- example/src/TextToSpeech.tsx | 80 ++++------ example/src/VoiceAgent.tsx | 179 ++++++--------------- ios/Deepgram.mm | 78 +++++++++ package.json | 5 +- src/helpers/audioUtils.ts | 32 ++++ src/helpers/createAgentSettings.ts | 76 +++++++++ src/helpers/index.ts | 3 + src/helpers/useAsyncCall.ts | 52 ++++++ src/index.ts | 6 + src/types/deepgram/management.ts | 38 +++++ src/types/deepgram/shared.ts | 8 + src/types/deepgram/speech-to-text.ts | 71 +++++++++ src/types/deepgram/text-intelligence.ts | 35 ++++ src/types/deepgram/text-to-speech.ts | 56 +++++++ src/types/deepgram/voice-agent.ts | 24 +++ src/useDeepgramSpeechToText.ts | 85 +++++++++- src/useDeepgramTextIntelligence.ts | 25 ++- src/useDeepgramTextToSpeech.ts | 74 +++++++-- src/useDeepgramVoiceAgent.ts | 202 +++++++++++++++++++++--- 22 files changed, 1006 insertions(+), 316 deletions(-) create mode 100644 src/helpers/audioUtils.ts create mode 100644 src/helpers/createAgentSettings.ts create mode 100644 src/helpers/useAsyncCall.ts diff --git a/README.md b/README.md index a676858..7047212 100644 --- a/README.md +++ b/README.md @@ -135,22 +135,33 @@ configure({ apiKey: 'YOUR_DEEPGRAM_API_KEY' }); | `useDeepgramTextIntelligence` | Text analysis (summaries, topics, intents, sentiment) | | `useDeepgramManagement` | Typed wrapper around the Management REST API | +> 💡 **Pro tip**: All hooks now export a `state` object (and other reactive values) so you can easily track connection status, errors, and transcripts without maintaining your own state. + --- ## Voice Agent (`useDeepgramVoiceAgent`) `useDeepgramVoiceAgent` connects to `wss://agent.deepgram.com/v1/agent/converse`, captures microphone audio, and optionally auto-plays the agent's streamed responses. It wraps the full Voice Agent messaging surface so you can react to conversation updates, function calls, warnings, and raw PCM audio. +> 🔊 **Audio Handling**: This hook uses `AVAudioEngine` on iOS for hardware-accelerated echo cancellation, ensuring the agent doesn't hear itself speak. It also manages the audio session automatically. + ### Quick start ```tsx const { connect, disconnect, + state, // { connectionState, error, warning } + agentStatus, // { thinking, latency } + conversation, // Array<{ role, content }> injectUserMessage, sendFunctionCallResponse, updatePrompt, } = useDeepgramVoiceAgent({ + trackState: true, // Enable reactive state tracking + trackConversation: true, // Enable conversation history tracking + trackAgentStatus: true, // Enable agent status tracking + autoPlayAudio: true, // Automatically play agent audio defaultSettings: { audio: { input: { encoding: 'linear16', sample_rate: 24_000 }, @@ -207,6 +218,7 @@ const rePrompt = () => { return ( <> + Status: {state.connectionState}