From 02207bc1757b0de50eb91e426b61827c80f8a5df Mon Sep 17 00:00:00 2001 From: Rares Date: Mon, 13 Jul 2026 10:41:35 -0400 Subject: [PATCH] ## [2.4.0] - 2026-07-08 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Added - **Caption export helpers.** New `toSRT()` and `toWebVTT()` exports that convert a Deepgram pre-recorded transcription response (word timings, or utterances when `utterances=true`) into SubRip / WebVTT subtitle text. Cue segmentation is tunable via `CaptionOptions` (`lineLength`, `lineCount`, `speakerLabels` for diarized `Speaker N:` prefixes). Pure functions — no network or native calls. - **Speaker-attributed transcripts.** New `toSpeakerSegments()` export that folds a diarized (`diarize=true`) response into ordered `SpeakerSegment { speaker, text, start, end, confidence }` chunks, merging consecutive words from the same speaker. Returns `[]` for non-diarized responses instead of throwing. - **Session telemetry.** `useDeepgramSpeechToText` and `useDeepgramVoiceAgent` now expose a `getStats()` method returning a `SessionStats` snapshot (`bytesSent`, `bytesReceived`, `framesDropped`, `reconnects`, `connectedAtMs`, `firstResultMs`). An opt-in `trackStats` prop additionally enables a reactive `stats` return value, throttled to at most one update per second so it never floods renders. - **Voice Agent barge-in.** Opt-in `bargeIn` prop on `useDeepgramVoiceAgent` flushes the native playback queue when the server reports `UserStartedSpeaking` while agent audio is playing, so the agent audibly stops when the user talks over it. Fires the new `onBargeIn` callback only when a flush actually happened; never flushes while muted, and the streaming player resumes seamlessly on the agent's next turn. Reliable barge-in depends on hardware echo cancellation — test on a physical device. --- CHANGELOG.md | 31 ++ README.md | 47 +++ example/src/App.tsx | 17 +- example/src/SpeechToText.tsx | 307 ++++++++++++++++- example/src/TextToSpeech.tsx | 11 +- example/src/VoiceAgent.tsx | 124 ++++++- ios/Deepgram+AudioSession.mm | 279 ++++++--------- ios/Deepgram+Playback.mm | 36 +- ios/Deepgram+Private.h | 73 ++-- ios/Deepgram+Recording.mm | 28 +- ios/Deepgram.mm | 26 +- package.json | 4 +- plugin/build/index.js | 7 +- plugin/src/index.ts | 7 +- src/__tests__/captions.test.ts | 325 ++++++++++++++++++ src/__tests__/diarization.test.ts | 158 +++++++++ src/__tests__/stats.test.ts | 254 ++++++++++++++ .../useDeepgramVoiceAgent.bargein.test.ts | 219 ++++++++++++ .../useDeepgramVoiceAgent.stats.test.ts | 248 +++++++++++++ src/helpers/captions.ts | 285 +++++++++++++++ src/helpers/diarization.ts | 64 ++++ src/helpers/index.ts | 2 + src/index.ts | 10 +- src/types/deepgram/index.ts | 1 + src/types/deepgram/speech-to-text.ts | 97 ++++++ src/types/deepgram/stats.ts | 36 ++ src/useDeepgramSpeechToText.ts | 196 +++++++---- src/useDeepgramTextToSpeech.ts | 1 - src/useDeepgramVoiceAgent.ts | 139 +++++++- tsconfig.json | 1 + 30 files changed, 2637 insertions(+), 396 deletions(-) create mode 100644 src/__tests__/captions.test.ts create mode 100644 src/__tests__/diarization.test.ts create mode 100644 src/__tests__/stats.test.ts create mode 100644 src/__tests__/useDeepgramVoiceAgent.bargein.test.ts create mode 100644 src/__tests__/useDeepgramVoiceAgent.stats.test.ts create mode 100644 src/helpers/captions.ts create mode 100644 src/helpers/diarization.ts create mode 100644 src/types/deepgram/stats.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index bbaea36..237d787 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,37 @@ All notable changes to this project are documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [2.4.0] - 2026-07-08 + +### Added + +- **Caption export helpers.** New `toSRT()` and `toWebVTT()` exports that + convert a Deepgram pre-recorded transcription response (word timings, or + utterances when `utterances=true`) into SubRip / WebVTT subtitle text. + Cue segmentation is tunable via `CaptionOptions` (`lineLength`, `lineCount`, + `speakerLabels` for diarized `Speaker N:` prefixes). Pure functions — no + network or native calls. +- **Speaker-attributed transcripts.** New `toSpeakerSegments()` export that + folds a diarized (`diarize=true`) response into ordered + `SpeakerSegment { speaker, text, start, end, confidence }` chunks, merging + consecutive words from the same speaker. Returns `[]` for non-diarized + responses instead of throwing. +- **Session telemetry.** `useDeepgramSpeechToText` and `useDeepgramVoiceAgent` + now expose a `getStats()` method returning a `SessionStats` snapshot + (`bytesSent`, `bytesReceived`, `framesDropped`, `reconnects`, + `connectedAtMs`, `firstResultMs`). An opt-in `trackStats` prop additionally + enables a reactive `stats` return value, throttled to at most one update per + second so it never floods renders. +- **Voice Agent barge-in.** Opt-in `bargeIn` prop on `useDeepgramVoiceAgent` + flushes the native playback queue when the server reports + `UserStartedSpeaking` while agent audio is playing, so the agent audibly + stops when the user talks over it. Fires the new `onBargeIn` callback only + when a flush actually happened; never flushes while muted, and the streaming + player resumes seamlessly on the agent's next turn. Reliable barge-in + depends on hardware echo cancellation — test on a physical device. + +All additions are backwards compatible and opt-in; no breaking changes. + ## [2.3.0] - 2026-07-06 ### Added diff --git a/README.md b/README.md index 2c51d24..0a905c2 100644 --- a/README.md +++ b/README.md @@ -590,6 +590,7 @@ const { | `defaultSettings` | `DeepgramVoiceAgentSettings` | – | Base `Settings` payload sent on connect; merge per-call overrides via `connect(override)`. | | `autoStartMicrophone` | `boolean` | `true` | Automatically requests mic access and starts streaming PCM. | | `autoPlayAudio` | `boolean` | `true` | Plays received audio using the native player. | +| `bargeIn` | `boolean` | `false` | Flush the agent's queued playback when the server reports `UserStartedSpeaking`, so the agent audibly stops when the user talks over it. Only flushes while agent audio is actually playing and the mic is not muted; the player stays usable for the next agent turn. Requires `autoPlayAudio`. Barge-in depends on hardware echo cancellation — test on a physical device (simulators have no VPIO, so the agent's own audio can trigger the VAD). | | `downsampleFactor` | `number` | heuristic | Manually override the downsample ratio applied to captured audio. | | `reconnect` | `DeepgramReconnectOptions` | `{ enabled: false }` | Auto-reconnect config for the agent socket. The stored `Settings` payload is re-sent on every successful reconnect. | @@ -614,6 +615,7 @@ const { | `onError` | `(error: unknown) => void` | Unexpected client-side error (mic, playback, socket send). | | `onReconnecting` | `(attempt: number) => void` | A reconnect attempt begins (1-based attempt number). Requires `reconnect.enabled`. | | `onReconnected` | `() => void` | The socket reconnects and the stored settings are re-sent. | +| `onBargeIn` | `() => void` | A `bargeIn` flush actually happened (agent audio was cut short by the user speaking). | | `onServerError` | `(message: DeepgramVoiceAgentErrorMessage) => void` | API reports a structured error (`description` + `code`). | | `onWarning` | `(message: DeepgramVoiceAgentWarningMessage) => void` | Non-fatal warning (e.g. degraded audio quality). | @@ -1528,6 +1530,51 @@ const pick = async () => { }; ``` +### Generate subtitles from a transcription + +`toSRT` / `toWebVTT` turn a raw pre-recorded `/listen` response into a +subtitle document — zero dependencies, pure functions. Request +`utterances: true` for the best cue boundaries (the helpers fall back to +word-level chunking otherwise), and pass `speakerLabels: true` alongside +`diarize: true` to prefix cues with `Speaker N:`. + +```tsx +import { toSRT, toWebVTT } from 'react-native-deepgram'; +import type { DeepgramPrerecordedResponse } from 'react-native-deepgram'; + +const res = await fetch( + 'https://api.deepgram.com/v1/listen?model=nova-3&smart_format=true&utterances=true&diarize=true', + { + method: 'POST', + headers: { + 'Authorization': `Token ${DEEPGRAM_API_KEY}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ url: 'https://dpgr.am/spacewalk.wav' }), + } +); +const response: DeepgramPrerecordedResponse = await res.json(); + +const srt = toSRT(response, { lineLength: 32, lineCount: 2 }); +const vtt = toWebVTT(response, { speakerLabels: true }); +``` + +### Split a transcript by speaker + +`toSpeakerSegments` folds the word-level diarization data of a response +requested with `diarize: true` into per-speaker segments (it returns `[]` +when diarization wasn't requested). Pairs naturally with the caption +helpers' `speakerLabels` option above. + +```tsx +import { toSpeakerSegments } from 'react-native-deepgram'; + +const segments = toSpeakerSegments(response); // same response as above +segments.forEach(({ speaker, text, start, end, confidence }) => { + console.log(`[${start}s–${end}s] Speaker ${speaker} (${confidence}): ${text}`); +}); +``` + ### Record the mic to a file while transcribing Persist the captured audio to a WAV file *and* get the live transcript at the diff --git a/example/src/App.tsx b/example/src/App.tsx index b763498..12cae8e 100644 --- a/example/src/App.tsx +++ b/example/src/App.tsx @@ -16,17 +16,12 @@ import TextToSpeech from './TextToSpeech'; import VoiceAgent from './VoiceAgent'; import { colors, radius, spacing, type } from './theme'; -// Initialize Deepgram once at startup. Provide your key via .env (EXPO_PUBLIC_DEEPGRAM_API_KEY). -// -// Custom endpoints (regional, Dedicated, or self-hosted) are opt-in: set any of -// the EXPO_PUBLIC_DEEPGRAM_BASE_URL / BASE_WSS / AGENT_URL vars in .env and they -// flow straight into configure(). Leave them unset to use Deepgram's defaults. -// -// Auth: for production, prefer ephemeral tokens over a raw API key. Set -// EXPO_PUBLIC_DEEPGRAM_TOKEN_URL to a backend endpoint that proxies Deepgram's -// /v1/auth/grant and returns { access_token, expires_in }. When set, `getToken` -// is used (and takes precedence over apiKey) so the key never ships in the app. -// See README → "Ephemeral / scoped tokens". +// Initialize Deepgram once at startup (key via EXPO_PUBLIC_DEEPGRAM_API_KEY). +// Custom endpoints are opt-in via EXPO_PUBLIC_DEEPGRAM_BASE_URL / BASE_WSS / +// AGENT_URL. For production, prefer ephemeral tokens: set +// EXPO_PUBLIC_DEEPGRAM_TOKEN_URL to a backend that proxies /v1/auth/grant and +// returns { access_token, expires_in } — getToken then takes precedence over +// apiKey so the key never ships in the app. See README → "Ephemeral / scoped tokens". const tokenUrl = process.env.EXPO_PUBLIC_DEEPGRAM_TOKEN_URL; configure({ diff --git a/example/src/SpeechToText.tsx b/example/src/SpeechToText.tsx index fb69083..6917bb4 100644 --- a/example/src/SpeechToText.tsx +++ b/example/src/SpeechToText.tsx @@ -10,7 +10,14 @@ import { } from 'react-native'; import * as DocumentPicker from 'expo-document-picker'; import * as FileSystem from 'expo-file-system'; -import { DeepgramError, useDeepgramSpeechToText } from 'react-native-deepgram'; +import { + DeepgramError, + toSRT, + toSpeakerSegments, + toWebVTT, + useDeepgramSpeechToText, + type SpeakerSegment, +} from 'react-native-deepgram'; import Button from './components/Button'; import Card from './components/Card'; import Field from './components/Field'; @@ -18,6 +25,19 @@ import OptionSelect from './components/OptionSelect'; import StatusBadge from './components/StatusBadge'; import { colors, radius, spacing, type } from './theme'; +const formatBytes = (n: number) => + n < 1024 + ? `${n} B` + : n < 1024 * 1024 + ? `${(n / 1024).toFixed(1)} KB` + : `${(n / (1024 * 1024)).toFixed(1)} MB`; + +const formatClock = (seconds: number) => { + const m = Math.floor(seconds / 60); + const s = seconds - m * 60; + return `${m}:${s.toFixed(1).padStart(4, '0')}`; +}; + export default function SpeechToText() { const [liveTranscript, setLiveTranscript] = useState(''); const [liveInterimTranscript, setLiveInterimTranscript] = useState(''); @@ -25,6 +45,17 @@ export default function SpeechToText() { const [pickedFileName, setPickedFileName] = useState(null); const [reconnectAttempt, setReconnectAttempt] = useState(null); + // Captions & speaker segments (2.4.0) + const [captionsLoading, setCaptionsLoading] = useState(false); + const [captionsError, setCaptionsError] = useState(null); + const [captionsFileName, setCaptionsFileName] = useState(null); + const [captionView, setCaptionView] = useState<'srt' | 'vtt' | 'speakers'>( + 'srt' + ); + const [captionSrt, setCaptionSrt] = useState(''); + const [captionVtt, setCaptionVtt] = useState(''); + const [speakerSegments, setSpeakerSegments] = useState([]); + // Record-to-file is opt-in: the user chooses whether to save and where. const [recordEnabled, setRecordEnabled] = useState(false); const [recordFolder, setRecordFolder] = useState<'documents' | 'cache'>( @@ -51,8 +82,10 @@ export default function SpeechToText() { isPaused, audioLevel, recordingUri, + stats, } = useDeepgramSpeechToText({ trackState: true, + trackStats: true, reconnect: { enabled: true }, metering: { enabled: true }, recordToFile: { enabled: recordEnabled, path: recordPath }, @@ -179,6 +212,60 @@ export default function SpeechToText() { stopListening(); }; + // Caption helpers need the raw prerecorded response (word timings, + // utterances, speakers), so hit /v1/listen directly — same request the + // hook makes, but with utterances + diarize enabled. + const pickForCaptions = async () => { + try { + const result = await DocumentPicker.getDocumentAsync({ + type: 'audio/*', + copyToCacheDirectory: true, + }); + if (!result.assets || result.assets.length === 0) return; + const asset = result.assets[0]!; + setCaptionsFileName(asset.name ?? 'audio file'); + setCaptionsLoading(true); + setCaptionsError(null); + setCaptionSrt(''); + setCaptionVtt(''); + setSpeakerSegments([]); + + const base = + process.env.EXPO_PUBLIC_DEEPGRAM_BASE_URL || + 'https://api.deepgram.com/v1'; + const formData = new FormData(); + formData.append('audio', { + uri: asset.uri, + name: asset.name || 'audio-file', + type: asset.mimeType || 'audio/mpeg', + } as any); + + const res = await fetch( + `${base}/listen?model=nova-3&smart_format=true&utterances=true&diarize=true`, + { + method: 'POST', + headers: { + Authorization: `Token ${process.env.EXPO_PUBLIC_DEEPGRAM_API_KEY}`, + }, + body: formData, + } + ); + if (!res.ok) { + throw new Error(`HTTP ${res.status}: ${await res.text()}`); + } + const json = await res.json(); + + setCaptionSrt(toSRT(json, { speakerLabels: true })); + setCaptionVtt(toWebVTT(json, { speakerLabels: true })); + setSpeakerSegments(toSpeakerSegments(json)); + setCaptionView('srt'); + } catch (err) { + setCaptionsError(err instanceof Error ? err.message : String(err)); + } finally { + setCaptionsLoading(false); + } + }; + const tone = liveState?.error ? 'error' : isReconnecting @@ -450,6 +537,55 @@ export default function SpeechToText() { )} + {/* Session stats (2.4.0) */} + + + + + {formatBytes(stats?.bytesSent ?? 0)} + + Audio sent + + + + {formatBytes(stats?.bytesReceived ?? 0)} + + Received + + + {stats?.framesDropped ?? 0} + Frames dropped + + + {stats?.reconnects ?? 0} + Reconnects + + + + {stats?.firstResultMs != null + ? `${stats.firstResultMs} ms` + : '—'} + + First result + + + + {stats?.connectedAtMs != null + ? new Date(stats.connectedAtMs).toLocaleTimeString() + : '—'} + + Connected at + + + + Counters reset on every start. Pause the mic to watch “frames + dropped” climb. + + + {/* File transcription */} + + {/* Captions & speaker segments (2.4.0) */} + +