From b3f505d68eb56a1f348c4f50d43e883f1b90a2d5 Mon Sep 17 00:00:00 2001 From: Manuel Torralbo Date: Wed, 23 Sep 2026 12:50:17 +0000 Subject: [PATCH] Add streaming and pcm response_format to /v1/audio/speech The OpenAI-compatible endpoint currently only returns a fully synthesized wav/opus/mp3 file, with no way to get raw PCM or to start receiving audio before the whole input finishes synthesizing. This blocks realtime voice-agent clients (e.g. pipecat OpenAITTSService, which requests response_format="pcm" and streams via with_streaming_response(...).iter_bytes()) from using this server as a drop-in OpenAI TTS backend. See #88. - OpenAISpeechRequest: add response_format="pcm" (headerless raw 16-bit signed little-endian mono PCM, matching the OpenAI pcm contract), plus stream (Optional[bool]) and chunk_size (optional) fields. - utils.encode_audio: add a pcm branch (used by both the non-streaming and streaming pcm paths). - utils.chunk_text_by_sentences_or_clauses: new sentence-aware chunker that additionally falls back to clause boundaries (comma, semicolon, colon) when a single sentence alone exceeds chunk_size, so a short one-sentence turn (60-90 chars, common in voice-agent prompts) can still be split and start streaming before the full sentence finishes synthesizing. chunk_text_by_sentences (used by the existing non-streaming paths) is untouched. - server.py: extract the crossfading chunk-streaming generator that /tts (stream=true) already used into a shared _stream_synthesized_audio() helper, parameterized by whether to emit a WAV header. /v1/audio/speech now calls the same helper when streaming is on and response_format is pcm or wav; opus/mp3 streaming falls back to a non-streaming response with a warning, since Opus/MP3 encoders need the complete signal. The existing non-streaming path is unchanged for requests that opt out. - stream defaults to unset (None), not false: most OpenAI-SDK clients (e.g. pipecat OpenAITTSService) call with_streaming_response.create(...) and read iter_bytes() without ever sending a "stream" field. When omitted, the effective value falls back to the new server.openai_stream_by_default config option (default false, so existing deployments see no change unless they opt in). An explicit "stream": true/false in the request always overrides that server default. - New server.openai_stream_chunk_size config default (50 chars), overridable per-request via chunk_size. - README.md: document the new request fields on /v1/audio/speech and the openai_stream_by_default config option. All changes are additive and backward compatible: response_format defaults to "wav", stream is unset by default, and openai_stream_by_default defaults to false, so existing callers of /v1/audio/speech see no behavior change unless the operator opts in. Co-Authored-By: Claude Opus 5.5 (1M context) --- README.md | 55 ++++++++++++- config.py | 34 ++++++++ server.py | 239 +++++++++++++++++++++++++++++++++++++++++------------- utils.py | 88 ++++++++++++++++++++ 4 files changed, 358 insertions(+), 58 deletions(-) diff --git a/README.md b/README.md index f31e77b..beac5c5 100644 --- a/README.md +++ b/README.md @@ -1103,7 +1103,7 @@ The primary endpoint for TTS generation is `/tts`. The OpenAI-compatible `/v1/au | Endpoint | Method | Purpose | |---|---|---| | `/tts` | POST | Custom TTS, full param set, supports `stream: true` (mp3 / wav / opus) | -| `/v1/audio/speech` | POST | OpenAI-compatible TTS | +| `/v1/audio/speech` | POST | OpenAI-compatible TTS, supports `stream: true` (`pcm` / `wav`) | | `/v1/audio/voices` | GET | OpenAI-compatible voice listing | | `/api/ui/initial-data` | GET | UI bootstrap + comprehensive health check | | `/api/model-info` | GET | Loaded model status, type, supported languages | @@ -1138,6 +1138,59 @@ curl -X POST http://localhost:8004/tts \ -d '{"text":"The first chunk arrives quickly, the rest stream behind.","stream":true}' \ --output stream.wav ``` + +**`/v1/audio/speech` request body (`OpenAISpeechRequest`):** + +- `model` (string, required) — accepted for OpenAI compatibility; not used to select a model. +- `input` (string, required) — text to synthesize. +- `voice` (string, required) — predefined voice filename or reference-audio filename. +- `response_format` ("wav" | "opus" | "mp3" | "pcm", default "wav"). +- `speed` (float, default `1.0`). +- `seed`, `language` — generation parameter overrides. +- `stream` (boolean, optional, default unset) — if true, returns a `StreamingResponse` and + yields audio as each sentence/clause chunk is synthesized, instead of waiting for the full + input. Supported for `response_format="pcm"` (headerless raw 16-bit PCM, little-endian mono + at the engine's sample rate — the format OpenAI's own `pcm` response uses, and what clients + such as pipecat's `OpenAITTSService` expect from `with_streaming_response(...).iter_bytes()`) + and `response_format="wav"` (a WAV header is sent once, followed by raw PCM16 frames, + mirroring `/tts`'s existing `stream=true` behavior). `stream=true` with `opus`/`mp3` is not + supported; the request falls back to a non-streaming response and logs a warning. If the + request omits `stream` entirely — the common case for OpenAI-SDK clients, which call + `with_streaming_response.create(...)` without ever setting this field — the server falls + back to the `server.openai_stream_by_default` config value (`false` unless the operator + opts in). An explicit `true`/`false` in the request always overrides that server default. +- `chunk_size` (integer 50–500, optional) — target character length per sentence/clause chunk + when `stream=true`. Defaults to `server.openai_stream_chunk_size` in `config.yaml` (`50`). + Chunking is sentence-aware first, then falls back to clause boundaries (`,`, `;`, `:`) for a + single sentence that alone exceeds `chunk_size`, so short one-sentence turns (60-90 chars, + common in voice-agent prompts) still get split and start streaming before the full sentence + finishes synthesizing. Ignored when streaming is off, where behavior is unchanged from + before (whole input synthesized, then encoded to the requested container format). + +**Streaming by default for clients that never send `stream` (config option):** + +Most OpenAI-SDK-based clients — including pipecat's `OpenAITTSService`, which only sends +`model`, `voice`, `input`, and `response_format` — never send the `stream` field at all. To +turn streaming on for such clients without changing their code, set in `config.yaml`: + +```yaml +server: + openai_stream_by_default: true +``` + +With that set, `POST /v1/audio/speech` streams (for `response_format="pcm"`/`"wav"`) even when +the request body omits `stream`. A request that explicitly sets `"stream": false` still gets a +non-streaming response; `"stream": true` still streams regardless of this setting. Default is +`false`, so existing deployments that don't add this key see no behavior change. + +**Example — streaming OpenAI-compatible TTS (raw PCM, e.g. for a realtime voice agent):** + +```bash +curl -X POST http://localhost:8004/v1/audio/speech \ + -H "Content-Type: application/json" \ + -d '{"model":"tts-1","input":"The first chunk arrives quickly.","voice":"Emily.wav","response_format":"pcm","stream":true}' \ + --output stream.pcm +``` # 🐳 Docker Installation Run Chatterbox TTS Server easily using Docker. The recommended method uses Docker Compose, which is pre-configured for different GPU types. diff --git a/config.py b/config.py index 4244a3d..6ceb908 100644 --- a/config.py +++ b/config.py @@ -49,6 +49,18 @@ "log_file_backup_count": 5, # Number of backup log files to keep. "ssl_certfile": None, # Path to SSL certificate file for HTTPS. None = HTTP only. "ssl_keyfile": None, # Path to SSL private key file for HTTPS. None = HTTP only. + "openai_stream_chunk_size": 50, # Default char length for sentence/clause chunks + # when streaming from /v1/audio/speech (stream=true). Small default so short, + # single-sentence turns (e.g. 60-90 chars, no internal '.'/'?'/'!') still get + # split at clause boundaries (',', ';', ':') and start streaming quickly. + # Overridable per-request via OpenAISpeechRequest.chunk_size. + "openai_stream_by_default": False, # If true, /v1/audio/speech streams + # (pcm/wav) even when the request omits the "stream" field entirely. + # Needed because most OpenAI-SDK-based clients (e.g. pipecat's + # OpenAITTSService) never send "stream" — they just call + # with_streaming_response.create(...) and read iter_bytes(). Per-request + # "stream": true/false always overrides this. Default false keeps + # existing deployments' behavior unchanged unless they opt in. }, "model": { # Added section for model source configuration "repo_id": "chatterbox-turbo", # UPDATED: Default to Turbo model @@ -883,6 +895,28 @@ def get_gen_default_language() -> str: ) +def get_openai_stream_chunk_size() -> int: + """Returns the default chunk size (characters) used to split text into + sentence/clause chunks for the streaming OpenAI-compatible endpoint + (POST /v1/audio/speech with stream=true). Can be overridden per-request + via OpenAISpeechRequest.chunk_size.""" + return config_manager.get_int( + "server.openai_stream_chunk_size", + _get_default_from_structure("server.openai_stream_chunk_size"), + ) + + +def get_openai_stream_by_default() -> bool: + """Returns whether POST /v1/audio/speech should stream (for pcm/wav) + when the request does not specify "stream" at all. Used because most + OpenAI-SDK clients never send that field. A request that explicitly sets + "stream": true or "stream": false always takes precedence over this.""" + return config_manager.get_bool( + "server.openai_stream_by_default", + _get_default_from_structure("server.openai_stream_by_default"), + ) + + # Audio Output Settings Accessors def get_audio_output_format() -> str: """Returns the default audio output format (e.g., 'wav').""" diff --git a/server.py b/server.py index e435185..9b10b30 100644 --- a/server.py +++ b/server.py @@ -60,6 +60,8 @@ get_audio_sample_rate, get_full_config_for_template, get_audio_output_format, + get_openai_stream_chunk_size, + get_openai_stream_by_default, ) import engine # TTS Engine interface @@ -77,10 +79,38 @@ class OpenAISpeechRequest(BaseModel): model: str input_: str = Field(..., alias="input") voice: str - response_format: Literal["wav", "opus", "mp3"] = "wav" # Add "mp3" + response_format: Literal["wav", "opus", "mp3", "pcm"] = "wav" # Add "mp3", "pcm" speed: float = 1.0 seed: Optional[int] = None language: Optional[str] = None + stream: Optional[bool] = Field( + None, + description=( + "If true, returns a StreamingResponse and yields audio as each " + "sentence/clause chunk is synthesized instead of waiting for the " + "full input to finish. Supported for response_format='pcm' " + "(headerless raw PCM16, matching the OpenAI 'pcm' contract used by " + "clients such as pipecat's OpenAITTSService) and response_format=" + "'wav' (a WAV header is sent once, followed by raw PCM16 frames, " + "mirroring the existing /tts stream=true behaviour). Streaming " + "with 'opus'/'mp3' is not supported; such requests fall back to " + "a non-streaming response. If omitted (the common case for " + "OpenAI-SDK clients, which never send this field), falls back to " + "server.openai_stream_by_default (false unless configured " + "otherwise); an explicit true/false here always overrides that " + "server default." + ), + ) + chunk_size: Optional[int] = Field( + None, + ge=50, + le=500, + description=( + "Approximate target character length for sentence/clause chunks " + "when stream=true. Defaults to server.openai_stream_chunk_size " + "(50 chars) if not set. Ignored when stream=false." + ), + ) # --- Logging Configuration --- @@ -408,6 +438,92 @@ def _float32_to_pcm16(audio_np: np.ndarray) -> bytes: return (clipped * 32767).astype(np.int16).tobytes() +_STREAM_CROSSFADE_MS = 20 + + +async def _stream_synthesized_audio( + text_chunks: List[str], + audio_prompt_path: Optional[str], + temperature: float, + exaggeration: float, + cfg_weight: float, + seeds: List[Optional[int]], + language: str, + speed_factor: float = 1.0, + include_wav_header: bool = True, +): + """ + Shared streaming generator used by both the custom /tts endpoint + (stream=true) and the OpenAI-compatible /v1/audio/speech endpoint + (stream=true). Synthesizes each text chunk sequentially and yields + 16-bit PCM audio bytes, crossfading chunk boundaries to avoid clicks at + the seams (the same approach /tts already used, before it was extracted + here for reuse). + + seeds must be the same length as text_chunks; each caller resolves its + own per-chunk seed convention before calling this (e.g. /tts reuses one + seed for every chunk; /v1/audio/speech increments the seed per chunk to + match its existing non-streaming behaviour). + + If include_wav_header is True, a WAV header with an unknown ("streaming") + data size is yielded first, then raw PCM16 frames — this is what /tts's + stream=true already returns. If False, only headerless raw PCM16 bytes + are yielded, matching the OpenAI response_format="pcm" contract (16-bit + signed little-endian mono, no container), used by clients such as + pipecat's OpenAITTSService. + """ + loop = asyncio.get_running_loop() + carry: Optional[np.ndarray] = None + header_sent = not include_wav_header + + for i, chunk_text in enumerate(text_chunks): + is_last = i == len(text_chunks) - 1 + chunk_seed = seeds[i] if i < len(seeds) else None + logger.info(f"Streaming chunk {i+1}/{len(text_chunks)}...") + + audio_tensor, chunk_sr = await loop.run_in_executor( + None, + lambda c=chunk_text, s=chunk_seed: engine.synthesize( + text=c, + audio_prompt_path=audio_prompt_path, + temperature=temperature, + exaggeration=exaggeration, + cfg_weight=cfg_weight, + seed=s, + language=language, + ), + ) + + if audio_tensor is None or chunk_sr is None: + logger.error(f"Streaming TTS: engine returned None for chunk {i+1}; stopping stream.") + return + + if speed_factor != 1.0: + audio_tensor, _ = utils.apply_speed_factor(audio_tensor, chunk_sr, speed_factor) + + audio_np = audio_tensor.cpu().numpy().squeeze().astype(np.float32) + + if not header_sent: + yield _create_wav_header(chunk_sr) + header_sent = True + + fade_samples = int(_STREAM_CROSSFADE_MS / 1000 * chunk_sr) + + if carry is not None: + # Crossfade the held-back tail of the previous chunk with the head of this one + audio_np = _crossfade_with_overlap(carry, audio_np, fade_samples) + carry = None + + if not is_last and len(audio_np) > fade_samples: + carry = audio_np[-fade_samples:].copy() + yield _float32_to_pcm16(audio_np[:-fade_samples]) + else: + yield _float32_to_pcm16(audio_np) + + if carry is not None: + yield _float32_to_pcm16(carry) + + # --- End Audio Stitching Helper Functions --- @@ -1004,65 +1120,20 @@ async def custom_tts_endpoint( request.language if request.language is not None else get_gen_default_language() ) - CROSSFADE_MS_STREAM = 20 - - async def _stream_generator(): - loop = asyncio.get_running_loop() - carry: Optional[np.ndarray] = None - header_sent = False - - for i, chunk_text in enumerate(text_chunks): - is_last = i == len(text_chunks) - 1 - logger.info(f"Streaming chunk {i+1}/{len(text_chunks)}...") - - audio_tensor, chunk_sr = await loop.run_in_executor( - None, - lambda c=chunk_text: engine.synthesize( - text=c, - audio_prompt_path=audio_prompt_str, - temperature=temperature_val, - exaggeration=exaggeration_val, - cfg_weight=cfg_weight_val, - seed=seed_val, - language=language_val, - ), - ) - - if audio_tensor is None or chunk_sr is None: - logger.error(f"Streaming TTS: engine returned None for chunk {i+1}; stopping stream.") - return - - if speed_factor_stream != 1.0: - audio_tensor, _ = utils.apply_speed_factor( - audio_tensor, chunk_sr, speed_factor_stream - ) - - audio_np = audio_tensor.cpu().numpy().squeeze().astype(np.float32) - - if not header_sent: - yield _create_wav_header(chunk_sr) - header_sent = True - - fade_samples = int(CROSSFADE_MS_STREAM / 1000 * chunk_sr) - - if carry is not None: - # Crossfade the held-back tail of the previous chunk with the head of this one - audio_np = _crossfade_with_overlap(carry, audio_np, fade_samples) - carry = None - - if not is_last and len(audio_np) > fade_samples: - carry = audio_np[-fade_samples:].copy() - yield _float32_to_pcm16(audio_np[:-fade_samples]) - else: - yield _float32_to_pcm16(audio_np) - - if carry is not None: - yield _float32_to_pcm16(carry) - timestamp_str = time.strftime("%Y%m%d_%H%M%S") stream_filename = utils.sanitize_filename(f"tts_stream_{timestamp_str}.wav") return StreamingResponse( - _stream_generator(), + _stream_synthesized_audio( + text_chunks=text_chunks, + audio_prompt_path=audio_prompt_str, + temperature=temperature_val, + exaggeration=exaggeration_val, + cfg_weight=cfg_weight_val, + seeds=[seed_val] * len(text_chunks), + language=language_val, + speed_factor=speed_factor_stream, + include_wav_header=True, + ), media_type="audio/wav", headers={"Content-Disposition": f'attachment; filename="{stream_filename}"'}, ) @@ -1423,6 +1494,60 @@ async def openai_speech_endpoint(request: OpenAISpeechRequest): request.seed if request.seed is not None else get_gen_default_seed() ) + # --- Streaming fork (pcm/wav only; opus/mp3 fall back to non-streaming) --- + # request.stream is Optional: most OpenAI-SDK clients (e.g. pipecat's + # OpenAITTSService) never send it at all, so an unset value falls back + # to the server-wide default instead of always meaning "false". + stream_requested = ( + request.stream if request.stream is not None else get_openai_stream_by_default() + ) + if stream_requested and request.response_format in ("pcm", "wav"): + stream_chunk_size = ( + request.chunk_size + if request.chunk_size is not None + else get_openai_stream_chunk_size() + ) + text_chunks = utils.chunk_text_by_sentences_or_clauses( + request.input_, stream_chunk_size + ) + if not text_chunks: + raise HTTPException( + status_code=400, detail="Text processing resulted in no usable chunks." + ) + + logger.info( + f"OpenAI speech (stream): processing {len(text_chunks)} chunk(s) for " + f"input of {len(request.input_)} chars, chunk_size={stream_chunk_size}, " + f"response_format={request.response_format}" + ) + + seeds = [ + seed_to_use + i if seed_to_use is not None and seed_to_use >= 0 else seed_to_use + for i in range(len(text_chunks)) + ] + + media_type = "audio/pcm" if request.response_format == "pcm" else "audio/wav" + return StreamingResponse( + _stream_synthesized_audio( + text_chunks=text_chunks, + audio_prompt_path=str(audio_prompt_path), + temperature=get_gen_default_temperature(), + exaggeration=get_gen_default_exaggeration(), + cfg_weight=get_gen_default_cfg_weight(), + seeds=seeds, + language=request.language or get_gen_default_language(), + speed_factor=request.speed, + include_wav_header=(request.response_format == "wav"), + ), + media_type=media_type, + ) + if stream_requested: + logger.warning( + f"stream=true is not supported for response_format=" + f"'{request.response_format}'; falling back to a non-streaming response." + ) + # --- End streaming fork --- + # Split long text into chunks for better quality (same as /tts endpoint) DEFAULT_CHUNK_SIZE = 120 text_chunks = utils.chunk_text_by_sentences(request.input_, DEFAULT_CHUNK_SIZE) diff --git a/utils.py b/utils.py index ec99d5f..2e0f505 100644 --- a/utils.py +++ b/utils.py @@ -349,6 +349,16 @@ def encode_audio( ) audio_segment.export(output_buffer, format="mp3") + elif output_format == "pcm": + # Raw headerless 16-bit signed little-endian mono PCM, matching + # the OpenAI /v1/audio/speech response_format="pcm" contract + # (used by clients such as pipecat's OpenAITTSService). No + # soundfile/pydub container is written; rate_to_write already + # reflects target_sample_rate from the resampling step above. + audio_clipped = np.clip(audio_array, -1.0, 1.0) + audio_int16 = (audio_clipped * 32767).astype(np.int16) + output_buffer.write(audio_int16.tobytes()) + else: logger.error( f"Unsupported output format requested for encoding: {output_format}" @@ -1023,6 +1033,84 @@ def _preprocess_and_segment_text(full_text: str) -> List[Tuple[Optional[str], st return segmented_with_tags +_CLAUSE_BOUNDARY_PATTERN = re.compile(r"(?<=[,;:])\s+") + + +def _split_segment_into_clauses(segment_text: str, chunk_size: int) -> List[str]: + """ + Fallback splitter for a single sentence/segment that alone exceeds + chunk_size: breaks it at clause boundaries (comma, semicolon, colon) so a + long single sentence can still start streaming before the whole sentence + is synthesized. Never splits mid-word. Returns [segment_text] unchanged + if it already fits or has no clause boundary to split on. + """ + if len(segment_text) <= chunk_size: + return [segment_text] + + parts = [p for p in _CLAUSE_BOUNDARY_PATTERN.split(segment_text) if p] + if len(parts) <= 1: + return [segment_text] + + clauses: List[str] = [] + current = "" + for part in parts: + candidate = f"{current} {part}".strip() if current else part + if current and len(candidate) > chunk_size: + clauses.append(current) + current = part + else: + current = candidate + if current: + clauses.append(current) + return clauses + + +def chunk_text_by_sentences_or_clauses(full_text: str, chunk_size: int) -> List[str]: + """ + Like chunk_text_by_sentences, but additionally splits at clause + boundaries (comma/semicolon/colon) when a single sentence alone exceeds + chunk_size. Used by the streaming OpenAI-compatible endpoint + (/v1/audio/speech, stream=true) so short turns made of one 60-90 char + sentence (no internal '.'/'?'/'!') can still start streaming before the + whole sentence finishes synthesizing. chunk_text_by_sentences itself + (used by the non-streaming /tts and /v1/audio/speech paths) is left + untouched, so their behaviour does not change for callers who do not + request streaming. + """ + if not full_text or full_text.isspace(): + return [] + if chunk_size <= 0: + chunk_size = float("inf") + + processed_segments = _preprocess_and_segment_text(full_text) + if not processed_segments: + return [] + + expanded_segments: List[str] = [] + for _, segment_text in processed_segments: + expanded_segments.extend(_split_segment_into_clauses(segment_text, chunk_size)) + + text_chunks: List[str] = [] + current_chunk_parts: List[str] = [] + current_chunk_length = 0 + for part in expanded_segments: + part_len = len(part) + if not current_chunk_parts: + current_chunk_parts = [part] + current_chunk_length = part_len + elif current_chunk_length + 1 + part_len <= chunk_size: + current_chunk_parts.append(part) + current_chunk_length += 1 + part_len + else: + text_chunks.append(" ".join(current_chunk_parts)) + current_chunk_parts = [part] + current_chunk_length = part_len + if current_chunk_parts: + text_chunks.append(" ".join(current_chunk_parts)) + + return text_chunks + + def chunk_text_by_sentences( full_text: str, chunk_size: int,