From 17324f8dcc59da23b7385e2b22d251f559c30ff0 Mon Sep 17 00:00:00 2001 From: JJ Eaton <58495321+jayleaton@users.noreply.github.com> Date: Wed, 28 Oct 2026 00:41:36 +0700 Subject: [PATCH] Voice: a long reply is watched for 1.6 s before it starts In the first second a long reply's speech often comes at real time and slows once the chat model writes its next sentences beside it, so the start the measured rate allowed then still paused in 16 of 84 long replies measured live on DGX-02. A reply whose text is still being written now waits OBSERVE_MS (1.6 s) from its first piece before the rule may start it: 3 of 84 then pause (0.07 to 0.4 s, each 35 s or more in). The page, the mobile app and voice_e2e's replay change together. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 5 ++++ apps/mobile/src/lib/voice/replyHold.test.ts | 27 ++++++++------------- apps/mobile/src/lib/voice/replyHold.ts | 6 +++++ docs/dev/VOICE.md | 7 ++++++ src/voice/voice.html | 6 ++++- tools/spark/jarvis/voice_e2e.py | 5 +++- 6 files changed, 37 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c89f60a..29637c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ ## Unreleased +- Voice: a long reply is watched for 1.6 s before it starts. In the first second its speech often comes at real time + and slows once the chat model writes its next sentences, so the start the rate allowed then still paused in 16 of + 84 long replies measured live on DGX-02; with the wait, 3 (each 0.07 to 0.4 s, 35 s or more in). Long replies start + about 0.3 s later (3.5 s after the request on average); short ones as before. + - Voice: speech first no longer stalls other sessions. The chat model's server is shared, so it is paused only while nothing else waits on the chat lane, and a request queued there (another session or device, a computer-use step, a prime) resumes it at once. Before, a second device's request waited behind a reply's paused chat model (6 and 17 s diff --git a/apps/mobile/src/lib/voice/replyHold.test.ts b/apps/mobile/src/lib/voice/replyHold.test.ts index b10bb31..45221a1 100644 --- a/apps/mobile/src/lib/voice/replyHold.test.ts +++ b/apps/mobile/src/lib/voice/replyHold.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest' -import { HOLD_CAP_MS, MARGIN_MS, ReplyHold, START_MS, type VoicePriors } from './replyHold' +import { HOLD_CAP_MS, MARGIN_MS, OBSERVE_MS, ReplyHold, START_MS, type VoicePriors } from './replyHold' /** Feeds `item` a first piece of `firstMs` at 0, then `ms` every 150 ms; returns when (and with what) it started. */ function stream(hold: ReplyHold, firstMs: number, ms: number, until = 200) { @@ -35,20 +35,16 @@ describe('the reply start hold', () => { expect(started!.have).toBe(1_287.5) }) - it('a stream at 1x or more still being written starts once the margin is in hand', () => { + it('a reply still being written is watched for OBSERVE_MS before it starts, however fast it comes', () => { const hold = new ReplyHold() hold.written('a', 2_000) hold.piece('a', 0, 46, 0) - expect(hold.piece('a', 1, 210, 150)).toEqual([]) // 1.4x, but 256 ms is short of the 300 ms margin - expect(hold.piece('a', 2, 210, 300)).toEqual([0, 1, 2]) // 466 ms in hand - - // While a picture is being made, GEN_START_MS: 466 ms is not enough, 676 is. - const busy = new ReplyHold(() => true) - busy.written('a', 2_000) - busy.piece('a', 0, 46, 0) - busy.piece('a', 1, 210, 150) - expect(busy.piece('a', 2, 210, 300)).toEqual([]) - expect(busy.piece('a', 3, 210, 450)).toEqual([0, 1, 2, 3]) + for (let k = 1; k * 150 < OBSERVE_MS; k++) expect(hold.piece('a', k, 210, 150 * k)).toEqual([]) // 1.4x + expect(hold.tick(OBSERVE_MS)).toHaveLength(11) // then the rate allows it at once + // Its text complete meanwhile: no wait. + const done = new ReplyHold() + done.text('a', 30, 1.4, false, 0) + expect(done.piece('a', 0, START_MS, 0)).toEqual([0]) }) it('all of the reply speech in starts it, and it never holds past the cap', () => { @@ -68,11 +64,8 @@ describe('the reply start hold', () => { it('after a pause, the session next replies need more in hand, until a new session', () => { const hold = new ReplyHold() hold.gap(100) // the margin grows by 200 ms: 500 ms - hold.written('a', 2_000) - hold.piece('a', 0, 46, 0) - hold.piece('a', 1, 210, 150) - expect(hold.piece('a', 2, 210, 300)).toEqual([]) // 466 ms is not enough now - expect(hold.piece('a', 3, 210, 450)).toEqual([0, 1, 2, 3]) + hold.text('a', 40, null, false, 0) // 2600 ms at 0.25x: in hand >= (2600 - in hand) x 0.75 + 500, about 1429 ms + expect(stream(hold, 50, 37.5)!.have).toBe(1_437.5) // 1287.5 with the usual margin expect(hold.margin).toBe(MARGIN_MS + 200) hold.reset() // a barge-in keeps it diff --git a/apps/mobile/src/lib/voice/replyHold.ts b/apps/mobile/src/lib/voice/replyHold.ts index 8481905..763bb69 100644 --- a/apps/mobile/src/lib/voice/replyHold.ts +++ b/apps/mobile/src/lib/voice/replyHold.ts @@ -12,6 +12,9 @@ * voice's rate on its last replies, less RATE_HEADROOM of its spread; once the text is complete with the GPU free, * the server's measure of speech with the GPU to itself (text_done's speech_rate, else the last reply's) counts; * - margin: MARGIN_MS, and more for the rest of the session after a pause (`gap`). + * While the text is still being written, the rate is first watched for OBSERVE_MS from the first piece: in the first + * second the speech of a long reply often comes at real time, and slows once the chat model writes its next sentences + * beside it; a start trusted on that first second ran dry in 16 of 84 long DGX-02 replies (3 of 84 with the wait). * A slow stream starts later, in proportion: at a quarter of real time, three quarters of the rest must be in hand. A * short reply written on a free GPU needs START_MS (no margin); all of a reply's speech in (speech_done) starts it. * Never more than HOLD_CAP_MS after the first piece. @@ -23,6 +26,8 @@ export const HOLD_CAP_MS = 30_000 /** A reply's audio is at least about this long per character. */ export const MS_PER_CHAR = 65 export const MARGIN_MS = 300 +/** A reply whose text is still being written is not started before its speech has come for this long. */ +export const OBSERVE_MS = 1_600 /** A pause adds twice its length to the margin for the rest of the session, up to this. */ export const MARGIN_UP_MAX_MS = 2_000 const RATE_WINDOW_MS = 150 @@ -183,6 +188,7 @@ export class ReplyHold { private need(item: string, held: Held, now: number) { const r = this.rateOf(item, held, now) const done = this.textDone.has(item) + if (!done && now - held.firstAt < OBSERVE_MS) return Infinity // still watching its rate const free = done && !this.generating() ? this.freeRate.get(item) || this.lastRate || 0 : 0 const chars = done ? (this.textDone.get(item) ?? 0) : (this.writtenChars.get(item) ?? 0) const rest = Math.max(0, chars * r.mpc - (this.got.get(item)?.ms ?? 0)) diff --git a/docs/dev/VOICE.md b/docs/dev/VOICE.md index 4b99653..ccad725 100644 --- a/docs/dev/VOICE.md +++ b/docs/dev/VOICE.md @@ -362,6 +362,13 @@ binary, and it works in desktop Chrome and Safari and in phone browsers. Chat an tips, one sentence; a fresh session each), parameters chosen among those with no piece over 50 ms late in any reply, validated on a third of them held out: long replies started 1.8 to 3.3 s after the request (the best start with hindsight was 1.1 to 2.3 s), two-tip replies 1.3 to 2.0 s, one sentence 0.15 s after its first audio. + - Measured live on DGX-02 the next night (84 long replies, Mae and Scarlette, typed and spoken, ten prompts, a fresh + session each; `voice_e2e.py`'s replay of the page on each), that rule left a pause in 16 of them (0.07 to 0.5 s + each): in the first second a long reply's speech often comes at real time, and slows once the chat model writes + its next sentences beside it. So a reply whose text is still being written is now watched for 1.6 s from its + first piece before it may start (`OBSERVE_MS`): 3 of the 84 then pause (0.07 to 0.4 s, each after 35 s or more of + speech), long replies start 3.5 s after the request on average instead of 3.2 (the best start with hindsight: 2.7), + and short ones, written before the wait ends, start as before (0.9 s on average after the turn). `voice_e2e.py` replays the same rule on its long reply and fails on any underrun over 50 ms. - **Speech first** (`voice.speech_first`, on by default; `src/sched/yielder.zig`). While a reply is written, the session pauses its chat model's server (SIGSTOP; SIGCONT resumes it) whenever the reply's speech is under 2 s ahead diff --git a/src/voice/voice.html b/src/voice/voice.html index 04130ea..11c27c3 100644 --- a/src/voice/voice.html +++ b/src/voice/voice.html @@ -2819,12 +2819,15 @@

Cloned voices

// voice's rate on its last replies, less RATE_HEADROOM of its spread; once the text is complete with the GPU free, the // server's measure of speech with the GPU to itself (text_done's speech_rate, else the last reply's) counts; // - margin: MARGIN_MS, and more for the rest of the session after a pause (V.marginUp; schedule). +// While the text is still being written, the rate is first watched for OBSERVE_MS from the first piece: in the first +// second the speech of a long reply often comes at real time, and slows once the chat model writes its next sentences +// beside it; a start trusted on that first second ran dry in 16 of 84 long DGX-02 replies (3 of 84 with the wait). // A slow stream starts later, in proportion: at a quarter of real time, three quarters of the rest must be in hand. A // short reply written on a free GPU needs START_MS (no margin); all of a reply's speech in (speech_done) starts it. // Replayed on 30 DGX-02 replies (Mae, Scarlette; one sentence to 46 s): no pause in any, long ones started 1.8 to 3.3 s // after the request (`voice_e2e.py` replays this rule). const START_MS = 200, GEN_START_MS = 500, HOLD_CAP_MS = 30000, MS_PER_CHAR = 65, MARGIN_MS = 300, RATE_WINDOW_MS = 150, - EWMA_ALPHA = 0.7, RATE_HEADROOM = 0.5, PRIOR_RATE = 1.0; + EWMA_ALPHA = 0.7, RATE_HEADROOM = 0.5, PRIOR_RATE = 1.0, OBSERVE_MS = 1600; const generating = () => !!cur && cur.msgs.some(m => m.role === 'tool' && !m.gone && !(m.media && m.media.length) && !m.error); const startMs = () => S.startMs ?? (generating() ? GEN_START_MS : START_MS); function decode(b64) { @@ -2908,6 +2911,7 @@

Cloned voices

// Audio in hand that plays the rest of the reply to its end without running dry (see the comment above START_MS). function holdNeed(item) { const r = rateOf(item), done = V.textDone.has(item); + if (!done && performance.now() - (V.firstAt.get(item) || performance.now()) < OBSERVE_MS) return Infinity; // still watching its rate const free = done && !generating() ? (V.freeRate.get(item) || V.lastRate || null) : null; const chars = done ? (V.chars.get(item) || 0) : (V.written.get(item) || 0); const rest = Math.max(0, chars * r.mpc - ((V.got.get(item) || {}).ms || 0)); diff --git a/tools/spark/jarvis/voice_e2e.py b/tools/spark/jarvis/voice_e2e.py index a81da39..3048aa0 100644 --- a/tools/spark/jarvis/voice_e2e.py +++ b/tools/spark/jarvis/voice_e2e.py @@ -352,7 +352,7 @@ def turn(sess, name, samples, results, checks, timeout=120, need_audio=True): LONG_ASK = "Tell me a short story about a lighthouse keeper, in about six sentences." # The /voice page's start hold (src/voice/voice.html: holdNeed, rateOf), replayed on a reply's recorded events, as a # fresh page has it (no voice priors learned, no pause yet in the session). -START_MS, HOLD_CAP_MS, MS_PER_CHAR, MARGIN_MS = 200, 30000, 65, 300 +START_MS, HOLD_CAP_MS, MS_PER_CHAR, MARGIN_MS, OBSERVE_MS = 200, 30000, 65, 300, 1600 RATE_WINDOW_MS, EWMA_ALPHA, RATE_HEADROOM, PRIOR_RATE = 150, 0.7, 0.5, 1.0 @@ -383,6 +383,9 @@ def page_start(span, rid, step=0.025): if any(e["type"] == "localrouter.response.speech_done" for e in seen) or (t - t0) * 1000 >= HOLD_CAP_MS: return t done = next((e for e in seen if e["type"] == "localrouter.response.text_done"), None) + if done is None and (t - t0) * 1000 < OBSERVE_MS: # still watching its rate + t += step + continue free = (done.get("speech_rate") or None) if done and not done.get("tool_calls") else None chars = (done.get("chars") or 0) if done else sum(len(e.get("delta", "")) for e in seen if e["type"] == "response.audio_transcript.delta") rest = max(0.0, chars * MS_PER_CHAR - have)