Voice: a long reply is watched for 1.6 s before it starts (16 → 3 of 84 replies with a pause)

CHANGELOG.md

@@ -2,6 +2,11 @@
 
 ## Unreleased
 
+- Voice: a long reply is watched for 1.6 s before it starts. In the first second its speech often comes at real time
+  and slows once the chat model writes its next sentences, so the start the rate allowed then still paused in 16 of
+  84 long replies measured live on DGX-02; with the wait, 3 (each 0.07 to 0.4 s, 35 s or more in). Long replies start
+  about 0.3 s later (3.5 s after the request on average); short ones as before.
+
 - Voice: speech first no longer stalls other sessions. The chat model's server is shared, so it is paused only while
   nothing else waits on the chat lane, and a request queued there (another session or device, a computer-use step, a
   prime) resumes it at once. Before, a second device's request waited behind a reply's paused chat model (6 and 17 s

apps/mobile/src/lib/voice/replyHold.test.ts

@@ -1,6 +1,6 @@
 import { describe, expect, it } from 'vitest'
 
-import { HOLD_CAP_MS, MARGIN_MS, ReplyHold, START_MS, type VoicePriors } from './replyHold'
+import { HOLD_CAP_MS, MARGIN_MS, OBSERVE_MS, ReplyHold, START_MS, type VoicePriors } from './replyHold'
 
 /** Feeds `item` a first piece of `firstMs` at 0, then `ms` every 150 ms; returns when (and with what) it started. */
 function stream(hold: ReplyHold<number>, firstMs: number, ms: number, until = 200) {
@@ -35,20 +35,16 @@ describe('the reply start hold', () => {
     expect(started!.have).toBe(1_287.5)
   })
 
-  it('a stream at 1x or more still being written starts once the margin is in hand', () => {
+  it('a reply still being written is watched for OBSERVE_MS before it starts, however fast it comes', () => {
     const hold = new ReplyHold<number>()
     hold.written('a', 2_000)
     hold.piece('a', 0, 46, 0)
-    expect(hold.piece('a', 1, 210, 150)).toEqual([]) // 1.4x, but 256 ms is short of the 300 ms margin
-    expect(hold.piece('a', 2, 210, 300)).toEqual([0, 1, 2]) // 466 ms in hand
-
-    // While a picture is being made, GEN_START_MS: 466 ms is not enough, 676 is.
-    const busy = new ReplyHold<number>(() => true)
-    busy.written('a', 2_000)
-    busy.piece('a', 0, 46, 0)
-    busy.piece('a', 1, 210, 150)
-    expect(busy.piece('a', 2, 210, 300)).toEqual([])
-    expect(busy.piece('a', 3, 210, 450)).toEqual([0, 1, 2, 3])
+    for (let k = 1; k * 150 < OBSERVE_MS; k++) expect(hold.piece('a', k, 210, 150 * k)).toEqual([]) // 1.4x
+    expect(hold.tick(OBSERVE_MS)).toHaveLength(11) // then the rate allows it at once
+    // Its text complete meanwhile: no wait.
+    const done = new ReplyHold<number>()
+    done.text('a', 30, 1.4, false, 0)
+    expect(done.piece('a', 0, START_MS, 0)).toEqual([0])
   })
 
   it('all of the reply speech in starts it, and it never holds past the cap', () => {
@@ -68,11 +64,8 @@ describe('the reply start hold', () => {
   it('after a pause, the session next replies need more in hand, until a new session', () => {
     const hold = new ReplyHold<number>()
     hold.gap(100) // the margin grows by 200 ms: 500 ms
-    hold.written('a', 2_000)
-    hold.piece('a', 0, 46, 0)
-    hold.piece('a', 1, 210, 150)
-    expect(hold.piece('a', 2, 210, 300)).toEqual([]) // 466 ms is not enough now
-    expect(hold.piece('a', 3, 210, 450)).toEqual([0, 1, 2, 3])
+    hold.text('a', 40, null, false, 0) // 2600 ms at 0.25x: in hand >= (2600 - in hand) x 0.75 + 500, about 1429 ms
+    expect(stream(hold, 50, 37.5)!.have).toBe(1_437.5) // 1287.5 with the usual margin
     expect(hold.margin).toBe(MARGIN_MS + 200)
 
     hold.reset() // a barge-in keeps it

apps/mobile/src/lib/voice/replyHold.ts

@@ -12,6 +12,9 @@
  *   voice's rate on its last replies, less RATE_HEADROOM of its spread; once the text is complete with the GPU free,
  *   the server's measure of speech with the GPU to itself (text_done's speech_rate, else the last reply's) counts;
  * - margin: MARGIN_MS, and more for the rest of the session after a pause (`gap`).
+ * While the text is still being written, the rate is first watched for OBSERVE_MS from the first piece: in the first
+ * second the speech of a long reply often comes at real time, and slows once the chat model writes its next sentences
+ * beside it; a start trusted on that first second ran dry in 16 of 84 long DGX-02 replies (3 of 84 with the wait).
  * A slow stream starts later, in proportion: at a quarter of real time, three quarters of the rest must be in hand. A
  * short reply written on a free GPU needs START_MS (no margin); all of a reply's speech in (speech_done) starts it.
  * Never more than HOLD_CAP_MS after the first piece.
@@ -23,6 +26,8 @@ export const HOLD_CAP_MS = 30_000
 /** A reply's audio is at least about this long per character. */
 export const MS_PER_CHAR = 65
 export const MARGIN_MS = 300
+/** A reply whose text is still being written is not started before its speech has come for this long. */
+export const OBSERVE_MS = 1_600
 /** A pause adds twice its length to the margin for the rest of the session, up to this. */
 export const MARGIN_UP_MAX_MS = 2_000
 const RATE_WINDOW_MS = 150
@@ -183,6 +188,7 @@ export class ReplyHold<T> {
   private need(item: string, held: Held<T>, now: number) {
     const r = this.rateOf(item, held, now)
     const done = this.textDone.has(item)
+    if (!done && now - held.firstAt < OBSERVE_MS) return Infinity // still watching its rate
     const free = done && !this.generating() ? this.freeRate.get(item) || this.lastRate || 0 : 0
     const chars = done ? (this.textDone.get(item) ?? 0) : (this.writtenChars.get(item) ?? 0)
     const rest = Math.max(0, chars * r.mpc - (this.got.get(item)?.ms ?? 0))

docs/dev/VOICE.md

@@ -362,6 +362,13 @@ binary, and it works in desktop Chrome and Safari and in phone browsers. Chat an
     tips, one sentence; a fresh session each), parameters chosen among those with no piece over 50 ms late in any
     reply, validated on a third of them held out: long replies started 1.8 to 3.3 s after the request (the best start
     with hindsight was 1.1 to 2.3 s), two-tip replies 1.3 to 2.0 s, one sentence 0.15 s after its first audio.
+  - Measured live on DGX-02 the next night (84 long replies, Mae and Scarlette, typed and spoken, ten prompts, a fresh
+    session each; `voice_e2e.py`'s replay of the page on each), that rule left a pause in 16 of them (0.07 to 0.5 s
+    each): in the first second a long reply's speech often comes at real time, and slows once the chat model writes
+    its next sentences beside it. So a reply whose text is still being written is now watched for 1.6 s from its
+    first piece before it may start (`OBSERVE_MS`): 3 of the 84 then pause (0.07 to 0.4 s, each after 35 s or more of
+    speech), long replies start 3.5 s after the request on average instead of 3.2 (the best start with hindsight: 2.7),
+    and short ones, written before the wait ends, start as before (0.9 s on average after the turn).
     `voice_e2e.py` replays the same rule on its long reply and fails on any underrun over 50 ms.
 - **Speech first** (`voice.speech_first`, on by default; `src/sched/yielder.zig`). While a reply is written, the
   session pauses its chat model's server (SIGSTOP; SIGCONT resumes it) whenever the reply's speech is under 2 s ahead

src/voice/voice.html

@@ -2819,12 +2819,15 @@ <h3 id="hMine">Cloned voices</h3>
 //   voice's rate on its last replies, less RATE_HEADROOM of its spread; once the text is complete with the GPU free, the
 //   server's measure of speech with the GPU to itself (text_done's speech_rate, else the last reply's) counts;
 // - margin: MARGIN_MS, and more for the rest of the session after a pause (V.marginUp; schedule).
+// While the text is still being written, the rate is first watched for OBSERVE_MS from the first piece: in the first
+// second the speech of a long reply often comes at real time, and slows once the chat model writes its next sentences
+// beside it; a start trusted on that first second ran dry in 16 of 84 long DGX-02 replies (3 of 84 with the wait).
 // A slow stream starts later, in proportion: at a quarter of real time, three quarters of the rest must be in hand. A
 // short reply written on a free GPU needs START_MS (no margin); all of a reply's speech in (speech_done) starts it.
 // Replayed on 30 DGX-02 replies (Mae, Scarlette; one sentence to 46 s): no pause in any, long ones started 1.8 to 3.3 s
 // after the request (`voice_e2e.py` replays this rule).
 const START_MS = 200, GEN_START_MS = 500, HOLD_CAP_MS = 30000, MS_PER_CHAR = 65, MARGIN_MS = 300, RATE_WINDOW_MS = 150,
-  EWMA_ALPHA = 0.7, RATE_HEADROOM = 0.5, PRIOR_RATE = 1.0;
+  EWMA_ALPHA = 0.7, RATE_HEADROOM = 0.5, PRIOR_RATE = 1.0, OBSERVE_MS = 1600;
 const generating = () => !!cur && cur.msgs.some(m => m.role === 'tool' && !m.gone && !(m.media && m.media.length) && !m.error);
 const startMs = () => S.startMs ?? (generating() ? GEN_START_MS : START_MS);
 function decode(b64) {
@@ -2908,6 +2911,7 @@ <h3 id="hMine">Cloned voices</h3>
 // Audio in hand that plays the rest of the reply to its end without running dry (see the comment above START_MS).
 function holdNeed(item) {
   const r = rateOf(item), done = V.textDone.has(item);
+  if (!done && performance.now() - (V.firstAt.get(item) || performance.now()) < OBSERVE_MS) return Infinity; // still watching its rate
   const free = done && !generating() ? (V.freeRate.get(item) || V.lastRate || null) : null;
   const chars = done ? (V.chars.get(item) || 0) : (V.written.get(item) || 0);
   const rest = Math.max(0, chars * r.mpc - ((V.got.get(item) || {}).ms || 0));

tools/spark/jarvis/voice_e2e.py

@@ -352,7 +352,7 @@ def turn(sess, name, samples, results, checks, timeout=120, need_audio=True):
 LONG_ASK = "Tell me a short story about a lighthouse keeper, in about six sentences."
 # The /voice page's start hold (src/voice/voice.html: holdNeed, rateOf), replayed on a reply's recorded events, as a
 # fresh page has it (no voice priors learned, no pause yet in the session).
-START_MS, HOLD_CAP_MS, MS_PER_CHAR, MARGIN_MS = 200, 30000, 65, 300
+START_MS, HOLD_CAP_MS, MS_PER_CHAR, MARGIN_MS, OBSERVE_MS = 200, 30000, 65, 300, 1600
 RATE_WINDOW_MS, EWMA_ALPHA, RATE_HEADROOM, PRIOR_RATE = 150, 0.7, 0.5, 1.0
 
 
@@ -383,6 +383,9 @@ def page_start(span, rid, step=0.025):
         if any(e["type"] == "localrouter.response.speech_done" for e in seen) or (t - t0) * 1000 >= HOLD_CAP_MS:
             return t
         done = next((e for e in seen if e["type"] == "localrouter.response.text_done"), None)
+        if done is None and (t - t0) * 1000 < OBSERVE_MS:  # still watching its rate
+            t += step
+            continue
         free = (done.get("speech_rate") or None) if done and not done.get("tool_calls") else None
         chars = (done.get("chars") or 0) if done else sum(len(e.get("delta", "")) for e in seen if e["type"] == "response.audio_transcript.delta")
         rest = max(0.0, chars * MS_PER_CHAR - have)