diff --git a/CHANGELOG.md b/CHANGELOG.md index c89f60a..29637c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ ## Unreleased +- Voice: a long reply is watched for 1.6 s before it starts. In the first second its speech often comes at real time + and slows once the chat model writes its next sentences, so the start the rate allowed then still paused in 16 of + 84 long replies measured live on DGX-02; with the wait, 3 (each 0.07 to 0.4 s, 35 s or more in). Long replies start + about 0.3 s later (3.5 s after the request on average); short ones as before. + - Voice: speech first no longer stalls other sessions. The chat model's server is shared, so it is paused only while nothing else waits on the chat lane, and a request queued there (another session or device, a computer-use step, a prime) resumes it at once. Before, a second device's request waited behind a reply's paused chat model (6 and 17 s diff --git a/apps/mobile/src/lib/voice/replyHold.test.ts b/apps/mobile/src/lib/voice/replyHold.test.ts index b10bb31..45221a1 100644 --- a/apps/mobile/src/lib/voice/replyHold.test.ts +++ b/apps/mobile/src/lib/voice/replyHold.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest' -import { HOLD_CAP_MS, MARGIN_MS, ReplyHold, START_MS, type VoicePriors } from './replyHold' +import { HOLD_CAP_MS, MARGIN_MS, OBSERVE_MS, ReplyHold, START_MS, type VoicePriors } from './replyHold' /** Feeds `item` a first piece of `firstMs` at 0, then `ms` every 150 ms; returns when (and with what) it started. */ function stream(hold: ReplyHold, firstMs: number, ms: number, until = 200) { @@ -35,20 +35,16 @@ describe('the reply start hold', () => { expect(started!.have).toBe(1_287.5) }) - it('a stream at 1x or more still being written starts once the margin is in hand', () => { + it('a reply still being written is watched for OBSERVE_MS before it starts, however fast it comes', () => { const hold = new ReplyHold() hold.written('a', 2_000) hold.piece('a', 0, 46, 0) - expect(hold.piece('a', 1, 210, 150)).toEqual([]) // 1.4x, but 256 ms is short of the 300 ms margin - expect(hold.piece('a', 2, 210, 300)).toEqual([0, 1, 2]) // 466 ms in hand - - // While a picture is being made, GEN_START_MS: 466 ms is not enough, 676 is. - const busy = new ReplyHold(() => true) - busy.written('a', 2_000) - busy.piece('a', 0, 46, 0) - busy.piece('a', 1, 210, 150) - expect(busy.piece('a', 2, 210, 300)).toEqual([]) - expect(busy.piece('a', 3, 210, 450)).toEqual([0, 1, 2, 3]) + for (let k = 1; k * 150 < OBSERVE_MS; k++) expect(hold.piece('a', k, 210, 150 * k)).toEqual([]) // 1.4x + expect(hold.tick(OBSERVE_MS)).toHaveLength(11) // then the rate allows it at once + // Its text complete meanwhile: no wait. + const done = new ReplyHold() + done.text('a', 30, 1.4, false, 0) + expect(done.piece('a', 0, START_MS, 0)).toEqual([0]) }) it('all of the reply speech in starts it, and it never holds past the cap', () => { @@ -68,11 +64,8 @@ describe('the reply start hold', () => { it('after a pause, the session next replies need more in hand, until a new session', () => { const hold = new ReplyHold() hold.gap(100) // the margin grows by 200 ms: 500 ms - hold.written('a', 2_000) - hold.piece('a', 0, 46, 0) - hold.piece('a', 1, 210, 150) - expect(hold.piece('a', 2, 210, 300)).toEqual([]) // 466 ms is not enough now - expect(hold.piece('a', 3, 210, 450)).toEqual([0, 1, 2, 3]) + hold.text('a', 40, null, false, 0) // 2600 ms at 0.25x: in hand >= (2600 - in hand) x 0.75 + 500, about 1429 ms + expect(stream(hold, 50, 37.5)!.have).toBe(1_437.5) // 1287.5 with the usual margin expect(hold.margin).toBe(MARGIN_MS + 200) hold.reset() // a barge-in keeps it diff --git a/apps/mobile/src/lib/voice/replyHold.ts b/apps/mobile/src/lib/voice/replyHold.ts index 8481905..763bb69 100644 --- a/apps/mobile/src/lib/voice/replyHold.ts +++ b/apps/mobile/src/lib/voice/replyHold.ts @@ -12,6 +12,9 @@ * voice's rate on its last replies, less RATE_HEADROOM of its spread; once the text is complete with the GPU free, * the server's measure of speech with the GPU to itself (text_done's speech_rate, else the last reply's) counts; * - margin: MARGIN_MS, and more for the rest of the session after a pause (`gap`). + * While the text is still being written, the rate is first watched for OBSERVE_MS from the first piece: in the first + * second the speech of a long reply often comes at real time, and slows once the chat model writes its next sentences + * beside it; a start trusted on that first second ran dry in 16 of 84 long DGX-02 replies (3 of 84 with the wait). * A slow stream starts later, in proportion: at a quarter of real time, three quarters of the rest must be in hand. A * short reply written on a free GPU needs START_MS (no margin); all of a reply's speech in (speech_done) starts it. * Never more than HOLD_CAP_MS after the first piece. @@ -23,6 +26,8 @@ export const HOLD_CAP_MS = 30_000 /** A reply's audio is at least about this long per character. */ export const MS_PER_CHAR = 65 export const MARGIN_MS = 300 +/** A reply whose text is still being written is not started before its speech has come for this long. */ +export const OBSERVE_MS = 1_600 /** A pause adds twice its length to the margin for the rest of the session, up to this. */ export const MARGIN_UP_MAX_MS = 2_000 const RATE_WINDOW_MS = 150 @@ -183,6 +188,7 @@ export class ReplyHold { private need(item: string, held: Held, now: number) { const r = this.rateOf(item, held, now) const done = this.textDone.has(item) + if (!done && now - held.firstAt < OBSERVE_MS) return Infinity // still watching its rate const free = done && !this.generating() ? this.freeRate.get(item) || this.lastRate || 0 : 0 const chars = done ? (this.textDone.get(item) ?? 0) : (this.writtenChars.get(item) ?? 0) const rest = Math.max(0, chars * r.mpc - (this.got.get(item)?.ms ?? 0)) diff --git a/docs/dev/VOICE.md b/docs/dev/VOICE.md index 4b99653..ccad725 100644 --- a/docs/dev/VOICE.md +++ b/docs/dev/VOICE.md @@ -362,6 +362,13 @@ binary, and it works in desktop Chrome and Safari and in phone browsers. Chat an tips, one sentence; a fresh session each), parameters chosen among those with no piece over 50 ms late in any reply, validated on a third of them held out: long replies started 1.8 to 3.3 s after the request (the best start with hindsight was 1.1 to 2.3 s), two-tip replies 1.3 to 2.0 s, one sentence 0.15 s after its first audio. + - Measured live on DGX-02 the next night (84 long replies, Mae and Scarlette, typed and spoken, ten prompts, a fresh + session each; `voice_e2e.py`'s replay of the page on each), that rule left a pause in 16 of them (0.07 to 0.5 s + each): in the first second a long reply's speech often comes at real time, and slows once the chat model writes + its next sentences beside it. So a reply whose text is still being written is now watched for 1.6 s from its + first piece before it may start (`OBSERVE_MS`): 3 of the 84 then pause (0.07 to 0.4 s, each after 35 s or more of + speech), long replies start 3.5 s after the request on average instead of 3.2 (the best start with hindsight: 2.7), + and short ones, written before the wait ends, start as before (0.9 s on average after the turn). `voice_e2e.py` replays the same rule on its long reply and fails on any underrun over 50 ms. - **Speech first** (`voice.speech_first`, on by default; `src/sched/yielder.zig`). While a reply is written, the session pauses its chat model's server (SIGSTOP; SIGCONT resumes it) whenever the reply's speech is under 2 s ahead diff --git a/src/voice/voice.html b/src/voice/voice.html index 04130ea..11c27c3 100644 --- a/src/voice/voice.html +++ b/src/voice/voice.html @@ -2819,12 +2819,15 @@

Cloned voices

// voice's rate on its last replies, less RATE_HEADROOM of its spread; once the text is complete with the GPU free, the // server's measure of speech with the GPU to itself (text_done's speech_rate, else the last reply's) counts; // - margin: MARGIN_MS, and more for the rest of the session after a pause (V.marginUp; schedule). +// While the text is still being written, the rate is first watched for OBSERVE_MS from the first piece: in the first +// second the speech of a long reply often comes at real time, and slows once the chat model writes its next sentences +// beside it; a start trusted on that first second ran dry in 16 of 84 long DGX-02 replies (3 of 84 with the wait). // A slow stream starts later, in proportion: at a quarter of real time, three quarters of the rest must be in hand. A // short reply written on a free GPU needs START_MS (no margin); all of a reply's speech in (speech_done) starts it. // Replayed on 30 DGX-02 replies (Mae, Scarlette; one sentence to 46 s): no pause in any, long ones started 1.8 to 3.3 s // after the request (`voice_e2e.py` replays this rule). const START_MS = 200, GEN_START_MS = 500, HOLD_CAP_MS = 30000, MS_PER_CHAR = 65, MARGIN_MS = 300, RATE_WINDOW_MS = 150, - EWMA_ALPHA = 0.7, RATE_HEADROOM = 0.5, PRIOR_RATE = 1.0; + EWMA_ALPHA = 0.7, RATE_HEADROOM = 0.5, PRIOR_RATE = 1.0, OBSERVE_MS = 1600; const generating = () => !!cur && cur.msgs.some(m => m.role === 'tool' && !m.gone && !(m.media && m.media.length) && !m.error); const startMs = () => S.startMs ?? (generating() ? GEN_START_MS : START_MS); function decode(b64) { @@ -2908,6 +2911,7 @@

Cloned voices

// Audio in hand that plays the rest of the reply to its end without running dry (see the comment above START_MS). function holdNeed(item) { const r = rateOf(item), done = V.textDone.has(item); + if (!done && performance.now() - (V.firstAt.get(item) || performance.now()) < OBSERVE_MS) return Infinity; // still watching its rate const free = done && !generating() ? (V.freeRate.get(item) || V.lastRate || null) : null; const chars = done ? (V.chars.get(item) || 0) : (V.written.get(item) || 0); const rest = Math.max(0, chars * r.mpc - ((V.got.get(item) || {}).ms || 0)); diff --git a/tools/spark/jarvis/voice_e2e.py b/tools/spark/jarvis/voice_e2e.py index a81da39..3048aa0 100644 --- a/tools/spark/jarvis/voice_e2e.py +++ b/tools/spark/jarvis/voice_e2e.py @@ -352,7 +352,7 @@ def turn(sess, name, samples, results, checks, timeout=120, need_audio=True): LONG_ASK = "Tell me a short story about a lighthouse keeper, in about six sentences." # The /voice page's start hold (src/voice/voice.html: holdNeed, rateOf), replayed on a reply's recorded events, as a # fresh page has it (no voice priors learned, no pause yet in the session). -START_MS, HOLD_CAP_MS, MS_PER_CHAR, MARGIN_MS = 200, 30000, 65, 300 +START_MS, HOLD_CAP_MS, MS_PER_CHAR, MARGIN_MS, OBSERVE_MS = 200, 30000, 65, 300, 1600 RATE_WINDOW_MS, EWMA_ALPHA, RATE_HEADROOM, PRIOR_RATE = 150, 0.7, 0.5, 1.0 @@ -383,6 +383,9 @@ def page_start(span, rid, step=0.025): if any(e["type"] == "localrouter.response.speech_done" for e in seen) or (t - t0) * 1000 >= HOLD_CAP_MS: return t done = next((e for e in seen if e["type"] == "localrouter.response.text_done"), None) + if done is None and (t - t0) * 1000 < OBSERVE_MS: # still watching its rate + t += step + continue free = (done.get("speech_rate") or None) if done and not done.get("tool_calls") else None chars = (done.get("chars") or 0) if done else sum(len(e.get("delta", "")) for e in seen if e["type"] == "response.audio_transcript.delta") rest = max(0.0, chars * MS_PER_CHAR - have)