Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions packages/tui/src/tui/controller/projection/turn-output-rate.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,14 @@ export class TuiTurnOutputRate {
const decodeDurationMs = event.message.usage?.decodeDurationMs;
// Missing timing is not a zero-duration sample and must not use the full request duration.
if (!nonnegativeFinite(outputTokens) || !nonnegativeFinite(decodeDurationMs)) return this.value;
if (
bufferedBurst(
outputTokens,
decodeDurationMs,
event.message.usage?.requestDurationMs,
)
)
return this.value;
this.samples.set(event.message.id, { outputTokens, decodeDurationMs });
let tokens = 0;
let duration = 0;
Expand Down Expand Up @@ -60,6 +68,20 @@ export class TuiTurnOutputRate {
}
}

/** Buffered upstream output arrives in one burst; its decode window says nothing about model speed. */
const BURST_MIN_TOKENS_PER_SECOND = 1_000;
const BURST_MAX_DECODE_SHARE = 0.1;

function bufferedBurst(
tokens: number,
decodeMs: number,
requestMs: number | undefined,
): boolean {
if (!nonnegativeFinite(requestMs) || decodeMs >= requestMs * BURST_MAX_DECODE_SHARE)
return false;
return tokens > (decodeMs / 1_000) * BURST_MIN_TOKENS_PER_SECOND;
}

function nonnegativeFinite(value: number | undefined): value is number {
return typeof value === 'number' && Number.isFinite(value) && value >= 0;
}
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,29 @@ describe("TuiTurnOutputRate", () => {
expect(rate.finalize()).toBeUndefined();
});

it("drops a buffered burst at turn start instead of reporting an inflated rate", () => {
const rate = new TuiTurnOutputRate();
rate.beginTurn("turn-1");
const apply = (
id: string,
outputTokens: number,
decodeDurationMs: number,
requestDurationMs: number,
) =>
rate.apply("turn-1", {
type: "message",
message: {
id,
role: "assistant",
usage: { outputTokens, decodeDurationMs, requestDurationMs },
},
});
expect(apply("burst", 107, 2, 2_221)).toBeUndefined();
expect(apply("instant-burst", 100, 0, 3_000)).toBeUndefined();
expect(apply("normal", 400, 2_000, 4_000)).toBe(200);
expect(apply("fast-provider", 1_500, 1_000, 1_200)).toBe(1_900 / 3);
});

it("ignores late events from an older turn after reset", () => {
const rate = new TuiTurnOutputRate();
rate.beginTurn("turn-old");
Expand Down
Loading