From 3385cafb620e7422619a8efa93dae52e2e51ed40 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Sun, 18 Oct 2026 00:25:50 +0200 Subject: [PATCH] mason: make quota-call pacing injectable and shrink the profile-rotation test QuotaManager gains an optional pauseBetweenQuotaCalls(ms). Unset, it uses pauseOnRealTimer, the same unreferenced native setTimeout promise that was inline before, returned directly so awaiting it takes the same turns. OpenCode (runtime override quotaCallPause) and Pi (__setPiQuotaCallPauseForTest) pass it only from test fixtures; production passes undefined. OpenCode and Pi provider-error rate-limit tests now record the requested wait instead of sleeping ~1 s, and assert each requested gap is in (0, 1000]. A new Core test asserts the second GET requests exactly the remaining gap. "completed profile hydration does not refetch later token generations" runs 3 rotations instead of 66. Same assertion (one profile call). The 66 came from an older token-keyed attempt map; today attempts and completed checks are keyed by route ID and account identity, a fresh stored profile skips the account first, and nothing in that path counts or caps rotations. mutations.toml: two Core rows (default pause skips the wait; gap set to 0), both proved CAUGHT with ckdev-mutate prove. --- mutations.toml | 32 +++++++++ packages/core/src/quota-manager.ts | 29 ++++++-- .../core/src/tests/quota-dispatch.test.ts | 39 ++++++++++ packages/opencode/src/index.ts | 4 ++ packages/opencode/src/tests/index.test.ts | 12 +++- .../opencode/src/tests/provider-error.test.ts | 42 ++++++++++- packages/pi/src/stream.ts | 14 ++++ packages/pi/src/tests/provider-error.test.ts | 71 +++++++++++++++---- 8 files changed, 219 insertions(+), 24 deletions(-) diff --git a/mutations.toml b/mutations.toml index 461c39fa7..791e51cb9 100644 --- a/mutations.toml +++ b/mutations.toml @@ -981,3 +981,35 @@ expect_message = "error: expect(received).toEqual(expected)" broad_command = ["bun", "scripts/run-mutation-bun.ts", "--broad", "packages/core", "tmp/mutations/core.xml"] broad_report = "tmp/mutations/core.xml" broad_id = "{name}" + +[[control]] +id = "quota-call-pause-default-waits" +guards = "Without an injected pause, QuotaManager waits on a real timer so physical quota GETs start at least one second apart." +file = "packages/core/src/quota-manager.ts" +old = "opts.pauseBetweenQuotaCalls ?? pauseOnRealTimer" +new = "opts.pauseBetweenQuotaCalls ?? (async () => {}) // NON-VACUITY BREAK: the default pause skips the wait" +test_file = "packages/core/src/tests/quota-dispatch.test.ts" +runner = "command" +command = ["bun", "scripts/run-mutation-bun.ts", "packages/core/src/tests/quota-dispatch.test.ts", "{test}"] +test_count_pattern = "CKDEV executed {count} tests" +expect_red = ["physical quota GET starts stay one second apart when the first GET fails"] +expect_message = "error: expect(received).toBeGreaterThanOrEqual(expected)" +broad_command = ["bun", "scripts/run-mutation-bun.ts", "--broad", "packages/core", "tmp/mutations/core.xml"] +broad_report = "tmp/mutations/core.xml" +broad_id = "{name}" + +[[control]] +id = "quota-call-gap-one-second" +guards = "Physical quota GETs from one QuotaManager start at least one second apart; a call inside that gap first waits out the rest of it." +file = "packages/core/src/quota-manager.ts" +old = "private static readonly API_CALL_GAP_MS = 1_000" +new = "private static readonly API_CALL_GAP_MS = 0 // NON-VACUITY BREAK: no spacing between quota calls" +test_file = "packages/core/src/tests/quota-dispatch.test.ts" +runner = "command" +command = ["bun", "scripts/run-mutation-bun.ts", "packages/core/src/tests/quota-dispatch.test.ts", "{test}"] +test_count_pattern = "CKDEV executed {count} tests" +expect_red = ["physical quota GET starts stay one second apart when the first GET fails", "the second physical quota GET first requests the rest of the one-second gap"] +expect_message = '/error: expect\(received\)\.(toBeGreaterThanOrEqual|toEqual)\(expected\)/' +broad_command = ["bun", "scripts/run-mutation-bun.ts", "--broad", "packages/core", "tmp/mutations/core.xml"] +broad_report = "tmp/mutations/core.xml" +broad_id = "{name}" diff --git a/packages/core/src/quota-manager.ts b/packages/core/src/quota-manager.ts index c3a716f41..7f3af20ab 100644 --- a/packages/core/src/quota-manager.ts +++ b/packages/core/src/quota-manager.ts @@ -37,6 +37,18 @@ import { tokenFingerprint } from './token-fingerprint.ts' // Capture real setTimeout before tests can mock globalThis.setTimeout const nativeSetTimeout = globalThis.setTimeout +/** + * The default pause between physical quota calls: a real timer that does not + * keep the process alive. It returns the timer's promise directly, without an + * async wrapper, so awaiting it takes the same turns as an inline timer. + */ +function pauseOnRealTimer(ms: number): Promise { + return new Promise((resolve) => { + const id = nativeSetTimeout(resolve, ms) + if (typeof id === 'object' && 'unref' in id) id.unref() + }) +} + // --------------------------------------------------------------------------- // Types // --------------------------------------------------------------------------- @@ -81,6 +93,14 @@ export type QuotaManagerOptions = { dispatch: QuotaRequestDispatch, ) => Promise now?: () => number + /** + * Waits before a physical usage GET until the minimum gap since the previous + * one has passed. It receives the milliseconds still to wait, measured with + * `now`, and is called only when that remainder is positive. Defaults to a + * real timer that does not keep the process alive. Tests pass one that + * records the requested wait and resolves at once. + */ + pauseBetweenQuotaCalls?: (ms: number) => Promise onMainQuotaFetched?: ( quota: OAuthQuotaSnapshot, checkedAt: number, @@ -180,6 +200,7 @@ export class QuotaManager { private readonly fetchImpl: typeof fetch private readonly fetchQuotaSnapshot: QuotaManagerOptions['fetchQuotaSnapshot'] private readonly now: () => number + private readonly pauseBetweenQuotaCalls: (ms: number) => Promise private readonly onMainQuotaFetched: QuotaManagerOptions['onMainQuotaFetched'] private readonly onApiError: QuotaManagerOptions['onApiError'] private readonly onFallbackQuotaFetched: QuotaManagerOptions['onFallbackQuotaFetched'] @@ -190,6 +211,8 @@ export class QuotaManager { this.fetchImpl = opts.fetchImpl ?? fetch this.fetchQuotaSnapshot = opts.fetchQuotaSnapshot this.now = opts.now ?? Date.now + this.pauseBetweenQuotaCalls = + opts.pauseBetweenQuotaCalls ?? pauseOnRealTimer this.onMainQuotaFetched = opts.onMainQuotaFetched this.onApiError = opts.onApiError this.onFallbackQuotaFetched = opts.onFallbackQuotaFetched @@ -945,11 +968,7 @@ export class QuotaManager { if (this.lastApiCallAt !== undefined) { const remaining = QuotaManager.API_CALL_GAP_MS - (this.now() - this.lastApiCallAt) - if (remaining > 0) - await new Promise((resolve) => { - const id = nativeSetTimeout(resolve, remaining) - if (typeof id === 'object' && 'unref' in id) id.unref() - }) + if (remaining > 0) await this.pauseBetweenQuotaCalls(remaining) } if (this.isClosed) throw new Error('QuotaManager is closed') // Authorization can precede the pacing wait. If custody changed or the diff --git a/packages/core/src/tests/quota-dispatch.test.ts b/packages/core/src/tests/quota-dispatch.test.ts index ff309def0..939d1b4b5 100644 --- a/packages/core/src/tests/quota-dispatch.test.ts +++ b/packages/core/src/tests/quota-dispatch.test.ts @@ -118,6 +118,45 @@ test('physical quota GET starts stay one second apart when the first GET fails', expect(starts[1]! - starts[0]!).toBeGreaterThanOrEqual(1_000) }) +test('the second physical quota GET first requests the rest of the one-second gap', async () => { + let clock = 5_000 + const pauses: number[] = [] + const starts: number[] = [] + const manager = ownManager( + new QuotaManager({ + storage: null, + now: () => clock, + // Record the requested wait and let the clock pass it, so no real + // time elapses and the gap is read from the request itself. + pauseBetweenQuotaCalls: async (ms) => { + pauses.push(ms) + clock += ms + }, + fetchImpl: Object.assign( + async () => { + starts.push(clock) + return Response.json(quotaResponse) + }, + { preconnect: fetch.preconnect }, + ), + }), + ) + await manager.refreshFallbackWithMetadata( + 'paced-first', + 'OAT-synthetic-paced-first', + undefined, + ) + expect(pauses).toEqual([]) + clock += 300 + await manager.refreshFallbackWithMetadata( + 'paced-second', + 'OAT-synthetic-paced-second', + undefined, + ) + expect(pauses).toEqual([700]) + expect(starts).toEqual([5_000, 6_000]) +}) + test('a quota lock loser neither sends a GET nor advances physical dispatch spacing', async () => { const held = await acquireRefreshFileLock({ name: 'opencode-fallback-quota-refresh-held', diff --git a/packages/opencode/src/index.ts b/packages/opencode/src/index.ts index 940284a30..95598f93b 100644 --- a/packages/opencode/src/index.ts +++ b/packages/opencode/src/index.ts @@ -171,6 +171,7 @@ import { type QuotaHeaderFeedPublishEntry, QuotaHeaderFeedRegistry, QuotaManager, + type QuotaManagerOptions, type QuotaState, quotaSnapshotCheckedAt, quotaSnapshotHasStandardWindows, @@ -1142,6 +1143,8 @@ type PluginRuntimeOverrides = Partial<{ claustrumScopedConnect: () => Promise nativeLocal: NativeAccountRuntimeOptions['local'] cacheKeepAggregateRefreshIntervalMs: number + /** Replaces the real-time wait between physical quota usage calls. */ + quotaCallPause: QuotaManagerOptions['pauseBetweenQuotaCalls'] }> function getConfiguredClaustrumConnectionFile(): string | undefined { @@ -1809,6 +1812,7 @@ const anthropicAuthPlugin = async ( // rejected refresh token became usable, so it must not clear refresh errors. const quotaManager = new QuotaManager({ storage: initialStorage, + pauseBetweenQuotaCalls: runtimeOverrides.quotaCallPause, fetchQuotaSnapshot: (request, dispatch) => { assertNativeEnvironment() const routeId = request.kind === 'main' ? 'main' : request.accountId diff --git a/packages/opencode/src/tests/index.test.ts b/packages/opencode/src/tests/index.test.ts index 585544b40..49033c372 100644 --- a/packages/opencode/src/tests/index.test.ts +++ b/packages/opencode/src/tests/index.test.ts @@ -12035,9 +12035,15 @@ describe('auth.loader', () => { (storage) => storage?.main?.profile?.accountIdentity === storage?.mainAccountId, ) - // Each generation is a refresh of the same account in another process, - // which rotates the pool's main access token. - for (let generation = 1; generation <= 66; generation++) { + // Each iteration simulates another process refreshing the same main + // account, which only rotates its access token. Profile hydration does + // not look at tokens: once the stored profile is fresh for this account + // identity the display skips the account, and a completed check is + // remembered under the route ID and account identity (storage row and + // login epoch are only consulted while a route's identity is unknown). + // Nothing there counts or caps rotations, so three rotations in a row + // cover what more would: none may cause a further profile fetch. + for (let generation = 1; generation <= 3; generation++) { await refreshPoolMainElsewhere({ access: `sk-ant-oat01-token-${generation}`, refresh: `refresh-sk-ant-oat01-token-${generation}`, diff --git a/packages/opencode/src/tests/provider-error.test.ts b/packages/opencode/src/tests/provider-error.test.ts index e5a1c57c9..b6a282382 100644 --- a/packages/opencode/src/tests/provider-error.test.ts +++ b/packages/opencode/src/tests/provider-error.test.ts @@ -465,6 +465,7 @@ async function localFixture(paid: boolean, paidKey = PAID_KEY) { async function createPlugin( owner: BodyOwner, client?: NativeCustodyClient | undefined, + quotaCallPause?: (ms: number) => Promise, ) { const creation = AnthropicAuthPlugin( { @@ -486,6 +487,7 @@ async function createPlugin( { scopedRosterPollIntervalMs: 0, cacheKeepAggregateRefreshIntervalMs: 0, + ...(quotaCallPause ? { quotaCallPause } : {}), ...(client ? { claustrumScopedConnect: async () => client } : {}), }, ) @@ -496,6 +498,34 @@ async function createPlugin( return plugin } +/** + * Stands in for the plugin's wait between quota usage polls. A route that hits + * a rate limit polls usage again soon after the startup poll, and the plugin + * spaces those polls one second apart. This records each requested wait and + * returns at once, so the test spends no real time on that spacing. + */ +function recordedQuotaPause() { + const requested: number[] = [] + return { + requested, + pause: async (ms: number) => { + requested.push(ms) + }, + } +} + +/** + * Each requested wait is what remains of the one-second gap since the previous + * poll, so it is positive and never longer than that gap. Whether a wait is + * requested at all depends on how long setup took, so none may be requested. + */ +function expectQuotaPollGaps(requested: readonly number[]) { + for (const ms of requested) { + expect(ms).toBeGreaterThan(0) + expect(ms).toBeLessThanOrEqual(1_000) + } +} + /** Sends one model request through the plugin's own request handler. */ async function sendNative( owner: BodyOwner, @@ -940,9 +970,11 @@ test('OC1 usage-limit echo still reaches the paid route, whose error omits the p ? jsonError(400, 'invalid_request_error', key) : jsonError(429, 'rate_limit_error', key) } - const plugin = await createPlugin(owner) + const quotaPause = recordedQuotaPause() + const plugin = await createPlugin(owner, undefined, quotaPause.pause) const seen = await visible(sendNative(owner, plugin, 'oc1-paid-error')) expect(modelRequests).toEqual([LOCAL_ROTATED, PAID_KEY]) + expectQuotaPollGaps(quotaPause.requested) expect(seen.status, seen.text).toBe(400) expectScrubbed(seen.text, [PAID_KEY, LOCAL_ROTATED]) expect(seen.text).toContain('invalid_request_error') @@ -1409,10 +1441,12 @@ async function streamedLimitRoute(primary: (key: string) => Response) { ? eventStream(healthy('paid route answered')) : primary(key) } - const plugin = await createPlugin(owner) + const quotaPause = recordedQuotaPause() + const plugin = await createPlugin(owner, undefined, quotaPause.pause) const seen = await visible( sendNative(owner, plugin, `oc1-limit-${Date.now()}`), ) + expectQuotaPollGaps(quotaPause.requested) return { routes, usagePolls, seen } } @@ -2006,10 +2040,12 @@ test('OC1 API lastUsed storage failure is detached, drained and warns without ch armed = true return eventStream(body) } - const plugin = await createPlugin(owner) + const quotaPause = recordedQuotaPause() + const plugin = await createPlugin(owner, undefined, quotaPause.pause) const response = await sendNative(owner, plugin, 'oc1-api-lastused-io') expect(response.status).toBe(200) expect(await response.text()).toBe(body) + expectQuotaPollGaps(quotaPause.requested) await entered.wait let disposed = false const disposing = Promise.resolve(plugin.dispose?.()).then(() => { diff --git a/packages/pi/src/stream.ts b/packages/pi/src/stream.ts index caa88323f..9b5f754a7 100644 --- a/packages/pi/src/stream.ts +++ b/packages/pi/src/stream.ts @@ -50,6 +50,7 @@ import { type ProviderErrorResponseScope, parseSseFrame, QuotaManager, + type QuotaManagerOptions, type QuotaState, quotaBackoffActive, quotaSnapshotHasStandardWindows, @@ -108,6 +109,18 @@ const quotaManagers = new Map< { identityKey: string; manager: QuotaManager } >() const PI_SERVICE_CACHE_LIMIT = 16 +/** + * Tests only: replaces the real-time wait between quota usage polls in quota + * managers created after it is set. Unset, managers keep Core's real timer. + */ +let quotaCallPauseForTest: QuotaManagerOptions['pauseBetweenQuotaCalls'] + +/** Set or clear the test replacement for the wait between quota usage polls. */ +export function __setPiQuotaCallPauseForTest( + pause: QuotaManagerOptions['pauseBetweenQuotaCalls'], +) { + quotaCallPauseForTest = pause +} function setBoundedService(map: Map, key: string, value: T) { map.delete(key) @@ -141,6 +154,7 @@ function getPiRoutingServices( if (!quotaManager) { quotaManager = new QuotaManager({ storage, + pauseBetweenQuotaCalls: quotaCallPauseForTest, fetchQuotaSnapshot: async (request, dispatch) => { const id = request.kind === 'main' ? 'main' : request.accountId if (!id) throw new Error('Native quota request has no account identity') diff --git a/packages/pi/src/tests/provider-error.test.ts b/packages/pi/src/tests/provider-error.test.ts index 986b7756a..4817ab8e0 100644 --- a/packages/pi/src/tests/provider-error.test.ts +++ b/packages/pi/src/tests/provider-error.test.ts @@ -22,7 +22,10 @@ import { nativeVaultPaths } from '../../../core/src/native-vault-paths.ts' import type { TestGate } from '../../../core/src/tests/test-lifetime.ts' import { createTestLifetimeSuite } from '../../../core/src/tests/test-lifetime.ts' import { closePiNativeRuntime, getPiNativeRuntime } from '../native.ts' -import { streamCortexKitAnthropic } from '../stream.ts' +import { + __setPiQuotaCallPauseForTest, + streamCortexKitAnthropic, +} from '../stream.ts' import { fixtureAccountIdentity, saveNativePiFixture, @@ -458,8 +461,37 @@ test('Pi thrown relay error omits the private relay token and the credential', a ) }) +/** + * Stands in for Pi's wait between quota usage polls in this test. A route that + * hits a rate limit polls usage again soon after the first poll, and Pi spaces + * those polls one second apart. This records each requested wait and returns + * at once, so the test spends no real time on that spacing. Set it before the + * first request: Pi reads it when it creates the storage's quota manager. + */ +function recordQuotaPauses() { + const requested: number[] = [] + __setPiQuotaCallPauseForTest(async (ms) => { + requested.push(ms) + }) + deferCleanup(() => __setPiQuotaCallPauseForTest(undefined)) + return requested +} + +/** + * Each requested wait is what remains of the one-second gap since the previous + * poll, so it is positive and never longer than that gap. Whether a wait is + * requested at all depends on how long setup took, so none may be requested. + */ +function expectQuotaPollGaps(requested: readonly number[]) { + for (const ms of requested) { + expect(ms).toBeGreaterThan(0) + expect(ms).toBeLessThanOrEqual(1_000) + } +} + /** Create a local OAuth account with a paid API fallback for general quota exhaustion. */ async function paidFixture(apiKey = PAID_KEY) { + const quotaPauses = recordQuotaPauses() const { directory, storagePath } = await ownDirectory( 'pi-provider-error-paid-', ) @@ -478,11 +510,11 @@ async function paidFixture(apiKey = PAID_KEY) { storagePath, { mainAccess: MAIN_ACCESS }, ) - return { directory } + return { directory, quotaPauses } } test('Pi usage-limit echo still reaches the paid route, whose error omits the paid key', async () => { - await paidFixture() + const { quotaPauses } = await paidFixture() const modelRequests: string[] = [] useFetch(async (url, init) => { if (url.includes('/api/oauth/usage')) @@ -503,10 +535,11 @@ test('Pi usage-limit echo still reaches the paid route, whose error omits the pa ['HTTP 400', 'invalid_request_error'], ) expect(modelRequests).toEqual([`Bearer ${MAIN_ACCESS}`, `Bearer ${PAID_KEY}`]) + expectQuotaPollGaps(quotaPauses) }) test('Pi usage-limit echo without exhaustion keeps the primary 429 and scrubs it', async () => { - await paidFixture() + const { quotaPauses } = await paidFixture() const modelRequests: string[] = [] useFetch(async (url, init) => { if (url.includes('/api/oauth/usage')) @@ -525,6 +558,7 @@ test('Pi usage-limit echo without exhaustion keeps the primary 429 and scrubs it ['HTTP 429', 'rate_limit_error'], ) expect(modelRequests).toEqual([`Bearer ${MAIN_ACCESS}`]) + expectQuotaPollGaps(quotaPauses) }) // A streamed rate limit arrives as an `error` event whose `error.type` is @@ -537,6 +571,7 @@ const BACKUP_ACCESS = 'sk-ant-oat01-synthetic-pi-backup-access' const FABLE = 'claude-fable-5-1' async function routingFixture(backup: boolean) { + const quotaPauses = recordQuotaPauses() const { storagePath } = await ownDirectory('pi-streamed-limit-') const paid: AccountStorage['accounts'][number] = { type: 'api', @@ -555,6 +590,7 @@ async function routingFixture(backup: boolean) { storagePath, { mainAccess: MAIN_ACCESS }, ) + return { quotaPauses } } const usageReading = (general: number, fable: number) => @@ -606,7 +642,7 @@ async function runModel(id: string, sessionId: string) { } test('Pi streamed rate_limit_error with the primary generally exhausted reaches the paid route', async () => { - await routingFixture(false) + const { quotaPauses } = await routingFixture(false) const record = routingFetch( () => eventStream(sse(errorEnvelope('rate_limit_error', MAIN_ACCESS))), () => usageReading(100, 100), @@ -618,10 +654,11 @@ test('Pi streamed rate_limit_error with the primary generally exhausted reaches expect(result.content).toEqual([ { type: 'text', text: 'another route answered' }, ]) + expectQuotaPollGaps(quotaPauses) }) test('Pi streamed rate_limit_error with only a model-scoped limit moves to OAuth, never the paid route', async () => { - await routingFixture(true) + const { quotaPauses } = await routingFixture(true) const record = routingFetch( () => eventStream(sse(errorEnvelope('rate_limit_error', MAIN_ACCESS))), () => usageReading(10, 100), @@ -634,10 +671,11 @@ test('Pi streamed rate_limit_error with only a model-scoped limit moves to OAuth expect(record.models).not.toContain(`Bearer ${PAID_KEY}`) expect(record.usagePolls).toBeGreaterThan(0) expect(result.stopReason).toBe('stop') + expectQuotaPollGaps(quotaPauses) }) test('Pi reply text that mentions rate limits neither refreshes quota nor reroutes', async () => { - await routingFixture(true) + const { quotaPauses } = await routingFixture(true) const text = "Notes: a rate_limit_error means you exceed your account's rate limit." const record = routingFetch( @@ -649,6 +687,7 @@ test('Pi reply text that mentions rate limits neither refreshes quota nor rerout expect(record.usagePolls).toBe(0) expect(result.stopReason).toBe('stop') expect(result.content).toEqual([{ type: 'text', text }]) + expectQuotaPollGaps(quotaPauses) }) for (const contentType of ['text/plain', undefined] as const) { @@ -875,7 +914,7 @@ const rateLimitFrame = (wire: Wire) => const manyComments = `: ${'k'.repeat(1_020)}\n\n`.repeat(80) test('Pi streamed rate limit on a kept-open primary cancels its provider body before the paid route answers', async () => { - await routingFixture(false) + const { quotaPauses } = await routingFixture(false) const cancelled = gate() const record = loopbackRouting((wire) => keptOpen(`: keepalive\n\n${rateLimitFrame(wire)}`, cancelled), @@ -887,10 +926,11 @@ test('Pi streamed rate limit on a kept-open primary cancels its provider body be { type: 'text', text: 'another route answered' }, ]) expect(await within(cancelled.wait)).toBe(true) + expectQuotaPollGaps(quotaPauses) }) test('Pi streamed rate limit on a kept-open fallback route is cancelled and the next route answers', async () => { - await routingFixture(true) + const { quotaPauses } = await routingFixture(true) const cancelled = gate() const record = loopbackRouting( (wire) => jsonError(429, 'rate_limit_error', wire.bearer), @@ -928,10 +968,11 @@ test('Pi streamed rate limit on a kept-open fallback route is cancelled and the await within(cancelled.wait), `phase: backup cancellation\n${trace()}`, ).toBe(true) + expectQuotaPollGaps(quotaPauses) }) test('Pi passes a stream through unclassified once comments exhaust the first-event budget', async () => { - await routingFixture(false) + const { quotaPauses } = await routingFixture(false) const record = loopbackRouting( (wire) => new Response(manyComments + rateLimitFrame(wire), { @@ -952,10 +993,11 @@ test('Pi passes a stream through unclassified once comments exhaust the first-ev expect(result.stopReason).toBe('error') expect(result.errorMessage).toContain('rate_limit_error') expectScrubbed(result.errorMessage ?? '', [MAIN_ACCESS]) + expectQuotaPollGaps(quotaPauses) }) test('Pi keeps a successful reply that follows comments past the first-event budget byte for byte', async () => { - await routingFixture(false) + const { quotaPauses } = await routingFixture(false) const text = `début 🧪 ${echoOf(MAIN_ACCESS)}` const record = loopbackRouting( () => @@ -968,6 +1010,7 @@ test('Pi keeps a successful reply that follows comments past the first-event bud expect(record.usagePolls).toBe(0) expect(result.stopReason).toBe('stop') expect(result.content).toEqual([{ type: 'text', text }]) + expectQuotaPollGaps(quotaPauses) }) function captureUsageLogs() { @@ -1153,7 +1196,7 @@ test('Pi serves first delta and full reply while the roster lock is held, then d }) test('Pi API lastUsed storage failure is detached, drained and warns without changing reply content', async () => { - await routingFixture(false) + const { quotaPauses } = await routingFixture(false) const native = getPiNativeRuntime() const service = await native.service() const logs = captureUsageLogs() @@ -1200,6 +1243,7 @@ test('Pi API lastUsed storage failure is detached, drained and warns without cha payload: { name: 'NativeRuntimeError', code: 'runtime-io' }, }, ]) + expectQuotaPollGaps(quotaPauses) }) test('Pi runtime eviction drains the evicted instance before closing its native service', async () => { @@ -1333,7 +1377,7 @@ test('Pi refuses to store a paid key with surrounding whitespace', async () => { }) test('Pi never sends a hand-padded stored paid key', async () => { - const { directory } = await paidFixture() + const { directory, quotaPauses } = await paidFixture() expect(await padStoredKey(directory)).toBeGreaterThan(0) const paidRequests: string[] = [] const models: string[] = [] @@ -1359,4 +1403,5 @@ test('Pi never sends a hand-padded stored paid key', async () => { ) expect(models).toEqual(['api.anthropic.com']) expect(paidRequests).toEqual([]) + expectQuotaPollGaps(quotaPauses) })