diff --git a/mutations.toml b/mutations.toml index 461c39fa..791e51cb 100644 --- a/mutations.toml +++ b/mutations.toml @@ -981,3 +981,35 @@ expect_message = "error: expect(received).toEqual(expected)" broad_command = ["bun", "scripts/run-mutation-bun.ts", "--broad", "packages/core", "tmp/mutations/core.xml"] broad_report = "tmp/mutations/core.xml" broad_id = "{name}" + +[[control]] +id = "quota-call-pause-default-waits" +guards = "Without an injected pause, QuotaManager waits on a real timer so physical quota GETs start at least one second apart." +file = "packages/core/src/quota-manager.ts" +old = "opts.pauseBetweenQuotaCalls ?? pauseOnRealTimer" +new = "opts.pauseBetweenQuotaCalls ?? (async () => {}) // NON-VACUITY BREAK: the default pause skips the wait" +test_file = "packages/core/src/tests/quota-dispatch.test.ts" +runner = "command" +command = ["bun", "scripts/run-mutation-bun.ts", "packages/core/src/tests/quota-dispatch.test.ts", "{test}"] +test_count_pattern = "CKDEV executed {count} tests" +expect_red = ["physical quota GET starts stay one second apart when the first GET fails"] +expect_message = "error: expect(received).toBeGreaterThanOrEqual(expected)" +broad_command = ["bun", "scripts/run-mutation-bun.ts", "--broad", "packages/core", "tmp/mutations/core.xml"] +broad_report = "tmp/mutations/core.xml" +broad_id = "{name}" + +[[control]] +id = "quota-call-gap-one-second" +guards = "Physical quota GETs from one QuotaManager start at least one second apart; a call inside that gap first waits out the rest of it." +file = "packages/core/src/quota-manager.ts" +old = "private static readonly API_CALL_GAP_MS = 1_000" +new = "private static readonly API_CALL_GAP_MS = 0 // NON-VACUITY BREAK: no spacing between quota calls" +test_file = "packages/core/src/tests/quota-dispatch.test.ts" +runner = "command" +command = ["bun", "scripts/run-mutation-bun.ts", "packages/core/src/tests/quota-dispatch.test.ts", "{test}"] +test_count_pattern = "CKDEV executed {count} tests" +expect_red = ["physical quota GET starts stay one second apart when the first GET fails", "the second physical quota GET first requests the rest of the one-second gap"] +expect_message = '/error: expect\(received\)\.(toBeGreaterThanOrEqual|toEqual)\(expected\)/' +broad_command = ["bun", "scripts/run-mutation-bun.ts", "--broad", "packages/core", "tmp/mutations/core.xml"] +broad_report = "tmp/mutations/core.xml" +broad_id = "{name}" diff --git a/packages/core/src/quota-manager.ts b/packages/core/src/quota-manager.ts index c3a716f4..7f3af20a 100644 --- a/packages/core/src/quota-manager.ts +++ b/packages/core/src/quota-manager.ts @@ -37,6 +37,18 @@ import { tokenFingerprint } from './token-fingerprint.ts' // Capture real setTimeout before tests can mock globalThis.setTimeout const nativeSetTimeout = globalThis.setTimeout +/** + * The default pause between physical quota calls: a real timer that does not + * keep the process alive. It returns the timer's promise directly, without an + * async wrapper, so awaiting it takes the same turns as an inline timer. + */ +function pauseOnRealTimer(ms: number): Promise { + return new Promise((resolve) => { + const id = nativeSetTimeout(resolve, ms) + if (typeof id === 'object' && 'unref' in id) id.unref() + }) +} + // --------------------------------------------------------------------------- // Types // --------------------------------------------------------------------------- @@ -81,6 +93,14 @@ export type QuotaManagerOptions = { dispatch: QuotaRequestDispatch, ) => Promise now?: () => number + /** + * Waits before a physical usage GET until the minimum gap since the previous + * one has passed. It receives the milliseconds still to wait, measured with + * `now`, and is called only when that remainder is positive. Defaults to a + * real timer that does not keep the process alive. Tests pass one that + * records the requested wait and resolves at once. + */ + pauseBetweenQuotaCalls?: (ms: number) => Promise onMainQuotaFetched?: ( quota: OAuthQuotaSnapshot, checkedAt: number, @@ -180,6 +200,7 @@ export class QuotaManager { private readonly fetchImpl: typeof fetch private readonly fetchQuotaSnapshot: QuotaManagerOptions['fetchQuotaSnapshot'] private readonly now: () => number + private readonly pauseBetweenQuotaCalls: (ms: number) => Promise private readonly onMainQuotaFetched: QuotaManagerOptions['onMainQuotaFetched'] private readonly onApiError: QuotaManagerOptions['onApiError'] private readonly onFallbackQuotaFetched: QuotaManagerOptions['onFallbackQuotaFetched'] @@ -190,6 +211,8 @@ export class QuotaManager { this.fetchImpl = opts.fetchImpl ?? fetch this.fetchQuotaSnapshot = opts.fetchQuotaSnapshot this.now = opts.now ?? Date.now + this.pauseBetweenQuotaCalls = + opts.pauseBetweenQuotaCalls ?? pauseOnRealTimer this.onMainQuotaFetched = opts.onMainQuotaFetched this.onApiError = opts.onApiError this.onFallbackQuotaFetched = opts.onFallbackQuotaFetched @@ -945,11 +968,7 @@ export class QuotaManager { if (this.lastApiCallAt !== undefined) { const remaining = QuotaManager.API_CALL_GAP_MS - (this.now() - this.lastApiCallAt) - if (remaining > 0) - await new Promise((resolve) => { - const id = nativeSetTimeout(resolve, remaining) - if (typeof id === 'object' && 'unref' in id) id.unref() - }) + if (remaining > 0) await this.pauseBetweenQuotaCalls(remaining) } if (this.isClosed) throw new Error('QuotaManager is closed') // Authorization can precede the pacing wait. If custody changed or the diff --git a/packages/core/src/tests/quota-dispatch.test.ts b/packages/core/src/tests/quota-dispatch.test.ts index ff309def..939d1b4b 100644 --- a/packages/core/src/tests/quota-dispatch.test.ts +++ b/packages/core/src/tests/quota-dispatch.test.ts @@ -118,6 +118,45 @@ test('physical quota GET starts stay one second apart when the first GET fails', expect(starts[1]! - starts[0]!).toBeGreaterThanOrEqual(1_000) }) +test('the second physical quota GET first requests the rest of the one-second gap', async () => { + let clock = 5_000 + const pauses: number[] = [] + const starts: number[] = [] + const manager = ownManager( + new QuotaManager({ + storage: null, + now: () => clock, + // Record the requested wait and let the clock pass it, so no real + // time elapses and the gap is read from the request itself. + pauseBetweenQuotaCalls: async (ms) => { + pauses.push(ms) + clock += ms + }, + fetchImpl: Object.assign( + async () => { + starts.push(clock) + return Response.json(quotaResponse) + }, + { preconnect: fetch.preconnect }, + ), + }), + ) + await manager.refreshFallbackWithMetadata( + 'paced-first', + 'OAT-synthetic-paced-first', + undefined, + ) + expect(pauses).toEqual([]) + clock += 300 + await manager.refreshFallbackWithMetadata( + 'paced-second', + 'OAT-synthetic-paced-second', + undefined, + ) + expect(pauses).toEqual([700]) + expect(starts).toEqual([5_000, 6_000]) +}) + test('a quota lock loser neither sends a GET nor advances physical dispatch spacing', async () => { const held = await acquireRefreshFileLock({ name: 'opencode-fallback-quota-refresh-held', diff --git a/packages/opencode/src/index.ts b/packages/opencode/src/index.ts index 940284a3..95598f93 100644 --- a/packages/opencode/src/index.ts +++ b/packages/opencode/src/index.ts @@ -171,6 +171,7 @@ import { type QuotaHeaderFeedPublishEntry, QuotaHeaderFeedRegistry, QuotaManager, + type QuotaManagerOptions, type QuotaState, quotaSnapshotCheckedAt, quotaSnapshotHasStandardWindows, @@ -1142,6 +1143,8 @@ type PluginRuntimeOverrides = Partial<{ claustrumScopedConnect: () => Promise nativeLocal: NativeAccountRuntimeOptions['local'] cacheKeepAggregateRefreshIntervalMs: number + /** Replaces the real-time wait between physical quota usage calls. */ + quotaCallPause: QuotaManagerOptions['pauseBetweenQuotaCalls'] }> function getConfiguredClaustrumConnectionFile(): string | undefined { @@ -1809,6 +1812,7 @@ const anthropicAuthPlugin = async ( // rejected refresh token became usable, so it must not clear refresh errors. const quotaManager = new QuotaManager({ storage: initialStorage, + pauseBetweenQuotaCalls: runtimeOverrides.quotaCallPause, fetchQuotaSnapshot: (request, dispatch) => { assertNativeEnvironment() const routeId = request.kind === 'main' ? 'main' : request.accountId diff --git a/packages/opencode/src/tests/index.test.ts b/packages/opencode/src/tests/index.test.ts index 585544b4..49033c37 100644 --- a/packages/opencode/src/tests/index.test.ts +++ b/packages/opencode/src/tests/index.test.ts @@ -12035,9 +12035,15 @@ describe('auth.loader', () => { (storage) => storage?.main?.profile?.accountIdentity === storage?.mainAccountId, ) - // Each generation is a refresh of the same account in another process, - // which rotates the pool's main access token. - for (let generation = 1; generation <= 66; generation++) { + // Each iteration simulates another process refreshing the same main + // account, which only rotates its access token. Profile hydration does + // not look at tokens: once the stored profile is fresh for this account + // identity the display skips the account, and a completed check is + // remembered under the route ID and account identity (storage row and + // login epoch are only consulted while a route's identity is unknown). + // Nothing there counts or caps rotations, so three rotations in a row + // cover what more would: none may cause a further profile fetch. + for (let generation = 1; generation <= 3; generation++) { await refreshPoolMainElsewhere({ access: `sk-ant-oat01-token-${generation}`, refresh: `refresh-sk-ant-oat01-token-${generation}`, diff --git a/packages/opencode/src/tests/provider-error.test.ts b/packages/opencode/src/tests/provider-error.test.ts index e5a1c57c..b6a28238 100644 --- a/packages/opencode/src/tests/provider-error.test.ts +++ b/packages/opencode/src/tests/provider-error.test.ts @@ -465,6 +465,7 @@ async function localFixture(paid: boolean, paidKey = PAID_KEY) { async function createPlugin( owner: BodyOwner, client?: NativeCustodyClient | undefined, + quotaCallPause?: (ms: number) => Promise, ) { const creation = AnthropicAuthPlugin( { @@ -486,6 +487,7 @@ async function createPlugin( { scopedRosterPollIntervalMs: 0, cacheKeepAggregateRefreshIntervalMs: 0, + ...(quotaCallPause ? { quotaCallPause } : {}), ...(client ? { claustrumScopedConnect: async () => client } : {}), }, ) @@ -496,6 +498,34 @@ async function createPlugin( return plugin } +/** + * Stands in for the plugin's wait between quota usage polls. A route that hits + * a rate limit polls usage again soon after the startup poll, and the plugin + * spaces those polls one second apart. This records each requested wait and + * returns at once, so the test spends no real time on that spacing. + */ +function recordedQuotaPause() { + const requested: number[] = [] + return { + requested, + pause: async (ms: number) => { + requested.push(ms) + }, + } +} + +/** + * Each requested wait is what remains of the one-second gap since the previous + * poll, so it is positive and never longer than that gap. Whether a wait is + * requested at all depends on how long setup took, so none may be requested. + */ +function expectQuotaPollGaps(requested: readonly number[]) { + for (const ms of requested) { + expect(ms).toBeGreaterThan(0) + expect(ms).toBeLessThanOrEqual(1_000) + } +} + /** Sends one model request through the plugin's own request handler. */ async function sendNative( owner: BodyOwner, @@ -940,9 +970,11 @@ test('OC1 usage-limit echo still reaches the paid route, whose error omits the p ? jsonError(400, 'invalid_request_error', key) : jsonError(429, 'rate_limit_error', key) } - const plugin = await createPlugin(owner) + const quotaPause = recordedQuotaPause() + const plugin = await createPlugin(owner, undefined, quotaPause.pause) const seen = await visible(sendNative(owner, plugin, 'oc1-paid-error')) expect(modelRequests).toEqual([LOCAL_ROTATED, PAID_KEY]) + expectQuotaPollGaps(quotaPause.requested) expect(seen.status, seen.text).toBe(400) expectScrubbed(seen.text, [PAID_KEY, LOCAL_ROTATED]) expect(seen.text).toContain('invalid_request_error') @@ -1409,10 +1441,12 @@ async function streamedLimitRoute(primary: (key: string) => Response) { ? eventStream(healthy('paid route answered')) : primary(key) } - const plugin = await createPlugin(owner) + const quotaPause = recordedQuotaPause() + const plugin = await createPlugin(owner, undefined, quotaPause.pause) const seen = await visible( sendNative(owner, plugin, `oc1-limit-${Date.now()}`), ) + expectQuotaPollGaps(quotaPause.requested) return { routes, usagePolls, seen } } @@ -2006,10 +2040,12 @@ test('OC1 API lastUsed storage failure is detached, drained and warns without ch armed = true return eventStream(body) } - const plugin = await createPlugin(owner) + const quotaPause = recordedQuotaPause() + const plugin = await createPlugin(owner, undefined, quotaPause.pause) const response = await sendNative(owner, plugin, 'oc1-api-lastused-io') expect(response.status).toBe(200) expect(await response.text()).toBe(body) + expectQuotaPollGaps(quotaPause.requested) await entered.wait let disposed = false const disposing = Promise.resolve(plugin.dispose?.()).then(() => { diff --git a/packages/pi/src/stream.ts b/packages/pi/src/stream.ts index caa88323..9b5f754a 100644 --- a/packages/pi/src/stream.ts +++ b/packages/pi/src/stream.ts @@ -50,6 +50,7 @@ import { type ProviderErrorResponseScope, parseSseFrame, QuotaManager, + type QuotaManagerOptions, type QuotaState, quotaBackoffActive, quotaSnapshotHasStandardWindows, @@ -108,6 +109,18 @@ const quotaManagers = new Map< { identityKey: string; manager: QuotaManager } >() const PI_SERVICE_CACHE_LIMIT = 16 +/** + * Tests only: replaces the real-time wait between quota usage polls in quota + * managers created after it is set. Unset, managers keep Core's real timer. + */ +let quotaCallPauseForTest: QuotaManagerOptions['pauseBetweenQuotaCalls'] + +/** Set or clear the test replacement for the wait between quota usage polls. */ +export function __setPiQuotaCallPauseForTest( + pause: QuotaManagerOptions['pauseBetweenQuotaCalls'], +) { + quotaCallPauseForTest = pause +} function setBoundedService(map: Map, key: string, value: T) { map.delete(key) @@ -141,6 +154,7 @@ function getPiRoutingServices( if (!quotaManager) { quotaManager = new QuotaManager({ storage, + pauseBetweenQuotaCalls: quotaCallPauseForTest, fetchQuotaSnapshot: async (request, dispatch) => { const id = request.kind === 'main' ? 'main' : request.accountId if (!id) throw new Error('Native quota request has no account identity') diff --git a/packages/pi/src/tests/provider-error.test.ts b/packages/pi/src/tests/provider-error.test.ts index 986b7756..4817ab8e 100644 --- a/packages/pi/src/tests/provider-error.test.ts +++ b/packages/pi/src/tests/provider-error.test.ts @@ -22,7 +22,10 @@ import { nativeVaultPaths } from '../../../core/src/native-vault-paths.ts' import type { TestGate } from '../../../core/src/tests/test-lifetime.ts' import { createTestLifetimeSuite } from '../../../core/src/tests/test-lifetime.ts' import { closePiNativeRuntime, getPiNativeRuntime } from '../native.ts' -import { streamCortexKitAnthropic } from '../stream.ts' +import { + __setPiQuotaCallPauseForTest, + streamCortexKitAnthropic, +} from '../stream.ts' import { fixtureAccountIdentity, saveNativePiFixture, @@ -458,8 +461,37 @@ test('Pi thrown relay error omits the private relay token and the credential', a ) }) +/** + * Stands in for Pi's wait between quota usage polls in this test. A route that + * hits a rate limit polls usage again soon after the first poll, and Pi spaces + * those polls one second apart. This records each requested wait and returns + * at once, so the test spends no real time on that spacing. Set it before the + * first request: Pi reads it when it creates the storage's quota manager. + */ +function recordQuotaPauses() { + const requested: number[] = [] + __setPiQuotaCallPauseForTest(async (ms) => { + requested.push(ms) + }) + deferCleanup(() => __setPiQuotaCallPauseForTest(undefined)) + return requested +} + +/** + * Each requested wait is what remains of the one-second gap since the previous + * poll, so it is positive and never longer than that gap. Whether a wait is + * requested at all depends on how long setup took, so none may be requested. + */ +function expectQuotaPollGaps(requested: readonly number[]) { + for (const ms of requested) { + expect(ms).toBeGreaterThan(0) + expect(ms).toBeLessThanOrEqual(1_000) + } +} + /** Create a local OAuth account with a paid API fallback for general quota exhaustion. */ async function paidFixture(apiKey = PAID_KEY) { + const quotaPauses = recordQuotaPauses() const { directory, storagePath } = await ownDirectory( 'pi-provider-error-paid-', ) @@ -478,11 +510,11 @@ async function paidFixture(apiKey = PAID_KEY) { storagePath, { mainAccess: MAIN_ACCESS }, ) - return { directory } + return { directory, quotaPauses } } test('Pi usage-limit echo still reaches the paid route, whose error omits the paid key', async () => { - await paidFixture() + const { quotaPauses } = await paidFixture() const modelRequests: string[] = [] useFetch(async (url, init) => { if (url.includes('/api/oauth/usage')) @@ -503,10 +535,11 @@ test('Pi usage-limit echo still reaches the paid route, whose error omits the pa ['HTTP 400', 'invalid_request_error'], ) expect(modelRequests).toEqual([`Bearer ${MAIN_ACCESS}`, `Bearer ${PAID_KEY}`]) + expectQuotaPollGaps(quotaPauses) }) test('Pi usage-limit echo without exhaustion keeps the primary 429 and scrubs it', async () => { - await paidFixture() + const { quotaPauses } = await paidFixture() const modelRequests: string[] = [] useFetch(async (url, init) => { if (url.includes('/api/oauth/usage')) @@ -525,6 +558,7 @@ test('Pi usage-limit echo without exhaustion keeps the primary 429 and scrubs it ['HTTP 429', 'rate_limit_error'], ) expect(modelRequests).toEqual([`Bearer ${MAIN_ACCESS}`]) + expectQuotaPollGaps(quotaPauses) }) // A streamed rate limit arrives as an `error` event whose `error.type` is @@ -537,6 +571,7 @@ const BACKUP_ACCESS = 'sk-ant-oat01-synthetic-pi-backup-access' const FABLE = 'claude-fable-5-1' async function routingFixture(backup: boolean) { + const quotaPauses = recordQuotaPauses() const { storagePath } = await ownDirectory('pi-streamed-limit-') const paid: AccountStorage['accounts'][number] = { type: 'api', @@ -555,6 +590,7 @@ async function routingFixture(backup: boolean) { storagePath, { mainAccess: MAIN_ACCESS }, ) + return { quotaPauses } } const usageReading = (general: number, fable: number) => @@ -606,7 +642,7 @@ async function runModel(id: string, sessionId: string) { } test('Pi streamed rate_limit_error with the primary generally exhausted reaches the paid route', async () => { - await routingFixture(false) + const { quotaPauses } = await routingFixture(false) const record = routingFetch( () => eventStream(sse(errorEnvelope('rate_limit_error', MAIN_ACCESS))), () => usageReading(100, 100), @@ -618,10 +654,11 @@ test('Pi streamed rate_limit_error with the primary generally exhausted reaches expect(result.content).toEqual([ { type: 'text', text: 'another route answered' }, ]) + expectQuotaPollGaps(quotaPauses) }) test('Pi streamed rate_limit_error with only a model-scoped limit moves to OAuth, never the paid route', async () => { - await routingFixture(true) + const { quotaPauses } = await routingFixture(true) const record = routingFetch( () => eventStream(sse(errorEnvelope('rate_limit_error', MAIN_ACCESS))), () => usageReading(10, 100), @@ -634,10 +671,11 @@ test('Pi streamed rate_limit_error with only a model-scoped limit moves to OAuth expect(record.models).not.toContain(`Bearer ${PAID_KEY}`) expect(record.usagePolls).toBeGreaterThan(0) expect(result.stopReason).toBe('stop') + expectQuotaPollGaps(quotaPauses) }) test('Pi reply text that mentions rate limits neither refreshes quota nor reroutes', async () => { - await routingFixture(true) + const { quotaPauses } = await routingFixture(true) const text = "Notes: a rate_limit_error means you exceed your account's rate limit." const record = routingFetch( @@ -649,6 +687,7 @@ test('Pi reply text that mentions rate limits neither refreshes quota nor rerout expect(record.usagePolls).toBe(0) expect(result.stopReason).toBe('stop') expect(result.content).toEqual([{ type: 'text', text }]) + expectQuotaPollGaps(quotaPauses) }) for (const contentType of ['text/plain', undefined] as const) { @@ -875,7 +914,7 @@ const rateLimitFrame = (wire: Wire) => const manyComments = `: ${'k'.repeat(1_020)}\n\n`.repeat(80) test('Pi streamed rate limit on a kept-open primary cancels its provider body before the paid route answers', async () => { - await routingFixture(false) + const { quotaPauses } = await routingFixture(false) const cancelled = gate() const record = loopbackRouting((wire) => keptOpen(`: keepalive\n\n${rateLimitFrame(wire)}`, cancelled), @@ -887,10 +926,11 @@ test('Pi streamed rate limit on a kept-open primary cancels its provider body be { type: 'text', text: 'another route answered' }, ]) expect(await within(cancelled.wait)).toBe(true) + expectQuotaPollGaps(quotaPauses) }) test('Pi streamed rate limit on a kept-open fallback route is cancelled and the next route answers', async () => { - await routingFixture(true) + const { quotaPauses } = await routingFixture(true) const cancelled = gate() const record = loopbackRouting( (wire) => jsonError(429, 'rate_limit_error', wire.bearer), @@ -928,10 +968,11 @@ test('Pi streamed rate limit on a kept-open fallback route is cancelled and the await within(cancelled.wait), `phase: backup cancellation\n${trace()}`, ).toBe(true) + expectQuotaPollGaps(quotaPauses) }) test('Pi passes a stream through unclassified once comments exhaust the first-event budget', async () => { - await routingFixture(false) + const { quotaPauses } = await routingFixture(false) const record = loopbackRouting( (wire) => new Response(manyComments + rateLimitFrame(wire), { @@ -952,10 +993,11 @@ test('Pi passes a stream through unclassified once comments exhaust the first-ev expect(result.stopReason).toBe('error') expect(result.errorMessage).toContain('rate_limit_error') expectScrubbed(result.errorMessage ?? '', [MAIN_ACCESS]) + expectQuotaPollGaps(quotaPauses) }) test('Pi keeps a successful reply that follows comments past the first-event budget byte for byte', async () => { - await routingFixture(false) + const { quotaPauses } = await routingFixture(false) const text = `début 🧪 ${echoOf(MAIN_ACCESS)}` const record = loopbackRouting( () => @@ -968,6 +1010,7 @@ test('Pi keeps a successful reply that follows comments past the first-event bud expect(record.usagePolls).toBe(0) expect(result.stopReason).toBe('stop') expect(result.content).toEqual([{ type: 'text', text }]) + expectQuotaPollGaps(quotaPauses) }) function captureUsageLogs() { @@ -1153,7 +1196,7 @@ test('Pi serves first delta and full reply while the roster lock is held, then d }) test('Pi API lastUsed storage failure is detached, drained and warns without changing reply content', async () => { - await routingFixture(false) + const { quotaPauses } = await routingFixture(false) const native = getPiNativeRuntime() const service = await native.service() const logs = captureUsageLogs() @@ -1200,6 +1243,7 @@ test('Pi API lastUsed storage failure is detached, drained and warns without cha payload: { name: 'NativeRuntimeError', code: 'runtime-io' }, }, ]) + expectQuotaPollGaps(quotaPauses) }) test('Pi runtime eviction drains the evicted instance before closing its native service', async () => { @@ -1333,7 +1377,7 @@ test('Pi refuses to store a paid key with surrounding whitespace', async () => { }) test('Pi never sends a hand-padded stored paid key', async () => { - const { directory } = await paidFixture() + const { directory, quotaPauses } = await paidFixture() expect(await padStoredKey(directory)).toBeGreaterThan(0) const paidRequests: string[] = [] const models: string[] = [] @@ -1359,4 +1403,5 @@ test('Pi never sends a hand-padded stored paid key', async () => { ) expect(models).toEqual(['api.anthropic.com']) expect(paidRequests).toEqual([]) + expectQuotaPollGaps(quotaPauses) })