diff --git a/CHANGELOG.md b/CHANGELOG.md index c1aa99e0..c6684c49 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ This repo is a CortexKit-maintained Anthropic auth monorepo for OpenCode and Pi. ### Dependencies - OpenTUI core and solid move to 0.6.6 together and the OpenCode 2 plugin SDK packages to 2.0.32, so each installs once. solid-js stays at 1.9.12, the exact version OpenTUI Solid 0.6.6 requires. +- Update common-auth to 0.22.0 so vault quota sweeps skip duplicate usage polls while insula starts or its upstream fetch has failed; requests also skip polls after an upstream failure. ## 2.1.2 diff --git a/bun.lock b/bun.lock index ed44fb02..86984cf0 100644 --- a/bun.lock +++ b/bun.lock @@ -34,7 +34,7 @@ "xxhash-wasm": "^1.1.0", }, "devDependencies": { - "@cortexkit/common-auth": "0.21.1", + "@cortexkit/common-auth": "0.22.0", }, }, "packages/e2e-tests": { @@ -248,7 +248,7 @@ "@cortexkit/claustrum-client": ["@cortexkit/claustrum-client@0.12.0", "", { "dependencies": { "@cortexkit/store": "^0.2.1", "@cortexkit/subc-client": "^0.22.0" } }, "sha512-Eqo2V8XgVzbZvolFYn11HX0TMRt21pnc9bWLYkcT4KW6kJ2fFLkzvXGzQ++MOmQpq8Bd3Jl5xA6yKkXTnimDOw=="], - "@cortexkit/common-auth": ["@cortexkit/common-auth@0.21.1", "", { "dependencies": { "@cortexkit/private-files": "^0.2.1", "jsonc-parser": "^3.3.1" }, "peerDependencies": { "@cortexkit/claustrum-client": ">=0.6.2", "@cortexkit/subc-client": "^0.16.0 || ^0.22.0", "@opencode/plugin": ">=2.0.21", "@opentui/core": ">=0.5.11", "@opentui/solid": ">=0.5.11", "solid-js": "1.9.12" }, "optionalPeers": ["@cortexkit/claustrum-client", "@cortexkit/subc-client", "@opencode/plugin", "@opentui/core", "@opentui/solid", "solid-js"] }, "sha512-DORasuyhF7FOTiw0kiL1DlIw9aPvMmBsdoxzkXJEIrKvNzUuv2lcOvHkflpot9KcoqPLCRJxyZLtMO+LVN61gw=="], + "@cortexkit/common-auth": ["@cortexkit/common-auth@0.22.0", "", { "dependencies": { "@cortexkit/private-files": "^0.2.1", "jsonc-parser": "^3.3.1" }, "peerDependencies": { "@cortexkit/claustrum-client": ">=0.6.2", "@cortexkit/subc-client": "^0.16.0 || ^0.22.0", "@opencode/plugin": ">=2.0.21", "@opentui/core": ">=0.5.11", "@opentui/solid": ">=0.5.11", "solid-js": "1.9.12" }, "optionalPeers": ["@cortexkit/claustrum-client", "@cortexkit/subc-client", "@opencode/plugin", "@opentui/core", "@opentui/solid", "solid-js"] }, "sha512-02yOfdzAI1QSwtofnjvSeFsELl6Lo1kp7jE4pCRFLRx0q0aFxlGeJgWacIVvM7y/DClkhstQvaRrtL/VSs8jUQ=="], "@cortexkit/opencode-anthropic-auth": ["@cortexkit/opencode-anthropic-auth@workspace:packages/opencode"], diff --git a/mutations.toml b/mutations.toml index e7eb8048..cdc613d5 100644 --- a/mutations.toml +++ b/mutations.toml @@ -1111,13 +1111,13 @@ broad_id = "{name}" id = "vault-insula-joins-vault-account-id" guards = "A background insula lookup must join on the roster row's vault account id (list_scoped account_id), never another row field such as the credential id." file = "packages/core/src/native-vault-runtime.ts" -old = "swept = insulaReading(row.accountIdentity)" -new = "swept = insulaReading(row.credentialId) // NON-VACUITY BREAK: join on the credential id" +old = "swept = insulaLookup(row.accountIdentity)" +new = "swept = insulaLookup(row.credentialId) // NON-VACUITY BREAK: join on the credential id" test_file = "packages/core/src/tests/native-vault-insula-quota.test.ts" runner = "command" command = ["bun", "scripts/run-mutation-bun.ts", "packages/core/src/tests/native-vault-insula-quota.test.ts", "{test}"] test_count_pattern = "CKDEV executed {count} tests" -expect_red = ["background sweep takes a fresh insula reading and sends no usage GET", "the primary account is looked up in insula by its roster row account id, not its logical id", "a stale, missing or failed insula entry falls back to one usage GET for that account only", "runtimes on one connection file and token path share one insula source until all release it", "the Fable window from extraRateWindows drives scoped routing and the scoped killswitch floor", "the source switch is logged once per change with fixed codes and no identity"] +expect_red = ["background sweep takes a fresh insula reading and sends no usage GET", "the primary account is looked up in insula by its roster row account id, not its logical id", "runtimes on one connection file and token path share one insula source until all release it", "the Fable window from extraRateWindows drives scoped routing and the scoped killswitch floor", "the source switch is logged once per change with fixed codes and no identity"] expect_message = '/error: expect\(received\)\.(toBe|toEqual)\(expected\)/' broad_command = ["bun", "scripts/run-mutation-bun.ts", "--broad", "packages/core", "tmp/mutations/core.xml"] broad_report = "tmp/mutations/core.xml" @@ -1140,7 +1140,7 @@ broad_report = "tmp/mutations/core.xml" broad_id = "{name}" [[control]] id = "vault-insula-reading-skips-usage-poll" -guards = "A current insula reading for a vault account must replace that account's usage GET; only a miss polls Anthropic." +guards = "A current insula reading for a vault account must replace that account's usage GET rather than polling Anthropic." file = "packages/core/src/native-vault-runtime.ts" old = ''' if (reading) { const stored''' @@ -1150,7 +1150,7 @@ test_file = "packages/core/src/tests/native-vault-insula-quota.test.ts" runner = "command" command = ["bun", "scripts/run-mutation-bun.ts", "packages/core/src/tests/native-vault-insula-quota.test.ts", "{test}"] test_count_pattern = "CKDEV executed {count} tests" -expect_red = ["background sweep takes a fresh insula reading and sends no usage GET", "the primary account is looked up in insula by its roster row account id, not its logical id", "a stale, missing or failed insula entry falls back to one usage GET for that account only", "admission reads only the insula cache: never refreshes it, polls on a miss", "runtimes on one connection file and token path share one insula source until all release it", "the Fable window from extraRateWindows drives scoped routing and the scoped killswitch floor", "the source switch is logged once per change with fixed codes and no identity"] +expect_red = ["background sweep takes a fresh insula reading and sends no usage GET", "the primary account is looked up in insula by its roster row account id, not its logical id", "admission reads only the insula cache: never refreshes it, polls on a miss", "runtimes on one connection file and token path share one insula source until all release it", "the Fable window from extraRateWindows drives scoped routing and the scoped killswitch floor", "the source switch is logged once per change with fixed codes and no identity"] expect_message = '/error: expect\(received\)\.(toBe|toEqual)\(expected\)/' broad_command = ["bun", "scripts/run-mutation-bun.ts", "--broad", "packages/core", "tmp/mutations/core.xml"] broad_report = "tmp/mutations/core.xml" @@ -1440,3 +1440,35 @@ expect_message = "error: expect(received).toBe(expected)" broad_command = ["bun", "scripts/run-mutation-bun.ts", "--broad", "packages/core", "tmp/mutations/core.xml"] broad_report = "tmp/mutations/core.xml" broad_id = "{name}" + +[[control]] +id = "vault-insula-cold-skips-usage-poll" +guards = "A cold insula cache must defer background usage polling without changing stored quota, its due time or poll bookkeeping." +file = "packages/core/src/native-vault-runtime.ts" +old = "if (swept.miss === 'cold' || swept.miss === 'upstream-failed') {" +new = "if (swept.miss === 'upstream-failed') { // NON-VACUITY BREAK: restore cold fallback polling" +test_file = "packages/core/src/tests/native-vault-insula-quota.test.ts" +runner = "command" +command = ["bun", "scripts/run-mutation-bun.ts", "packages/core/src/tests/native-vault-insula-quota.test.ts", "{test}"] +test_count_pattern = "CKDEV executed {count} tests" +expect_red = ["background cold defers without a usage GET or poll bookkeeping"] +expect_message = "error: expect(received).toEqual(expected)" +broad_command = ["bun", "scripts/run-mutation-bun.ts", "--broad", "packages/core", "tmp/mutations/core.xml"] +broad_report = "tmp/mutations/core.xml" +broad_id = "{name}" + +[[control]] +id = "vault-insula-upstream-failed-skips-usage-poll" +guards = "An insula upstream failure must defer background usage polling without changing stored quota, its due time or poll bookkeeping." +file = "packages/core/src/native-vault-runtime.ts" +old = "if (swept.miss === 'cold' || swept.miss === 'upstream-failed') {" +new = "if (swept.miss === 'cold') { // NON-VACUITY BREAK: restore upstream-failed fallback polling" +test_file = "packages/core/src/tests/native-vault-insula-quota.test.ts" +runner = "command" +command = ["bun", "scripts/run-mutation-bun.ts", "packages/core/src/tests/native-vault-insula-quota.test.ts", "{test}"] +test_count_pattern = "CKDEV executed {count} tests" +expect_red = ["background upstream-failed defers without a usage GET or poll bookkeeping"] +expect_message = "error: expect(received).toEqual(expected)" +broad_command = ["bun", "scripts/run-mutation-bun.ts", "--broad", "packages/core", "tmp/mutations/core.xml"] +broad_report = "tmp/mutations/core.xml" +broad_id = "{name}" diff --git a/packages/core/package.json b/packages/core/package.json index dafe6ff6..a8d01ff9 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -30,7 +30,7 @@ "prepublishOnly": "bun run build" }, "devDependencies": { - "@cortexkit/common-auth": "0.21.1" + "@cortexkit/common-auth": "0.22.0" }, "dependencies": { "@cortexkit/claustrum-client": "0.12.0", diff --git a/packages/core/scripts/check-native-type-closure.test.ts b/packages/core/scripts/check-native-type-closure.test.ts index a400f72a..4b2ace7f 100644 --- a/packages/core/scripts/check-native-type-closure.test.ts +++ b/packages/core/scripts/check-native-type-closure.test.ts @@ -110,7 +110,7 @@ test('rejects unsupported published producer versions before copying declaration producer, 'package.json', JSON.stringify({ - version: '0.21.0', + version: '0.21.1', exports: { './store': { types: './dist/store.d.ts' } }, }), ) @@ -124,7 +124,7 @@ test('rejects unsupported published producer versions before copying declaration ) await expect(closeNativeTypeDeclarations(dist, producer)).rejects.toThrow( - 'Expected published common-auth 0.21.1, got 0.21.0', + 'Expected published common-auth 0.22.0, got 0.21.1', ) expect(await readdir(dist)).not.toContain('internal-types') }) @@ -137,7 +137,7 @@ test('copies the complete published relative declaration graph including cycles producer, 'package.json', JSON.stringify({ - version: '0.21.1', + version: '0.22.0', exports: { './store': { types: './dist/store/index.d.ts' } }, }), ) @@ -273,7 +273,7 @@ test('packed Core types and Node runtime work in a fresh consumer without common const manifest = JSON.parse( await readFile(join(core, 'package.json'), 'utf8'), ) - expect(manifest.devDependencies['@cortexkit/common-auth']).toBe('0.21.1') + expect(manifest.devDependencies['@cortexkit/common-auth']).toBe('0.22.0') expect(manifest.dependencies['@cortexkit/claustrum-client']).toBe('0.12.0') expect(manifest.dependencies['@cortexkit/subc-client']).toBe('^0.22.0') expect(manifest.dependencies['jsonc-parser']).toBe('^3.3.1') diff --git a/packages/core/scripts/check-native-type-closure.ts b/packages/core/scripts/check-native-type-closure.ts index c1eaf046..c94cb103 100644 --- a/packages/core/scripts/check-native-type-closure.ts +++ b/packages/core/scripts/check-native-type-closure.ts @@ -185,9 +185,9 @@ export async function closeNativeTypeDeclarations( version: string exports: Record } - if (manifest.version !== '0.21.1') + if (manifest.version !== '0.22.0') throw new Error( - `Expected published common-auth 0.21.1, got ${manifest.version}`, + `Expected published common-auth 0.22.0, got ${manifest.version}`, ) const producerDist = join(producerRoot, 'dist') const assets = join(distRoot, TYPE_ASSETS) diff --git a/packages/core/src/native-vault-runtime.ts b/packages/core/src/native-vault-runtime.ts index 0b8193b6..760bddf8 100644 --- a/packages/core/src/native-vault-runtime.ts +++ b/packages/core/src/native-vault-runtime.ts @@ -6,6 +6,8 @@ import { ClaustrumConsumerError, connectClaustrumScopedClient, declineAccount, + type InsulaQuotaMiss, + type InsulaQuotaResult, type InsulaQuotaSource, mutateVaultRoster, readClaustrumEnrollmentToken, @@ -94,9 +96,10 @@ import { fetchOAuthAccountProfile } from './oauth-profile.ts' import { NativeAuthorityError } from './pool-authority.ts' import type { NativePoolPaths } from './pool-paths.ts' import { createNativePoolStore } from './pool-store.ts' -import type { - QuotaReadingPurpose, - QuotaRequestDispatch, +import { + type QuotaReadingPurpose, + QuotaRefreshDeferred, + type QuotaRequestDispatch, } from './quota-manager.ts' export interface NativeVaultRuntimeOptions { @@ -131,10 +134,12 @@ export interface NativeVaultRuntimeOptions { * background quota cache may answer instead of Anthropic's usage endpoint. * - 'background': a periodic sweep. Refresh insula's cache (at most once a * minute per daemon and enrollment), look the route's account up in it, - * and skip the usage GET when it holds a current reading. + * and skip the usage GET for a current reading, a cold cache or an upstream + * failure. Skipped misses leave the account due for the next sweep. * - 'admission': a request needs a reading. Look the served credential's * account up in insula's cache only, never refreshing it, then poll on a - * miss. + * miss except an upstream failure, which uses a dated reading if attached + * and otherwise leaves quota unknown. * Omitted: poll Anthropic directly (explicit user refreshes and other * callers that must see the provider's own reading). */ @@ -517,27 +522,31 @@ export function createNativeVaultRuntime( /** * Cache-only lookup by the vault's account id for this account (the * list_scoped or get_scoped account_id), never an email or label. - * common-auth returns only a current entry; anything we cannot map is a miss. + * Preserve insula's miss reason, even when the lookup also contains a dated reading. */ - function insulaReading( + function insulaLookup( accountId: string | undefined, - ): InsulaClaudeQuota | undefined { - if (!insula || !accountId || shutdown.signal.aborted) return undefined - const current = insula.source.current('claude', accountId, clock()) - return current - ? normalizeInsulaClaudeQuota(current.entry, current.context) - : undefined + ): InsulaQuotaResult { + if (!insula || !accountId || shutdown.signal.aborted) + return { miss: 'unreachable' } + const result = insula.source.lookup('claude', accountId, clock()) + const reading = + result.reading && + normalizeInsulaClaudeQuota(result.reading.entry, result.reading.context) + if (result.miss) return { miss: result.miss, ...(reading && { reading }) } + return reading ? { reading } : { miss: 'unreachable' } } // The last source each route's guided quota reading came from, so a change // is logged once rather than on every sweep. - const quotaSources = new Map() + const quotaSources = new Map() function noteQuotaSource( routeId: string, - source: 'insula' | 'fallback', - reason: 'insula-reading' | 'no-insula-reading' | 'account-changed', + source: 'insula' | 'fallback' | 'skipped', + reason: 'insula-reading' | 'account-changed' | InsulaQuotaMiss, ) { - if (quotaSources.get(routeId) === source) return - quotaSources.set(routeId, source) + const transition = `${source}:${reason}` + if (quotaSources.get(routeId) === transition) return + quotaSources.set(routeId, transition) if (quotaSources.size > 1000) { const oldest = quotaSources.keys().next().value if (oldest !== undefined) quotaSources.delete(oldest) @@ -1050,11 +1059,15 @@ export function createNativeVaultRuntime( const row = await rowFor(await runtime.read(), routeId) // A background sweep asks insula about the roster row's account (its // list_scoped account_id) before any credential is fetched. - let swept: InsulaClaudeQuota | undefined + let swept: InsulaQuotaResult | undefined if (purpose === 'background' && row) { await refreshInsula() check(signal) - swept = insulaReading(row.accountIdentity) + swept = insulaLookup(row.accountIdentity) + if (swept.miss === 'cold' || swept.miss === 'upstream-failed') { + noteQuotaSource(row.routeId, 'skipped', swept.miss) + throw new QuotaRefreshDeferred() + } } const state = await readNativeRuntime(paths.runtime, paths.storageId) const before = @@ -1082,8 +1095,10 @@ export function createNativeVaultRuntime( // A served receipt is still needed with an insula reading: the // reading is published bound to this credential version. let reading: InsulaClaudeQuota | undefined - let miss: 'no-insula-reading' | 'account-changed' = - 'no-insula-reading' + let lookup: InsulaQuotaResult = { + miss: 'unreachable', + } + let accountChanged = false if (purpose === 'background') { // The sweep joined the roster row's account; publish it only for // a credential served for that same account. @@ -1092,17 +1107,30 @@ export function createNativeVaultRuntime( row?.accountIdentity && receipt.assertedAccountIdentity === row.accountIdentity ) - reading = swept - else if (swept) miss = 'account-changed' + lookup = swept + else if (swept) accountChanged = true } else if (purpose === 'admission') { // Only the account id the vault asserted for this credential. - reading = insulaReading(receipt.assertedAccountIdentity) + lookup = insulaLookup(receipt.assertedAccountIdentity) } + // Admission polls on every miss except an upstream failure. A dated + // reading attached to that failure avoids another GET to a refusing account. + reading = + !lookup.miss || + (purpose === 'admission' && lookup.miss === 'upstream-failed') + ? lookup.reading + : undefined if (purpose) noteQuotaSource( - routeId, - reading ? 'insula' : 'fallback', - reading ? 'insula-reading' : miss, + row?.routeId ?? routeId, + reading + ? 'insula' + : lookup.miss === 'upstream-failed' + ? 'skipped' + : 'fallback', + accountChanged + ? 'account-changed' + : (lookup.miss ?? 'insula-reading'), ) if (reading) { const stored = matchingBefore(receipt)?.quota @@ -1111,6 +1139,8 @@ export function createNativeVaultRuntime( stored ? fromNativeQuotaMap(stored) : undefined, ) } + if (purpose === 'admission' && lookup.miss === 'upstream-failed') + return undefined return fetchOAuthQuotaSnapshot({ accessToken: receipt.accessToken, fetchImpl: transport, @@ -1133,6 +1163,7 @@ export function createNativeVaultRuntime( }, dispatch, ) + if (!result.value) throw new QuotaRefreshDeferred() const quota = { ...result.value, accountIdentity: result.receipt.assertedAccountIdentity, diff --git a/packages/core/src/quota-manager.ts b/packages/core/src/quota-manager.ts index f1494ae2..3004b1a5 100644 --- a/packages/core/src/quota-manager.ts +++ b/packages/core/src/quota-manager.ts @@ -75,6 +75,9 @@ export type QuotaRefreshResult = { */ export type QuotaReadingPurpose = 'background' | 'admission' +/** Insula is still fetching or its upstream failed; preserve quota without recording a poll. */ +export class QuotaRefreshDeferred extends Error {} + /** Pace a physical usage GET and verify the selected credential is still eligible before sending. */ export type QuotaRequestDispatch = ( send: () => Promise, @@ -1171,6 +1174,14 @@ export class QuotaManager { ) return { quota: completedQuota, fetched: true } } catch (error) { + if (error instanceof QuotaRefreshDeferred) { + return { + quota: this.getMain(mainAccountId)?.quota ?? { + accountIdentity: mainAccountId, + }, + fetched: false, + } + } this._handleMainFetchError(error, generation) throw error } finally { @@ -1292,6 +1303,14 @@ export class QuotaManager { await fileLock.release() } } catch (error) { + if (error instanceof QuotaRefreshDeferred) { + return { + quota: this.getFallback(accountId, account)?.quota ?? { + accountIdentity: accountId, + }, + fetched: false, + } + } this._handleFallbackFetchError(accountId, error, generation) throw error } diff --git a/packages/core/src/tests/native-vault-insula-quota.test.ts b/packages/core/src/tests/native-vault-insula-quota.test.ts index c46e556d..2267f961 100644 --- a/packages/core/src/tests/native-vault-insula-quota.test.ts +++ b/packages/core/src/tests/native-vault-insula-quota.test.ts @@ -1,8 +1,10 @@ -import { expect } from 'bun:test' +import { expect, spyOn } from 'bun:test' import { mkdtemp, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import { + acquireInsulaQuotaSource, + type InsulaQuotaLookup, type InsulaUsageEntry, projectVaultRoster, } from '@cortexkit/common-auth/claustrum' @@ -32,6 +34,7 @@ import { nativeVaultPaths } from '../native-vault-paths.ts' import { createNativeVaultRuntime } from '../native-vault-runtime.ts' import { resolveNativePoolPaths } from '../pool-paths.ts' import { createNativePoolStore } from '../pool-store.ts' +import { QuotaManager, QuotaRefreshDeferred } from '../quota-manager.ts' import { initializeNativeTestAuthority } from './native-authority-fixture.ts' import { createTestLifetimeSuite } from './test-lifetime.ts' @@ -107,7 +110,10 @@ const usageBody = { seven_day: { utilization: 20, resets_at: '1970-01-08T00:00:00.000Z' }, } -async function fixture(entries: () => unknown[]) { +async function fixture( + entries: () => unknown[], + completeProviders = ['claude'], +) { const root = await mkdtemp(join(tmpdir(), 'native-vault-insula-')) deferCleanup(() => rm(root, { recursive: true, force: true })) const paths = await resolveNativePoolPaths(join(root, 'anthropic-auth.json')) @@ -183,7 +189,7 @@ async function fixture(entries: () => unknown[]) { connectionFile: join(root, 'daemon.json'), getUsage: async () => { counts.insula++ - return { result: entries() } + return { result: entries(), completeProviders } }, }, } @@ -219,7 +225,16 @@ async function fixture(entries: () => unknown[]) { if (!id) throw new Error('Missing fixture route') return id } - return { runtime, make, counts, usage, vaultPaths, routeOf, clock } + const source = () => { + if (!client.insula) throw new Error('Missing fixture insula transport') + const lease = acquireInsulaQuotaSource( + client.insula, + env.OPENCODE_ANTHROPIC_AUTH_CLAUSTRUM_ENROLLMENT_FILE, + ) + deferCleanup(() => lease.release()) + return lease.source + } + return { runtime, make, counts, usage, vaultPaths, routeOf, clock, source } } const bearer = (credentialId: string) => @@ -297,14 +312,237 @@ test('the primary account is looked up in insula by its roster row account id, n expect(quota.five_hour?.checkedAt).toBe(fetchedAt) }) -test('a stale, missing or failed insula entry falls back to one usage GET for that account only', async () => { - for (const work of [ - entry(uuid2, { stale: true }), - entry(uuid2, { errorClass: 'rate_limited' }), - entry(uuid2, { fetchedAt: new Date(now - 5 * 60_000).toISOString() }), +for (const [miss, entries] of [ + ['cold', () => []], + ['upstream-failed', () => [entry(uuid2, { errorClass: 'upstream_failed' })]], +] as const) { + test(`background ${miss} defers without a usage GET or poll bookkeeping`, async () => { + const records = captureLogs() + const f = await fixture(entries, miss === 'cold' ? [] : ['claude']) + const route = f.routeOf(uuid2) + const stored = { + checkedAt: fetchedAt, + five_hour: { + usedPercent: 40, + remainingPercent: 60, + checkedAt: fetchedAt, + }, + } + await f.runtime.fetchQuota(route, f.usage) + await f.runtime.drainQuotaPublications() + const before = await readNativeRuntime( + f.vaultPaths.runtime, + f.vaultPaths.storageId, + ) + f.counts.usage.length = 0 + const fetched: unknown[] = [] + const failed: unknown[] = [] + const manager = new QuotaManager({ + storage: null, + now: () => f.clock.now, + fetchQuotaSnapshot: (request, dispatch) => + f.runtime.fetchQuota( + request.accountId ?? 'main', + f.usage, + undefined, + dispatch, + request.purpose, + ), + onMainQuotaFetched: (...args) => { + fetched.push(args) + }, + onFallbackQuotaFetched: (...args) => { + fetched.push(args) + }, + onApiError: (...args) => { + failed.push(args) + }, + onFallbackApiError: (...args) => { + failed.push(args) + }, + }) + deferCleanup(() => manager.close()) + const cached = { + quota: stored, + checkedAt: fetchedAt, + refreshAfter: now - 1, + } + manager.setMain(route, cached) + manager.setFallback(route, cached, undefined) + for (let tick = 0; tick < 2; tick++) { + expect( + await manager.refreshMainWithMetadata( + route, + '', + undefined, + 'background', + ), + ).toEqual({ quota: stored, fetched: false }) + expect( + await manager.refreshFallbackWithMetadata( + route, + '', + undefined, + 'background', + ), + ).toEqual({ quota: stored, fetched: false }) + expect(manager.getMain(route)).toEqual(cached) + expect(manager.getFallback(route)).toEqual(cached) + expect(manager.isBackedOff()).toBe(false) + expect(manager.isFallbackBackedOff(route)).toBe(false) + expect(manager.isMainStale()).toBe(true) + expect(manager.isFallbackStale(route)).toBe(true) + } + await expect( + f.runtime.fetchQuota(route, f.usage, undefined, undefined, 'background'), + ).rejects.toThrow(QuotaRefreshDeferred) + expect(f.counts.usage).toEqual([]) + expect(fetched).toEqual([]) + expect(failed).toEqual([]) + await f.runtime.drainQuotaPublications() + expect( + await readNativeRuntime(f.vaultPaths.runtime, f.vaultPaths.storageId), + ).toEqual(before) + expect( + records + .filter((record) => record.message === 'vault quota source changed') + .map((record) => record.payload), + ).toEqual([{ source: 'skipped', reason: miss }]) + if (miss === 'cold') { + // Insula's cold miss becomes unreachable after two minutes, allowing a + // fallback GET without a plugin-side backoff or a successful poll. + f.clock.now += 120_000 + await f.runtime.fetchQuota( + route, + f.usage, + undefined, + undefined, + 'background', + ) + expect(f.counts.usage).toEqual([bearer(workRow.id)]) + } + }) +} + +for (const [miss, entries] of [ + ['stale', () => [entry(uuid2, { stale: true })]], + ['credential', () => [entry(uuid2, { errorClass: 'credential_rejected' })]], + ['absent', () => [entry(uuid)]], + [ + 'unreachable', + () => { + throw new Error('insula unavailable') + }, + ], +] as const) { + test(`background ${miss} falls back to one usage GET for that account only`, async () => { + const f = await fixture(entries) + const quota = await f.runtime.fetchQuota( + f.routeOf(uuid2), + f.usage, + undefined, + undefined, + 'background', + ) + expect(f.counts.usage).toEqual([bearer(workRow.id)]) + expect(quota.five_hour).toMatchObject({ usedPercent: 10, checkedAt: now }) + expect(quota.scoped).toEqual([]) + }) +} + +test('admission upstream-failed without a reading uses unknown quota without a usage GET', async () => { + const f = await fixture(() => [ + entry(uuid), + entry(uuid2, { errorClass: 'upstream_failed' }), + ]) + await f.runtime.fetchQuota( + f.routeOf(uuid), + f.usage, + undefined, + undefined, + 'background', + ) + const manager = new QuotaManager({ + storage: null, + now: () => f.clock.now, + fetchQuotaSnapshot: (request, dispatch) => + f.runtime.fetchQuota( + request.accountId ?? 'main', + f.usage, + undefined, + dispatch, + request.purpose, + ), + }) + deferCleanup(() => manager.close()) + const result = await manager.refreshFallbackWithMetadata( + f.routeOf(uuid2), + '', + undefined, + 'admission', + ) + expect(f.counts.usage).toEqual([]) + expect(result.fetched).toBe(false) + expect(result.quota.five_hour).toBeUndefined() + expect( + killswitchPassesPolicy(result.quota, { + version: 1, + accounts: [], + killswitch: { enabled: true, main: { five_hour: 5, seven_day: 0 } }, + }), + ).toBe(false) + expect(manager.isFallbackBackedOff(f.routeOf(uuid2))).toBe(false) +}) + +test('admission upstream-failed uses an attached dated reading without a usage GET', async () => { + const f = await fixture(() => [entry(uuid)]) + await f.runtime.fetchQuota( + f.routeOf(uuid), + f.usage, + undefined, + undefined, + 'background', + ) + const lookup = spyOn(f.source(), 'lookup').mockReturnValue({ + miss: 'upstream-failed', + reading: { entry: entry(uuid2), context: { fetchedAt } }, + }) + deferCleanup(() => lookup.mockRestore()) + const quota = await f.runtime.fetchQuota( + f.routeOf(uuid2), + f.usage, undefined, - ]) { - const f = await fixture(() => [entry(uuid), ...(work ? [work] : [])]) + undefined, + 'admission', + ) + expect(f.counts.usage).toEqual([]) + expect(quota.five_hour).toMatchObject({ + remainingPercent: 60, + checkedAt: fetchedAt, + }) + const policy: AccountStorage = { + version: 1, + accounts: [], + killswitch: { enabled: true, main: { five_hour: 5, seven_day: 0 } }, + } + expect(killswitchPassesPolicy(quota, policy)).toBe(true) + expect( + killswitchPassesPolicy(quota, { + ...policy, + killswitch: { enabled: true, main: { five_hour: 65, seven_day: 0 } }, + }), + ).toBe(false) +}) + +for (const miss of [ + 'cold', + 'stale', + 'credential', + 'absent', + 'unreachable', +] as const) { + test(`admission ${miss} retains one usage GET fallback`, async () => { + const f = await fixture(() => [entry(uuid)]) await f.runtime.fetchQuota( f.routeOf(uuid), f.usage, @@ -312,18 +550,23 @@ test('a stale, missing or failed insula entry falls back to one usage GET for th undefined, 'background', ) - expect(f.counts.usage).toEqual([]) + const lookup = spyOn(f.source(), 'lookup').mockReturnValue({ + miss, + reading: { entry: entry(uuid2), context: { fetchedAt } }, + }) + deferCleanup(() => lookup.mockRestore()) const quota = await f.runtime.fetchQuota( f.routeOf(uuid2), f.usage, undefined, undefined, - 'background', + 'admission', ) expect(f.counts.usage).toEqual([bearer(workRow.id)]) expect(quota.five_hour).toMatchObject({ usedPercent: 10, checkedAt: now }) - } -}) + expect(quota.scoped).toEqual([]) + }) +} test('an insula entry for a different account never applies', async () => { const f = await fixture(() => [ @@ -587,7 +830,7 @@ test('the source switch is logged once per change with fixed codes and no identi expect(f.counts.usage).toEqual([bearer(workRow.id), bearer(workRow.id)]) expect(switches()).toEqual([ { source: 'insula', reason: 'insula-reading' }, - { source: 'fallback', reason: 'no-insula-reading' }, + { source: 'fallback', reason: 'absent' }, ]) const after = records.filter( (record) => record.message === 'vault quota source changed', @@ -604,6 +847,45 @@ test('the source switch is logged once per change with fixed codes and no identi expect(text).not.toContain(secret) }) +test('miss reason transitions each log one source-change line per account', async () => { + const records = captureLogs() + const f = await fixture(() => [entry(uuid)]) + let result: InsulaQuotaLookup = { miss: 'stale' } + const lookup = spyOn(f.source(), 'lookup').mockImplementation(() => result) + deferCleanup(() => lookup.mockRestore()) + for (const miss of [ + 'stale', + 'credential', + 'absent', + 'unreachable', + ] as const) { + result = { miss, reading: { entry: entry(uuid2), context: { fetchedAt } } } + for (const account of [uuid, uuid2]) { + for (let repeat = 0; repeat < 2; repeat++) { + await f.runtime.fetchQuota( + f.routeOf(account), + f.usage, + undefined, + undefined, + 'background', + ) + } + } + } + expect( + records + .filter((record) => record.message === 'vault quota source changed') + .map((record) => record.payload), + ).toEqual( + ['stale', 'credential', 'absent', 'unreachable'].flatMap((reason) => + Array.from({ length: 2 }, () => ({ + source: 'fallback', + reason, + })), + ), + ) +}) + test('normalizer: unknown shapes return undefined so the poll stays in charge', () => { expect( normalizeInsulaClaudeQuota(