From 3bfaa25bdeeffdebd0c2c1201791dd740823b6c5 Mon Sep 17 00:00:00 2001 From: ualtinok <94532+ualtinok@users.noreply.github.com> Date: Wed, 21 Oct 2026 03:31:11 +0200 Subject: [PATCH] mason: keep saved refresh usable across quota publication Known-account validation publishes only credential proof, so compare its captured refresh-error reset without rejecting unrelated quota marker changes. Preserve current quota metadata and all credential, binding and first-identity reset checks. The two known-validation quota-drift refusal cases intentionally change contract: they encoded the overbroad fence that caused after-persist failures. They now require current reset state to survive proof publication; malformed marker schemas, refresh-error resets and first-identity drift remain refused. Controlled Core regression is red first; restoring the old fence also reddens OC1, OC2 and Pi checks. Direct catalogue replay catches the named mutant with refresh-reset companion green. Scoped Core tests: 236 pass; host regressions: 3 pass; all three packages typecheck. Original Prime unchanged, 30 fresh-process runs: 28 pass and 2 existing-deadline timeouts, no saved-refresh refusal. Full breadth retained unrelated journal timeout and vault-routing assertion failure; no rerun-until-green or deadline changes. --- CHANGELOG.md | 1 + mutations.toml | 16 +++ packages/core/src/native-runtime.ts | 8 +- .../src/tests/native-account-runtime.test.ts | 111 ++++++++++++++++++ .../src/tests/native-known-validation.test.ts | 35 +++++- .../src/tests/native-refresh-quota-fixture.ts | 79 +++++++++++++ .../opencode/src/tests/native-serving.test.ts | 54 +++++++++ .../src/tests/opencode2-native-source.test.ts | 51 ++++++++ .../pi/src/tests/native-request-locks.test.ts | 44 ++++++- 9 files changed, 389 insertions(+), 10 deletions(-) create mode 100644 packages/core/src/tests/native-refresh-quota-fixture.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index c6684c490..aab0ff1f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ This repo is a CortexKit-maintained Anthropic auth monorepo for OpenCode and Pi. ### Accounts +- A successful token refresh no longer fails a request just because a quota check finished while it was refreshing. - Valid tokens no longer wait for proactive refreshes; requests waiting for a rejected-token refresh fail retryably after two seconds while the refresh continues. ### Dependencies diff --git a/mutations.toml b/mutations.toml index cdc613d50..69100b890 100644 --- a/mutations.toml +++ b/mutations.toml @@ -1472,3 +1472,19 @@ expect_message = "error: expect(received).toEqual(expected)" broad_command = ["bun", "scripts/run-mutation-bun.ts", "--broad", "packages/core", "tmp/mutations/core.xml"] broad_report = "tmp/mutations/core.xml" broad_id = "{name}" + +[[control]] +id = "known-validation-ignores-quota-only-marker-drift" +guards = "A saved refresh remains usable when an existing quota writer advances its clear marker before credential validation." +file = "packages/core/src/native-runtime.ts" +old = "if ((old?.refreshErrorClearedAt ?? null) !== context.refreshErrorClearedAt)" +new = "if (!capturedResetContextMatches(old, context)) // NON-VACUITY BREAK: fence credential proof on unrelated quota markers" +test_file = "packages/core/src/tests/native-account-runtime.test.ts" +runner = "command" +command = ["bun", "scripts/run-mutation-bun.ts", "packages/core/src/tests/native-account-runtime.test.ts", "{test}"] +test_count_pattern = "CKDEV executed {count} tests" +expect_red = ["saved refresh remains usable when a pending quota writer clears its error"] +expect_message = "error: expect(received).toMatchObject(expected)" +broad_command = ["bun", "scripts/run-mutation-bun.ts", "--broad", "packages/core", "tmp/mutations/core.xml"] +broad_report = "tmp/mutations/core.xml" +broad_id = "{name}" diff --git a/packages/core/src/native-runtime.ts b/packages/core/src/native-runtime.ts index 54bc9bb87..af3c9b6c3 100644 --- a/packages/core/src/native-runtime.ts +++ b/packages/core/src/native-runtime.ts @@ -1155,7 +1155,13 @@ function knownValidationEntry( publication: LocalValidationPublication, ): NativeRuntimeEntry { const { context, proof } = publication - if (!capturedResetContextMatches(old, context)) + // Known-account validation replaces only credential proof, not quota state. + // A quota poll or reset may advance its markers while the provider is pending; + // preserve those current fields without invalidating the checked credentials. + // A refresh-error reset still prevents this validation from being published. + // First-identity publication checks all reset markers because it also discards + // metadata that has not been confirmed to belong to the identified account. + if ((old?.refreshErrorClearedAt ?? null) !== context.refreshErrorClearedAt) throw new NativeRuntimeError('publication-refused') if (!old) { if (context.runtimeBinding !== null) diff --git a/packages/core/src/tests/native-account-runtime.test.ts b/packages/core/src/tests/native-account-runtime.test.ts index 43d046b1b..9d9760a4e 100644 --- a/packages/core/src/tests/native-account-runtime.test.ts +++ b/packages/core/src/tests/native-account-runtime.test.ts @@ -21,6 +21,7 @@ import { createNativeAccountRuntime, nativeAccountPolicy, } from '../native-account-runtime.ts' +import { nativeLocalCredentialValidationMatches } from '../native-credential-validation.ts' import { createNativeLocalCredentialService, type NativeLocalCredentialServiceOptions, @@ -2183,6 +2184,116 @@ test('slow superseded poll publication cannot overwrite newer quota on disk', as expect(await f.store.read()).toEqual(before) }) +for (const resetRefresh of [false, true]) { + test( + resetRefresh + ? 'saved refresh still refuses a refresh-error reset during provider exchange' + : 'saved refresh remains usable when a pending quota writer clears its error', + async () => { + const held = gate() + const arrived = gate() + let exchanges = 0 + const f = await fixture({ + refreshToken: async () => { + exchanges++ + // Complete the real poll's pending writer after context capture, but + // before the coordinator saves and validates the successor. + held.open() + await f.runtime.drainQuotaPublications() + if (resetRefresh) { + const subject = await f.runtime.captureLocalSubject('main') + expect( + await f.runtime.resetBackoff('main', subject, 'refresh'), + ).toBe(true) + } + return { + access: 'sk-ant-oat01-synthetic-successor', + refresh: 'synthetic-new-refresh', + expires: now + 4_000_000, + expiresIn: 4000, + } + }, + }) + const publish = f.runtime.publishLocal.bind(f.runtime) + const delayed = spyOn(f.runtime, 'publishLocal').mockImplementation( + async (subject, patch) => { + if (patch.quota) { + arrived.open() + await held.wait + } + return publish(subject, patch) + }, + ) + deferCleanup(() => delayed.mockRestore()) + await f.runtime.fetchQuota( + 'main', + Object.assign(async () => Response.json({}), { + preconnect: fetch.preconnect, + }), + ) + await arrived.wait + const before = await readNativeRuntime(f.paths.runtime, f.paths.storageId) + expect( + before.status === 'ready' && + before.state.accounts['imported-main']?.quotaErrorClearedAt, + ).toBeUndefined() + const result = await f.runtime.authorizeLocal('main', { + intent: 'refresh', + }) + const pool = await f.store.read() + const saved = await readNativeRuntime(f.paths.runtime, f.paths.storageId) + if (pool.status !== 'ready' || saved.status !== 'ready') + throw new Error('Missing saved refresh') + const entry = saved.state.accounts['imported-main']! + const row = pool.rows.find((row) => row.id === 'imported-main')! + if (entry.binding.kind !== 'local') + throw new Error('Expected local binding') + expect(row.credential).toMatchObject({ + access: 'sk-ant-oat01-synthetic-successor', + refresh: 'synthetic-new-refresh', + }) + expect(entry.quotaErrorClearedAt).toBe(now) + expect(entry.lastRefreshError).toBeUndefined() + expect(entry.validationRetry).toBeUndefined() + if (resetRefresh) { + expect(result).toMatchObject({ + status: 'failed', + persisted: true, + reconciliationFailed: true, + }) + expect(entry.refreshErrorClearedAt).toBe(now) + expect( + nativeLocalCredentialValidationMatches( + entry.credentialValidation, + entry.binding, + row.credential, + ), + ).toBe(false) + } else { + expect(result).toMatchObject({ + status: 'usable', + source: 'rotated', + access: 'sk-ant-oat01-synthetic-successor', + }) + expect( + nativeLocalCredentialValidationMatches( + entry.credentialValidation, + entry.binding, + row.credential, + ), + ).toBe(true) + } + // A refused publication must not poison the durable successor. The next + // serve either uses its proof or validates it without another exchange. + expect(await f.runtime.authorizeLocal('main')).toMatchObject({ + status: 'usable', + access: 'sk-ant-oat01-synthetic-successor', + }) + expect(exchanges).toBe(1) + }, + ) +} + test('refused poll publication drops only its own pending reading with a fixed warning', async () => { const f = await fixture() const held = gate() diff --git a/packages/core/src/tests/native-known-validation.test.ts b/packages/core/src/tests/native-known-validation.test.ts index fe8e2995e..c9654fc89 100644 --- a/packages/core/src/tests/native-known-validation.test.ts +++ b/packages/core/src/tests/native-known-validation.test.ts @@ -491,17 +491,30 @@ for (const key of [ 'quotaErrorClearedAt', 'quotaErrorGeneration', ]) { - test(`known publication keeps null zero and safe ${key} semantics`, async () => { + test(`known publication validates ${key} and preserves current reset state`, async () => { for (const [captured, current] of [ [null, 0], [0, null], [1, 2], ]) { const f = await fixture(current === null ? {} : { [key]: current }) - await refused(f, { + const event = { ...f.event, context: { ...f.event.context, [key]: captured }, - }) + } + if (key === 'refreshErrorClearedAt') { + await refused(f, event) + } else { + const next = await publishNativeLocalCredentialValidation( + f.paths, + event, + ) + if (!f.runtime.accounts.row) throw new Error('Missing metadata') + expect(next.accounts.row).toEqual({ + ...f.runtime.accounts.row, + credentialValidation: f.subject, + }) + } } for (const invalid of [0.5, -1, Number.MAX_SAFE_INTEGER + 1, undefined]) { const f = await fixture() @@ -511,10 +524,22 @@ for (const key of [ }) } const missing = await fixture({}, true) - await refused(missing, { + const event = { ...missing.event, context: { ...missing.event.context, [key]: 0 }, - }) + } + if (key === 'refreshErrorClearedAt') { + await refused(missing, event) + } else { + const next = await publishNativeLocalCredentialValidation( + missing.paths, + event, + ) + expect(next.accounts.row).toEqual({ + binding: missing.subject.binding, + credentialValidation: missing.subject, + }) + } }) } test('known publication preserves recorded zero floors and generation', async () => { diff --git a/packages/core/src/tests/native-refresh-quota-fixture.ts b/packages/core/src/tests/native-refresh-quota-fixture.ts new file mode 100644 index 000000000..d569dd1aa --- /dev/null +++ b/packages/core/src/tests/native-refresh-quota-fixture.ts @@ -0,0 +1,79 @@ +import { expect, spyOn } from 'bun:test' +import type { NativeQuotaObservation } from '../native-quota-observations.ts' +import type { createTestLifetimeSuite } from './test-lifetime.ts' + +export const quotaRefreshAccess = 'sk-ant-oat01-quota-refresh-successor' + +interface QuotaRuntime { + read(): Promise<{ + accounts: Array<{ id: string; quotaErrorClearedAt?: number }> + }> + fetchQuota(routeId: string, transport: typeof fetch): Promise + drainQuotaPublications(): Promise + quotaObservations: { + publish( + accountId: string, + observation: NativeQuotaObservation, + write: () => Promise, + ): Promise + } +} + +/** Order a real pending quota write between refresh context capture and validation. */ +export function holdQuotaPublicationForRefresh( + lifetime: ReturnType, +) { + const held = lifetime.gate() + const arrived = lifetime.gate() + let runtime: QuotaRuntime + let previousMarker: number | undefined + let exchanges = 0 + return { + exchanges: () => exchanges, + async poll(service: QuotaRuntime) { + runtime = service + previousMarker = (await runtime.read()).accounts.find( + (row) => row.id === 'main', + )?.quotaErrorClearedAt + // The observation queue is shared by leased runtime facades as well as + // direct runtimes. Wrap its existing writer, not a facade's forwarding method. + const observations = runtime.quotaObservations + const publish = observations.publish.bind(observations) + const delayed = spyOn(observations, 'publish').mockImplementation( + (accountId, observation, write) => + publish(accountId, observation, async () => { + arrived.open() + await held.wait + return write() + }), + ) + lifetime.deferCleanup(async () => { + held.open() + await runtime.drainQuotaPublications() + delayed.mockRestore() + }) + await runtime.fetchQuota( + 'main', + Object.assign(async () => Response.json({}), { + preconnect: fetch.preconnect, + }), + ) + await arrived.wait + }, + refreshToken: async () => { + exchanges++ + held.open() + await runtime.drainQuotaPublications() + const account = (await runtime.read()).accounts.find( + (row) => row.id === 'main', + ) + expect(account?.quotaErrorClearedAt).toBeGreaterThan(previousMarker ?? -1) + return { + access: quotaRefreshAccess, + refresh: 'synthetic-quota-refresh-successor', + expires: Date.now() + 3600_000, + expiresIn: 3600, + } + }, + } +} diff --git a/packages/opencode/src/tests/native-serving.test.ts b/packages/opencode/src/tests/native-serving.test.ts index a5d69f5e6..d3c6d7fe0 100644 --- a/packages/opencode/src/tests/native-serving.test.ts +++ b/packages/opencode/src/tests/native-serving.test.ts @@ -48,6 +48,10 @@ import { refreshedJoinAccess, refreshJoinMessage, } from '../../../core/src/tests/native-refresh-join-fixture.ts' +import { + holdQuotaPublicationForRefresh, + quotaRefreshAccess, +} from '../../../core/src/tests/native-refresh-quota-fixture.ts' import { holdNativeRelayLock } from '../../../core/src/tests/native-relay-lock-fixture.ts' import { requestLockProbe } from '../../../core/src/tests/native-request-lock-fixture.ts' import { createTestLifetimeSuite } from '../../../core/src/tests/test-lifetime.ts' @@ -1035,6 +1039,56 @@ test('supervised content after real migration refuses foreground and background expect(fixture.records).toHaveLength(0) }) +test('OC1 saved refresh serves the successor across a pending quota publication', async () => { + const fixture = await migrateServingFixture('local') + const warmup = createNativeAccountRuntime({ + paths: fixture.paths, + host: 'opencode', + }) + const admitted = await warmup.authorizeLocal('main') + await warmup.close() + if (admitted.status !== 'usable') + throw new Error('Missing initial authorization') + const held = holdQuotaPublicationForRefresh(lifetimes) + const local = { + refreshToken: held.refreshToken, + resolveIdentity: async () => ({ + accountUuid: mainIdentity as ProviderAccountUuid, + deviceId: 'fixture', + sessionId: 'fixture', + }), + } + const runtime = createNativeAccountRuntime({ + paths: fixture.paths, + host: 'opencode', + local, + }) + lifetimes.deferCleanup(() => runtime.close()) + await held.poll(runtime) + const sent: string[] = [] + network.mockImplementation((input, init) => { + const url = input instanceof Request ? input.url : input.toString() + if (url.includes('/v1/messages')) { + sent.push(new Headers(init?.headers).get('authorization') ?? '') + if (sent.length === 1) + return Promise.resolve(Response.json({}, { status: 401 })) + } + return fixture.fetchProvider(input, init) + }) + const plugin = await createPlugin({ nativeLocal: local }) + expect((await sendNative(plugin)).status).toBe(200) + expect(sent).toEqual([ + `Bearer ${admitted.access}`, + `Bearer ${quotaRefreshAccess}`, + ]) + expect(held.exchanges()).toBe(1) + expect(await runtime.authorizeLocal('main')).toMatchObject({ + status: 'usable', + source: 'current', + access: quotaRefreshAccess, + }) +}) + for (const scenario of ['proactive', '401', 'deadline'] as const) { test(`OC1 refresh join ${scenario} preserves valid-token admission`, async () => { const fixture = await migrateServingFixture('local') diff --git a/packages/opencode/src/tests/opencode2-native-source.test.ts b/packages/opencode/src/tests/opencode2-native-source.test.ts index 1bf37d877..75db3d39b 100644 --- a/packages/opencode/src/tests/opencode2-native-source.test.ts +++ b/packages/opencode/src/tests/opencode2-native-source.test.ts @@ -38,6 +38,10 @@ import { refreshedJoinAccess, refreshJoinMessage, } from '../../../core/src/tests/native-refresh-join-fixture.ts' +import { + holdQuotaPublicationForRefresh, + quotaRefreshAccess, +} from '../../../core/src/tests/native-refresh-quota-fixture.ts' import { holdNativeRelayLock } from '../../../core/src/tests/native-relay-lock-fixture.ts' import { requestLockProbe } from '../../../core/src/tests/native-request-lock-fixture.ts' import { createTestLifetimeSuite } from '../../../core/src/tests/test-lifetime.ts' @@ -170,6 +174,53 @@ usageTests.test( }, ) +usageTests.test( + 'OC2 saved refresh serves the successor across a pending quota publication', + async () => { + const owned: Array<() => Promise | void> = [] + usageTests.deferCleanup(async () => { + for (const cleanup of owned.reverse()) await cleanup() + }) + const held = holdQuotaPublicationForRefresh(usageTests) + const fixture = await local( + { + refreshToken: held.refreshToken, + resolveIdentity: async () => ({ + accountUuid: identity as ProviderAccountUuid, + deviceId: 'fixture', + sessionId: 'fixture', + }), + }, + owned, + ) + const native = source(fixture.runtime, fixture.paths, owned) + const first = await native.authorizeAccount(scope, 'main') + if (first?.route !== 'oauth') throw new Error('Missing first authorization') + await held.poll(fixture.runtime) + const limit = await native.limitFromResponse!({ + ...context(first.provenance), + status: 401, + headers: new Headers(), + body: async () => '', + }) + expect(limit).toMatchObject({ status: 401 }) + if (!limit) throw new Error('Missing retry limit') + await native.chooseAccount({ + ...scope, + rerouteFrom: { accountId: 'main', limit }, + }) + expect(await native.authorizeAccount(scope, 'main')).toMatchObject({ + accessToken: quotaRefreshAccess, + }) + expect(held.exchanges()).toBe(1) + expect(await fixture.runtime.authorizeLocal('main')).toMatchObject({ + status: 'usable', + source: 'current', + access: quotaRefreshAccess, + }) + }, +) + for (const scenario of ['proactive', '401', 'deadline'] as const) { usageTests.test( `OC2 refresh join ${scenario} preserves valid-token admission`, diff --git a/packages/pi/src/tests/native-request-locks.test.ts b/packages/pi/src/tests/native-request-locks.test.ts index f4fabaebc..7fb13c6d4 100644 --- a/packages/pi/src/tests/native-request-locks.test.ts +++ b/packages/pi/src/tests/native-request-locks.test.ts @@ -13,6 +13,10 @@ import { refreshedJoinAccess, refreshJoinMessage, } from '../../../core/src/tests/native-refresh-join-fixture.ts' +import { + holdQuotaPublicationForRefresh, + quotaRefreshAccess, +} from '../../../core/src/tests/native-refresh-quota-fixture.ts' import { requestLockProbe } from '../../../core/src/tests/native-request-lock-fixture.ts' import { createTestLifetimeSuite } from '../../../core/src/tests/test-lifetime.ts' import { closePiNativeRuntime, getPiNativeRuntime } from '../native.ts' @@ -62,11 +66,14 @@ for (const scenario of [ 'join-proactive', 'join-401', 'join-deadline', + 'quota-refresh', ] as const) { lifetimes.test( - scenario.startsWith('join-') - ? `Pi refresh join ${scenario.slice(5)} preserves valid-token admission` - : `Pi ${scenario} request observes only required foreground locks`, + scenario === 'quota-refresh' + ? 'Pi saved refresh serves the successor across a pending quota publication' + : scenario.startsWith('join-') + ? `Pi refresh join ${scenario.slice(5)} preserves valid-token admission` + : `Pi ${scenario} request observes only required foreground locks`, () => trackPiTestBody( (async () => { @@ -102,6 +109,10 @@ for (const scenario of [ ) const probe = requestLockProbe() const held = holdRefreshJoin(lifetimes) + const quotaRefresh = + scenario === 'quota-refresh' + ? holdQuotaPublicationForRefresh(lifetimes) + : undefined const identity = fixtureAccountIdentity('main') const client: NativeCustodyClient = { async listScoped() { @@ -155,6 +166,13 @@ for (const scenario of [ : {}), local: { onLockEvent: probe.onLockEvent, + ...(quotaRefresh + ? { + refreshToken: quotaRefresh.refreshToken, + resolveIdentity: async () => + getClaudeCodeIdentityForVerifiedAccount('main', identity), + } + : {}), ...(scenario.startsWith('join-') ? { refreshToken: held.refreshToken, @@ -202,7 +220,9 @@ for (const scenario of [ completedAtDispatch.push(held.completed()) probe.dispatch() if ( - (scenario === 'join-401' || scenario === 'join-deadline') && + (scenario === 'join-401' || + scenario === 'join-deadline' || + quotaRefresh) && sent.length === 1 ) return Response.json( @@ -221,6 +241,22 @@ for (const scenario of [ }, { preconnect: originalFetch.preconnect }, ) + if (quotaRefresh) { + await quotaRefresh.poll(runtime) + const result = await streamCortexKitAnthropic(model, context, { + sessionId: 'quota-refresh', + }).result() + expect(result.stopReason).toBe('stop') + expect(sent).toHaveLength(2) + expect(sent[1]).toBe(`Bearer ${quotaRefreshAccess}`) + expect(quotaRefresh.exchanges()).toBe(1) + expect(await runtime.authorizeLocal('main')).toMatchObject({ + status: 'usable', + source: 'current', + access: quotaRefreshAccess, + }) + return + } if (scenario.startsWith('join-')) { const send = (sessionId: string) => streamCortexKitAnthropic(model, context, { sessionId }).result()