diff --git a/bots/quoter-bot/src/infrastructure/setup-state/http-json.utils.ts b/bots/quoter-bot/src/infrastructure/setup-state/http-json.utils.ts index 73d9770d..865ae601 100644 --- a/bots/quoter-bot/src/infrastructure/setup-state/http-json.utils.ts +++ b/bots/quoter-bot/src/infrastructure/setup-state/http-json.utils.ts @@ -1,4 +1,5 @@ import { SafeProviderError } from '../../application/setup/safe-provider.error' +import { retryTransientProviderRead } from './provider-retry.utils' /** Stable HTTP provider identifiers safe to include in reports. */ export type ProviderId = 'morpho-api' | 'router-api' @@ -9,17 +10,7 @@ export type JsonRequest = ( timeoutMs?: number ) => Promise -/** - * Fetches and decodes JSON under a per-request timeout while redacting unsafe failure details. - * @param url - Provider endpoint; it is used for the request but never copied into thrown metadata. - * @param provider - Fixed provider identifier safe for reports. - * @param timeoutMs - Abort timeout in milliseconds, defaulting to 10 seconds. - * @returns Parsed JSON response value. - * @throws `SafeProviderError` with allowlisted provider/status/context metadata on HTTP, - * timeout, network, or JSON failures; raw URLs and response bodies are not exposed. - * @remarks Performs one read-only HTTP GET and has no chain or filesystem side effects. - */ -export const requestJson = async (url: string, provider: ProviderId, timeoutMs = 10_000) => { +const attemptJson = async (url: string, provider: ProviderId, timeoutMs: number) => { const signal = AbortSignal.timeout(timeoutMs) try { const response = await fetch(url, { headers: { accept: 'application/json' }, signal }) @@ -45,6 +36,20 @@ export const requestJson = async (url: string, provider: ProviderId, timeoutMs = } } +/** + * Fetches and decodes JSON under a per-attempt timeout while redacting unsafe failure details. + * @param url - Provider endpoint; it is used for the request but never copied into thrown metadata. + * @param provider - Fixed provider identifier safe for reports. + * @param timeoutMs - Abort timeout applied to each attempt, defaulting to 10 seconds. + * @returns Parsed JSON response value. + * @throws `SafeProviderError` with allowlisted provider/status/context metadata on HTTP, + * timeout, network, or JSON failures; raw URLs and response bodies are not exposed. + * @remarks Performs read-only HTTP GETs and has no chain or filesystem side effects, so transient + * failures are retried by {@link retryTransientProviderRead} before the error reaches the caller. + */ +export const requestJson = async (url: string, provider: ProviderId, timeoutMs = 10_000) => + retryTransientProviderRead(() => attemptJson(url, provider, timeoutMs)) + /** * Adapts the JSON transport for the legacy SDK books endpoint. * @param request - Existing provider-safe JSON transport. diff --git a/bots/quoter-bot/src/infrastructure/setup-state/provider-retry.utils.ts b/bots/quoter-bot/src/infrastructure/setup-state/provider-retry.utils.ts new file mode 100644 index 00000000..d341fa89 --- /dev/null +++ b/bots/quoter-bot/src/infrastructure/setup-state/provider-retry.utils.ts @@ -0,0 +1,47 @@ +import { delay } from '@repo/utils' + +import { SafeProviderError } from '../../application/setup/safe-provider.error' + +const MAX_ATTEMPTS = 3 +const BASE_DELAY_MS = 500 +const MAX_DELAY_MS = 4_000 +const JITTER_SHARE = 0.5 +const RETRYABLE_STATUSES = new Set([408, 429]) +const SERVER_ERROR_MINIMUM_STATUS = 500 + +const isTransientFailure = (error: unknown) => { + if (!(error instanceof SafeProviderError)) return false + const { name, status } = error.failure + if (name === 'TimeoutError' || name === 'NetworkError') return true + if (status === undefined) return false + return RETRYABLE_STATUSES.has(status) || status >= SERVER_ERROR_MINIMUM_STATUS +} + +const backoffDelayMs = (failures: number) => { + const capped = Math.min(MAX_DELAY_MS, BASE_DELAY_MS * 2 ** (failures - 1)) + const fixed = capped * (1 - JITTER_SHARE) + return fixed + Math.random() * (capped - fixed) +} + +/** + * Re-runs an idempotent read-only provider request while its failures look transient. + * @param attempt - Deferred read-only request; it is invoked at most three times and must be safe + * to repeat, since no attempt is cancelled or compensated. + * @returns The first successful attempt's value. + * @throws The final attempt's error unchanged — a sanitized `SafeProviderError` for provider + * failures — so callers keep today's halt behavior and operator-visible metadata. Non-transient + * failures (any error that is not a timeout, network fault, or HTTP 408/429/5xx `SafeProviderError`) + * are rethrown immediately without a retry. + * @remarks Waits a half-jittered exponential backoff between attempts (500 ms base, 4 s cap), so + * the worst-case added latency stays around 1.5 s and well inside the monitor cycle interval. + */ +export const retryTransientProviderRead = async (attempt: () => Promise) => { + for (let failures = 1; ; failures += 1) { + try { + return await attempt() + } catch (error) { + if (failures >= MAX_ATTEMPTS || !isTransientFailure(error)) throw error + await delay(backoffDelayMs(failures)) + } + } +} diff --git a/bots/quoter-bot/test/infrastructure/setup-state/provider-retry.utils.test.ts b/bots/quoter-bot/test/infrastructure/setup-state/provider-retry.utils.test.ts new file mode 100644 index 00000000..a3562e17 --- /dev/null +++ b/bots/quoter-bot/test/infrastructure/setup-state/provider-retry.utils.test.ts @@ -0,0 +1,120 @@ +import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest' + +import type { SafeProviderFailure } from '../../../src/application/setup/safe-provider.error' + +import { SafeProviderError } from '../../../src/application/setup/safe-provider.error' +import { retryTransientProviderRead } from '../../../src/infrastructure/setup-state/provider-retry.utils' + +const providerError = (failure: Omit) => + new SafeProviderError({ kind: 'provider-error', provider: 'router-api', ...failure }) + +const settle = async (pending: Promise) => { + const outcome = pending.then( + value => ({ value }), + (error: unknown) => ({ error }) + ) + await vi.advanceTimersByTimeAsync(60_000) + return outcome +} + +describe('retryTransientProviderRead', () => { + beforeEach(() => { + vi.useFakeTimers() + }) + + afterEach(() => { + vi.useRealTimers() + vi.restoreAllMocks() + }) + + test.each([ + ['a 5xx response', providerError({ name: 'HttpError', status: 503, context: 'request' })], + ['a 429 response', providerError({ name: 'HttpError', status: 429, context: 'request' })], + ['a 408 response', providerError({ name: 'HttpError', status: 408, context: 'request' })], + [ + 'a request timeout', + providerError({ name: 'TimeoutError', code: 'REQUEST_TIMEOUT', context: 'request' }) + ], + ['a network fault', providerError({ name: 'NetworkError', context: 'request' })] + ])('retries after %s and returns the eventual success', async (_label, error) => { + const attempt = vi.fn<() => Promise>() + attempt.mockRejectedValueOnce(error).mockResolvedValueOnce('recovered') + + await expect(settle(retryTransientProviderRead(attempt))).resolves.toEqual({ + value: 'recovered' + }) + expect(attempt).toHaveBeenCalledTimes(2) + }) + + test.each([ + ['400', 400], + ['404', 404], + ['403', 403] + ])('does not retry an HTTP %s response', async (_label, status) => { + const error = providerError({ name: 'HttpError', status, context: 'request' }) + const attempt = vi.fn<() => Promise>().mockRejectedValue(error) + + await expect(settle(retryTransientProviderRead(attempt))).resolves.toEqual({ error }) + expect(attempt).toHaveBeenCalledTimes(1) + }) + + test('does not retry a failure that is not a sanitized provider failure', async () => { + const error = new TypeError('unexpected') + const attempt = vi.fn<() => Promise>().mockRejectedValue(error) + + await expect(settle(retryTransientProviderRead(attempt))).resolves.toEqual({ error }) + expect(attempt).toHaveBeenCalledTimes(1) + }) + + test('exhausts three attempts and rethrows the final sanitized provider error', async () => { + const error = providerError({ name: 'HttpError', status: 502, context: 'request' }) + const attempt = vi.fn<() => Promise>().mockRejectedValue(error) + + const outcome = await settle(retryTransientProviderRead(attempt)) + + expect(attempt).toHaveBeenCalledTimes(3) + expect(outcome).toEqual({ error }) + expect(error).toBeInstanceOf(SafeProviderError) + expect(error.failure).toStrictEqual({ + kind: 'provider-error', + provider: 'router-api', + name: 'HttpError', + status: 502, + context: 'request' + }) + }) + + test('waits a half-jittered exponential backoff between attempts', async () => { + vi.spyOn(Math, 'random').mockReturnValue(0) + const error = providerError({ name: 'NetworkError', context: 'request' }) + const attempt = vi.fn<() => Promise>().mockRejectedValue(error) + const pending = retryTransientProviderRead(attempt).catch(() => 'failed') + + await vi.advanceTimersByTimeAsync(249) + expect(attempt).toHaveBeenCalledTimes(1) + await vi.advanceTimersByTimeAsync(1) + expect(attempt).toHaveBeenCalledTimes(2) + await vi.advanceTimersByTimeAsync(499) + expect(attempt).toHaveBeenCalledTimes(2) + await vi.advanceTimersByTimeAsync(1) + expect(attempt).toHaveBeenCalledTimes(3) + await expect(pending).resolves.toBe('failed') + }) + + test('keeps each jittered backoff between half and the full exponential delay', async () => { + vi.spyOn(Math, 'random').mockReturnValue(0.999_999) + const error = providerError({ name: 'NetworkError', context: 'request' }) + const attempt = vi.fn<() => Promise>().mockRejectedValue(error) + const pending = retryTransientProviderRead(attempt).catch(() => 'failed') + + await vi.advanceTimersByTimeAsync(249) + expect(attempt).toHaveBeenCalledTimes(1) + await vi.advanceTimersByTimeAsync(251) + expect(attempt).toHaveBeenCalledTimes(2) + await vi.advanceTimersByTimeAsync(499) + expect(attempt).toHaveBeenCalledTimes(2) + await vi.advanceTimersByTimeAsync(501) + expect(attempt).toHaveBeenCalledTimes(3) + await expect(pending).resolves.toBe('failed') + }) +}) diff --git a/bots/quoter-bot/test/infrastructure/setup-state/viem-setup-state.service.test.ts b/bots/quoter-bot/test/infrastructure/setup-state/viem-setup-state.service.test.ts index 08fcd873..136f1351 100644 --- a/bots/quoter-bot/test/infrastructure/setup-state/viem-setup-state.service.test.ts +++ b/bots/quoter-bot/test/infrastructure/setup-state/viem-setup-state.service.test.ts @@ -466,6 +466,45 @@ describe('ViemSetupStateService', () => { } }) + test('retries a transient 5xx and returns the recovered payload', async () => { + let requests = 0 + const server = await startFixtureServer(() => { + requests += 1 + return requests < 3 ? new Response('flaky', { status: 503 }) : Response.json({ ok: true }) + }) + + try { + await expect( + requestJson(`http://localhost:${server.port}/flaky`, 'router-api') + ).resolves.toEqual({ ok: true }) + expect(requests).toBe(3) + } finally { + await server.stop() + } + }) + + test('does not retry a client error that is not a rate limit or request timeout', async () => { + let requests = 0 + const server = await startFixtureServer(() => { + requests += 1 + return new Response('bad request', { status: 400 }) + }) + + try { + const error = await requestJson(`http://localhost:${server.port}/bad`, 'morpho-api').catch( + value => value + ) + + expect(error).toBeInstanceOf(SafeProviderError) + expect(error).toMatchObject({ + failure: { kind: 'provider-error', provider: 'morpho-api', name: 'HttpError', status: 400 } + }) + expect(requests).toBe(1) + } finally { + await server.stop() + } + }) + test('classifies a bounded request timeout without exposing URL credentials', async () => { const server = await startFixtureServer(async () => { await sleep(100) diff --git a/bots/quoter-bot/typedoc.json b/bots/quoter-bot/typedoc.json index 17c14eee..1987381c 100644 --- a/bots/quoter-bot/typedoc.json +++ b/bots/quoter-bot/typedoc.json @@ -87,6 +87,7 @@ "src/infrastructure/setup-state/provider-pagination.error.ts", "src/infrastructure/setup-state/provider-read.error.ts", "src/infrastructure/setup-state/provider-read.utils.ts", + "src/infrastructure/setup-state/provider-retry.utils.ts", "src/infrastructure/setup-state/provider-response.error.ts", "src/infrastructure/setup-state/viem-setup-state.service.ts", "src/infrastructure/setup-state/viem-setup-state.utils.ts",