fix(quoter-bot): retry transient provider reads with backoff

bots/quoter-bot/src/infrastructure/setup-state/http-json.utils.ts

@@ -1,4 +1,5 @@
 import { SafeProviderError } from '../../application/setup/safe-provider.error'
+import { retryTransientProviderRead } from './provider-retry.utils'
 
 /** Stable HTTP provider identifiers safe to include in reports. */
 export type ProviderId = 'morpho-api' | 'router-api'
@@ -9,17 +10,7 @@ export type JsonRequest = (
   timeoutMs?: number
 ) => Promise<unknown>
 
-/**
- * Fetches and decodes JSON under a per-request timeout while redacting unsafe failure details.
- * @param url - Provider endpoint; it is used for the request but never copied into thrown metadata.
- * @param provider - Fixed provider identifier safe for reports.
- * @param timeoutMs - Abort timeout in milliseconds, defaulting to 10 seconds.
- * @returns Parsed JSON response value.
- * @throws `SafeProviderError` with allowlisted provider/status/context metadata on HTTP,
- * timeout, network, or JSON failures; raw URLs and response bodies are not exposed.
- * @remarks Performs one read-only HTTP GET and has no chain or filesystem side effects.
- */
-export const requestJson = async (url: string, provider: ProviderId, timeoutMs = 10_000) => {
+const attemptJson = async (url: string, provider: ProviderId, timeoutMs: number) => {
   const signal = AbortSignal.timeout(timeoutMs)
   try {
     const response = await fetch(url, { headers: { accept: 'application/json' }, signal })
@@ -45,6 +36,20 @@ export const requestJson = async (url: string, provider: ProviderId, timeoutMs =
   }
 }
 
+/**
+ * Fetches and decodes JSON under a per-attempt timeout while redacting unsafe failure details.
+ * @param url - Provider endpoint; it is used for the request but never copied into thrown metadata.
+ * @param provider - Fixed provider identifier safe for reports.
+ * @param timeoutMs - Abort timeout applied to each attempt, defaulting to 10 seconds.
+ * @returns Parsed JSON response value.
+ * @throws `SafeProviderError` with allowlisted provider/status/context metadata on HTTP,
+ * timeout, network, or JSON failures; raw URLs and response bodies are not exposed.
+ * @remarks Performs read-only HTTP GETs and has no chain or filesystem side effects, so transient
+ * failures are retried by {@link retryTransientProviderRead} before the error reaches the caller.
+ */
+export const requestJson = async (url: string, provider: ProviderId, timeoutMs = 10_000) =>
+  retryTransientProviderRead(() => attemptJson(url, provider, timeoutMs))
+
 /**
  * Adapts the JSON transport for the legacy SDK books endpoint.
  * @param request - Existing provider-safe JSON transport.

bots/quoter-bot/src/infrastructure/setup-state/provider-retry.utils.ts

@@ -0,0 +1,47 @@
+import { delay } from '@repo/utils'
+
+import { SafeProviderError } from '../../application/setup/safe-provider.error'
+
+const MAX_ATTEMPTS = 3
+const BASE_DELAY_MS = 500
+const MAX_DELAY_MS = 4_000
+const JITTER_SHARE = 0.5
+const RETRYABLE_STATUSES = new Set([408, 429])
+const SERVER_ERROR_MINIMUM_STATUS = 500
+
+const isTransientFailure = (error: unknown) => {
+  if (!(error instanceof SafeProviderError)) return false
+  const { name, status } = error.failure
+  if (name === 'TimeoutError' || name === 'NetworkError') return true
+  if (status === undefined) return false
+  return RETRYABLE_STATUSES.has(status) || status >= SERVER_ERROR_MINIMUM_STATUS
+}
+
+const backoffDelayMs = (failures: number) => {
+  const capped = Math.min(MAX_DELAY_MS, BASE_DELAY_MS * 2 ** (failures - 1))
+  const fixed = capped * (1 - JITTER_SHARE)
+  return fixed + Math.random() * (capped - fixed)
+}
+
+/**
+ * Re-runs an idempotent read-only provider request while its failures look transient.
+ * @param attempt - Deferred read-only request; it is invoked at most three times and must be safe
+ * to repeat, since no attempt is cancelled or compensated.
+ * @returns The first successful attempt's value.
+ * @throws The final attempt's error unchanged — a sanitized `SafeProviderError` for provider
+ * failures — so callers keep today's halt behavior and operator-visible metadata. Non-transient
+ * failures (any error that is not a timeout, network fault, or HTTP 408/429/5xx `SafeProviderError`)
+ * are rethrown immediately without a retry.
+ * @remarks Waits a half-jittered exponential backoff between attempts (500 ms base, 4 s cap), so
+ * the worst-case added latency stays around 1.5 s and well inside the monitor cycle interval.
+ */
+export const retryTransientProviderRead = async <Result>(attempt: () => Promise<Result>) => {
+  for (let failures = 1; ; failures += 1) {
+    try {
+      return await attempt()
+    } catch (error) {
+      if (failures >= MAX_ATTEMPTS || !isTransientFailure(error)) throw error
+      await delay(backoffDelayMs(failures))
+    }
+  }
+}

bots/quoter-bot/test/infrastructure/setup-state/provider-retry.utils.test.ts

@@ -0,0 +1,120 @@
+import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest'
+
+import type { SafeProviderFailure } from '../../../src/application/setup/safe-provider.error'
+
+import { SafeProviderError } from '../../../src/application/setup/safe-provider.error'
+import { retryTransientProviderRead } from '../../../src/infrastructure/setup-state/provider-retry.utils'
+
+const providerError = (failure: Omit<SafeProviderFailure, 'kind' | 'provider'>) =>
+  new SafeProviderError({ kind: 'provider-error', provider: 'router-api', ...failure })
+
+const settle = async <Result>(pending: Promise<Result>) => {
+  const outcome = pending.then(
+    value => ({ value }),
+    (error: unknown) => ({ error })
+  )
+  await vi.advanceTimersByTimeAsync(60_000)
+  return outcome
+}
+
+describe('retryTransientProviderRead', () => {
+  beforeEach(() => {
+    vi.useFakeTimers()
+  })
+
+  afterEach(() => {
+    vi.useRealTimers()
+    vi.restoreAllMocks()
+  })
+
+  test.each([
+    ['a 5xx response', providerError({ name: 'HttpError', status: 503, context: 'request' })],
+    ['a 429 response', providerError({ name: 'HttpError', status: 429, context: 'request' })],
+    ['a 408 response', providerError({ name: 'HttpError', status: 408, context: 'request' })],
+    [
+      'a request timeout',
+      providerError({ name: 'TimeoutError', code: 'REQUEST_TIMEOUT', context: 'request' })
+    ],
+    ['a network fault', providerError({ name: 'NetworkError', context: 'request' })]
+  ])('retries after %s and returns the eventual success', async (_label, error) => {
+    const attempt = vi.fn<() => Promise<string>>()
+    attempt.mockRejectedValueOnce(error).mockResolvedValueOnce('recovered')
+
+    await expect(settle(retryTransientProviderRead(attempt))).resolves.toEqual({
+      value: 'recovered'
+    })
+    expect(attempt).toHaveBeenCalledTimes(2)
+  })
+
+  test.each([
+    ['400', 400],
+    ['404', 404],
+    ['403', 403]
+  ])('does not retry an HTTP %s response', async (_label, status) => {
+    const error = providerError({ name: 'HttpError', status, context: 'request' })
+    const attempt = vi.fn<() => Promise<string>>().mockRejectedValue(error)
+
+    await expect(settle(retryTransientProviderRead(attempt))).resolves.toEqual({ error })
+    expect(attempt).toHaveBeenCalledTimes(1)
+  })
+
+  test('does not retry a failure that is not a sanitized provider failure', async () => {
+    const error = new TypeError('unexpected')
+    const attempt = vi.fn<() => Promise<string>>().mockRejectedValue(error)
+
+    await expect(settle(retryTransientProviderRead(attempt))).resolves.toEqual({ error })
+    expect(attempt).toHaveBeenCalledTimes(1)
+  })
+
+  test('exhausts three attempts and rethrows the final sanitized provider error', async () => {
+    const error = providerError({ name: 'HttpError', status: 502, context: 'request' })
+    const attempt = vi.fn<() => Promise<string>>().mockRejectedValue(error)
+
+    const outcome = await settle(retryTransientProviderRead(attempt))
+
+    expect(attempt).toHaveBeenCalledTimes(3)
+    expect(outcome).toEqual({ error })
+    expect(error).toBeInstanceOf(SafeProviderError)
+    expect(error.failure).toStrictEqual({
+      kind: 'provider-error',
+      provider: 'router-api',
+      name: 'HttpError',
+      status: 502,
+      context: 'request'
+    })
+  })
+
+  test('waits a half-jittered exponential backoff between attempts', async () => {
+    vi.spyOn(Math, 'random').mockReturnValue(0)
+    const error = providerError({ name: 'NetworkError', context: 'request' })
+    const attempt = vi.fn<() => Promise<string>>().mockRejectedValue(error)
+    const pending = retryTransientProviderRead(attempt).catch(() => 'failed')
+
+    await vi.advanceTimersByTimeAsync(249)
+    expect(attempt).toHaveBeenCalledTimes(1)
+    await vi.advanceTimersByTimeAsync(1)
+    expect(attempt).toHaveBeenCalledTimes(2)
+    await vi.advanceTimersByTimeAsync(499)
+    expect(attempt).toHaveBeenCalledTimes(2)
+    await vi.advanceTimersByTimeAsync(1)
+    expect(attempt).toHaveBeenCalledTimes(3)
+    await expect(pending).resolves.toBe('failed')
+  })
+
+  test('keeps each jittered backoff between half and the full exponential delay', async () => {
+    vi.spyOn(Math, 'random').mockReturnValue(0.999_999)
+    const error = providerError({ name: 'NetworkError', context: 'request' })
+    const attempt = vi.fn<() => Promise<string>>().mockRejectedValue(error)
+    const pending = retryTransientProviderRead(attempt).catch(() => 'failed')
+
+    await vi.advanceTimersByTimeAsync(249)
+    expect(attempt).toHaveBeenCalledTimes(1)
+    await vi.advanceTimersByTimeAsync(251)
+    expect(attempt).toHaveBeenCalledTimes(2)
+    await vi.advanceTimersByTimeAsync(499)
+    expect(attempt).toHaveBeenCalledTimes(2)
+    await vi.advanceTimersByTimeAsync(501)
+    expect(attempt).toHaveBeenCalledTimes(3)
+    await expect(pending).resolves.toBe('failed')
+  })
+})

bots/quoter-bot/test/infrastructure/setup-state/viem-setup-state.service.test.ts

@@ -466,6 +466,45 @@ describe('ViemSetupStateService', () => {
     }
   })
 
+  test('retries a transient 5xx and returns the recovered payload', async () => {
+    let requests = 0
+    const server = await startFixtureServer(() => {
+      requests += 1
+      return requests < 3 ? new Response('flaky', { status: 503 }) : Response.json({ ok: true })
+    })
+
+    try {
+      await expect(
+        requestJson(`http://localhost:${server.port}/flaky`, 'router-api')
+      ).resolves.toEqual({ ok: true })
+      expect(requests).toBe(3)
+    } finally {
+      await server.stop()
+    }
+  })
+
+  test('does not retry a client error that is not a rate limit or request timeout', async () => {
+    let requests = 0
+    const server = await startFixtureServer(() => {
+      requests += 1
+      return new Response('bad request', { status: 400 })
+    })
+
+    try {
+      const error = await requestJson(`http://localhost:${server.port}/bad`, 'morpho-api').catch(
+        value => value
+      )
+
+      expect(error).toBeInstanceOf(SafeProviderError)
+      expect(error).toMatchObject({
+        failure: { kind: 'provider-error', provider: 'morpho-api', name: 'HttpError', status: 400 }
+      })
+      expect(requests).toBe(1)
+    } finally {
+      await server.stop()
+    }
+  })
+
   test('classifies a bounded request timeout without exposing URL credentials', async () => {
     const server = await startFixtureServer(async () => {
       await sleep(100)

bots/quoter-bot/typedoc.json

@@ -87,6 +87,7 @@
     "src/infrastructure/setup-state/provider-pagination.error.ts",
     "src/infrastructure/setup-state/provider-read.error.ts",
     "src/infrastructure/setup-state/provider-read.utils.ts",
+    "src/infrastructure/setup-state/provider-retry.utils.ts",
     "src/infrastructure/setup-state/provider-response.error.ts",
     "src/infrastructure/setup-state/viem-setup-state.service.ts",
     "src/infrastructure/setup-state/viem-setup-state.utils.ts",