Docs/horizon timeout readme

README.md

@@ -239,6 +239,27 @@ BASE_URL=http://localhost:3001/api/v1 k6 run loadtest/k6/pool-stress.js
 
 ---
 
+## Stellar Horizon timeouts & retries
+
+Horizon requests made by `StellarService` (currently the account-balance fallback in `getAccountBalance`) go through a resilience wrapper defined in `src/stellar/horizon-resilience.ts`, so a transient network issue no longer surfaces as an unhandled exception or a 500.
+
+**Retry with exponential backoff.** Each attempt has a 10s timeout (callers may pass a shorter one). Up to 3 attempts are made, with backoff of 500ms → 1s (capped at 5s); rate-limited responses (429) back off longer, 5s → 10s (capped at 30s).
+
+| Failure | Retried? |
+|---|---|
+| Timeout (per-attempt deadline, `ETIMEDOUT`, `ECONNABORTED`) | Yes |
+| Network errors (`ECONNRESET`, `ECONNREFUSED`, `EAI_AGAIN`, …) | Yes |
+| `429` / `5xx` responses | Yes |
+| Other `4xx` (e.g. `404` account not found) | No — returned to the caller immediately |
+
+**Typed errors.** When every attempt times out, the service throws `HorizonTimeoutError` (HTTP 504, `errorCode: HORIZON_TIMEOUT`). Non-timeout failures that exhaust their retries rethrow the original error.
+
+**Timeout circuit breaker.** After 3 consecutive calls exhaust their retries on timeouts, the circuit opens and Horizon calls fail fast with `HorizonUnavailableError` (HTTP 503, `errorCode: HORIZON_UNAVAILABLE`) for 30s without touching the network. After that window, a single probe request is allowed through: any response within the deadline closes the circuit, another timeout reopens it.
+
+**Health checks** make a single Horizon attempt (no retries) with the 3s health-check timeout, so a slow Horizon can't multiply probe latency and trigger pod restarts.
+
+---
+
 ## Error handling (RFC 7807 Problem Details)
 
 All error responses use `Content-Type: application/problem+json` conforming to [RFC 7807](https://datatracker.ietf.org/doc/html/rfc7807). Legacy fields remain present in the response body for backward compatibility, but are deprecated:
@@ -298,6 +319,8 @@ All error responses use `Content-Type: application/problem+json` conforming to [
 | `GONE` | 410 | Resource existed but is no longer accessible (e.g. an expired policy). |
 | `TOO_MANY_REQUESTS` | 429 | Rate limit exceeded (60 requests per minute per IP). Back off and retry after the `Retry-After` header value. |
 | `SERVICE_UNAVAILABLE` | 503 | Downstream dependency unavailable (database, Redis, Stellar RPC). |
+| `HORIZON_TIMEOUT` | 504 | Stellar Horizon did not respond in time after all retry attempts. Transient — safe to retry. See [Stellar Horizon timeouts](#stellar-horizon-timeouts--retries). |
+| `HORIZON_UNAVAILABLE` | 503 | The Horizon circuit breaker is open after repeated timeouts, so the request was not sent. Retry after ~30s. |
 | `BAD_REQUEST` | 400 | Generic bad request not covered by validation (malformed path param, unsupported value, etc.). |
 | `INTERNAL_ERROR` | 500 | Unexpected server-side failure. The error is logged server-side; the response body intentionally omits internal details. |
 

src/common/errors/error-codes.ts

@@ -38,6 +38,12 @@ export enum ErrorCode {
 
   // ── Service availability ──────────────────────────────────────────────────
   SERVICE_UNAVAILABLE   = 'SERVICE_UNAVAILABLE',
+
+  // ── Stellar Horizon ───────────────────────────────────────────────────────
+  // Horizon did not respond in time after retries; transient, safe to retry.
+  HORIZON_TIMEOUT       = 'HORIZON_TIMEOUT',
+  // Horizon circuit is open after repeated timeouts; retry after a short wait.
+  HORIZON_UNAVAILABLE   = 'HORIZON_UNAVAILABLE',
 }
 
 /**

src/health/health.controller.ts

@@ -481,6 +481,8 @@ export class HealthController {
           keeperBalanceXlm = await this.stellar.getAccountBalance(
             this.stellar.keeperPublicKey,
             HEALTH_CHECK_RPC_TIMEOUT_MS,
+            // One Horizon attempt: retries would multiply probe latency (#338).
+            1,
           );
 
           // #191 — RPC reachability alone isn't enough: a keeper account

src/stellar/horizon-resilience.ts

@@ -0,0 +1,132 @@
+import { HttpException, HttpStatus, Logger } from "@nestjs/common";
+import { ErrorCode } from "../common/errors/error-codes";
+
+/**
+ * Horizon timeout handling — retry with exponential backoff, a typed error on
+ * final failure, and a circuit breaker that fails fast once Horizon has timed
+ * out repeatedly. Previously a slow Horizon surfaced as an untyped timeout
+ * exception on the first attempt, turning a transient network blip into a 500.
+ */
+
+/** Thrown when every attempt of a Horizon call timed out. Maps to 504. */
+export class HorizonTimeoutError extends HttpException {
+  constructor(
+    readonly operation: string,
+    readonly attempts: number,
+    readonly timeoutMs: number,
+  ) {
+    super(
+      {
+        message: `Stellar Horizon request "${operation}" timed out after ${attempts} attempt(s) (${timeoutMs}ms each)`,
+        errorCode: ErrorCode.HORIZON_TIMEOUT,
+      },
+      HttpStatus.GATEWAY_TIMEOUT,
+    );
+    this.name = "HorizonTimeoutError";
+  }
+}
+
+/** Thrown without calling Horizon while the timeout circuit is open. Maps to 503. */
+export class HorizonUnavailableError extends HttpException {
+  constructor(readonly operation: string, readonly retryAfterMs: number) {
+    super(
+      {
+        message: `Stellar Horizon is temporarily unavailable (circuit open after repeated timeouts) — "${operation}" not attempted`,
+        errorCode: ErrorCode.HORIZON_UNAVAILABLE,
+      },
+      HttpStatus.SERVICE_UNAVAILABLE,
+    );
+    this.name = "HorizonUnavailableError";
+  }
+}
+
+const NETWORK_ERROR_CODES = new Set([
+  "ECONNABORTED",
+  "ECONNRESET",
+  "ECONNREFUSED",
+  "ETIMEDOUT",
+  "ESOCKETTIMEDOUT",
+  "EAI_AGAIN",
+  "EPIPE",
+]);
+
+function statusOf(err: unknown): number | undefined {
+  return (err as { response?: { status?: number } })?.response?.status
+    ?? (err as { status?: number })?.status;
+}
+
+/** True for our own withTimeout rejection and for socket/axios-level timeouts. */
+export function isTimeoutError(err: unknown): boolean {
+  if (err instanceof HorizonTimeoutError) return true;
+  if (err instanceof HttpException && err.getStatus() === HttpStatus.GATEWAY_TIMEOUT) return true;
+  const code = (err as { code?: string })?.code;
+  if (code === "ECONNABORTED" || code === "ETIMEDOUT" || code === "ESOCKETTIMEDOUT") return true;
+  const message = err instanceof Error ? err.message : String(err);
+  return /timed? ?out/i.test(message);
+}
+
+/**
+ * Timeouts, network failures, 429s and 5xx responses are worth retrying.
+ * Other 4xx responses (e.g. 404 account not found) are deterministic and are
+ * rethrown immediately so callers can handle them.
+ */
+export function isRetryableHorizonError(err: unknown): boolean {
+  if (isTimeoutError(err)) return true;
+  const status = statusOf(err);
+  if (typeof status === "number") return status === 429 || status >= 500;
+  const code = (err as { code?: string })?.code;
+  return typeof code === "string" && NETWORK_ERROR_CODES.has(code);
+}
+
+/**
+ * Counts consecutive Horizon calls that exhausted their retries on timeouts.
+ * At `threshold` the circuit opens and calls fail fast for `resetTimeoutMs`;
+ * after that a single probe is let through — its outcome closes or reopens
+ * the circuit.
+ */
+export class HorizonCircuitBreaker {
+  private consecutiveTimeouts = 0;
+  private openedAt: number | null = null;
+  private probeInFlight = false;
+  private readonly logger = new Logger("CircuitBreaker:horizon");
+
+  constructor(
+    private readonly threshold: number = 3,
+    private readonly resetTimeoutMs: number = 30_000,
+  ) {}
+
+  get isOpen(): boolean {
+    return this.openedAt !== null;
+  }
+
+  /** Throws HorizonUnavailableError if the call must not reach Horizon. */
+  acquire(operation: string): void {
+    if (this.openedAt === null) return;
+    const elapsed = Date.now() - this.openedAt;
+    if (elapsed < this.resetTimeoutMs || this.probeInFlight) {
+      throw new HorizonUnavailableError(operation, Math.max(this.resetTimeoutMs - elapsed, 0));
+    }
+    this.probeInFlight = true;
+    this.logger.warn(`Circuit half-open — allowing probe "${operation}"`);
+  }
+
+  /** Horizon answered within the deadline (success or a non-timeout error). */
+  recordResponse(): void {
+    if (this.openedAt !== null) this.logger.log("Probe succeeded — circuit closed");
+    this.consecutiveTimeouts = 0;
+    this.openedAt = null;
+    this.probeInFlight = false;
+  }
+
+  /** A call exhausted its retries on timeouts. */
+  recordTimeout(): void {
+    this.probeInFlight = false;
+    this.consecutiveTimeouts++;
+    if (this.openedAt !== null || this.consecutiveTimeouts >= this.threshold) {
+      this.openedAt = Date.now();
+      this.logger.error(
+        `Circuit opened after ${this.consecutiveTimeouts} consecutive Horizon timeouts — failing fast for ${this.resetTimeoutMs}ms`,
+      );
+    }
+  }
+}

src/stellar/stellar.service.spec.ts

@@ -1,6 +1,8 @@
 import { Test, TestingModule } from "@nestjs/testing";
 import { ConfigService } from "@nestjs/config";
 import { StellarService } from "./stellar.service";
+import { HorizonTimeoutError, HorizonUnavailableError } from "./horizon-resilience";
+import { ErrorCode } from "../common/errors/error-codes";
 
 jest.mock("@stellar/stellar-sdk", () => ({
   Networks: {
@@ -753,6 +755,99 @@ describe("StellarService", () => {
       mockHorizon.loadAccount.mockRejectedValue(new Error("account not found"));
 
       await expect(service.getAccountBalance("GSOMEACCOUNT")).rejects.toThrow("account not found");
+      // Deterministic errors are not retried.
+      expect(mockHorizon.loadAccount).toHaveBeenCalledTimes(1);
+    });
+
+    describe("Horizon timeouts", () => {
+      let sleepSpy: jest.SpyInstance;
+      const hang = () => new Promise<never>(() => {});
+
+      beforeEach(() => {
+        sleepSpy = jest
+          .spyOn(service as unknown as { sleep: (ms: number) => Promise<void> }, "sleep")
+          .mockResolvedValue(undefined);
+        jest.spyOn((service as any).logger, "warn").mockImplementation();
+        jest.spyOn((service as any).logger, "error").mockImplementation();
+      });
+
+      it("retries a timed-out request with exponential backoff and returns on recovery", async () => {
+        mockHorizon.loadAccount
+          .mockImplementationOnce(hang)
+          .mockImplementationOnce(hang)
+          .mockResolvedValueOnce({ balances: [{ asset_type: "native", balance: "42.0000000" }] });
+
+        const balance = await service.getAccountBalance("GSOMEACCOUNT", 5);
+
+        expect(balance).toBe("42.0000000");
+        expect(mockHorizon.loadAccount).toHaveBeenCalledTimes(3);
+        expect(sleepSpy.mock.calls.map(([ms]) => ms)).toEqual([500, 1000]);
+      });
+
+      it("throws a typed HorizonTimeoutError (504) once every attempt times out", async () => {
+        mockHorizon.loadAccount.mockImplementation(hang);
+
+        const err = await service.getAccountBalance("GSOMEACCOUNT", 5).catch((e) => e);
+
+        expect(err).toBeInstanceOf(HorizonTimeoutError);
+        expect(err.getStatus()).toBe(504);
+        expect(err.getResponse()).toMatchObject({ errorCode: ErrorCode.HORIZON_TIMEOUT });
+        expect(err.attempts).toBe(3);
+        expect(mockHorizon.loadAccount).toHaveBeenCalledTimes(3);
+      });
+
+      it("retries transient 5xx responses but not 404s", async () => {
+        mockHorizon.loadAccount
+          .mockRejectedValueOnce({ response: { status: 503 } })
+          .mockRejectedValueOnce({ response: { status: 404 } });
+
+        await expect(service.getAccountBalance("GSOMEACCOUNT", 5)).resolves.toBe("0");
+        expect(mockHorizon.loadAccount).toHaveBeenCalledTimes(2);
+      });
+
+      it("honours maxAttempts (health checks make a single attempt)", async () => {
+        mockHorizon.loadAccount.mockImplementation(hang);
+
+        await expect(service.getAccountBalance("GSOMEACCOUNT", 5, 1)).rejects.toBeInstanceOf(HorizonTimeoutError);
+        expect(mockHorizon.loadAccount).toHaveBeenCalledTimes(1);
+        expect(sleepSpy).not.toHaveBeenCalled();
+      });
+
+      it("opens the circuit after repeated timeouts and fails fast with HorizonUnavailableError (503)", async () => {
+        mockHorizon.loadAccount.mockImplementation(hang);
+
+        for (let i = 0; i < 3; i++) {
+          await expect(service.getAccountBalance("GSOMEACCOUNT", 5, 1)).rejects.toBeInstanceOf(HorizonTimeoutError);
+        }
+        mockHorizon.loadAccount.mockClear();
+
+        const err = await service.getAccountBalance("GSOMEACCOUNT", 5).catch((e) => e);
+        expect(err).toBeInstanceOf(HorizonUnavailableError);
+        expect(err.getStatus()).toBe(503);
+        expect(err.getResponse()).toMatchObject({ errorCode: ErrorCode.HORIZON_UNAVAILABLE });
+        expect(mockHorizon.loadAccount).not.toHaveBeenCalled();
+      });
+
+      it("lets a probe through after the reset window and closes the circuit on success", async () => {
+        const nowSpy = jest.spyOn(Date, "now");
+        let now = 1_000_000;
+        nowSpy.mockImplementation(() => now);
+        try {
+          mockHorizon.loadAccount.mockImplementation(hang);
+          for (let i = 0; i < 3; i++) {
+            await service.getAccountBalance("GSOMEACCOUNT", 5, 1).catch(() => undefined);
+          }
+          await expect(service.getAccountBalance("GSOMEACCOUNT", 5)).rejects.toBeInstanceOf(HorizonUnavailableError);
+
+          now += 30_000;
+          mockHorizon.loadAccount.mockResolvedValue({ balances: [{ asset_type: "native", balance: "1.0000000" }] });
+
+          await expect(service.getAccountBalance("GSOMEACCOUNT", 5)).resolves.toBe("1.0000000");
+          await expect(service.getAccountBalance("GSOMEACCOUNT", 5)).resolves.toBe("1.0000000");
+        } finally {
+          nowSpy.mockRestore();
+        }
+      });
     });
   });