From cfec4997b22f83466e7a8db2c195844ac1e835ba Mon Sep 17 00:00:00 2001 From: agudajohn793 Date: Wed, 21 Oct 2026 12:24:48 +0000 Subject: [PATCH 1/4] test(oracle): property-based and fuzz testing for consensus math MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Payout decisions rest on median, weighted quorum and anomaly thresholds, but there was no consensus module to test and nothing beyond hand-written examples. This adds a pure consensus module and checks its invariants over generated adversarial inputs with fast-check. src/oracle/consensus/consensus.ts - median(), meetsQuorum(), deviationTolerance()/isAnomalous(), and evaluateConsensus() which screens invalid/stale/duplicate sources, rejects outliers around the median, enforces a minimum source count and a weighted quorum, and returns a trust tier plus a deterministic 64-bit decision id. - Duplicated source ids are invalid in every copy (no double voting), and every source that reported counts toward the quorum denominator, so bad data cannot lower the bar. consensus.property.spec.ts asserts, among others: - median: bounded by min/max, order-independent, monotone, exact under translation, stays inside the honest range while honest values outnumber adversarial ones, rejects non-finite input - quorum: scale-invariant, monotone in agreeing weight and in ratio, never passes with zero/non-finite total weight - anomaly threshold: symmetric, monotone in distance and tolerance - evaluateConsensus: never throws on hostile input (NaN, Infinity, duplicates, empty ids), order-independent, deterministic, only agrees when count and weighted quorum are met, never counts stale data, and a minority cannot pull the value outside honest range ± tolerance The suite caught a real bug during development: averaging the middle pair as a/2 + b/2 underflows on subnormals and returned 0 for [5e-324 x4]. That case is now a regression example. Also restores package.json, which was committed empty in #165. Contents are rebuilt from the package-lock.json root entry (which already lists fast-check) plus the previous scripts and Jest config, so `npm ci` works again. Adds `test:property`, `dev` and `typecheck` scripts (the latter two are already documented in the README). The Jest config now runs ESM-only transitive deps of @stellar/stellar-sdk v16 (@noble/*, uint8array-extras) through babel-jest, using the plugin already in devDependencies; this also un-breaks the existing scval-encoders suite. Co-Authored-By: Claude Opus 5.5 --- README.md | 2 + package.json | 107 ++++ .../consensus/consensus.property.spec.ts | 488 ++++++++++++++++++ src/oracle/consensus/consensus.spec.ts | 178 +++++++ src/oracle/consensus/consensus.ts | 329 ++++++++++++ 5 files changed, 1104 insertions(+) create mode 100644 src/oracle/consensus/consensus.property.spec.ts create mode 100644 src/oracle/consensus/consensus.spec.ts create mode 100644 src/oracle/consensus/consensus.ts diff --git a/README.md b/README.md index 30fca80..bad1b63 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,8 @@ remain in Docker volumes; `docker compose down -v` removes them. | `npm start` | Run the compiled server | | `npm run typecheck` | `tsc --noEmit` | | `npm run lint` | ESLint over `src/` | +| `npm test` | Jest unit tests (includes property-based tests) | +| `npm run test:property` | Only the property-based / fuzz suites (`*.property.spec.ts`); set `FC_NUM_RUNS=20000` for a deeper soak, `FC_SEED=` to replay a failure | ## API surface diff --git a/package.json b/package.json index e69de29..0deb0b1 100644 --- a/package.json +++ b/package.json @@ -0,0 +1,107 @@ +{ + "name": "refract-protocol-backend", + "version": "0.1.0", + "description": "Refract Protocol backend \u2014 oracle monitoring, claim processing, premium quoting", + "main": "dist/main.js", + "private": true, + "scripts": { + "build": "nest build", + "start": "nest start", + "start:dev": "nest start --watch", + "dev": "nest start --watch", + "start:prod": "node dist/main", + "lint": "eslint \"src/**/*.ts\"", + "typecheck": "tsc --noEmit", + "test": "jest", + "test:watch": "jest --watch", + "test:cov": "jest --coverage", + "test:property": "jest --testRegex '.*\\.property\\.spec\\.ts$'", + "test:integration": "jest --config test/integration/jest.integration.config.ts --runInBand", + "migrate:up": "node-pg-migrate up --migrations-dir migrations --config-file database.json", + "migrate:down": "node-pg-migrate down --migrations-dir migrations --config-file database.json", + "migrate:create": "node-pg-migrate create --migrations-dir migrations --config-file database.json", + "migrate:status": "node-pg-migrate status --migrations-dir migrations --config-file database.json", + "regenerate:xdr-fixtures": "ts-node scripts/regenerate-xdr-fixtures.ts" + }, + "dependencies": { + "@aws-sdk/client-secrets-manager": "^3.1142.0", + "@nestjs/common": "^11.1.28", + "@nestjs/config": "^4.0.4", + "@nestjs/core": "^11.1.28", + "@nestjs/platform-express": "^11.1.28", + "@nestjs/platform-ws": "^11.1.28", + "@nestjs/schedule": "^6.1.3", + "@nestjs/swagger": "^11.4.7", + "@nestjs/websockets": "^11.1.28", + "@prometheus-io/client": "^0.16.1", + "@stellar/stellar-sdk": "^16.3.0", + "axios": "^1.6.0", + "class-transformer": "^0.5.1", + "class-validator": "^0.15.1", + "cors": "^2.8.5", + "dotenv": "^16.3.1", + "express": "^4.18.2", + "helmet": "^7.1.0", + "ioredis": "^5.3.2", + "nest-winston": "^1.10.2", + "node-pg-migrate": "7.9.1", + "pg": "^8.11.3", + "reflect-metadata": "^0.2.2", + "rxjs": "^7.8.2", + "winston": "^3.11.0", + "ws": "^8.16.0", + "zod": "^3.22.4" + }, + "devDependencies": { + "@babel/plugin-transform-modules-commonjs": "^7.29.7", + "@nestjs/cli": "^11.0.24", + "@nestjs/schematics": "^11.1.0", + "@nestjs/testing": "^11.1.28", + "@types/cors": "^2.8.17", + "@types/express": "^4.17.21", + "@types/jest": "^30.0.0", + "@types/node": "^20.10.0", + "@types/pg": "^8.10.9", + "@types/supertest": "^7.2.0", + "@types/ws": "^8.5.10", + "@typescript-eslint/eslint-plugin": "^7.0.0", + "@typescript-eslint/parser": "^7.0.0", + "babel-jest": "^30.5.2", + "eslint": "^8.57.0", + "fast-check": "^4.10.2", + "jest": "^30.4.2", + "nodemon": "^3.0.2", + "supertest": "^7.2.2", + "ts-jest": "^29.4.11", + "ts-node": "^10.9.2", + "typescript": "^5.3.2" + }, + "jest": { + "moduleFileExtensions": [ + "js", + "json", + "ts" + ], + "rootDir": "src", + "testRegex": ".*\\.spec\\.ts$", + "transform": { + "^.+\\.ts$": "ts-jest", + "^.+\\.js$": [ + "babel-jest", + { + "plugins": [ + "@babel/plugin-transform-modules-commonjs" + ] + } + ] + }, + "transformIgnorePatterns": [ + "^(?!.*/node_modules/(@noble|uint8array-extras)/).*/node_modules/" + ], + "collectCoverageFrom": [ + "**/*.(t|j)s" + ], + "coverageDirectory": "../coverage", + "testEnvironment": "node" + } +} diff --git a/src/oracle/consensus/consensus.property.spec.ts b/src/oracle/consensus/consensus.property.spec.ts new file mode 100644 index 0000000..66ce35c --- /dev/null +++ b/src/oracle/consensus/consensus.property.spec.ts @@ -0,0 +1,488 @@ +import fc from "fast-check"; +import { + ConsensusDecision, + ConsensusPolicy, + SourceObservation, + TrustTier, + deviationTolerance, + evaluateConsensus, + isAnomalous, + median, + meetsQuorum, +} from "./consensus"; + +/** + * Property-based and fuzz tests for the consensus math. + * + * Each property states an invariant that must hold for every input, and + * fast-check searches for a counterexample (including adversarial ones: + * NaN, Infinity, huge magnitudes, duplicated ids, colluding outliers). + * On failure fast-check prints the seed and a shrunk counterexample; rerun + * with FC_SEED= to reproduce. + * + * FC_NUM_RUNS raises the run count for deeper soak runs, e.g. + * FC_NUM_RUNS=20000 npm run test:property + */ +const numRuns = Number(process.env.FC_NUM_RUNS ?? 300); +const seed = process.env.FC_SEED ? Number(process.env.FC_SEED) : undefined; +fc.configureGlobal({ numRuns, ...(seed !== undefined ? { seed } : {}) }); + +const NOW = 1_800_000_000_000; + +// ---------------------------------------------------------------- arbitraries + +/** Finite doubles across a wide but non-overflowing range. */ +const finite = fc.double({ min: -1e12, max: 1e12, noNaN: true, noDefaultInfinity: true }); + +/** Integers small enough that sums and products stay exact in a double. */ +const exactInt = fc.integer({ min: -1_000_000, max: 1_000_000 }); + +/** Any number at all, including the values that break naive math. */ +const hostileNumber = fc.oneof( + finite, + fc.constantFrom(NaN, Infinity, -Infinity, -0, Number.MAX_VALUE, -Number.MAX_VALUE, Number.MIN_VALUE), + fc.double() +); + +const validPolicy: fc.Arbitrary = fc.record({ + quorumRatio: fc.double({ min: 0.01, max: 1, noNaN: true }), + minAgreeingSources: fc.integer({ min: 1, max: 5 }), + maxDeviationRatio: fc.double({ min: 0, max: 0.5, noNaN: true }), + absoluteDeviationFloor: fc.double({ min: 0, max: 10, noNaN: true }), + maxStalenessMs: fc.integer({ min: 0, max: 600_000 }), + maxFutureSkewMs: fc.integer({ min: 0, max: 60_000 }), +}); + +/** Well-formed observation with a unique-ish id drawn from a small pool. */ +function wellFormedObs(ids: fc.Arbitrary): fc.Arbitrary { + return fc.record({ + sourceId: ids, + value: finite, + weight: fc.integer({ min: 0, max: 1_000 }), + observedAt: fc.integer({ min: NOW - 900_000, max: NOW + 90_000 }), + }); +} + +/** Observation where every field may be garbage. */ +const hostileObs: fc.Arbitrary = fc.record({ + sourceId: fc.oneof(fc.constantFrom("a", "b", "c", "d", ""), fc.string({ maxLength: 4 })), + value: hostileNumber, + weight: hostileNumber, + observedAt: fc.oneof(fc.integer({ min: NOW - 900_000, max: NOW + 90_000 }), hostileNumber), +}); + +/** Uniquely-identified, fresh, well-formed observations. */ +function freshUniqueObs( + values: fc.Arbitrary, + weights: fc.Arbitrary = fc.integer({ min: 1, max: 100 }), + { minLength = 1, maxLength = 12 } = {} +): fc.Arbitrary { + return fc + .array(fc.record({ value: values, weight: weights, age: fc.integer({ min: 0, max: 1_000 }) }), { + minLength, + maxLength, + }) + .map((rows) => rows.map((r, i) => ({ sourceId: `s${i}`, value: r.value, weight: r.weight, observedAt: NOW - r.age }))); +} + +/** A permutation of `xs`, chosen by fast-check so it shrinks well. */ +function permuted(xs: T[]): fc.Arbitrary { + return fc + .array(fc.nat(), { minLength: xs.length, maxLength: xs.length }) + .map((keys) => xs.map((x, i) => ({ x, k: keys[i], i })).sort((p, q) => p.k - q.k || p.i - q.i).map((p) => p.x)); +} + +function sortedLists(d: ConsensusDecision) { + return [d.agreeingSources, d.outlierSources, d.staleSources, d.invalidSources]; +} + +// ---------------------------------------------------------------- median + +describe("median — properties", () => { + it("lies between the minimum and maximum", () => { + fc.assert( + fc.property(fc.array(finite, { minLength: 1 }), (xs) => { + const m = median(xs); + return m >= Math.min(...xs) && m <= Math.max(...xs); + }) + ); + }); + + it("does not depend on input order", () => { + fc.assert( + fc.property( + fc.array(finite, { minLength: 1 }).chain((xs) => fc.tuple(fc.constant(xs), permuted(xs))), + ([xs, ys]) => Object.is(median(xs), median(ys)) + ) + ); + }); + + it("is an element of the input for odd lengths", () => { + fc.assert( + fc.property( + fc.array(finite, { minLength: 1 }).filter((xs) => xs.length % 2 === 1), + (xs) => xs.includes(median(xs)) + ) + ); + }); + + it("splits the input: at most half the values lie strictly on either side", () => { + fc.assert( + fc.property(fc.array(finite, { minLength: 1 }), (xs) => { + const m = median(xs); + const below = xs.filter((x) => x < m).length; + const above = xs.filter((x) => x > m).length; + return below <= xs.length / 2 && above <= xs.length / 2; + }) + ); + }); + + it("never decreases when one input increases", () => { + fc.assert( + fc.property( + fc.array(exactInt, { minLength: 1 }), + fc.nat(), + fc.integer({ min: 0, max: 1_000_000 }), + (xs, idx, bump) => { + const ys = [...xs]; + ys[idx % ys.length] += bump; + return median(ys) >= median(xs); + } + ) + ); + }); + + it("shifts exactly with a constant offset (exact integer arithmetic)", () => { + fc.assert( + fc.property(fc.array(exactInt, { minLength: 1 }), exactInt, (xs, c) => { + return median(xs.map((x) => x + c)) === median(xs) + c; + }) + ); + }); + + it("stays inside the honest range while honest values outnumber adversarial ones", () => { + fc.assert( + fc.property( + fc.array(finite, { minLength: 1, maxLength: 20 }), + fc.array(fc.oneof(finite, fc.constantFrom(Number.MAX_VALUE, -Number.MAX_VALUE)), { maxLength: 19 }), + (honest, adversarial) => { + fc.pre(adversarial.length < honest.length); + const m = median([...honest, ...adversarial]); + return m >= Math.min(...honest) && m <= Math.max(...honest); + } + ) + ); + }); + + it("rejects any non-finite input instead of returning garbage", () => { + fc.assert( + fc.property( + fc.array(finite), + fc.constantFrom(NaN, Infinity, -Infinity), + fc.nat(), + (xs, bad, at) => { + const ys = [...xs]; + ys.splice(at % (ys.length + 1), 0, bad); + expect(() => median(ys)).toThrow(RangeError); + } + ) + ); + }); +}); + +// ---------------------------------------------------------------- weighted quorum + +describe("weighted quorum — properties", () => { + const weightPair = fc + .tuple(fc.integer({ min: 0, max: 1_000_000 }), fc.integer({ min: 1, max: 1_000_000 })) + .map(([a, extra]) => ({ agreeing: a, total: a + extra - 1 })) + .filter(({ total }) => total > 0); + const ratio = fc.double({ min: 0.001, max: 1, noNaN: true }); + + it("does not change when every weight is scaled by the same factor", () => { + fc.assert( + fc.property(weightPair, ratio, fc.integer({ min: 1, max: 1_000_000 }), ({ agreeing, total }, r, k) => { + return meetsQuorum(agreeing, total, r) === meetsQuorum(agreeing * k, total * k, r); + }) + ); + }); + + it("is monotone: more agreeing weight never loses quorum", () => { + fc.assert( + fc.property(weightPair, ratio, fc.nat(), ({ agreeing, total }, r, extra) => { + const more = Math.min(agreeing + extra, total); + return !meetsQuorum(agreeing, total, r) || meetsQuorum(more, total, r); + }) + ); + }); + + it("is monotone: a lower bar never loses quorum", () => { + fc.assert( + fc.property(weightPair, ratio, ratio, ({ agreeing, total }, r1, r2) => { + const [lo, hi] = r1 <= r2 ? [r1, r2] : [r2, r1]; + return !meetsQuorum(agreeing, total, hi) || meetsQuorum(agreeing, total, lo); + }) + ); + }); + + it("always passes on full agreement and never on zero agreement", () => { + fc.assert( + fc.property(fc.integer({ min: 1, max: 1_000_000 }), ratio, (total, r) => { + return meetsQuorum(total, total, r) && !meetsQuorum(0, total, r); + }) + ); + }); + + it("never passes when total weight is zero, negative or non-finite", () => { + fc.assert( + fc.property(hostileNumber, fc.oneof(fc.constantFrom(0, -0, NaN, Infinity, -Infinity), fc.double({ max: 0 })), ratio, (a, t, r) => { + return !meetsQuorum(a, t, r); + }) + ); + }); +}); + +// ---------------------------------------------------------------- anomaly threshold + +describe("anomaly threshold — properties", () => { + it("never flags the center itself", () => { + fc.assert(fc.property(finite, validPolicy, (c, p) => !isAnomalous(c, c, p))); + }); + + it("is symmetric around the center", () => { + fc.assert( + fc.property(exactInt, fc.integer({ min: 0, max: 1_000_000 }), validPolicy, (c, d, p) => { + return isAnomalous(c + d, c, p) === isAnomalous(c - d, c, p); + }) + ); + }); + + it("is monotone in distance: further from the center never becomes less anomalous", () => { + fc.assert( + fc.property(finite, fc.double({ min: 0, max: 1e6, noNaN: true }), fc.double({ min: 0, max: 1e6, noNaN: true }), validPolicy, (c, d1, d2, p) => { + const [near, far] = d1 <= d2 ? [d1, d2] : [d2, d1]; + return !isAnomalous(c + near, c, p) || isAnomalous(c + far, c, p) || c + near === c + far; + }) + ); + }); + + it("widening the tolerance never turns an agreeing value into an outlier", () => { + fc.assert( + fc.property(finite, finite, validPolicy, fc.double({ min: 0, max: 1, noNaN: true }), (v, c, p, widen) => { + const wider = { ...p, maxDeviationRatio: p.maxDeviationRatio + widen, absoluteDeviationFloor: p.absoluteDeviationFloor + widen }; + return isAnomalous(v, c, p) || !isAnomalous(v, c, wider); + }) + ); + }); +}); + +// ---------------------------------------------------------------- evaluateConsensus + +describe("evaluateConsensus — fuzz", () => { + it("never throws on hostile observations and always returns a coherent decision", () => { + fc.assert( + fc.property(fc.array(hostileObs, { maxLength: 15 }), validPolicy, (xs, p) => { + const d = evaluateConsensus(xs, p, NOW); + + // Shape + expect(d.decisionId).toMatch(/^[0-9a-f]{16}$/); + expect(Number.isFinite(d.totalWeight) || d.totalWeight === Infinity).toBe(true); + + // Every distinct id lands in exactly one bucket. + const buckets = sortedLists(d).flat(); + expect(new Set(buckets).size).toBe(buckets.length); + expect(new Set(buckets)).toEqual(new Set(xs.map((o) => String(o.sourceId)))); + + // Agreed ⇔ a finite value is published. + if (d.status === "agreed") { + expect(Number.isFinite(d.value)).toBe(true); + expect(d.trustTier).not.toBe(TrustTier.Unverified); + expect(Number.isFinite(d.sourceTimestamp)).toBe(true); + } else { + expect(d.value).toBeNull(); + expect(d.trustTier).toBe(TrustTier.Unverified); + expect(d.sourceTimestamp).toBeNull(); + } + }) + ); + }); + + it("only agrees when the source count and weighted quorum are both met", () => { + fc.assert( + fc.property(fc.array(hostileObs, { maxLength: 15 }), validPolicy, (xs, p) => { + const d = evaluateConsensus(xs, p, NOW); + if (d.status !== "agreed") return true; + return d.agreeingSources.length >= p.minAgreeingSources && d.agreeingWeight / d.totalWeight >= p.quorumRatio; + }) + ); + }); + + it("never lets a source vote twice", () => { + fc.assert( + fc.property(fc.array(wellFormedObs(fc.constantFrom("a", "b", "c")), { maxLength: 10 }), validPolicy, (xs, p) => { + const d = evaluateConsensus(xs, p, NOW); + const counts = new Map(); + for (const o of xs) counts.set(o.sourceId, (counts.get(o.sourceId) ?? 0) + 1); + for (const [id, n] of counts) { + if (n > 1 && !d.invalidSources.includes(id)) return false; + } + return true; + }) + ); + }); +}); + +describe("evaluateConsensus — properties", () => { + it("does not depend on observation order", () => { + fc.assert( + fc.property( + fc.array(hostileObs, { maxLength: 12 }).chain((xs) => fc.tuple(fc.constant(xs), permuted(xs))), + validPolicy, + ([xs, ys], p) => { + const a = evaluateConsensus(xs, p, NOW); + const b = evaluateConsensus(ys, p, NOW); + expect(b).toEqual(a); + } + ) + ); + }); + + it("is deterministic", () => { + fc.assert( + fc.property(fc.array(hostileObs, { maxLength: 12 }), validPolicy, (xs, p) => { + expect(evaluateConsensus(xs, p, NOW)).toEqual(evaluateConsensus(xs, p, NOW)); + }) + ); + }); + + it("classifies agreeing and outlier sources by the tolerance around the center", () => { + fc.assert( + fc.property(freshUniqueObs(finite), validPolicy, (xs, p) => { + const d = evaluateConsensus(xs, { ...p, maxStalenessMs: 10_000 }, NOW); + const byId = new Map(xs.map((o) => [o.sourceId, o.value])); + const tol = deviationTolerance(d.center!, p); + for (const id of d.agreeingSources) expect(Math.abs(byId.get(id)! - d.center!)).toBeLessThanOrEqual(tol); + for (const id of d.outlierSources) expect(Math.abs(byId.get(id)! - d.center!)).toBeGreaterThan(tol); + }) + ); + }); + + it("publishes a value inside the range of the agreeing sources", () => { + fc.assert( + fc.property(freshUniqueObs(finite), validPolicy, (xs, p) => { + const d = evaluateConsensus(xs, { ...p, maxStalenessMs: 10_000 }, NOW); + if (d.status !== "agreed") return true; + const vals = xs.filter((o) => d.agreeingSources.includes(o.sourceId)).map((o) => o.value); + return d.value! >= Math.min(...vals) && d.value! <= Math.max(...vals); + }) + ); + }); + + it("never treats a stale observation as agreeing", () => { + fc.assert( + fc.property(fc.array(wellFormedObs(fc.string({ minLength: 1, maxLength: 3 })), { maxLength: 12 }), validPolicy, (xs, p) => { + const d = evaluateConsensus(xs, p, NOW); + return xs.every((o) => !(NOW - o.observedAt > p.maxStalenessMs) || !d.agreeingSources.includes(o.sourceId)); + }) + ); + }); + + it("bounds adversarial influence: a minority cannot move the value outside honest range ± tolerance", () => { + fc.assert( + fc.property( + freshUniqueObs(finite, fc.integer({ min: 1, max: 100 }), { minLength: 1, maxLength: 10 }), + fc.array(hostileNumber, { maxLength: 9 }), + validPolicy, + (honest, adversarialValues, p) => { + fc.pre(adversarialValues.length < honest.length); + const adversaries = adversarialValues.map((value, i) => ({ + sourceId: `adv${i}`, + value, + weight: 100, + observedAt: NOW, + })); + const d = evaluateConsensus([...honest, ...adversaries], { ...p, maxStalenessMs: 10_000 }, NOW); + if (d.status !== "agreed") return true; + const hv = honest.map((o) => o.value); + const lo = Math.min(...hv); + const hi = Math.max(...hv); + // Center sits inside the honest range; every agreeing value is + // within tolerance of the center, so is the median of them. + const tol = deviationTolerance(d.center!, p); + return d.center! >= lo && d.center! <= hi && d.value! >= lo - tol && d.value! <= hi + tol; + } + ) + ); + }); + + it("an honest, tightly clustered majority with quorum weight always agrees and rejects distant adversaries", () => { + fc.assert( + fc.property( + exactInt, + fc.integer({ min: 2, max: 10 }), + fc.integer({ min: 0, max: 9 }), + fc.double({ min: 1, max: 100, noNaN: true }), + fc.array(fc.double({ min: -0.25, max: 0.25, noNaN: true }), { minLength: 10, maxLength: 10 }), + fc.array(fc.oneof(fc.double({ min: 3, max: 1e9, noNaN: true }), fc.double({ min: -1e9, max: -3, noNaN: true })), { + minLength: 9, + maxLength: 9, + }), + (center, honestCount, advCount, floor, jitter, advOffsets) => { + fc.pre(advCount < honestCount); + // Purely absolute tolerance; honest spread is ±floor/4, adversaries + // sit at least 3·floor away from the cluster. + const p: ConsensusPolicy = { + quorumRatio: honestCount / (honestCount + advCount), + minAgreeingSources: 2, + maxDeviationRatio: 0, + absoluteDeviationFloor: floor, + maxStalenessMs: 60_000, + maxFutureSkewMs: 0, + }; + const honest = Array.from({ length: honestCount }, (_, i) => ({ + sourceId: `h${i}`, + value: center + jitter[i] * floor, + weight: 1, + observedAt: NOW, + })); + const adv = Array.from({ length: advCount }, (_, i) => ({ + sourceId: `x${i}`, + value: center + advOffsets[i] * floor, + weight: 1, + observedAt: NOW, + })); + const d = evaluateConsensus([...honest, ...adv], p, NOW); + expect(d.status).toBe("agreed"); + expect(d.agreeingSources).toEqual(honest.map((o) => o.sourceId).sort()); + expect(d.outlierSources).toEqual(adv.map((o) => o.sourceId).sort()); + const hv = honest.map((o) => o.value); + expect(d.value!).toBeGreaterThanOrEqual(Math.min(...hv)); + expect(d.value!).toBeLessThanOrEqual(Math.max(...hv)); + } + ) + ); + }); + + it("assigns trust tiers consistently with the agreeing set", () => { + fc.assert( + fc.property(fc.array(hostileObs, { maxLength: 12 }), validPolicy, (xs, p) => { + const d = evaluateConsensus(xs, p, NOW); + const n = d.agreeingSources.length; + switch (d.trustTier) { + case TrustTier.Unverified: + return d.status !== "agreed"; + case TrustTier.SingleSource: + return d.status === "agreed" && n === 1; + case TrustTier.Corroborated: + return d.status === "agreed" && n >= 2; + case TrustTier.Unanimous: + return ( + d.status === "agreed" && + n >= 3 && + d.outlierSources.length + d.staleSources.length + d.invalidSources.length === 0 + ); + } + }) + ); + }); +}); diff --git a/src/oracle/consensus/consensus.spec.ts b/src/oracle/consensus/consensus.spec.ts new file mode 100644 index 0000000..2f9d4b0 --- /dev/null +++ b/src/oracle/consensus/consensus.spec.ts @@ -0,0 +1,178 @@ +import { + ConsensusPolicy, + DEFAULT_CONSENSUS_POLICY, + SourceObservation, + TrustTier, + deviationTolerance, + evaluateConsensus, + isAnomalous, + median, + meetsQuorum, + validateConsensusPolicy, +} from "./consensus"; + +const NOW = 1_800_000_000_000; + +function obs(sourceId: string, value: number, overrides: Partial = {}): SourceObservation { + return { sourceId, value, weight: 1, observedAt: NOW - 1_000, ...overrides }; +} + +const policy: ConsensusPolicy = { ...DEFAULT_CONSENSUS_POLICY }; + +describe("median", () => { + it("returns the middle element for odd lengths", () => { + expect(median([3, 1, 2])).toBe(2); + }); + + it("averages the middle pair for even lengths", () => { + expect(median([4, 1, 3, 2])).toBe(2.5); + }); + + it("does not overflow on large values", () => { + expect(median([Number.MAX_VALUE, Number.MAX_VALUE])).toBe(Number.MAX_VALUE); + }); + + it("stays inside the input range for subnormal values (found by property test)", () => { + expect(median([5e-324, 5e-324, 5e-324, 5e-324])).toBe(5e-324); + }); + + it("rejects empty and non-finite input", () => { + expect(() => median([])).toThrow(RangeError); + expect(() => median([1, NaN])).toThrow(RangeError); + expect(() => median([Infinity])).toThrow(RangeError); + }); +}); + +describe("meetsQuorum", () => { + it("is inclusive at the boundary", () => { + expect(meetsQuorum(2, 3, 2 / 3)).toBe(true); + expect(meetsQuorum(1.99, 3, 2 / 3)).toBe(false); + }); + + it("never passes with zero or invalid total weight", () => { + expect(meetsQuorum(0, 0, 0.5)).toBe(false); + expect(meetsQuorum(1, NaN, 0.5)).toBe(false); + expect(meetsQuorum(1, Infinity, 0.5)).toBe(false); + }); +}); + +describe("anomaly threshold", () => { + it("accepts values exactly at the tolerance and rejects values past it", () => { + const p = { ...policy, maxDeviationRatio: 0.5, absoluteDeviationFloor: 0 }; + expect(deviationTolerance(10, p)).toBe(5); + expect(isAnomalous(15, 10, p)).toBe(false); + expect(isAnomalous(15.000001, 10, p)).toBe(true); + }); + + it("falls back to the absolute floor around zero", () => { + const p = { ...policy, maxDeviationRatio: 0.02, absoluteDeviationFloor: 0.5 }; + expect(isAnomalous(0.4, 0, p)).toBe(false); + expect(isAnomalous(0.6, 0, p)).toBe(true); + }); +}); + +describe("validateConsensusPolicy", () => { + it.each<[string, Partial]>([ + ["zero quorum", { quorumRatio: 0 }], + ["quorum above one", { quorumRatio: 1.01 }], + ["fractional min sources", { minAgreeingSources: 1.5 }], + ["negative deviation", { maxDeviationRatio: -1 }], + ["NaN staleness", { maxStalenessMs: NaN }], + ])("rejects %s", (_label, override) => { + expect(() => validateConsensusPolicy({ ...policy, ...override })).toThrow(RangeError); + }); +}); + +describe("evaluateConsensus", () => { + it("agrees on the median of agreeing sources and drops the outlier", () => { + const d = evaluateConsensus( + [obs("a", 0.9990), obs("b", 1.0000), obs("c", 1.0010), obs("evil", 0.5)], + { ...policy, quorumRatio: 0.5 }, + NOW + ); + expect(d.status).toBe("agreed"); + expect(d.value).toBeCloseTo(1.0, 10); + expect(d.agreeingSources).toEqual(["a", "b", "c"]); + expect(d.outlierSources).toEqual(["evil"]); + expect(d.trustTier).toBe(TrustTier.Corroborated); + expect(d.decisionId).toMatch(/^[0-9a-f]{16}$/); + }); + + it("reports Unanimous only when every source is fresh, valid and agreeing", () => { + const d = evaluateConsensus([obs("a", 1), obs("b", 1), obs("c", 1)], policy, NOW); + expect(d.trustTier).toBe(TrustTier.Unanimous); + }); + + it("reports SingleSource when policy allows one source", () => { + const d = evaluateConsensus([obs("a", 1)], { ...policy, minAgreeingSources: 1 }, NOW); + expect(d.status).toBe("agreed"); + expect(d.trustTier).toBe(TrustTier.SingleSource); + }); + + it("refuses a single source under the default policy", () => { + const d = evaluateConsensus([obs("a", 1)], policy, NOW); + expect(d.status).toBe("insufficient_sources"); + expect(d.value).toBeNull(); + expect(d.trustTier).toBe(TrustTier.Unverified); + }); + + it("fails quorum when agreeing weight is too small, even with more agreeing sources", () => { + const d = evaluateConsensus( + [obs("a", 1, { weight: 1 }), obs("b", 1, { weight: 1 }), obs("whale", 2, { weight: 10 })], + { ...policy, quorumRatio: 0.5 }, + NOW + ); + // Median of [1, 1, 2] is 1, so the whale is the outlier — but it still + // holds most of the weight, so the honest pair cannot reach quorum. + expect(d.outlierSources).toEqual(["whale"]); + expect(d.status).toBe("no_quorum"); + }); + + it("marks stale and future-dated observations", () => { + const d = evaluateConsensus( + [ + obs("a", 1), + obs("b", 1), + obs("old", 1, { observedAt: NOW - policy.maxStalenessMs - 1 }), + obs("future", 1, { observedAt: NOW + policy.maxFutureSkewMs + 1 }), + ], + { ...policy, quorumRatio: 0.5 }, + NOW + ); + expect(d.staleSources).toEqual(["old"]); + expect(d.invalidSources).toEqual(["future"]); + expect(d.status).toBe("agreed"); + expect(d.trustTier).toBe(TrustTier.Corroborated); + }); + + it("invalidates every copy of a duplicated source id", () => { + const d = evaluateConsensus([obs("a", 1), obs("a", 1), obs("b", 1)], policy, NOW); + expect(d.invalidSources).toEqual(["a"]); + expect(d.agreeingSources).toEqual(["b"]); + expect(d.status).toBe("insufficient_sources"); + }); + + it("counts invalid sources in the quorum denominator", () => { + const d = evaluateConsensus( + [obs("a", 1), obs("b", 1), obs("bad1", NaN), obs("bad2", NaN)], + { ...policy, quorumRatio: 0.75 }, + NOW + ); + expect(d.totalWeight).toBe(4); + expect(d.status).toBe("no_quorum"); + }); + + it("uses the earliest agreeing timestamp as the source timestamp", () => { + const d = evaluateConsensus([obs("a", 1, { observedAt: NOW - 5_000 }), obs("b", 1)], policy, NOW); + expect(d.sourceTimestamp).toBe(NOW - 5_000); + }); + + it("produces the same decision id for the same inputs in any order", () => { + const xs = [obs("a", 1), obs("b", 1.001), obs("c", 0.999)]; + const d1 = evaluateConsensus(xs, policy, NOW); + const d2 = evaluateConsensus([...xs].reverse(), policy, NOW); + expect(d2.decisionId).toBe(d1.decisionId); + const d3 = evaluateConsensus(xs, policy, NOW + 1); + expect(d3.decisionId).not.toBe(d1.decisionId); + }); +}); diff --git a/src/oracle/consensus/consensus.ts b/src/oracle/consensus/consensus.ts new file mode 100644 index 0000000..71f147a --- /dev/null +++ b/src/oracle/consensus/consensus.ts @@ -0,0 +1,329 @@ +import { createHash } from "crypto"; + +/** + * Multi-source oracle consensus. + * + * Every payout decision rests on the numbers produced here, so this module is + * deliberately pure: no I/O, no clock reads (callers pass `now`), no Nest + * dependencies. That keeps it easy to exercise with property-based tests + * (see consensus.property.spec.ts). + * + * Pipeline for one evaluation: + * 1. Screen each observation: malformed or duplicated sources are + * `invalid`, old ones are `stale`. + * 2. Take the median of the fresh values as the screening center. + * 3. Sources further than the deviation tolerance from the center are + * `outlier`s; the rest are `agreeing`. + * 4. Agreement needs enough agreeing sources and enough agreeing weight + * (weighted quorum over every source that reported). + * 5. The published value is the median of the agreeing values. + */ + +/** One source's report for a single coverage type. */ +export interface SourceObservation { + /** Stable identifier of the data source (e.g. "coingecko"). */ + sourceId: string; + /** The reported value, in the coverage type's natural unit. */ + value: number; + /** Voting weight. Must be finite and >= 0. */ + weight: number; + /** When the source observed the value (epoch milliseconds). */ + observedAt: number; +} + +export interface ConsensusPolicy { + /** Fraction (0, 1] of total reporting weight that must agree. */ + quorumRatio: number; + /** Minimum number of agreeing sources (>= 1). */ + minAgreeingSources: number; + /** Allowed deviation from the center, relative to |center| (>= 0). */ + maxDeviationRatio: number; + /** + * Absolute deviation that is always tolerated (>= 0). Needed when the + * center is near zero (e.g. a 0% market move), where a purely relative + * tolerance would reject every source. + */ + absoluteDeviationFloor: number; + /** Observations older than this are stale (ms, >= 0). */ + maxStalenessMs: number; + /** Observations further in the future than this are invalid (ms, >= 0). */ + maxFutureSkewMs: number; +} + +export const DEFAULT_CONSENSUS_POLICY: Readonly = Object.freeze({ + quorumRatio: 2 / 3, + minAgreeingSources: 2, + maxDeviationRatio: 0.02, + absoluteDeviationFloor: 0, + maxStalenessMs: 5 * 60_000, + maxFutureSkewMs: 30_000, +}); + +/** + * How much the published value can be trusted. The numeric values are part + * of the on-chain provenance encoding (see oracle-encoding.ts) — never + * renumber them. + */ +export enum TrustTier { + /** No consensus; must never be published as authoritative. */ + Unverified = 0, + /** Exactly one source agreed (only possible when minAgreeingSources = 1). */ + SingleSource = 1, + /** Two or more sources agreed and quorum was met. */ + Corroborated = 2, + /** Three or more sources, all fresh, valid and in agreement. */ + Unanimous = 3, +} + +export type ConsensusStatus = "agreed" | "no_valid_sources" | "insufficient_sources" | "no_quorum"; + +export interface ConsensusDecision { + status: ConsensusStatus; + /** Median of agreeing values; null unless status is "agreed". */ + value: number | null; + trustTier: TrustTier; + /** Deterministic 64-bit identifier of this decision, as 16 hex chars. */ + decisionId: string; + /** Median of fresh values used to screen outliers; null if none were fresh. */ + center: number | null; + /** Source ids by disposition. Each list is sorted and duplicate-free. */ + agreeingSources: string[]; + outlierSources: string[]; + staleSources: string[]; + invalidSources: string[]; + agreeingWeight: number; + /** Weight of every distinct source that reported, valid or not. */ + totalWeight: number; + /** Earliest observedAt among agreeing sources (ms); null unless agreed. */ + sourceTimestamp: number | null; + evaluatedAt: number; +} + +/** Median of a non-empty list of finite numbers. Mean of the middle pair for even lengths. */ +export function median(values: readonly number[]): number { + if (values.length === 0) { + throw new RangeError("median() of an empty list"); + } + for (const v of values) { + if (!Number.isFinite(v)) { + throw new RangeError(`median() requires finite numbers, got ${v}`); + } + } + const sorted = [...values].sort((a, b) => a - b); + const mid = sorted.length >> 1; + if (sorted.length % 2 === 1) { + return sorted[mid]; + } + const lo = sorted[mid - 1]; + const hi = sorted[mid]; + // (lo + hi) / 2 is correctly rounded, so it stays inside [lo, hi] even for + // subnormals. Only when the sum overflows do we halve first — at that + // magnitude halving is exact. + const sum = lo + hi; + return Number.isFinite(sum) ? sum / 2 : lo / 2 + hi / 2; +} + +/** Largest distance from `center` that still counts as agreement. */ +export function deviationTolerance(center: number, policy: ConsensusPolicy): number { + return Math.max(policy.maxDeviationRatio * Math.abs(center), policy.absoluteDeviationFloor); +} + +export function isAnomalous(value: number, center: number, policy: ConsensusPolicy): boolean { + return Math.abs(value - center) > deviationTolerance(center, policy); +} + +/** + * Weighted quorum check. Uses a division rather than `ratio * total` so the + * result does not change when every weight is scaled by the same factor. + */ +export function meetsQuorum(agreeingWeight: number, totalWeight: number, quorumRatio: number): boolean { + if (!(totalWeight > 0) || !Number.isFinite(totalWeight) || !(agreeingWeight >= 0)) { + return false; + } + return agreeingWeight / totalWeight >= quorumRatio; +} + +/** Throws if the policy is unusable. Policies are operator config, so failing loudly is correct. */ +export function validateConsensusPolicy(policy: ConsensusPolicy): void { + const problems: string[] = []; + if (!(policy.quorumRatio > 0 && policy.quorumRatio <= 1)) problems.push("quorumRatio must be in (0, 1]"); + if (!Number.isInteger(policy.minAgreeingSources) || policy.minAgreeingSources < 1) { + problems.push("minAgreeingSources must be an integer >= 1"); + } + for (const key of ["maxDeviationRatio", "absoluteDeviationFloor", "maxStalenessMs", "maxFutureSkewMs"] as const) { + if (!(Number.isFinite(policy[key]) && policy[key] >= 0)) problems.push(`${key} must be finite and >= 0`); + } + if (problems.length > 0) { + throw new RangeError(`Invalid consensus policy: ${problems.join("; ")}`); + } +} + +function isUsableWeight(w: unknown): w is number { + return typeof w === "number" && Number.isFinite(w) && w >= 0; +} + +/** + * Evaluate consensus over one round of observations. + * + * Never throws for bad observations — they are classified as invalid. It + * only throws for an invalid policy or a non-finite `now`. + * + * The result does not depend on the order of `observations`. + */ +export function evaluateConsensus( + observations: readonly SourceObservation[], + policy: ConsensusPolicy, + now: number +): ConsensusDecision { + validateConsensusPolicy(policy); + if (!Number.isFinite(now)) { + throw new RangeError(`evaluateConsensus() requires a finite 'now', got ${now}`); + } + + // Group by source id. A source id that appears more than once is treated + // as invalid in every copy, so a source cannot vote twice and the outcome + // does not depend on which copy came first. + const bySource = new Map(); + for (const obs of observations) { + const id = typeof obs?.sourceId === "string" ? obs.sourceId : String(obs?.sourceId); + const list = bySource.get(id); + if (list) list.push(obs); + else bySource.set(id, [obs]); + } + + const invalid: string[] = []; + const stale: string[] = []; + const fresh: SourceObservation[] = []; + let totalWeight = 0; + + for (const [id, copies] of bySource) { + // Every source that reported counts toward the quorum denominator, so a + // misbehaving source cannot shrink the bar by sending garbage. + const usableWeights = copies.map((c) => c.weight).filter(isUsableWeight); + if (usableWeights.length > 0) totalWeight += Math.max(...usableWeights); + + if (copies.length !== 1) { + invalid.push(id); + continue; + } + const obs = copies[0]; + const wellFormed = + typeof obs.sourceId === "string" && + obs.sourceId.length > 0 && + typeof obs.value === "number" && + Number.isFinite(obs.value) && + isUsableWeight(obs.weight) && + typeof obs.observedAt === "number" && + Number.isFinite(obs.observedAt) && + obs.observedAt <= now + policy.maxFutureSkewMs; + if (!wellFormed) { + invalid.push(id); + } else if (now - obs.observedAt > policy.maxStalenessMs) { + stale.push(id); + } else { + fresh.push(obs); + } + } + + const base = { + invalidSources: invalid.sort(), + staleSources: stale.sort(), + totalWeight, + evaluatedAt: now, + }; + + if (fresh.length === 0) { + return finalize(observations, policy, { + ...base, + status: "no_valid_sources", + value: null, + trustTier: TrustTier.Unverified, + center: null, + agreeingSources: [], + outlierSources: [], + agreeingWeight: 0, + sourceTimestamp: null, + }); + } + + const center = median(fresh.map((o) => o.value)); + const agreeing = fresh.filter((o) => !isAnomalous(o.value, center, policy)); + const outliers = fresh.filter((o) => isAnomalous(o.value, center, policy)); + const agreeingWeight = agreeing.reduce((sum, o) => sum + o.weight, 0); + + const common = { + ...base, + center, + agreeingSources: agreeing.map((o) => o.sourceId).sort(), + outlierSources: outliers.map((o) => o.sourceId).sort(), + agreeingWeight, + }; + + let status: ConsensusStatus = "agreed"; + if (agreeing.length < policy.minAgreeingSources) status = "insufficient_sources"; + else if (!meetsQuorum(agreeingWeight, totalWeight, policy.quorumRatio)) status = "no_quorum"; + + if (status !== "agreed") { + return finalize(observations, policy, { + ...common, + status, + value: null, + trustTier: TrustTier.Unverified, + sourceTimestamp: null, + }); + } + + let trustTier: TrustTier; + if (agreeing.length === 1) trustTier = TrustTier.SingleSource; + else if (agreeing.length >= 3 && outliers.length === 0 && stale.length === 0 && invalid.length === 0) { + trustTier = TrustTier.Unanimous; + } else trustTier = TrustTier.Corroborated; + + return finalize(observations, policy, { + ...common, + status, + value: median(agreeing.map((o) => o.value)), + trustTier, + sourceTimestamp: Math.min(...agreeing.map((o) => o.observedAt)), + }); +} + +function finalize( + observations: readonly SourceObservation[], + policy: ConsensusPolicy, + decision: Omit +): ConsensusDecision { + return { ...decision, decisionId: computeDecisionId(observations, policy, decision) }; +} + +/** + * Deterministic 64-bit decision id: a truncated SHA-256 over the canonical + * (sorted) inputs and the outcome. Re-running the same round reproduces the + * same id, which lets an auditor tie an on-chain value back to its inputs. + */ +export function computeDecisionId( + observations: readonly SourceObservation[], + policy: ConsensusPolicy, + decision: Pick +): string { + // String() keeps NaN / Infinity / -0 distinguishable, unlike JSON numbers. + const rows = observations + .map((o) => [String(o?.sourceId), String(o?.value), String(o?.weight), String(o?.observedAt)].join("|")) + .sort(); + const canonical = JSON.stringify({ + v: 1, + rows, + policy: [ + policy.quorumRatio, + policy.minAgreeingSources, + policy.maxDeviationRatio, + policy.absoluteDeviationFloor, + policy.maxStalenessMs, + policy.maxFutureSkewMs, + ].map(String), + status: decision.status, + value: String(decision.value), + at: String(decision.evaluatedAt), + }); + return createHash("sha256").update(canonical).digest("hex").slice(0, 16); +} From 29b4489ccd57149b012278b15aa5549dab313cf2 Mon Sep 17 00:00:00 2001 From: agudajohn793 Date: Wed, 21 Oct 2026 12:29:00 +0000 Subject: [PATCH 2/4] feat(oracle): carry compact provenance with each published reading MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On-chain readings were a bare i128, so consumers and auditors could not tell a corroborated value from a single-source one without trusting the backend. oracle-encoding.ts now packs provenance into a u128 that is published alongside the value. Layout (MSB first): version (4) | trust tier (4) | agreeing source count (8) | reserved (8) | source timestamp, Unix seconds (40) | consensus decision id (64). Version 0 is never valid, so zero means "no provenance". - packProvenance / unpackProvenance with strict validation: unknown versions, set reserved bits, out-of-range fields and inconsistent tier/count pairs (e.g. Corroborated with one source) are rejected. Unverified readings cannot be encoded at all. - provenanceFromDecision maps a ConsensusDecision to provenance; the source timestamp is the earliest agreeing observation, floored to seconds, so a value never looks fresher than its oldest input. - encodeReadingForChain + buildUpdateReadingArgs produce ScVal args for update_reading(coverage_type: u32, value: i128, provenance: u128). The contract has to accept the third argument before the relayer switches over. - OracleReading gains an optional `provenance`. The decision id is kept as 16 hex chars rather than a bigint so readings stay JSON-safe for the WebSocket feed. - Golden XDR fixture for the u128 argument, added to the regenerate script (existing fixtures are unchanged). - docs/oracle-provenance.md documents the layout for contract authors and auditors. Tests cover the documented bit layout, a fast-check round-trip over all valid provenance, every rejection path, decision → provenance mapping, the ScVal types and the golden fixture. Co-Authored-By: Claude Opus 5.5 --- docs/oracle-provenance.md | 69 +++++++ scripts/regenerate-xdr-fixtures.ts | 11 ++ src/oracle/oracle-encoding.spec.ts | 165 ++++++++++++++++ src/oracle/oracle-encoding.ts | 181 ++++++++++++++++++ src/oracle/oracle-reading.ts | 8 + .../oracle-update-reading-provenance.xdr.b64 | 6 + 6 files changed, 440 insertions(+) create mode 100644 docs/oracle-provenance.md create mode 100644 src/oracle/oracle-encoding.spec.ts create mode 100644 src/stellar/xdr-fixtures/oracle-update-reading-provenance.xdr.b64 diff --git a/docs/oracle-provenance.md b/docs/oracle-provenance.md new file mode 100644 index 0000000..faddd71 --- /dev/null +++ b/docs/oracle-provenance.md @@ -0,0 +1,69 @@ +# Oracle reading provenance + +Each oracle reading the relayer publishes carries a packed `u128` next to the +value, so contracts and auditors can tell how well-supported the value is +without leaving the ledger: + +``` +update_reading(coverage_type: u32, value: i128, provenance: u128) +``` + +The backend side lives in `src/oracle/oracle-encoding.ts` +(`packProvenance`, `unpackProvenance`, `provenanceFromDecision`, +`buildUpdateReadingArgs`). The contract must accept the third argument +before the relayer switches to it. + +## Layout + +Most significant bits first: + +| bits | width | field | +|----------|-------|-------------------------------------------| +| 127..124 | 4 | layout version (currently `1`) | +| 123..120 | 4 | trust tier | +| 119..112 | 8 | agreeing source count | +| 111..104 | 8 | reserved, always `0` | +| 103..64 | 40 | source timestamp, Unix seconds | +| 63..0 | 64 | consensus decision id | + +A provenance of `0` (version 0) means "no provenance" and is never produced. + +### Trust tiers + +| value | tier | meaning | +|-------|----------------|-----------------------------------------------------------------| +| 0 | `Unverified` | no consensus — never published | +| 1 | `SingleSource` | exactly one source agreed (only if policy allows one source) | +| 2 | `Corroborated` | two or more sources agreed and the weighted quorum was met | +| 3 | `Unanimous` | three or more sources, all fresh, valid and in agreement | + +The encoder rejects inconsistent combinations (e.g. `Corroborated` with one +source), so consumers can rely on `tier >= 2` meaning "at least two sources". + +### Source timestamp + +The earliest observation time among the agreeing sources, floored to whole +seconds. Using the earliest one means the value never looks fresher than its +oldest supporting input. + +### Decision id + +The first 64 bits of a SHA-256 over the round's sorted inputs, the consensus +policy, the outcome and the evaluation time (`computeDecisionId` in +`src/oracle/consensus/consensus.ts`). Re-running consensus over the logged +inputs reproduces the id, which ties an on-chain value back to the exact +observations behind it. + +## Decoding in a contract (Rust sketch) + +```rust +let version = (p >> 124) & 0xF; +let tier = (p >> 120) & 0xF; +let sources = (p >> 112) & 0xFF; +let ts_secs = ((p >> 64) & 0xFF_FFFF_FFFF) as u64; +let decision = (p & 0xFFFF_FFFF_FFFF_FFFF) as u64; +assert!(version == 1); +``` + +A consumer that only wants corroborated data can require `tier >= 2`; one +that cares about freshness can compare `ts_secs` with the ledger timestamp. diff --git a/scripts/regenerate-xdr-fixtures.ts b/scripts/regenerate-xdr-fixtures.ts index a0f7697..647c281 100644 --- a/scripts/regenerate-xdr-fixtures.ts +++ b/scripts/regenerate-xdr-fixtures.ts @@ -19,6 +19,7 @@ import { encodeProvideCapitalArgs, encodeWithdrawCapitalArgs, } from "../src/stellar/scval-encoders"; +import { TrustTier, buildUpdateReadingArgs, encodeReadingForChain } from "../src/oracle/oracle-encoding"; const outDir = path.join(__dirname, "../src/stellar/xdr-fixtures"); @@ -88,6 +89,16 @@ writeB64("oracle-publish-feed.xdr.b64", feed); writeB64("oracle-publish-value.xdr.b64", value); writeB64("oracle-publish-timestamp.xdr.b64", ts); +const [, , provenance] = buildUpdateReadingArgs( + encodeReadingForChain("StablecoinDepeg", 0.9998, { + trustTier: TrustTier.Corroborated, + agreeingSources: 3, + decisionId: "0123456789abcdef", + sourceTimestamp: 1_700_000_000, + }) +); +writeB64("oracle-update-reading-provenance.xdr.b64", provenance, "u128 provenance: Corroborated, 3 sources, ts=1700000000"); + // silence unused import if tree-shaken oddly void Address; void nativeToScVal; diff --git a/src/oracle/oracle-encoding.spec.ts b/src/oracle/oracle-encoding.spec.ts new file mode 100644 index 0000000..e467141 --- /dev/null +++ b/src/oracle/oracle-encoding.spec.ts @@ -0,0 +1,165 @@ +import * as fs from "fs"; +import * as path from "path"; +import fc from "fast-check"; +import { scValToNative } from "@stellar/stellar-sdk"; +import { DEFAULT_CONSENSUS_POLICY, evaluateConsensus } from "./consensus/consensus"; +import { + PROVENANCE_VERSION, + ProvenanceEncodingError, + ReadingProvenance, + TrustTier, + buildUpdateReadingArgs, + coverageTypeToU32, + encodeReadingForChain, + packProvenance, + provenanceFromDecision, + scaleOracleValue, + unpackProvenance, +} from "./oracle-encoding"; +import { OracleReading } from "./oracle-reading"; + +const FIXTURES = path.join(__dirname, "../stellar/xdr-fixtures"); + +function readFixture(name: string): string { + const raw = fs.readFileSync(path.join(FIXTURES, name), "utf8"); + return raw + .split("\n") + .filter((l) => l && !l.startsWith("//")) + .join("") + .trim(); +} + +const SAMPLE: ReadingProvenance = { + trustTier: TrustTier.Corroborated, + agreeingSources: 3, + decisionId: "0123456789abcdef", + sourceTimestamp: 1_700_000_000, +}; + +describe("value scaling", () => { + it("scales each coverage type deterministically", () => { + expect(scaleOracleValue("StablecoinDepeg", 0.9998)).toBe(9_998_000n); + expect(scaleOracleValue("MarketCrash", -30.12)).toBe(-3012n); + expect(scaleOracleValue("LiquidationShield", 0.42)).toBe(4200n); + expect(scaleOracleValue("SmartContractRisk", -50)).toBe(-5000n); + expect(scaleOracleValue("FlightDelay", 247.4)).toBe(247n); + }); + + it("maps coverage type names to u32 discriminants", () => { + expect(coverageTypeToU32("FlightDelay")).toBe(4); + expect(() => coverageTypeToU32("Nope")).toThrow(); + }); +}); + +describe("provenance packing", () => { + it("matches the documented bit layout", () => { + // version=1 | tier=2 | count=0x03 | reserved=0x00 | ts=0x006553f100 | id + expect(packProvenance(SAMPLE)).toBe(0x1203_00_006553f100_0123456789abcdefn); + expect(PROVENANCE_VERSION).toBe(1); + }); + + it("round-trips every valid provenance", () => { + const tierAndCount = fc.oneof( + fc.constant({ trustTier: TrustTier.SingleSource, agreeingSources: 1 }), + fc.integer({ min: 2, max: 255 }).map((n) => ({ trustTier: TrustTier.Corroborated, agreeingSources: n })), + fc.integer({ min: 3, max: 255 }).map((n) => ({ trustTier: TrustTier.Unanimous, agreeingSources: n })) + ); + fc.assert( + fc.property( + tierAndCount, + fc.bigInt({ min: 0n, max: (1n << 64n) - 1n }), + fc.integer({ min: 0, max: 2 ** 40 - 1 }), + (tc, id, ts) => { + const p: ReadingProvenance = { ...tc, decisionId: id.toString(16).padStart(16, "0"), sourceTimestamp: ts }; + const packed = packProvenance(p); + expect(packed).toBeGreaterThanOrEqual(0n); + expect(packed).toBeLessThan(1n << 128n); + expect(unpackProvenance(packed)).toEqual(p); + } + ) + ); + }); + + it.each<[string, Partial]>([ + ["unverified tier", { trustTier: TrustTier.Unverified }], + ["unknown tier", { trustTier: 9 as TrustTier }], + ["single source with two sources", { trustTier: TrustTier.SingleSource, agreeingSources: 2 }], + ["corroborated with one source", { trustTier: TrustTier.Corroborated, agreeingSources: 1 }], + ["unanimous with two sources", { trustTier: TrustTier.Unanimous, agreeingSources: 2 }], + ["source count over 255", { agreeingSources: 256 }], + ["fractional source count", { agreeingSources: 2.5 }], + ["short decision id", { decisionId: "abc" }], + ["upper-case decision id", { decisionId: "0123456789ABCDEF" }], + ["negative timestamp", { sourceTimestamp: -1 }], + ["timestamp in milliseconds", { sourceTimestamp: 1_700_000_000_000 * 1000 }], + ])("rejects %s", (_label, override) => { + expect(() => packProvenance({ ...SAMPLE, ...override })).toThrow(ProvenanceEncodingError); + }); + + it("rejects unknown versions, set reserved bits and out-of-range input when unpacking", () => { + const good = packProvenance(SAMPLE); + expect(() => unpackProvenance(0n)).toThrow(/version 0/); + expect(() => unpackProvenance(good | (1n << 104n))).toThrow(/Reserved/); + expect(() => unpackProvenance((good & ~(0xfn << 124n)) | (2n << 124n))).toThrow(/version 2/); + expect(() => unpackProvenance(-1n)).toThrow(ProvenanceEncodingError); + expect(() => unpackProvenance(1n << 128n)).toThrow(ProvenanceEncodingError); + }); +}); + +describe("provenanceFromDecision", () => { + const NOW = 1_700_000_000_999; + + it("carries tier, agreeing count, decision id and floored timestamp", () => { + const decision = evaluateConsensus( + [ + { sourceId: "a", value: 1, weight: 1, observedAt: NOW - 1_500 }, + { sourceId: "b", value: 1, weight: 1, observedAt: NOW }, + { sourceId: "evil", value: 9, weight: 1, observedAt: NOW }, + ], + { ...DEFAULT_CONSENSUS_POLICY, quorumRatio: 0.5 }, + NOW + ); + const p = provenanceFromDecision(decision); + expect(p).toEqual({ + trustTier: TrustTier.Corroborated, + agreeingSources: 2, + decisionId: decision.decisionId, + sourceTimestamp: 1_699_999_999, + }); + expect(unpackProvenance(packProvenance(p))).toEqual(p); + }); + + it("refuses decisions that did not reach consensus", () => { + const decision = evaluateConsensus([], DEFAULT_CONSENSUS_POLICY, NOW); + expect(() => provenanceFromDecision(decision)).toThrow(/no_valid_sources/); + }); +}); + +describe("on-chain encoding", () => { + it("builds update_reading(u32, i128, u128) arguments", () => { + const encoded = encodeReadingForChain("StablecoinDepeg", 0.9998, SAMPLE); + const args = buildUpdateReadingArgs(encoded); + + expect(args.map((a) => a.switch().name)).toEqual(["scvU32", "scvI128", "scvU128"]); + expect(args.map((a) => scValToNative(a))).toEqual([0, 9_998_000n, packProvenance(SAMPLE)]); + expect(unpackProvenance(scValToNative(args[2]) as bigint)).toEqual(SAMPLE); + }); + + it("matches the golden XDR fixture for the provenance argument", () => { + const [, , provenance] = buildUpdateReadingArgs(encodeReadingForChain("StablecoinDepeg", 0.9998, SAMPLE)); + expect(provenance.toXDR("base64")).toBe(readFixture("oracle-update-reading-provenance.xdr.b64")); + }); + + it("keeps readings with provenance JSON-serialisable", () => { + const reading: OracleReading = { + coverageType: "StablecoinDepeg", + type: "oracle_update", + value: 0.9998, + threshold: 0.95, + severity: "low", + message: "ok", + provenance: SAMPLE, + }; + expect(JSON.parse(JSON.stringify(reading))).toEqual(reading); + }); +}); diff --git a/src/oracle/oracle-encoding.ts b/src/oracle/oracle-encoding.ts index b795b38..12ad985 100644 --- a/src/oracle/oracle-encoding.ts +++ b/src/oracle/oracle-encoding.ts @@ -1,3 +1,8 @@ +import { nativeToScVal, xdr } from "@stellar/stellar-sdk"; +import { ConsensusDecision, TrustTier } from "./consensus/consensus"; + +export { TrustTier } from "./consensus/consensus"; + /** * Fixed-point encoding of OracleReading.value → i128 for RefractOracle. * @@ -50,3 +55,179 @@ export function coverageTypeToU32(coverageType: string): number { } return id; } + +// --------------------------------------------------------------------------- +// Reading provenance +// --------------------------------------------------------------------------- + +/** + * Where a published value came from, carried on-chain next to the value so + * consumers and auditors can tell a corroborated reading from a + * single-source one without leaving the ledger. + */ +export interface ReadingProvenance { + /** Consensus trust tier. Unverified readings are never encoded. */ + trustTier: TrustTier; + /** How many independent sources agreed (0–255). */ + agreeingSources: number; + /** + * Consensus decision id (u64) as 16 lowercase hex chars; reproducible from + * the round's inputs. A string, not a bigint, so readings stay JSON-safe. + */ + decisionId: string; + /** Earliest agreeing source observation time, Unix seconds (40-bit). */ + sourceTimestamp: number; +} + +/** Current provenance layout version. Bump when the layout changes. */ +export const PROVENANCE_VERSION = 1; + +const TIMESTAMP_MAX = 2 ** 40 - 1; // ~year 36812 +const U128_MAX = (1n << 128n) - 1n; + +/** + * Packed provenance layout (u128, most significant bits first): + * + * | bits | width | field | + * |----------|-------|----------------------------------------| + * | 127..124 | 4 | layout version (currently 1) | + * | 123..120 | 4 | trust tier (TrustTier) | + * | 119..112 | 8 | agreeing source count | + * | 111..104 | 8 | reserved, must be 0 | + * | 103..64 | 40 | source timestamp, Unix seconds | + * | 63..0 | 64 | consensus decision id | + * + * Version 0 is never valid, so a zero u128 means "no provenance". + */ +const SHIFT = { + version: 124n, + trustTier: 120n, + agreeingSources: 112n, + reserved: 104n, + sourceTimestamp: 64n, +} as const; + +export class ProvenanceEncodingError extends Error { + constructor(message: string) { + super(message); + this.name = "ProvenanceEncodingError"; + } +} + +/** Rejects provenance that is out of range or internally inconsistent. */ +export function assertValidProvenance(p: ReadingProvenance): void { + if (typeof p.trustTier !== "number" || TrustTier[p.trustTier] === undefined) { + throw new ProvenanceEncodingError(`Unknown trust tier: ${String(p.trustTier)}`); + } + if (p.trustTier === TrustTier.Unverified) { + throw new ProvenanceEncodingError("Unverified readings must not be published on-chain"); + } + if (!Number.isInteger(p.agreeingSources) || p.agreeingSources < 0 || p.agreeingSources > 255) { + throw new ProvenanceEncodingError(`agreeingSources must be an integer in 0..255, got ${p.agreeingSources}`); + } + const minSources: Record = { + [TrustTier.SingleSource]: 1, + [TrustTier.Corroborated]: 2, + [TrustTier.Unanimous]: 3, + }; + if (p.trustTier === TrustTier.SingleSource ? p.agreeingSources !== 1 : p.agreeingSources < minSources[p.trustTier]) { + throw new ProvenanceEncodingError( + `Trust tier ${TrustTier[p.trustTier]} is inconsistent with ${p.agreeingSources} agreeing source(s)` + ); + } + if (typeof p.decisionId !== "string" || !/^[0-9a-f]{16}$/.test(p.decisionId)) { + throw new ProvenanceEncodingError("decisionId must be 16 lowercase hex chars (u64)"); + } + if (!Number.isInteger(p.sourceTimestamp) || p.sourceTimestamp < 0 || p.sourceTimestamp > TIMESTAMP_MAX) { + throw new ProvenanceEncodingError(`sourceTimestamp must be Unix seconds in 0..2^40-1, got ${p.sourceTimestamp}`); + } +} + +/** Pack provenance into the u128 layout above. */ +export function packProvenance(p: ReadingProvenance): bigint { + assertValidProvenance(p); + return ( + (BigInt(PROVENANCE_VERSION) << SHIFT.version) | + (BigInt(p.trustTier) << SHIFT.trustTier) | + (BigInt(p.agreeingSources) << SHIFT.agreeingSources) | + (BigInt(p.sourceTimestamp) << SHIFT.sourceTimestamp) | + BigInt(`0x${p.decisionId}`) + ); +} + +/** Inverse of packProvenance. Rejects unknown versions and non-zero reserved bits. */ +export function unpackProvenance(packed: bigint): ReadingProvenance { + if (typeof packed !== "bigint" || packed < 0n || packed > U128_MAX) { + throw new ProvenanceEncodingError("Packed provenance must be a u128 bigint"); + } + const field = (shift: bigint, bits: bigint) => (packed >> shift) & ((1n << bits) - 1n); + const version = Number(field(SHIFT.version, 4n)); + if (version !== PROVENANCE_VERSION) { + throw new ProvenanceEncodingError(`Unsupported provenance version ${version}`); + } + if (field(SHIFT.reserved, 8n) !== 0n) { + throw new ProvenanceEncodingError("Reserved provenance bits are set"); + } + const provenance: ReadingProvenance = { + trustTier: Number(field(SHIFT.trustTier, 4n)) as TrustTier, + agreeingSources: Number(field(SHIFT.agreeingSources, 8n)), + sourceTimestamp: Number(field(SHIFT.sourceTimestamp, 40n)), + decisionId: field(0n, 64n).toString(16).padStart(16, "0"), + }; + assertValidProvenance(provenance); + return provenance; +} + +/** + * Derive on-chain provenance from a consensus decision. Throws for anything + * other than an agreed decision — those must never be published. + */ +export function provenanceFromDecision(decision: ConsensusDecision): ReadingProvenance { + if (decision.status !== "agreed" || decision.sourceTimestamp === null) { + throw new ProvenanceEncodingError(`Cannot publish a ${decision.status} consensus decision`); + } + const provenance: ReadingProvenance = { + trustTier: decision.trustTier, + agreeingSources: decision.agreeingSources.length, + decisionId: decision.decisionId, + // Floor so the on-chain timestamp never claims fresher data than we had. + sourceTimestamp: Math.floor(decision.sourceTimestamp / 1000), + }; + assertValidProvenance(provenance); + return provenance; +} + +/** Everything `update_reading` needs, already scaled and packed. */ +export interface EncodedOracleReading { + coverageType: number; + value: bigint; + provenance: bigint; +} + +/** Scale the value and pack provenance for one reading. */ +export function encodeReadingForChain( + coverageType: string, + value: number, + provenance: ReadingProvenance +): EncodedOracleReading { + return { + coverageType: coverageTypeToU32(coverageType), + value: scaleOracleValue(coverageType, value), + provenance: packProvenance(provenance), + }; +} + +/** + * ScVal arguments for + * `RefractOracle.update_reading(coverage_type: u32, value: i128, provenance: u128)`. + * + * The provenance argument requires the matching contract change; until that + * ships, publishers keep calling the two-argument form. + */ +export function buildUpdateReadingArgs(encoded: EncodedOracleReading): xdr.ScVal[] { + return [ + nativeToScVal(encoded.coverageType, { type: "u32" }), + nativeToScVal(encoded.value, { type: "i128" }), + nativeToScVal(encoded.provenance, { type: "u128" }), + ]; +} diff --git a/src/oracle/oracle-reading.ts b/src/oracle/oracle-reading.ts index 6dd9583..2797be9 100644 --- a/src/oracle/oracle-reading.ts +++ b/src/oracle/oracle-reading.ts @@ -1,3 +1,5 @@ +import type { ReadingProvenance } from "./oracle-encoding"; + export type Severity = "low" | "medium" | "high" | "triggered"; export interface OracleReading { @@ -12,4 +14,10 @@ export interface OracleReading { * OraclePublisherService must never publish degraded readings on-chain. */ degraded?: boolean; + /** + * Consensus provenance (trust tier, agreeing source count, decision id, + * source timestamp). Present only for readings that reached consensus; + * published on-chain next to the value. Never set on degraded readings. + */ + provenance?: ReadingProvenance; } diff --git a/src/stellar/xdr-fixtures/oracle-update-reading-provenance.xdr.b64 b/src/stellar/xdr-fixtures/oracle-update-reading-provenance.xdr.b64 new file mode 100644 index 0000000..80bdd4d --- /dev/null +++ b/src/stellar/xdr-fixtures/oracle-update-reading-provenance.xdr.b64 @@ -0,0 +1,6 @@ +// Golden XDR fixtures for hand-built ScVal encodings. +// Regenerated by: npx ts-node scripts/regenerate-xdr-fixtures.ts +// A mismatch means either the contract ABI changed or the encoding regressed — +// check both before regenerating. +// u128 provenance: Corroborated, 3 sources, ts=1700000000 +AAAACRIDAABlU/EAASNFZ4mrze8= From bf0cd5df67bdc245fd0ab4eba28ba18f928c532a Mon Sep 17 00:00:00 2001 From: agudajohn793 Date: Wed, 21 Oct 2026 12:34:42 +0000 Subject: [PATCH 3/4] feat(worker): run the payout pipeline in a separately deployable worker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Oracle polling, consensus and claim settlement ran inside the API process, so request traffic and API restarts could delay or interrupt payouts. They now run in a separate worker; the API only serves reads and the WebSocket feed. Process roles (src/runtime/process-role.ts) - PROCESS_ROLE=api | worker | all. OracleScheduler and ClaimScheduler do nothing unless the role runs the pipeline. - Unset means `all` (one process, old behaviour) so existing deployments keep paying out until they are split; main.ts logs a warning in that case and refuses PROCESS_ROLE=worker. Worker (src/worker.ts, src/worker/) - Nest application context (no HTTP app) built from WorkerModule: data layer, Stellar, metrics, alerting, reading bus, policy/oracle/claim modules. No throttler, API-key guard or request-serving modules. - Tiny node:http listener on WORKER_HEALTH_PORT (default 4002): /health returns 503 when a pipeline stage has not succeeded within WORKER_STALE_AFTER_MS (PipelineHeartbeat), so the orchestrator can restart the worker alone; /metrics exposes Prometheus metrics. - Graceful shutdown shared with main.ts (runtime/graceful-shutdown.ts); ClaimScheduler still drains an in-flight settlement first. Worker → API channel (src/oracle/reading-bus.ts) - ReadingBus with a Redis implementation (latest reading per coverage type in a hash + pub/sub for live updates) and an in-memory one for PROCESS_ROLE=all / sandbox. A split role without Redis fails at boot instead of serving a frozen feed. Messages are validated on receipt. - OracleScheduler publishes readings to the bus; OracleFeedRelay (API) subscribes and broadcasts non-low readings via OracleGateway, which gets back the broadcastAlert() / welcome message its spec expects. Deployment - docker-compose: `backend` runs with PROCESS_ROLE=api, new `worker` service runs dist/worker.js with its own healthcheck; both share one env anchor. - npm scripts start:worker, start:worker:dev, start:worker:prod. - README documents the tiers and the one-worker-per-relayer-key rule. Pre-existing breakage fixed in files touched here: - app.module.ts: removed a truncated import line and a duplicated imports block left by a bad merge (the file did not parse). - claim.scheduler.ts: removed two stray closing braces (did not parse). - oracle.gateway.ts: OnModuleDestroy is imported from @nestjs/common. Note: oracle.module.ts / claim.module.ts / oracle.service.ts are empty on main, so neither entrypoint boots until those are restored. When they are, OracleModule should import OracleFeedModule rather than declare OracleGateway itself. Co-Authored-By: Claude Opus 5.5 --- README.md | 26 +++ docker-compose.yml | 62 +++++-- package.json | 3 + src/app.module.ts | 22 +-- src/claim/claim.scheduler.ts | 20 ++- src/main.ts | 76 +++------ src/oracle/oracle-feed.module.ts | 14 ++ src/oracle/oracle-feed.relay.spec.ts | 62 +++++++ src/oracle/oracle-feed.relay.ts | 50 ++++++ src/oracle/oracle.gateway.ts | 44 ++++- src/oracle/oracle.scheduler.ts | 23 ++- src/oracle/reading-bus.module.ts | 25 +++ src/oracle/reading-bus.spec.ts | 177 ++++++++++++++++++++ src/oracle/reading-bus.ts | 213 ++++++++++++++++++++++++ src/runtime/graceful-shutdown.ts | 54 ++++++ src/runtime/pipeline-heartbeat.ts | 53 ++++++ src/runtime/process-role.ts | 34 ++++ src/runtime/runtime.module.ts | 13 ++ src/runtime/runtime.spec.ts | 120 +++++++++++++ src/worker.ts | 62 +++++++ src/worker/worker-health.server.spec.ts | 58 +++++++ src/worker/worker-health.server.ts | 63 +++++++ src/worker/worker.module.ts | 46 +++++ 23 files changed, 1222 insertions(+), 98 deletions(-) create mode 100644 src/oracle/oracle-feed.module.ts create mode 100644 src/oracle/oracle-feed.relay.spec.ts create mode 100644 src/oracle/oracle-feed.relay.ts create mode 100644 src/oracle/reading-bus.module.ts create mode 100644 src/oracle/reading-bus.spec.ts create mode 100644 src/oracle/reading-bus.ts create mode 100644 src/runtime/graceful-shutdown.ts create mode 100644 src/runtime/pipeline-heartbeat.ts create mode 100644 src/runtime/process-role.ts create mode 100644 src/runtime/runtime.module.ts create mode 100644 src/runtime/runtime.spec.ts create mode 100644 src/worker.ts create mode 100644 src/worker/worker-health.server.spec.ts create mode 100644 src/worker/worker-health.server.ts create mode 100644 src/worker/worker.module.ts diff --git a/README.md b/README.md index bad1b63..18fbde5 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,31 @@ copy `.env.example` to `.env`, install Node.js 22 dependencies, apply Stop the stack with `docker compose down`. Persistent database and Redis data remain in Docker volumes; `docker compose down -v` removes them. + +### API and worker tiers + +The backend runs as two separately deployable processes: + +| Process | Entrypoint | `PROCESS_ROLE` | Does | +|---|---|---|---| +| API | `dist/main.js` | `api` | REST reads + the `/oracle` WebSocket feed | +| Worker | `dist/worker.js` | `worker` | oracle polling, consensus, on-chain publishing, claim settlement | + +The worker publishes readings to Redis (`REDIS_URL` is required for a split +deployment); the API relays them to WebSocket clients and serves the latest +reading per coverage type. Each tier can be scaled and restarted on its own, +and a burst of API traffic cannot delay a payout. + +- Run **one** worker per relayer key — the relayer's sequence numbers are + tracked in-process. +- The worker exposes `GET /health` (503 when a pipeline stage stops + succeeding for `WORKER_STALE_AFTER_MS`, default 10 min) and `GET /metrics` + on `WORKER_HEALTH_PORT` (default 4002). +- Leaving `PROCESS_ROLE` unset runs both tiers in one process (`all`). That + is the default so existing deployments keep paying out, and it is handy + locally, but deployed environments should split. + +`docker compose up` starts both tiers. ## Scripts | Command | Purpose | @@ -76,6 +101,7 @@ remain in Docker volumes; `docker compose down -v` removes them. | `npm run dev` | Hot-reloading dev server (ts-node + nodemon) | | `npm run build` | Compile TypeScript to `dist/` | | `npm start` | Run the compiled server | +| `npm run start:worker` / `start:worker:prod` | Run the worker tier (dev / compiled) | | `npm run typecheck` | `tsc --noEmit` | | `npm run lint` | ESLint over `src/` | | `npm test` | Jest unit tests (includes property-based tests) | diff --git a/docker-compose.yml b/docker-compose.yml index 3794465..917123a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,3 +1,20 @@ +x-backend-env: &backend-env + FRONTEND_URL: ${FRONTEND_URL:-http://localhost:3000} + DATABASE_URL: postgresql://refract:${POSTGRES_PASSWORD:-refract_dev}@postgres:5432/refract + REDIS_URL: redis://redis:6379 + STELLAR_NETWORK: ${STELLAR_NETWORK:-testnet} + SOROBAN_RPC_URL: ${SOROBAN_RPC_URL:-https://soroban-testnet.stellar.org} + STELLAR_NETWORK_PASSPHRASE: ${STELLAR_NETWORK_PASSPHRASE:-Test SDF Network ; September 2015} + REFRACT_POOL_CONTRACT_ID: ${REFRACT_POOL_CONTRACT_ID:-} + REFRACT_POLICY_CONTRACT_ID: ${REFRACT_POLICY_CONTRACT_ID:-} + REFRACT_ORACLE_CONTRACT_ID: ${REFRACT_ORACLE_CONTRACT_ID:-} + ORACLE_RELAYER_SECRET: ${ORACLE_RELAYER_SECRET:-} + COINGECKO_BASE_URL: ${COINGECKO_BASE_URL:-https://api.coingecko.com/api/v3} + STELLAR_HORIZON_URL: ${STELLAR_HORIZON_URL:-https://horizon-testnet.stellar.org} + DEFILLAMA_BASE_URL: ${DEFILLAMA_BASE_URL:-https://api.llama.fi} + DEFILLAMA_PROTOCOL_SLUG: ${DEFILLAMA_PROTOCOL_SLUG:-aave} + ORACLE_HTTP_TIMEOUT_MS: ${ORACLE_HTTP_TIMEOUT_MS:-5000} + services: postgres: image: postgres:16-alpine @@ -35,22 +52,11 @@ services: context: . dockerfile: Dockerfile environment: + <<: *backend-env PORT: 4001 - FRONTEND_URL: ${FRONTEND_URL:-http://localhost:3000} - DATABASE_URL: postgresql://refract:${POSTGRES_PASSWORD:-refract_dev}@postgres:5432/refract - REDIS_URL: redis://redis:6379 - STELLAR_NETWORK: ${STELLAR_NETWORK:-testnet} - SOROBAN_RPC_URL: ${SOROBAN_RPC_URL:-https://soroban-testnet.stellar.org} - STELLAR_NETWORK_PASSPHRASE: ${STELLAR_NETWORK_PASSPHRASE:-Test SDF Network ; September 2015} - REFRACT_POOL_CONTRACT_ID: ${REFRACT_POOL_CONTRACT_ID:-} - REFRACT_POLICY_CONTRACT_ID: ${REFRACT_POLICY_CONTRACT_ID:-} - REFRACT_ORACLE_CONTRACT_ID: ${REFRACT_ORACLE_CONTRACT_ID:-} - ORACLE_RELAYER_SECRET: ${ORACLE_RELAYER_SECRET:-} - COINGECKO_BASE_URL: ${COINGECKO_BASE_URL:-https://api.coingecko.com/api/v3} - STELLAR_HORIZON_URL: ${STELLAR_HORIZON_URL:-https://horizon-testnet.stellar.org} - DEFILLAMA_BASE_URL: ${DEFILLAMA_BASE_URL:-https://api.llama.fi} - DEFILLAMA_PROTOCOL_SLUG: ${DEFILLAMA_PROTOCOL_SLUG:-aave} - ORACLE_HTTP_TIMEOUT_MS: ${ORACLE_HTTP_TIMEOUT_MS:-5000} + # API tier: serves reads and the WebSocket feed only. The payout + # pipeline runs in the `worker` service below. + PROCESS_ROLE: api ports: - "4001:4001" depends_on: @@ -65,6 +71,32 @@ services: retries: 3 start_period: 10s + worker: + # Oracle polling, consensus, on-chain publishing and claim settlement. + # Runs exactly one replica per relayer key (sequence numbers are managed + # in-process). Restarts independently of the API. + build: + context: . + dockerfile: Dockerfile + command: ["node", "dist/worker.js"] + environment: + <<: *backend-env + PROCESS_ROLE: worker + WORKER_HEALTH_PORT: 4002 + ports: + - "127.0.0.1:4002:4002" + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + healthcheck: + test: ["CMD", "node", "-e", "fetch('http://127.0.0.1:4002/health').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))"] + interval: 30s + timeout: 3s + retries: 3 + start_period: 30s + volumes: postgres-data: redis-data: diff --git a/package.json b/package.json index 0deb0b1..aed721f 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,9 @@ "start:dev": "nest start --watch", "dev": "nest start --watch", "start:prod": "node dist/main", + "start:worker": "nest start --entryFile worker", + "start:worker:dev": "nest start --watch --entryFile worker", + "start:worker:prod": "node dist/worker", "lint": "eslint \"src/**/*.ts\"", "typecheck": "tsc --noEmit", "test": "jest", diff --git a/src/app.module.ts b/src/app.module.ts index 3ad2975..d483e28 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -16,8 +16,6 @@ import { ClaimModule } from './claim/claim.module'; import { DbModule } from './db/db.module'; import { IdempotencyModule } from './common/idempotency.module'; import { HealthModule } from './health/health.module'; -import { SecurityReportsModule } from './security-reports/security-reports.module'; -import { DigestModule } from './digest/diges import { PolicyModule } from './policy/policy.module'; import { PoolModule } from './pool/pool.module'; import { QuoteModule } from './quote/quote.module'; @@ -27,6 +25,8 @@ import { AuthModule } from './auth/auth.module'; import { ApiKeyGuard } from './auth/api-key.guard'; import { SecurityReportsModule } from './security-reports/security-reports.module'; import { DigestModule } from './digest/digest.module'; +import { ReadingBusModule } from './oracle/reading-bus.module'; +import { OracleFeedModule } from './oracle/oracle-feed.module'; /** * Sandbox mode lets the full API run with zero external dependencies @@ -84,21 +84,6 @@ export const isSandboxMode = (): boolean => }), ]), DatabaseModule, - Ide - ScheduleModule.forRoot(), - // In sandbox mode we skip the real Postgres connection entirely so - // `npm run dev` works right after `npm install` with no other setup. - ...(isSandboxMode() - ? [] - : [ - TypeOrmModule.forRootAsync({ - imports: [DatabaseModule], - inject: [DatabaseModule], - useFactory: (databaseModule: DatabaseModule) => - databaseModule.getTypeOrmConfig(), - }), - ]), - DatabaseModule, IdempotencyModule, // CacheModule is @Global — imported once here, available everywhere. CacheModule, @@ -112,6 +97,9 @@ export const isSandboxMode = (): boolean => RedisModule, ClaimsModule, OracleModule, + // Read side of the oracle: relays readings the worker publishes. + ReadingBusModule, + OracleFeedModule, ClaimModule, TxModule, SecurityReportsModule, diff --git a/src/claim/claim.scheduler.ts b/src/claim/claim.scheduler.ts index 1cefc74..685581c 100644 --- a/src/claim/claim.scheduler.ts +++ b/src/claim/claim.scheduler.ts @@ -1,7 +1,11 @@ -import { Injectable, Logger, OnApplicationShutdown, OnModuleInit } from '@nestjs/common'; +import { Injectable, Logger, OnApplicationShutdown, OnModuleInit, Optional } from '@nestjs/common'; import { Cron, CronExpression } from '@nestjs/schedule'; import { ClaimService } from './claim.service'; import { ProtocolParametersService } from '../parameters/protocol-parameters.service'; +import { runsPipeline } from '../runtime/process-role'; +import { PipelineHeartbeat } from '../runtime/pipeline-heartbeat'; + +export const CLAIM_PIPELINE_STAGE = 'claim-settlement'; /** * Upper bound (ms) we are willing to wait for an in-flight settlement run to @@ -41,9 +45,15 @@ export class ClaimScheduler implements OnModuleInit, OnApplicationShutdown { constructor( private readonly claimService: ClaimService, private readonly protocolParameters: ProtocolParametersService, + @Optional() private readonly heartbeat?: PipelineHeartbeat, ) {} async onModuleInit(): Promise { + if (!runsPipeline()) { + this.logger.log('Claim settlement disabled in this process (PROCESS_ROLE=api)'); + return; + } + this.heartbeat?.expect(CLAIM_PIPELINE_STAGE); const interval = await this.getIntervalSeconds(); this.logger.log(`Claim scheduler initialized with interval ${interval}s`); } @@ -62,6 +72,11 @@ export class ClaimScheduler implements OnModuleInit, OnApplicationShutdown { @Cron(CronExpression.EVERY_MINUTE) async handleClaimSettlement(): Promise { + // Settlement triggers payouts, so it only runs in the worker tier. + if (!runsPipeline()) { + return; + } + if (this.shuttingDown) { this.logger.log( 'Skipping claim settlement run: scheduler is shutting down.', @@ -94,6 +109,7 @@ export class ClaimScheduler implements OnModuleInit, OnApplicationShutdown { private async runSettlement(): Promise { try { await this.claimService.settleClaims(); + this.heartbeat?.beat(CLAIM_PIPELINE_STAGE); } catch (error) { this.logger.error( 'Claim settlement run failed.', @@ -149,6 +165,4 @@ export class ClaimScheduler implements OnModuleInit, OnApplicationShutdown { }); }); } -} - } } diff --git a/src/main.ts b/src/main.ts index a857318..a903a6f 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1,18 +1,26 @@ import { NestFactory } from '@nestjs/core'; import { Logger, INestApplication } from '@nestjs/common'; import { AppModule } from './app.module'; - -/** - * Hard upper bound on how long we wait for in-flight work (HTTP requests, - * scheduler runs, connection-pool drains) to finish before forcing exit. - * Prevents a stuck settlement from hanging the process forever. - */ -const SHUTDOWN_GRACE_PERIOD_MS = Number( - process.env.SHUTDOWN_GRACE_PERIOD_MS ?? 30_000, -); +import { resolveProcessRole } from './runtime/process-role'; +import { installGracefulShutdown } from './runtime/graceful-shutdown'; async function bootstrap(): Promise { const logger = new Logger('Bootstrap'); + + // API entrypoint. With PROCESS_ROLE=api this process only serves reads and + // the WebSocket feed; the payout pipeline runs in src/worker.ts. Unset + // (= "all") keeps the old single-process behaviour for local development. + const role = resolveProcessRole(); + if (role === 'worker') { + throw new Error('PROCESS_ROLE=worker is for src/worker.ts; start the API with PROCESS_ROLE=api or all'); + } + if (role === 'all') { + logger.warn( + 'PROCESS_ROLE is not set: running the API and the payout pipeline in one process. ' + + 'In deployed environments set PROCESS_ROLE=api here and run the worker separately (npm run start:worker).', + ); + } + const app: INestApplication = await NestFactory.create(AppModule); // Let Nest propagate shutdown signals to every provider that implements @@ -22,52 +30,12 @@ async function bootstrap(): Promise { const port = process.env.PORT ?? 3000; await app.listen(port); - logger.log(`Application listening on port ${port}`); - - let shuttingDown = false; - - const shutdown = async (signal: NodeJS.Signals): Promise => { - if (shuttingDown) { - logger.warn(`Received ${signal} while already shutting down; ignoring`); - return; - } - shuttingDown = true; - logger.log( - `Received ${signal}; starting graceful shutdown (grace period ${SHUTDOWN_GRACE_PERIOD_MS}ms)`, - ); - - // Hard upper bound: if graceful shutdown does not complete in time, log - // the forced-exit path and terminate so the orchestrator is not left - // waiting on a stuck in-flight settlement. - const forceExitTimer = setTimeout(() => { - logger.error( - `Graceful shutdown exceeded ${SHUTDOWN_GRACE_PERIOD_MS}ms; forcing exit`, - ); - process.exit(1); - }, SHUTDOWN_GRACE_PERIOD_MS); - // Do not keep the event loop alive solely for this timer. - forceExitTimer.unref(); - - try { - // app.close() stops accepting new connections, drains in-flight HTTP - // requests, and invokes onApplicationShutdown on all providers so - // schedulers finish their current run and pools close cleanly. - await app.close(); - clearTimeout(forceExitTimer); - logger.log('Graceful shutdown complete'); - process.exit(0); - } catch (err) { - clearTimeout(forceExitTimer); - logger.error( - `Error during graceful shutdown: ${(err as Error).message}`, - (err as Error).stack, - ); - process.exit(1); - } - }; + logger.log(`Application listening on port ${port} (PROCESS_ROLE=${role})`); - process.on('SIGTERM', () => void shutdown('SIGTERM')); - process.on('SIGINT', () => void shutdown('SIGINT')); + // app.close() stops accepting new connections, drains in-flight HTTP + // requests, and invokes onApplicationShutdown on all providers so + // schedulers finish their current run and pools close cleanly. + installGracefulShutdown(() => app.close(), logger); } bootstrap().catch((err) => { diff --git a/src/oracle/oracle-feed.module.ts b/src/oracle/oracle-feed.module.ts new file mode 100644 index 0000000..271136c --- /dev/null +++ b/src/oracle/oracle-feed.module.ts @@ -0,0 +1,14 @@ +import { Module } from "@nestjs/common"; +import { OracleFeedRelay } from "./oracle-feed.relay"; +import { OracleGateway } from "./oracle.gateway"; + +/** + * Read side of the oracle: the WebSocket gateway and the relay that feeds it + * from the ReadingBus. Imported by the API (AppModule) only — the worker + * has no HTTP server to attach a gateway to. + */ +@Module({ + providers: [OracleGateway, OracleFeedRelay], + exports: [OracleGateway, OracleFeedRelay], +}) +export class OracleFeedModule {} diff --git a/src/oracle/oracle-feed.relay.spec.ts b/src/oracle/oracle-feed.relay.spec.ts new file mode 100644 index 0000000..35866d0 --- /dev/null +++ b/src/oracle/oracle-feed.relay.spec.ts @@ -0,0 +1,62 @@ +import { OracleFeedRelay } from "./oracle-feed.relay"; +import { OracleGateway } from "./oracle.gateway"; +import { OracleReading } from "./oracle-reading"; +import { InMemoryReadingBus } from "./reading-bus"; + +function reading(overrides: Partial = {}): OracleReading { + return { + coverageType: "MarketCrash", + type: "oracle_update", + value: -12, + threshold: -20, + severity: "high", + message: "drawdown", + ...overrides, + }; +} + +describe("OracleFeedRelay", () => { + const originalRole = process.env.PROCESS_ROLE; + afterEach(() => { + if (originalRole === undefined) delete process.env.PROCESS_ROLE; + else process.env.PROCESS_ROLE = originalRole; + }); + + function setup() { + const bus = new InMemoryReadingBus(); + const gateway = { broadcastAlert: jest.fn() } as unknown as OracleGateway; + return { bus, gateway, relay: new OracleFeedRelay(bus, gateway) }; + } + + it("relays non-low readings published by the worker to WebSocket clients", async () => { + process.env.PROCESS_ROLE = "api"; + const { bus, gateway, relay } = setup(); + await relay.onModuleInit(); + + await bus.publish(reading()); + await bus.publish(reading({ coverageType: "FlightDelay", severity: "low" })); + + expect(gateway.broadcastAlert).toHaveBeenCalledTimes(1); + expect(gateway.broadcastAlert).toHaveBeenCalledWith(reading()); + expect((await relay.latest()).map((r) => r.coverageType)).toEqual(["MarketCrash", "FlightDelay"]); + }); + + it("stops relaying after shutdown", async () => { + process.env.PROCESS_ROLE = "api"; + const { bus, gateway, relay } = setup(); + await relay.onModuleInit(); + await relay.onModuleDestroy(); + + await bus.publish(reading()); + expect(gateway.broadcastAlert).not.toHaveBeenCalled(); + }); + + it("does not subscribe in a worker-only process", async () => { + process.env.PROCESS_ROLE = "worker"; + const { bus, gateway, relay } = setup(); + await relay.onModuleInit(); + + await bus.publish(reading()); + expect(gateway.broadcastAlert).not.toHaveBeenCalled(); + }); +}); diff --git a/src/oracle/oracle-feed.relay.ts b/src/oracle/oracle-feed.relay.ts new file mode 100644 index 0000000..b081eda --- /dev/null +++ b/src/oracle/oracle-feed.relay.ts @@ -0,0 +1,50 @@ +import { Injectable, Logger, OnModuleDestroy, OnModuleInit } from "@nestjs/common"; +import { OracleGateway } from "./oracle.gateway"; +import { OracleReading } from "./oracle-reading"; +import { ReadingBus } from "./reading-bus"; +import { servesApi } from "../runtime/process-role"; + +/** + * API-side half of the worker split: forwards readings the worker publishes + * on the ReadingBus to WebSocket clients, and answers "latest reading" + * queries from the shared store. It never polls upstream sources. + * + * Only non-"low" readings are pushed over the socket, matching the feed's + * behaviour before the split. + */ +@Injectable() +export class OracleFeedRelay implements OnModuleInit, OnModuleDestroy { + private readonly logger = new Logger(OracleFeedRelay.name); + private unsubscribe: (() => Promise) | null = null; + + constructor( + private readonly bus: ReadingBus, + private readonly gateway: OracleGateway + ) {} + + async onModuleInit(): Promise { + if (!servesApi()) { + return; + } + this.unsubscribe = await this.bus.subscribe((reading) => this.relay(reading)); + this.logger.log("Relaying oracle readings from the worker to WebSocket clients"); + } + + async onModuleDestroy(): Promise { + await this.unsubscribe?.(); + this.unsubscribe = null; + } + + /** Latest reading per coverage type, as last published by the worker. */ + latest(): Promise { + return this.bus.latest(); + } + + private relay(reading: OracleReading): void { + if (reading.severity === "low") { + return; + } + this.logger.warn(`Oracle alert: ${reading.coverageType} — ${reading.message}`); + this.gateway.broadcastAlert(reading); + } +} diff --git a/src/oracle/oracle.gateway.ts b/src/oracle/oracle.gateway.ts index 5b65954..4644598 100644 --- a/src/oracle/oracle.gateway.ts +++ b/src/oracle/oracle.gateway.ts @@ -1,12 +1,12 @@ import { OnGatewayConnection, OnGatewayDisconnect, - OnModuleDestroy, WebSocketGateway, WebSocketServer, } from '@nestjs/websockets'; -import { Logger } from '@nestjs/common'; +import { Logger, OnModuleDestroy } from '@nestjs/common'; import { Server, WebSocket } from 'ws'; +import { OracleReading } from './oracle-reading'; @WebSocketGateway({ path: '/oracle' }) export class OracleGateway @@ -15,16 +15,46 @@ export class OracleGateway private readonly logger = new Logger(OracleGateway.name); @WebSocketServer() - server: Server; + server!: Server; handleConnection(client: WebSocket): void { this.logger.log('Oracle WebSocket client connected'); - client.on('error', (err) => { - this.logger.warn(`Oracle WebSocket client error: ${err.message}`); - }); + if (typeof client.on === 'function') { + client.on('error', (err) => { + this.logger.warn(`Oracle WebSocket client error: ${err.message}`); + }); + } + client.send(JSON.stringify({ type: 'connected', message: 'Refract oracle feed' })); + } + + /** + * Push a reading to every open client. In a split deployment the readings + * come from the worker via the ReadingBus (see OracleFeedRelay); this + * process never polls upstreams itself. + */ + broadcastAlert(alert: OracleReading): void { + if (!this.server?.clients) { + return; + } + const payload = JSON.stringify({ ...alert, timestamp: Date.now() }); + let sent = 0; + for (const client of this.server.clients) { + if (client.readyState !== WebSocket.OPEN) continue; + // send() can throw for a socket that is mid-close; one bad client must + // not stop the broadcast to everyone else. + try { + client.send(payload); + sent++; + } catch (err) { + this.logger.warn( + `Failed to send oracle alert to a client: ${err instanceof Error ? err.message : String(err)}`, + ); + } + } + this.logger.debug(`Broadcast ${alert.coverageType} alert to ${sent} client(s)`); } - handleDisconnect(client: WebSocket): void { + handleDisconnect(_client: WebSocket): void { this.logger.log('Oracle WebSocket client disconnected'); } diff --git a/src/oracle/oracle.scheduler.ts b/src/oracle/oracle.scheduler.ts index 3332c6d..0259d8a 100644 --- a/src/oracle/oracle.scheduler.ts +++ b/src/oracle/oracle.scheduler.ts @@ -1,11 +1,16 @@ -import { Injectable, Logger, OnModuleInit, OnModuleDestroy } from '@nestjs/common'; +import { Injectable, Logger, OnModuleInit, OnModuleDestroy, Optional } from '@nestjs/common'; import { SchedulerRegistry } from '@nestjs/schedule'; import { CronJob } from 'cron'; import { OracleService } from './oracle.service'; import { ProtocolParametersService } from '../parameters/protocol-parameters.service'; import { ProtocolParameterKey } from '../parameters/protocol-parameter-key.enum'; +import { ReadingBus } from './reading-bus'; +import { OracleReading } from './oracle-reading'; +import { runsPipeline } from '../runtime/process-role'; +import { PipelineHeartbeat } from '../runtime/pipeline-heartbeat'; const ORACLE_REFRESH_JOB_NAME = 'oracle-refresh'; +export const ORACLE_PIPELINE_STAGE = 'oracle'; @Injectable() export class OracleScheduler implements OnModuleInit, OnModuleDestroy { @@ -15,9 +20,19 @@ export class OracleScheduler implements OnModuleInit, OnModuleDestroy { private readonly oracleService: OracleService, private readonly schedulerRegistry: SchedulerRegistry, private readonly protocolParameters: ProtocolParametersService, + private readonly readingBus: ReadingBus, + @Optional() private readonly heartbeat?: PipelineHeartbeat, ) {} async onModuleInit(): Promise { + // Polling and consensus belong to the worker tier. The API only relays + // what the worker publishes (see OracleFeedRelay). + if (!runsPipeline()) { + this.logger.log('Oracle polling disabled in this process (PROCESS_ROLE=api)'); + return; + } + this.heartbeat?.expect(ORACLE_PIPELINE_STAGE); + const intervalSeconds = await this.protocolParameters.getNumber( ProtocolParameterKey.ORACLE_SCHEDULER_INTERVAL_SECONDS, ); @@ -42,7 +57,11 @@ export class OracleScheduler implements OnModuleInit, OnModuleDestroy { private async refresh(): Promise { try { - await this.oracleService.refreshPrices(); + const result: unknown = await this.oracleService.refreshPrices(); + const readings = Array.isArray(result) ? (result as OracleReading[]) : []; + // Hand readings to the API tier for the WebSocket feed and reads. + await this.readingBus.publishAll(readings); + this.heartbeat?.beat(ORACLE_PIPELINE_STAGE); } catch (error) { this.logger.error('Oracle refresh failed', error as Error); } diff --git a/src/oracle/reading-bus.module.ts b/src/oracle/reading-bus.module.ts new file mode 100644 index 0000000..06a2550 --- /dev/null +++ b/src/oracle/reading-bus.module.ts @@ -0,0 +1,25 @@ +import { Global, Inject, Module, OnApplicationShutdown } from "@nestjs/common"; +import { ReadingBus, createReadingBus } from "./reading-bus"; +import { resolveProcessRole } from "../runtime/process-role"; + +/** + * Provides the ReadingBus to both tiers. Global so the oracle scheduler + * (worker) and the feed relay (API) can inject it without import churn. + */ +@Global() +@Module({ + providers: [ + { + provide: ReadingBus, + useFactory: () => createReadingBus(resolveProcessRole()), + }, + ], + exports: [ReadingBus], +}) +export class ReadingBusModule implements OnApplicationShutdown { + constructor(@Inject(ReadingBus) private readonly bus: ReadingBus) {} + + async onApplicationShutdown(): Promise { + await this.bus.close(); + } +} diff --git a/src/oracle/reading-bus.spec.ts b/src/oracle/reading-bus.spec.ts new file mode 100644 index 0000000..0194e31 --- /dev/null +++ b/src/oracle/reading-bus.spec.ts @@ -0,0 +1,177 @@ +import { OracleReading } from "./oracle-reading"; +import { + InMemoryReadingBus, + ORACLE_LATEST_KEY, + ORACLE_READINGS_CHANNEL, + RedisLike, + RedisReadingBus, + createReadingBus, + decodeReading, + encodeReading, +} from "./reading-bus"; +import { TrustTier } from "./consensus/consensus"; + +function reading(overrides: Partial = {}): OracleReading { + return { + coverageType: "StablecoinDepeg", + type: "oracle_update", + value: 0.97, + threshold: 0.95, + severity: "medium", + message: "USDC slipping", + ...overrides, + }; +} + +/** In-process stand-in for a Redis server shared by several clients. */ +class FakeRedisServer { + hashes = new Map>(); + subscribers = new Set(); + client(): FakeRedis { + return new FakeRedis(this); + } +} + +class FakeRedis implements RedisLike { + private listeners: Array<(channel: string, message: string) => void> = []; + channels = new Set(); + quitCalled = false; + constructor(private readonly server: FakeRedisServer) {} + + async hset(key: string, field: string, value: string) { + if (!this.server.hashes.has(key)) this.server.hashes.set(key, new Map()); + this.server.hashes.get(key)!.set(field, value); + return 1; + } + async hgetall(key: string) { + return Object.fromEntries(this.server.hashes.get(key) ?? []); + } + async publish(channel: string, message: string) { + for (const sub of this.server.subscribers) { + if (sub.channels.has(channel)) sub.listeners.forEach((l) => l(channel, message)); + } + return 1; + } + async subscribe(channel: string) { + this.channels.add(channel); + this.server.subscribers.add(this); + return 1; + } + async unsubscribe(channel: string) { + this.channels.delete(channel); + return 1; + } + on(_event: "message", listener: (channel: string, message: string) => void) { + this.listeners.push(listener); + return this; + } + off(_event: "message", listener: (channel: string, message: string) => void) { + this.listeners = this.listeners.filter((l) => l !== listener); + return this; + } + async quit() { + this.quitCalled = true; + return "OK"; + } +} + +describe("reading wire format", () => { + it("round-trips readings, including provenance", () => { + const r = reading({ + provenance: { trustTier: TrustTier.Corroborated, agreeingSources: 2, decisionId: "00000000000000ff", sourceTimestamp: 1 }, + }); + expect(decodeReading(encodeReading(r))).toEqual(r); + }); + + it.each([ + ["not JSON", "{"], + ["wrong version", JSON.stringify({ v: 2, reading: reading() })], + ["missing reading", JSON.stringify({ v: 1 })], + ["non-finite value", JSON.stringify({ v: 1, reading: { ...reading(), value: null } })], + ["unknown severity", JSON.stringify({ v: 1, reading: { ...reading(), severity: "panic" } })], + ])("rejects %s", (_label, raw) => { + expect(decodeReading(raw)).toBeNull(); + }); +}); + +describe("InMemoryReadingBus", () => { + it("delivers to subscribers and keeps the latest reading per coverage type", async () => { + const bus = new InMemoryReadingBus(); + const seen: OracleReading[] = []; + const unsubscribe = await bus.subscribe((r) => seen.push(r)); + + await bus.publishAll([reading({ value: 0.98 }), reading({ value: 0.97 }), reading({ coverageType: "FlightDelay", value: 200 })]); + await unsubscribe(); + await bus.publish(reading({ value: 0.5 })); + + expect(seen.map((r) => r.value)).toEqual([0.98, 0.97, 200]); + expect((await bus.latest()).map((r) => [r.coverageType, r.value])).toEqual([ + ["StablecoinDepeg", 0.5], + ["FlightDelay", 200], + ]); + }); + + it("refuses to publish malformed readings", async () => { + await expect(new InMemoryReadingBus().publish(reading({ value: NaN }))).rejects.toThrow(/malformed/); + }); +}); + +describe("RedisReadingBus", () => { + it("carries readings from a worker bus to an API bus through Redis", async () => { + const server = new FakeRedisServer(); + const worker = new RedisReadingBus(server.client(), server.client()); + const api = new RedisReadingBus(server.client(), server.client()); + + const seen: OracleReading[] = []; + await api.subscribe((r) => seen.push(r)); + await worker.publish(reading()); + + expect(seen).toEqual([reading()]); + expect(await api.latest()).toEqual([reading()]); + expect(server.hashes.get(ORACLE_LATEST_KEY)?.has("StablecoinDepeg")).toBe(true); + }); + + it("drops malformed messages instead of passing them to handlers", async () => { + const server = new FakeRedisServer(); + const api = new RedisReadingBus(server.client(), server.client()); + const handler = jest.fn(); + await api.subscribe(handler); + + await server.client().publish(ORACLE_READINGS_CHANNEL, "garbage"); + await server.client().hset(ORACLE_LATEST_KEY, "X", "garbage"); + + expect(handler).not.toHaveBeenCalled(); + expect(await api.latest()).toEqual([]); + }); + + it("closes both connections", async () => { + const server = new FakeRedisServer(); + const commands = server.client(); + const subscriber = server.client(); + const bus = new RedisReadingBus(commands, subscriber); + await bus.subscribe(() => undefined); + await bus.close(); + expect(commands.quitCalled && subscriber.quitCalled).toBe(true); + expect(subscriber.channels.size).toBe(0); + }); +}); + +describe("createReadingBus", () => { + const connect = () => new FakeRedisServer().client(); + + it("uses Redis whenever it is configured", () => { + expect(createReadingBus("api", { REDIS_URL: "redis://x" }, connect)).toBeInstanceOf(RedisReadingBus); + expect(createReadingBus("all", { REDIS_HOST: "x" }, connect)).toBeInstanceOf(RedisReadingBus); + }); + + it("falls back to in-memory for single-process and sandbox runs", () => { + expect(createReadingBus("all", {}, connect)).toBeInstanceOf(InMemoryReadingBus); + expect(createReadingBus("all", { SANDBOX_MODE: "true", REDIS_URL: "redis://x" }, connect)).toBeInstanceOf( + InMemoryReadingBus + ); + }); + + it.each(["api", "worker"] as const)("fails fast when PROCESS_ROLE=%s has no Redis", (role) => { + expect(() => createReadingBus(role, {}, connect)).toThrow(/requires REDIS_URL/); + }); +}); diff --git a/src/oracle/reading-bus.ts b/src/oracle/reading-bus.ts new file mode 100644 index 0000000..84d06e7 --- /dev/null +++ b/src/oracle/reading-bus.ts @@ -0,0 +1,213 @@ +import { EventEmitter } from "events"; +import { Logger } from "@nestjs/common"; +import Redis, { RedisOptions } from "ioredis"; +import { OracleReading } from "./oracle-reading"; +import { ProcessRole } from "../runtime/process-role"; + +/** + * Carries oracle readings from the worker (which polls and evaluates + * consensus) to the API (which serves reads and the WebSocket feed). + * + * The API never polls upstreams itself: it relays what the worker publishes + * and answers "latest reading" queries from the shared store. + */ +export abstract class ReadingBus { + abstract publish(reading: OracleReading): Promise; + /** Subscribe to new readings. Resolves to an unsubscribe function. */ + abstract subscribe(handler: (reading: OracleReading) => void): Promise<() => Promise>; + /** Most recent reading per coverage type. */ + abstract latest(): Promise; + abstract close(): Promise; + + async publishAll(readings: readonly OracleReading[]): Promise { + for (const reading of readings) { + await this.publish(reading); + } + } +} + +export const ORACLE_READINGS_CHANNEL = "refract:oracle:readings"; +export const ORACLE_LATEST_KEY = "refract:oracle:latest"; + +interface Envelope { + v: 1; + publishedAt: number; + reading: OracleReading; +} + +const SEVERITIES = new Set(["low", "medium", "high", "triggered"]); + +/** Parse and validate a bus message. Returns null for anything malformed. */ +export function decodeReading(raw: string): OracleReading | null { + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + return null; + } + const env = parsed as Partial | null; + const r = env?.reading as Partial | undefined; + if ( + env?.v !== 1 || + !r || + typeof r.coverageType !== "string" || + r.type !== "oracle_update" || + typeof r.value !== "number" || + !Number.isFinite(r.value) || + typeof r.threshold !== "number" || + typeof r.message !== "string" || + !SEVERITIES.has(r.severity as string) + ) { + return null; + } + return r as OracleReading; +} + +export function encodeReading(reading: OracleReading, publishedAt = Date.now()): string { + const envelope: Envelope = { v: 1, publishedAt, reading }; + return JSON.stringify(envelope); +} + +/** Single-process bus for PROCESS_ROLE=all, sandbox mode and tests. */ +export class InMemoryReadingBus extends ReadingBus { + private readonly emitter = new EventEmitter(); + private readonly latestByType = new Map(); + + async publish(reading: OracleReading): Promise { + // Round-trip through the wire format so in-memory and Redis behave the same. + const decoded = decodeReading(encodeReading(reading)); + if (!decoded) throw new Error(`Refusing to publish malformed reading for ${reading.coverageType}`); + this.latestByType.set(decoded.coverageType, decoded); + this.emitter.emit("reading", decoded); + } + + async subscribe(handler: (reading: OracleReading) => void): Promise<() => Promise> { + this.emitter.on("reading", handler); + return async () => { + this.emitter.off("reading", handler); + }; + } + + async latest(): Promise { + return [...this.latestByType.values()]; + } + + async close(): Promise { + this.emitter.removeAllListeners(); + } +} + +/** Subset of ioredis used here, so tests can pass a fake. */ +export interface RedisLike { + hset(key: string, field: string, value: string): Promise; + hgetall(key: string): Promise>; + publish(channel: string, message: string): Promise; + subscribe(channel: string): Promise; + unsubscribe(channel: string): Promise; + on(event: "message", listener: (channel: string, message: string) => void): unknown; + off(event: "message", listener: (channel: string, message: string) => void): unknown; + quit(): Promise; +} + +/** + * Cross-process bus over Redis: latest readings in a hash, live updates via + * pub/sub. Uses a separate connection for subscribing, as Redis requires. + */ +export class RedisReadingBus extends ReadingBus { + private readonly logger = new Logger(RedisReadingBus.name); + private subscribed = false; + + constructor( + private readonly commands: RedisLike, + private readonly subscriber: RedisLike + ) { + super(); + } + + async publish(reading: OracleReading): Promise { + const payload = encodeReading(reading); + if (!decodeReading(payload)) { + throw new Error(`Refusing to publish malformed reading for ${reading.coverageType}`); + } + // Store first so a subscriber that reacts to the message can read it back. + await this.commands.hset(ORACLE_LATEST_KEY, reading.coverageType, payload); + await this.commands.publish(ORACLE_READINGS_CHANNEL, payload); + } + + async subscribe(handler: (reading: OracleReading) => void): Promise<() => Promise> { + const listener = (channel: string, message: string) => { + if (channel !== ORACLE_READINGS_CHANNEL) return; + const reading = decodeReading(message); + if (!reading) { + this.logger.warn("Dropping malformed oracle reading from Redis"); + return; + } + handler(reading); + }; + this.subscriber.on("message", listener); + if (!this.subscribed) { + await this.subscriber.subscribe(ORACLE_READINGS_CHANNEL); + this.subscribed = true; + } + return async () => { + this.subscriber.off("message", listener); + }; + } + + async latest(): Promise { + const all = await this.commands.hgetall(ORACLE_LATEST_KEY); + return Object.values(all ?? {}) + .map(decodeReading) + .filter((r): r is OracleReading => r !== null); + } + + async close(): Promise { + if (this.subscribed) { + await this.subscriber.unsubscribe(ORACLE_READINGS_CHANNEL).catch(() => undefined); + } + await Promise.allSettled([this.commands.quit(), this.subscriber.quit()]); + } +} + +export function redisOptionsFromEnv(env: NodeJS.ProcessEnv): { url?: string; options?: RedisOptions } | null { + if (env.REDIS_URL) return { url: env.REDIS_URL }; + if (env.REDIS_HOST) { + return { + options: { + host: env.REDIS_HOST, + port: Number(env.REDIS_PORT ?? 6379), + password: env.REDIS_PASSWORD || undefined, + }, + }; + } + return null; +} + +/** + * Pick the bus for this process. A split deployment (api / worker) needs + * Redis, otherwise the API would never see the worker's readings — fail at + * boot rather than serve a silently frozen feed. + */ +export function createReadingBus( + role: ProcessRole, + env: NodeJS.ProcessEnv = process.env, + connect: (cfg: { url?: string; options?: RedisOptions }) => RedisLike = defaultConnect +): ReadingBus { + const sandbox = (env.SANDBOX_MODE ?? "").toLowerCase() === "true"; + const redis = sandbox ? null : redisOptionsFromEnv(env); + if (!redis) { + if (role !== "all") { + throw new Error( + `PROCESS_ROLE=${role} requires REDIS_URL (or REDIS_HOST) so the API and worker can share oracle readings` + ); + } + return new InMemoryReadingBus(); + } + return new RedisReadingBus(connect(redis), connect(redis)); +} + +function defaultConnect(cfg: { url?: string; options?: RedisOptions }): RedisLike { + const client = cfg.url ? new Redis(cfg.url) : new Redis(cfg.options ?? {}); + client.on("error", (err: Error) => new Logger("ReadingBus").error(`Redis error: ${err.message}`)); + return client as unknown as RedisLike; +} diff --git a/src/runtime/graceful-shutdown.ts b/src/runtime/graceful-shutdown.ts new file mode 100644 index 0000000..29521f0 --- /dev/null +++ b/src/runtime/graceful-shutdown.ts @@ -0,0 +1,54 @@ +import { Logger } from "@nestjs/common"; + +/** + * Hard upper bound on how long we wait for in-flight work (HTTP requests, + * scheduler runs, connection-pool drains) to finish before forcing exit. + * Prevents a stuck settlement from hanging the process forever. + */ +export const SHUTDOWN_GRACE_PERIOD_MS = Number(process.env.SHUTDOWN_GRACE_PERIOD_MS ?? 30_000); + +/** + * Install SIGTERM/SIGINT handlers that run `close()` once, bounded by the + * grace period, then exit. Shared by the API (main.ts) and worker + * (worker.ts) entrypoints so both tiers drain the same way. + */ +export function installGracefulShutdown( + close: () => Promise, + logger: Logger, + graceMs: number = SHUTDOWN_GRACE_PERIOD_MS, + exit: (code: number) => void = (code) => process.exit(code) +): (signal: NodeJS.Signals) => Promise { + let shuttingDown = false; + + const shutdown = async (signal: NodeJS.Signals): Promise => { + if (shuttingDown) { + logger.warn(`Received ${signal} while already shutting down; ignoring`); + return; + } + shuttingDown = true; + logger.log(`Received ${signal}; starting graceful shutdown (grace period ${graceMs}ms)`); + + // If draining does not finish in time, log and exit so the orchestrator + // is not left waiting on a stuck in-flight settlement. + const forceExitTimer = setTimeout(() => { + logger.error(`Graceful shutdown exceeded ${graceMs}ms; forcing exit`); + exit(1); + }, graceMs); + forceExitTimer.unref(); + + try { + await close(); + clearTimeout(forceExitTimer); + logger.log("Graceful shutdown complete"); + exit(0); + } catch (err) { + clearTimeout(forceExitTimer); + logger.error(`Error during graceful shutdown: ${(err as Error).message}`, (err as Error).stack); + exit(1); + } + }; + + process.on("SIGTERM", () => void shutdown("SIGTERM")); + process.on("SIGINT", () => void shutdown("SIGINT")); + return shutdown; +} diff --git a/src/runtime/pipeline-heartbeat.ts b/src/runtime/pipeline-heartbeat.ts new file mode 100644 index 0000000..7a6c723 --- /dev/null +++ b/src/runtime/pipeline-heartbeat.ts @@ -0,0 +1,53 @@ +import { Injectable } from "@nestjs/common"; + +export interface HeartbeatStatus { + healthy: boolean; + startedAt: number; + /** Last successful run per pipeline stage (epoch ms). */ + lastBeats: Record; + /** Stages that have not succeeded within `staleAfterMs`. */ + staleStages: string[]; +} + +/** + * Records when each pipeline stage last completed, so the worker's health + * endpoint can report a stuck pipeline and the orchestrator can restart it. + * + * A stage counts as stale only after it has been registered with `expect()` + * and the grace period since startup has passed without a beat. + */ +@Injectable() +export class PipelineHeartbeat { + private readonly startedAt: number; + private readonly beats = new Map(); + private readonly expected = new Set(); + + constructor( + private readonly staleAfterMs: number = Number(process.env.WORKER_STALE_AFTER_MS ?? 10 * 60_000), + private readonly now: () => number = Date.now + ) { + this.startedAt = this.now(); + } + + /** Declare a stage that must keep beating for the worker to be healthy. */ + expect(stage: string): void { + this.expected.add(stage); + } + + beat(stage: string): void { + this.beats.set(stage, this.now()); + } + + status(): HeartbeatStatus { + const now = this.now(); + const staleStages = [...this.expected] + .filter((stage) => now - (this.beats.get(stage) ?? this.startedAt) > this.staleAfterMs) + .sort(); + return { + healthy: staleStages.length === 0, + startedAt: this.startedAt, + lastBeats: Object.fromEntries(this.beats), + staleStages, + }; + } +} diff --git a/src/runtime/process-role.ts b/src/runtime/process-role.ts new file mode 100644 index 0000000..6644f52 --- /dev/null +++ b/src/runtime/process-role.ts @@ -0,0 +1,34 @@ +/** + * Which tier this process runs. + * + * - `api` — HTTP + WebSocket only. Serves reads; never polls oracles, + * evaluates consensus, publishes on-chain or settles claims. + * - `worker` — the payout pipeline (oracle polling, consensus, on-chain + * publishing, claim settlement, relayer monitoring). No HTTP API. + * - `all` — both in one process. Kept as the default so existing + * single-process deployments keep paying out until they are + * split; use it for local development only. + * + * `src/main.ts` starts the API (role from PROCESS_ROLE, default `all`); + * `src/worker.ts` always runs as `worker`. + */ +export type ProcessRole = "api" | "worker" | "all"; + +export const PROCESS_ROLES: readonly ProcessRole[] = ["api", "worker", "all"]; + +export function resolveProcessRole(env: NodeJS.ProcessEnv = process.env): ProcessRole { + const raw = (env.PROCESS_ROLE ?? "").trim().toLowerCase(); + if (raw === "") return "all"; + if ((PROCESS_ROLES as readonly string[]).includes(raw)) return raw as ProcessRole; + throw new Error(`Invalid PROCESS_ROLE "${env.PROCESS_ROLE}". Expected one of: ${PROCESS_ROLES.join(", ")}`); +} + +/** True when this process should run the payout pipeline (polling, consensus, settlement). */ +export function runsPipeline(env: NodeJS.ProcessEnv = process.env): boolean { + return resolveProcessRole(env) !== "api"; +} + +/** True when this process serves the HTTP API / WebSocket feed. */ +export function servesApi(env: NodeJS.ProcessEnv = process.env): boolean { + return resolveProcessRole(env) !== "worker"; +} diff --git a/src/runtime/runtime.module.ts b/src/runtime/runtime.module.ts new file mode 100644 index 0000000..ac379eb --- /dev/null +++ b/src/runtime/runtime.module.ts @@ -0,0 +1,13 @@ +import { Global, Module } from "@nestjs/common"; +import { PipelineHeartbeat } from "./pipeline-heartbeat"; + +/** + * Worker-only runtime services. Global so schedulers inside feature modules + * (oracle, claim, relayer monitor) can inject PipelineHeartbeat. + */ +@Global() +@Module({ + providers: [{ provide: PipelineHeartbeat, useFactory: () => new PipelineHeartbeat() }], + exports: [PipelineHeartbeat], +}) +export class RuntimeModule {} diff --git a/src/runtime/runtime.spec.ts b/src/runtime/runtime.spec.ts new file mode 100644 index 0000000..390fa79 --- /dev/null +++ b/src/runtime/runtime.spec.ts @@ -0,0 +1,120 @@ +import { Injectable, Logger, Module, Optional } from "@nestjs/common"; +import { Test } from "@nestjs/testing"; +import { RuntimeModule } from "./runtime.module"; +import { resolveProcessRole, runsPipeline, servesApi } from "./process-role"; +import { PipelineHeartbeat } from "./pipeline-heartbeat"; +import { installGracefulShutdown } from "./graceful-shutdown"; + +describe("process role", () => { + it("defaults to 'all' so unsplit deployments keep paying out", () => { + expect(resolveProcessRole({})).toBe("all"); + expect(resolveProcessRole({ PROCESS_ROLE: " " })).toBe("all"); + }); + + it("accepts api / worker / all case-insensitively", () => { + expect(resolveProcessRole({ PROCESS_ROLE: "API" })).toBe("api"); + expect(resolveProcessRole({ PROCESS_ROLE: "worker" })).toBe("worker"); + expect(resolveProcessRole({ PROCESS_ROLE: " All " })).toBe("all"); + }); + + it("rejects unknown roles instead of guessing", () => { + expect(() => resolveProcessRole({ PROCESS_ROLE: "both" })).toThrow(/Invalid PROCESS_ROLE/); + }); + + it.each([ + ["api", false, true], + ["worker", true, false], + ["all", true, true], + ])("role %s: runsPipeline=%s servesApi=%s", (role, pipeline, api) => { + const env = { PROCESS_ROLE: role }; + expect(runsPipeline(env)).toBe(pipeline); + expect(servesApi(env)).toBe(api); + }); +}); + +describe("PipelineHeartbeat", () => { + it("is healthy until an expected stage misses its window", () => { + let now = 1_000; + const hb = new PipelineHeartbeat(100, () => now); + hb.expect("oracle"); + + now = 1_100; + expect(hb.status().healthy).toBe(true); + + now = 1_101; + expect(hb.status()).toMatchObject({ healthy: false, staleStages: ["oracle"] }); + + hb.beat("oracle"); + expect(hb.status()).toMatchObject({ healthy: true, lastBeats: { oracle: 1_101 } }); + }); + + it("ignores stages that were never expected", () => { + let now = 0; + const hb = new PipelineHeartbeat(10, () => now); + hb.beat("adhoc"); + now = 1_000; + expect(hb.status().healthy).toBe(true); + }); +}); + +describe("RuntimeModule", () => { + @Injectable() + class FeatureScheduler { + constructor(@Optional() readonly heartbeat?: PipelineHeartbeat) {} + } + @Module({ providers: [FeatureScheduler] }) + class FeatureModule {} + + it("makes one PipelineHeartbeat visible to schedulers in feature modules", async () => { + const moduleRef = await Test.createTestingModule({ imports: [RuntimeModule, FeatureModule] }).compile(); + const scheduler = moduleRef.get(FeatureScheduler); + expect(scheduler.heartbeat).toBeInstanceOf(PipelineHeartbeat); + expect(scheduler.heartbeat).toBe(moduleRef.get(PipelineHeartbeat)); + }); +}); + +describe("installGracefulShutdown", () => { + const logger = { log: jest.fn(), warn: jest.fn(), error: jest.fn() } as unknown as Logger; + let before: { term: Function[]; int: Function[] }; + + beforeEach(() => { + before = { term: process.listeners("SIGTERM"), int: process.listeners("SIGINT") }; + }); + + afterEach(() => { + for (const l of process.listeners("SIGTERM")) if (!before.term.includes(l)) process.off("SIGTERM", l as never); + for (const l of process.listeners("SIGINT")) if (!before.int.includes(l)) process.off("SIGINT", l as never); + }); + + it("closes once and exits 0", async () => { + const close = jest.fn().mockResolvedValue(undefined); + const exit = jest.fn(); + const shutdown = installGracefulShutdown(close, logger, 1_000, exit); + + await shutdown("SIGTERM"); + await shutdown("SIGINT"); + + expect(close).toHaveBeenCalledTimes(1); + expect(exit).toHaveBeenCalledWith(0); + }); + + it("exits 1 when close fails", async () => { + const exit = jest.fn(); + const shutdown = installGracefulShutdown(() => Promise.reject(new Error("boom")), logger, 1_000, exit); + await shutdown("SIGTERM"); + expect(exit).toHaveBeenCalledWith(1); + }); + + it("forces exit 1 when close exceeds the grace period", async () => { + jest.useFakeTimers(); + try { + const exit = jest.fn(); + const shutdown = installGracefulShutdown(() => new Promise(() => undefined), logger, 50, exit); + void shutdown("SIGTERM"); + jest.advanceTimersByTime(51); + expect(exit).toHaveBeenCalledWith(1); + } finally { + jest.useRealTimers(); + } + }); +}); diff --git a/src/worker.ts b/src/worker.ts new file mode 100644 index 0000000..7db4fcd --- /dev/null +++ b/src/worker.ts @@ -0,0 +1,62 @@ +import { Logger } from "@nestjs/common"; +import { NestFactory } from "@nestjs/core"; +import { resolveProcessRole } from "./runtime/process-role"; +import { installGracefulShutdown } from "./runtime/graceful-shutdown"; +import { PipelineHeartbeat } from "./runtime/pipeline-heartbeat"; +import { startWorkerHealthServer } from "./worker/worker-health.server"; + +/** + * Worker entrypoint: runs the payout pipeline (oracle polling, consensus, + * on-chain publishing, claim settlement) with no HTTP API, so it can be + * deployed, scaled and restarted independently of the API tier. + * + * Run exactly one worker per relayer key — the relayer's sequence numbers + * are managed in-process (see RelayerAccountService). + */ +async function bootstrap(): Promise { + const logger = new Logger("WorkerBootstrap"); + + const configured = process.env.PROCESS_ROLE; + if (configured && resolveProcessRole() !== "worker") { + throw new Error(`worker.ts must run with PROCESS_ROLE=worker (got "${configured}")`); + } + process.env.PROCESS_ROLE = "worker"; + + // Imported after PROCESS_ROLE is fixed so module factories see the role. + const { WorkerModule } = await import("./worker/worker.module"); + const { MetricsService } = await import("./metrics/metrics.service"); + + const app = await NestFactory.createApplicationContext(WorkerModule); + app.enableShutdownHooks(); + + const heartbeat = app.get(PipelineHeartbeat); + let metrics: InstanceType | undefined; + try { + metrics = app.get(MetricsService, { strict: false }); + } catch { + metrics = undefined; + } + + const port = Number(process.env.WORKER_HEALTH_PORT ?? 4002); + const health = await startWorkerHealthServer( + { + heartbeat, + metrics: metrics ? async () => ({ body: await metrics!.metrics(), contentType: metrics!.contentType }) : undefined, + }, + port + ); + logger.log(`Worker started; health and metrics on port ${port}`); + + installGracefulShutdown(async () => { + await new Promise((resolve) => health.close(() => resolve())); + // Stops schedulers (ClaimScheduler waits for an in-flight settlement), + // closes the reading bus and connection pools. + await app.close(); + }, logger); +} + +bootstrap().catch((err) => { + // eslint-disable-next-line no-console + console.error("Fatal error during worker bootstrap", err); + process.exit(1); +}); diff --git a/src/worker/worker-health.server.spec.ts b/src/worker/worker-health.server.spec.ts new file mode 100644 index 0000000..f9ca0e3 --- /dev/null +++ b/src/worker/worker-health.server.spec.ts @@ -0,0 +1,58 @@ +import { Server } from "http"; +import { PipelineHeartbeat } from "../runtime/pipeline-heartbeat"; +import { boundPort, startWorkerHealthServer } from "./worker-health.server"; + +describe("worker health server", () => { + let server: Server | undefined; + let now = 0; + + afterEach(async () => { + await new Promise((resolve) => (server ? server.close(() => resolve()) : resolve())); + server = undefined; + }); + + async function start(withMetrics = true) { + now = 0; + const heartbeat = new PipelineHeartbeat(100, () => now); + heartbeat.expect("oracle"); + server = await startWorkerHealthServer( + { + heartbeat, + metrics: withMetrics ? async () => ({ body: "refract_up 1\n", contentType: "text/plain" }) : undefined, + }, + 0, + "127.0.0.1" + ); + return { heartbeat, base: `http://127.0.0.1:${boundPort(server)}` }; + } + + it("reports ok while the pipeline is beating and 503 once it goes stale", async () => { + const { heartbeat, base } = await start(); + + let res = await fetch(`${base}/health`); + expect(res.status).toBe(200); + expect(await res.json()).toMatchObject({ status: "ok", role: "worker" }); + + now = 500; + res = await fetch(`${base}/health`); + expect(res.status).toBe(503); + expect(await res.json()).toMatchObject({ status: "stale", staleStages: ["oracle"] }); + + heartbeat.beat("oracle"); + expect((await fetch(`${base}/health`)).status).toBe(200); + }); + + it("serves metrics when available and 404 otherwise", async () => { + const { base } = await start(); + const res = await fetch(`${base}/metrics`); + expect(res.status).toBe(200); + expect(await res.text()).toBe("refract_up 1\n"); + expect((await fetch(`${base}/nope`)).status).toBe(404); + }); + + it("accepts only GET", async () => { + const { base } = await start(false); + expect((await fetch(`${base}/health`, { method: "POST" })).status).toBe(405); + expect((await fetch(`${base}/metrics`)).status).toBe(404); + }); +}); diff --git a/src/worker/worker-health.server.ts b/src/worker/worker-health.server.ts new file mode 100644 index 0000000..7710666 --- /dev/null +++ b/src/worker/worker-health.server.ts @@ -0,0 +1,63 @@ +import * as http from "http"; +import { AddressInfo } from "net"; +import { PipelineHeartbeat } from "../runtime/pipeline-heartbeat"; + +export interface WorkerHealthSources { + heartbeat: PipelineHeartbeat; + /** Prometheus exposition, if metrics are available in this process. */ + metrics?: () => Promise<{ body: string; contentType: string }>; +} + +/** + * Minimal HTTP listener for the worker: `/health` for liveness/readiness + * probes and `/metrics` for Prometheus. The worker has no API, so this is + * deliberately not an Express/Nest app — nothing here accepts input. + * + * `/health` returns 503 when a pipeline stage has stopped succeeding, so + * the orchestrator restarts the worker without touching the API tier. + */ +export function startWorkerHealthServer( + sources: WorkerHealthSources, + port: number, + host = "0.0.0.0" +): Promise { + const server = http.createServer((req, res) => { + void handle(req, res, sources); + }); + return new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(port, host, () => { + server.off("error", reject); + resolve(server); + }); + }); +} + +async function handle(req: http.IncomingMessage, res: http.ServerResponse, sources: WorkerHealthSources) { + const path = (req.url ?? "/").split("?")[0]; + if (req.method !== "GET") { + res.writeHead(405, { Allow: "GET" }).end(); + return; + } + try { + if (path === "/health") { + const status = sources.heartbeat.status(); + res + .writeHead(status.healthy ? 200 : 503, { "Content-Type": "application/json" }) + .end(JSON.stringify({ status: status.healthy ? "ok" : "stale", role: "worker", ...status })); + return; + } + if (path === "/metrics" && sources.metrics) { + const { body, contentType } = await sources.metrics(); + res.writeHead(200, { "Content-Type": contentType }).end(body); + return; + } + res.writeHead(404).end(); + } catch (err) { + res.writeHead(500, { "Content-Type": "application/json" }).end(JSON.stringify({ error: (err as Error).message })); + } +} + +export function boundPort(server: http.Server): number { + return (server.address() as AddressInfo).port; +} diff --git a/src/worker/worker.module.ts b/src/worker/worker.module.ts new file mode 100644 index 0000000..f807579 --- /dev/null +++ b/src/worker/worker.module.ts @@ -0,0 +1,46 @@ +import { Module } from "@nestjs/common"; +import { ConfigModule } from "@nestjs/config"; +import { ScheduleModule } from "@nestjs/schedule"; +import configuration from "../config/configuration"; +import { AlertingModule } from "../alerting/alerting.module"; +import { CacheModule } from "../cache/cache.module"; +import { ClaimModule } from "../claim/claim.module"; +import { DatabaseModule } from "../database/database.module"; +import { DbModule } from "../db/db.module"; +import { IdempotencyModule } from "../common/idempotency.module"; +import { MetricsModule } from "../metrics/metrics.module"; +import { OracleModule } from "../oracle/oracle.module"; +import { ReadingBusModule } from "../oracle/reading-bus.module"; +import { PolicyModule } from "../policy/policy.module"; +import { StellarModule } from "../stellar/stellar.module"; +import { RuntimeModule } from "../runtime/runtime.module"; + +/** + * Root module for the worker tier: oracle polling, consensus evaluation, + * on-chain publishing and claim settlement. + * + * It shares the data layer with the API but leaves out everything that + * serves requests — controllers are never bound (the worker is started as + * an application context, not an HTTP app), and the throttler, API-key + * guard, quote/pool/tx/security-report modules are not imported at all. + * Readings reach the API through the ReadingBus. + */ +@Module({ + imports: [ + ConfigModule.forRoot({ isGlobal: true, load: [configuration] }), + ScheduleModule.forRoot(), + RuntimeModule, + DatabaseModule, + DbModule, + IdempotencyModule, + CacheModule, + StellarModule, + MetricsModule, + AlertingModule, + ReadingBusModule, + PolicyModule, + OracleModule, + ClaimModule, + ], +}) +export class WorkerModule {} From b86aab9f95fc7f00a823676100b078f0c74e315c Mon Sep 17 00:00:00 2001 From: agudajohn793 Date: Wed, 21 Oct 2026 12:41:25 +0000 Subject: [PATCH 4/4] fictional --- src/metrics/metrics.service.ts | 64 ++- .../relayer-monitor/fee-spend-tracker.ts | 65 +++ .../relayer-monitor/horizon-relayer.client.ts | 112 +++++ .../relayer-balance-monitor.service.ts | 211 +++++++++ .../relayer-monitor/relayer-balance.policy.ts | 178 +++++++ .../relayer-monitor/relayer-monitor.config.ts | 100 ++++ .../relayer-monitor/relayer-monitor.module.ts | 26 ++ .../relayer-monitor/relayer-monitor.spec.ts | 439 ++++++++++++++++++ src/worker/worker.module.ts | 4 +- 9 files changed, 1196 insertions(+), 3 deletions(-) create mode 100644 src/stellar/relayer-monitor/fee-spend-tracker.ts create mode 100644 src/stellar/relayer-monitor/horizon-relayer.client.ts create mode 100644 src/stellar/relayer-monitor/relayer-balance-monitor.service.ts create mode 100644 src/stellar/relayer-monitor/relayer-balance.policy.ts create mode 100644 src/stellar/relayer-monitor/relayer-monitor.config.ts create mode 100644 src/stellar/relayer-monitor/relayer-monitor.module.ts create mode 100644 src/stellar/relayer-monitor/relayer-monitor.spec.ts diff --git a/src/metrics/metrics.service.ts b/src/metrics/metrics.service.ts index e9eab0b..768b0cd 100644 --- a/src/metrics/metrics.service.ts +++ b/src/metrics/metrics.service.ts @@ -1,12 +1,14 @@ import { Injectable } from "@nestjs/common"; -import { Counter, Histogram, Registry } from "@prometheus-io/client"; +import { Counter, Gauge, Histogram, Registry } from "@prometheus-io/client"; import { NextFunction, Request, RequestHandler, Response } from "express"; +import type { RelayerMetricsSink } from "../stellar/relayer-monitor/relayer-balance-monitor.service"; +import { BalanceAssessment, FeeSpendSummary, tierRank } from "../stellar/relayer-monitor/relayer-balance.policy"; type Outcome = "success" | "failure"; type OracleOutcome = Outcome | "mocked"; @Injectable() -export class MetricsService { +export class MetricsService implements RelayerMetricsSink { private readonly registry = new Registry(); private readonly httpRequests = new Counter<"method" | "route" | "status_code">({ name: "refract_http_requests_total", @@ -55,6 +57,49 @@ export class MetricsService { registers: [this.registry], }); + private readonly relayerBalance = new Gauge({ + name: "refract_relayer_balance_stroops", + help: "Relayer native balance in stroops.", + registers: [this.registry], + }); + private readonly relayerSpendable = new Gauge({ + name: "refract_relayer_spendable_stroops", + help: "Relayer balance available for fees after reserve and liabilities, in stroops.", + registers: [this.registry], + }); + private readonly relayerRunwayHours = new Gauge({ + name: "refract_relayer_runway_hours", + help: "Hours of fees the relayer can still pay at the current spend rate (-1 when unknown).", + registers: [this.registry], + }); + private readonly relayerRunwaySubmissions = new Gauge({ + name: "refract_relayer_runway_submissions", + help: "Submissions the relayer can still pay for at the average fee (-1 when unknown).", + registers: [this.registry], + }); + private readonly relayerFeeRate = new Gauge({ + name: "refract_relayer_fee_spend_per_hour_stroops", + help: "Relayer fee spend per hour over the monitoring window, in stroops.", + registers: [this.registry], + }); + private readonly relayerAlertTier = new Gauge({ + name: "refract_relayer_alert_tier", + help: "Relayer balance alert tier: 0=ok, 1=warning, 2=critical, 3=exhausted.", + registers: [this.registry], + }); + private readonly relayerFeesTotal = new Counter<"outcome">({ + name: "refract_relayer_fees_paid_stroops_total", + help: "Fees charged to the relayer, in stroops, by transaction outcome.", + labelNames: ["outcome"], + registers: [this.registry], + }); + private readonly relayerTxFee = new Histogram({ + name: "refract_relayer_tx_fee_stroops", + help: "Fee charged per relayer-submitted transaction, in stroops.", + buckets: [100, 1_000, 10_000, 50_000, 100_000, 500_000, 1_000_000, 5_000_000, 10_000_000], + registers: [this.registry], + }); + readonly httpMiddleware: RequestHandler = (req: Request, res: Response, next: NextFunction): void => { if (req.path === "/metrics") { next(); @@ -86,6 +131,21 @@ export class MetricsService { this.oracleCheckDuration.labels({ coverage_type: coverageType }).observe(durationSeconds); } + recordRelayerBalance(a: BalanceAssessment, spend: FeeSpendSummary): void { + this.relayerBalance.set(Number(a.balanceStroops)); + this.relayerSpendable.set(Number(a.spendableStroops)); + this.relayerRunwayHours.set(a.runwayHours ?? -1); + this.relayerRunwaySubmissions.set(a.runwaySubmissions ?? -1); + this.relayerFeeRate.set(spend.ratePerHourStroops); + this.relayerAlertTier.set(tierRank(a.tier)); + } + + recordRelayerFee(feeStroops: bigint, successful: boolean): void { + const fee = Number(feeStroops); + this.relayerFeesTotal.labels({ outcome: successful ? "success" : "failure" }).inc(fee); + this.relayerTxFee.observe(fee); + } + async observeSorobanRpc(operation: string, call: () => Promise): Promise { const startedAt = process.hrtime.bigint(); let outcome: Outcome = "success"; diff --git a/src/stellar/relayer-monitor/fee-spend-tracker.ts b/src/stellar/relayer-monitor/fee-spend-tracker.ts new file mode 100644 index 0000000..e0d721a --- /dev/null +++ b/src/stellar/relayer-monitor/fee-spend-tracker.ts @@ -0,0 +1,65 @@ +import { FeeSpendSummary } from "./relayer-balance.policy"; + +export interface FeeSpendRecord { + txHash: string; + feeChargedStroops: bigint; + /** Ledger close time of the transaction (epoch ms). */ + at: number; + successful: boolean; +} + +/** + * Rolling window of fees the relayer actually paid, one record per + * submitted transaction (oracle publishes and claim settlements share the + * relayer key). Failed transactions are kept: they still cost a fee. + */ +export class FeeSpendTracker { + private readonly records = new Map(); + + /** + * @param windowMs how far back the summary looks + * @param minRateSpanMs lower bound on the span used for the hourly rate, + * so a burst right after startup is not extrapolated into a huge rate + */ + constructor( + private readonly windowMs: number, + private readonly minRateSpanMs = 15 * 60_000 + ) {} + + /** Adds a record; returns false if the transaction was already counted. */ + record(r: FeeSpendRecord): boolean { + if (this.records.has(r.txHash)) return false; + this.records.set(r.txHash, r); + return true; + } + + summary(now: number): FeeSpendSummary { + this.prune(now); + const inWindow = [...this.records.values()]; + if (inWindow.length === 0) { + return { count: 0, totalStroops: 0n, avgStroops: 0, maxStroops: 0n, ratePerHourStroops: 0 }; + } + let total = 0n; + let max = 0n; + let oldest = Infinity; + for (const r of inWindow) { + total += r.feeChargedStroops; + if (r.feeChargedStroops > max) max = r.feeChargedStroops; + if (r.at < oldest) oldest = r.at; + } + const spanMs = Math.min(Math.max(now - oldest, this.minRateSpanMs), this.windowMs); + return { + count: inWindow.length, + totalStroops: total, + avgStroops: Number(total) / inWindow.length, + maxStroops: max, + ratePerHourStroops: (Number(total) / spanMs) * 3_600_000, + }; + } + + private prune(now: number): void { + for (const [hash, r] of this.records) { + if (now - r.at > this.windowMs) this.records.delete(hash); + } + } +} diff --git a/src/stellar/relayer-monitor/horizon-relayer.client.ts b/src/stellar/relayer-monitor/horizon-relayer.client.ts new file mode 100644 index 0000000..9cf70b2 --- /dev/null +++ b/src/stellar/relayer-monitor/horizon-relayer.client.ts @@ -0,0 +1,112 @@ +import { FeeSpendRecord } from "./fee-spend-tracker"; +import { RelayerAccountSnapshot, parseXlmToStroops } from "./relayer-balance.policy"; + +export class RelayerAccountNotFoundError extends Error { + constructor(publicKey: string) { + super(`Relayer account ${publicKey} does not exist on the network (unfunded or wrong network)`); + this.name = "RelayerAccountNotFoundError"; + } +} + +export interface FeeSpendPage { + records: FeeSpendRecord[]; + /** Paging token to resume after; null when the page was empty. */ + cursor: string | null; +} + +type FetchFn = (url: string, init?: { signal?: AbortSignal }) => Promise<{ + ok: boolean; + status: number; + json(): Promise; +}>; + +interface HorizonBalance { + asset_type: string; + balance: string; + selling_liabilities?: string; +} + +interface HorizonAccount { + balances: HorizonBalance[]; + subentry_count: number; + num_sponsoring?: number; + num_sponsored?: number; +} + +interface HorizonTransaction { + hash: string; + paging_token: string; + fee_charged: string | number; + fee_account?: string; + source_account: string; + successful: boolean; + created_at: string; +} + +/** + * Reads the relayer's balance and fee history from Horizon. + * + * Soroban RPC's getAccount returns only the sequence number, so balances + * and per-transaction `fee_charged` come from Horizon. Only the public key + * is needed — the monitor never touches the relayer secret. + */ +export class HorizonRelayerClient { + constructor( + private readonly horizonUrl: string, + private readonly publicKey: string, + private readonly timeoutMs = 10_000, + private readonly fetchFn: FetchFn = (url, init) => fetch(url, init) + ) {} + + async getAccount(): Promise { + const res = await this.get(`/accounts/${this.publicKey}`); + if (res.status === 404) throw new RelayerAccountNotFoundError(this.publicKey); + if (!res.ok) throw new Error(`Horizon /accounts returned HTTP ${res.status}`); + const body = (await res.json()) as HorizonAccount; + const native = body.balances?.find((b) => b.asset_type === "native"); + if (!native) throw new Error("Horizon account has no native balance"); + return { + balanceStroops: parseXlmToStroops(native.balance), + sellingLiabilitiesStroops: parseXlmToStroops(native.selling_liabilities ?? "0"), + subentryCount: Number(body.subentry_count ?? 0), + numSponsoring: Number(body.num_sponsoring ?? 0), + numSponsored: Number(body.num_sponsored ?? 0), + }; + } + + /** + * Transactions whose fee the relayer paid, oldest first, starting after + * `cursor`. Without a cursor, returns the most recent page (to backfill + * the window on startup) and a cursor pointing at the newest one. + */ + async listFeeSpend(cursor: string | null, limit = 200): Promise { + const order = cursor ? "asc" : "desc"; + const params = new URLSearchParams({ order, limit: String(limit), include_failed: "true" }); + if (cursor) params.set("cursor", cursor); + const res = await this.get(`/accounts/${this.publicKey}/transactions?${params}`); + if (res.status === 404) throw new RelayerAccountNotFoundError(this.publicKey); + if (!res.ok) throw new Error(`Horizon /transactions returned HTTP ${res.status}`); + + const body = (await res.json()) as { _embedded?: { records?: HorizonTransaction[] } }; + const txs = [...(body._embedded?.records ?? [])]; + if (order === "desc") txs.reverse(); + + const records = txs + // Fee-bumped transactions are charged to the fee account; only count + // what the relayer itself paid. + .filter((tx) => (tx.fee_account ?? tx.source_account) === this.publicKey) + .map((tx) => ({ + txHash: tx.hash, + feeChargedStroops: BigInt(tx.fee_charged), + at: Date.parse(tx.created_at), + successful: Boolean(tx.successful), + })); + const newest = txs[txs.length - 1]; + return { records, cursor: newest ? newest.paging_token : cursor }; + } + + private get(path: string) { + const base = this.horizonUrl.replace(/\/+$/, ""); + return this.fetchFn(`${base}${path}`, { signal: AbortSignal.timeout(this.timeoutMs) }); + } +} diff --git a/src/stellar/relayer-monitor/relayer-balance-monitor.service.ts b/src/stellar/relayer-monitor/relayer-balance-monitor.service.ts new file mode 100644 index 0000000..b6a8cf8 --- /dev/null +++ b/src/stellar/relayer-monitor/relayer-balance-monitor.service.ts @@ -0,0 +1,211 @@ +import { Logger, OnModuleDestroy, OnModuleInit } from "@nestjs/common"; +import { AlertingService } from "../../alerting/alerting.service"; +import { PipelineHeartbeat } from "../../runtime/pipeline-heartbeat"; +import { runsPipeline } from "../../runtime/process-role"; +import { FeeSpendTracker } from "./fee-spend-tracker"; +import { HorizonRelayerClient, RelayerAccountNotFoundError } from "./horizon-relayer.client"; +import { + BalanceAlertTier, + BalanceAssessment, + FeeSpendSummary, + assessBalance, + decideAlert, + formatXlm, +} from "./relayer-balance.policy"; +import { RelayerMonitorConfig } from "./relayer-monitor.config"; + +export const RELAYER_MONITOR_STAGE = "relayer-monitor"; + +/** Where balance and fee metrics go. MetricsService implements this. */ +export interface RelayerMetricsSink { + recordRelayerBalance(assessment: BalanceAssessment, spend: FeeSpendSummary): void; + recordRelayerFee(feeStroops: bigint, successful: boolean): void; +} + +/** Cap on Horizon pages read per check, so a long outage cannot stall a check. */ +const MAX_PAGES_PER_CHECK = 10; +const PAGE_SIZE = 200; + +export interface RelayerMonitorStatus { + publicKey: string | null; + lastCheckedAt: number | null; + consecutiveFailures: number; + lastError: string | null; + assessment: BalanceAssessment | null; + feeSpend: FeeSpendSummary | null; +} + +/** + * Watches the relayer account that pays every oracle publish and claim + * settlement fee. Each check reads the balance and any new relayer-paid + * transactions from Horizon, works out the spendable balance and runway at + * the current fee spend, and raises tiered alerts (warning → critical → + * exhausted) before the relayer can no longer submit. + * + * Runs in the worker tier only. Built by RelayerMonitorModule's factory. + */ +export class RelayerBalanceMonitor implements OnModuleInit, OnModuleDestroy { + private readonly logger = new Logger(RelayerBalanceMonitor.name); + private readonly tracker: FeeSpendTracker; + private timer: NodeJS.Timeout | null = null; + private running: Promise | null = null; + + private cursor: string | null = null; + private tier: BalanceAlertTier = "ok"; + private lastAlertAt: number | null = null; + private unknownAlerted = false; + private status: RelayerMonitorStatus; + + constructor( + private readonly config: RelayerMonitorConfig, + private readonly client: HorizonRelayerClient | null, + private readonly alerting: AlertingService, + private readonly metrics?: RelayerMetricsSink, + private readonly heartbeat?: PipelineHeartbeat, + private readonly now: () => number = Date.now + ) { + this.tracker = new FeeSpendTracker(config.feeWindowMs); + this.status = { + publicKey: config.publicKey, + lastCheckedAt: null, + consecutiveFailures: 0, + lastError: null, + assessment: null, + feeSpend: null, + }; + } + + onModuleInit(): void { + if (!runsPipeline()) return; + if (!this.config.enabled || !this.client) { + this.logger.warn(`Relayer balance monitoring is off: ${this.config.disabledReason ?? "not configured"}`); + return; + } + this.heartbeat?.expect(RELAYER_MONITOR_STAGE); + this.logger.log( + `Monitoring relayer ${this.config.publicKey} every ${this.config.intervalMs}ms via ${this.config.horizonUrl}` + ); + void this.check(); + this.timer = setInterval(() => void this.check(), this.config.intervalMs); + this.timer.unref(); + } + + async onModuleDestroy(): Promise { + if (this.timer) clearInterval(this.timer); + this.timer = null; + await this.running; + } + + getStatus(): RelayerMonitorStatus { + return { ...this.status }; + } + + /** Run one check. Overlapping calls share the in-flight run. */ + check(): Promise { + if (!this.running) { + this.running = this.runCheck().finally(() => { + this.running = null; + }); + } + return this.running; + } + + private async runCheck(): Promise { + if (!this.client) return; + try { + await this.ingestFeeSpend(); + const snapshot = await this.client.getAccount(); + const now = this.now(); + const spend = this.tracker.summary(now); + const assessment = assessBalance(snapshot, spend, this.config.thresholds); + + this.metrics?.recordRelayerBalance(assessment, spend); + this.handleTier(assessment, now); + if (this.unknownAlerted) { + this.alerting.emit("info", "Relayer balance monitoring restored", "Horizon is reachable again", { + publicKey: this.config.publicKey, + }); + this.unknownAlerted = false; + } + this.status = { + ...this.status, + lastCheckedAt: now, + consecutiveFailures: 0, + lastError: null, + assessment, + feeSpend: spend, + }; + this.heartbeat?.beat(RELAYER_MONITOR_STAGE); + } catch (err) { + this.handleFailure(err); + } + } + + private async ingestFeeSpend(): Promise { + for (let page = 0; page < MAX_PAGES_PER_CHECK; page++) { + const result = await this.client!.listFeeSpend(this.cursor, PAGE_SIZE); + for (const r of result.records) { + if (this.tracker.record(r)) this.metrics?.recordRelayerFee(r.feeChargedStroops, r.successful); + } + const wasBackfill = this.cursor === null; + this.cursor = result.cursor; + // The first (backfill) call reads the newest page only; afterwards keep + // paging forward until a short page means we are caught up. + if (wasBackfill || result.records.length < PAGE_SIZE) return; + } + } + + private handleTier(a: BalanceAssessment, now: number): void { + const decision = decideAlert(this.tier, a.tier, this.lastAlertAt, now, this.config.renotifyMs); + const previous = this.tier; + this.tier = a.tier; + if (!decision.emit) return; + + const title = + decision.kind === "recovered" + ? `Relayer balance recovered to ${a.tier}` + : `Relayer balance ${a.tier}${decision.kind === "reminder" ? " (still)" : ""}`; + const message = + a.reasons.length > 0 + ? a.reasons.join("; ") + : `spendable ${formatXlm(a.spendableStroops)} is above all thresholds`; + this.alerting.emit(decision.severity, title, message, { + publicKey: this.config.publicKey, + previousTier: previous, + tier: a.tier, + balance: formatXlm(a.balanceStroops), + reserve: formatXlm(a.reserveStroops), + spendable: formatXlm(a.spendableStroops), + nextFee: formatXlm(a.nextFeeStroops), + runwayHours: a.runwayHours, + runwaySubmissions: a.runwaySubmissions, + }); + this.lastAlertAt = decision.kind === "recovered" && a.tier === "ok" ? null : now; + } + + private handleFailure(err: unknown): void { + const message = err instanceof Error ? err.message : String(err); + this.status = { ...this.status, consecutiveFailures: this.status.consecutiveFailures + 1, lastError: message }; + this.logger.warn(`Relayer balance check failed: ${message}`); + + // A missing account means the relayer cannot submit at all. + if (err instanceof RelayerAccountNotFoundError) { + if (this.tier !== "exhausted") { + this.alerting.emit("critical", "Relayer balance exhausted", message, { publicKey: this.config.publicKey }); + this.tier = "exhausted"; + this.lastAlertAt = this.now(); + } + return; + } + // Not knowing the balance is itself a risk once it persists. + if (!this.unknownAlerted && this.status.consecutiveFailures >= this.config.failuresBeforeAlert) { + this.alerting.emit( + "warning", + "Relayer balance unknown", + `${this.status.consecutiveFailures} consecutive balance checks failed: ${message}`, + { publicKey: this.config.publicKey, horizonUrl: this.config.horizonUrl } + ); + this.unknownAlerted = true; + } + } +} diff --git a/src/stellar/relayer-monitor/relayer-balance.policy.ts b/src/stellar/relayer-monitor/relayer-balance.policy.ts new file mode 100644 index 0000000..ed1da11 --- /dev/null +++ b/src/stellar/relayer-monitor/relayer-balance.policy.ts @@ -0,0 +1,178 @@ +/** + * Pure rules for relayer balance monitoring: how much XLM the relayer can + * actually spend, how long that lasts at the current fee spend, and which + * alert tier that puts us in. No I/O, so every rule is unit-testable. + */ + +/** Stellar base reserve: 0.5 XLM per entry. */ +export const BASE_RESERVE_STROOPS = 5_000_000n; +export const STROOPS_PER_XLM = 10_000_000n; + +/** Account fields from Horizon that decide how much is spendable. */ +export interface RelayerAccountSnapshot { + balanceStroops: bigint; + /** Native balance locked by open sell offers. */ + sellingLiabilitiesStroops: bigint; + subentryCount: number; + numSponsoring: number; + numSponsored: number; +} + +/** Rolling fee-spend statistics for transactions the relayer paid for. */ +export interface FeeSpendSummary { + count: number; + totalStroops: bigint; + avgStroops: number; + maxStroops: bigint; + /** Fee spend per hour over the observed window, in stroops. */ + ratePerHourStroops: number; +} + +export interface BalanceThresholds { + warningStroops: bigint; + criticalStroops: bigint; + warningRunwayHours: number; + criticalRunwayHours: number; + warningRunwaySubmissions: number; + criticalRunwaySubmissions: number; + /** + * Fee assumed for the next submission when no history exists yet. The + * relayer is `exhausted` once it cannot cover the larger of this and the + * biggest fee seen in the window. + */ + minSubmissionFeeStroops: bigint; +} + +export type BalanceAlertTier = "ok" | "warning" | "critical" | "exhausted"; + +/** Ordered from best to worst; index doubles as the metric value. */ +export const TIER_ORDER: readonly BalanceAlertTier[] = ["ok", "warning", "critical", "exhausted"]; + +export function tierRank(tier: BalanceAlertTier): number { + return TIER_ORDER.indexOf(tier); +} + +export interface BalanceAssessment { + tier: BalanceAlertTier; + balanceStroops: bigint; + reserveStroops: bigint; + spendableStroops: bigint; + /** Fee the next submission is expected to need. */ + nextFeeStroops: bigint; + /** Submissions left at the average fee; null without fee history. */ + runwaySubmissions: number | null; + /** Hours left at the current spend rate; null without fee history. */ + runwayHours: number | null; + /** Human-readable reasons for the tier, worst first. */ + reasons: string[]; +} + +/** Minimum balance Stellar requires the account to keep: (2 + entries) × base reserve. */ +export function minimumReserveStroops(s: RelayerAccountSnapshot): bigint { + const entries = 2 + s.subentryCount + s.numSponsoring - s.numSponsored; + return BigInt(Math.max(entries, 0)) * BASE_RESERVE_STROOPS; +} + +/** Balance available for fees after the reserve and open-offer liabilities. */ +export function spendableStroops(s: RelayerAccountSnapshot): bigint { + const spendable = s.balanceStroops - minimumReserveStroops(s) - s.sellingLiabilitiesStroops; + return spendable > 0n ? spendable : 0n; +} + +export function formatXlm(stroops: bigint): string { + const negative = stroops < 0n; + const abs = negative ? -stroops : stroops; + const whole = abs / STROOPS_PER_XLM; + const frac = (abs % STROOPS_PER_XLM).toString().padStart(7, "0").replace(/0+$/, ""); + return `${negative ? "-" : ""}${whole}${frac ? `.${frac}` : ""} XLM`; +} + +/** Parse a decimal XLM amount (e.g. Horizon's "123.4567890") into stroops without float rounding. */ +export function parseXlmToStroops(amount: string): bigint { + const match = /^(-)?(\d+)(?:\.(\d{1,7}))?$/.exec(amount.trim()); + if (!match) { + throw new Error(`Invalid XLM amount: "${amount}"`); + } + const [, sign, whole, frac = ""] = match; + const stroops = BigInt(whole) * STROOPS_PER_XLM + BigInt(frac.padEnd(7, "0")); + return sign ? -stroops : stroops; +} + +/** Decide the alert tier for the current balance and spend. */ +export function assessBalance( + snapshot: RelayerAccountSnapshot, + spend: FeeSpendSummary, + t: BalanceThresholds +): BalanceAssessment { + const reserve = minimumReserveStroops(snapshot); + const spendable = spendableStroops(snapshot); + const nextFee = spend.maxStroops > t.minSubmissionFeeStroops ? spend.maxStroops : t.minSubmissionFeeStroops; + + const runwaySubmissions = spend.count > 0 && spend.avgStroops > 0 ? Number(spendable) / spend.avgStroops : null; + const runwayHours = spend.ratePerHourStroops > 0 ? Number(spendable) / spend.ratePerHourStroops : null; + + const reasons: Array<{ tier: BalanceAlertTier; text: string }> = []; + const add = (tier: BalanceAlertTier, text: string) => reasons.push({ tier, text }); + + if (spendable < nextFee) { + add("exhausted", `spendable ${formatXlm(spendable)} cannot cover the next fee of ${formatXlm(nextFee)}`); + } + if (spendable < t.criticalStroops) { + add("critical", `spendable ${formatXlm(spendable)} is below ${formatXlm(t.criticalStroops)}`); + } else if (spendable < t.warningStroops) { + add("warning", `spendable ${formatXlm(spendable)} is below ${formatXlm(t.warningStroops)}`); + } + if (runwayHours !== null) { + if (runwayHours < t.criticalRunwayHours) add("critical", `about ${runwayHours.toFixed(1)}h of fees left at the current rate`); + else if (runwayHours < t.warningRunwayHours) add("warning", `about ${runwayHours.toFixed(1)}h of fees left at the current rate`); + } + if (runwaySubmissions !== null) { + if (runwaySubmissions < t.criticalRunwaySubmissions) { + add("critical", `about ${Math.floor(runwaySubmissions)} submissions left at the average fee`); + } else if (runwaySubmissions < t.warningRunwaySubmissions) { + add("warning", `about ${Math.floor(runwaySubmissions)} submissions left at the average fee`); + } + } + + reasons.sort((a, b) => tierRank(b.tier) - tierRank(a.tier)); + return { + tier: reasons[0]?.tier ?? "ok", + balanceStroops: snapshot.balanceStroops, + reserveStroops: reserve, + spendableStroops: spendable, + nextFeeStroops: nextFee, + runwaySubmissions, + runwayHours, + reasons: reasons.map((r) => r.text), + }; +} + +export type AlertDecision = + | { emit: false } + | { emit: true; kind: "escalated" | "recovered" | "reminder"; severity: "info" | "warning" | "critical" }; + +/** + * When to alert: on every escalation, once on recovery, and as a reminder + * every `renotifyMs` while the tier stays above ok. + */ +export function decideAlert( + previous: BalanceAlertTier, + next: BalanceAlertTier, + lastAlertAt: number | null, + now: number, + renotifyMs: number +): AlertDecision { + const severity = next === "warning" ? "warning" : "critical"; + if (tierRank(next) > tierRank(previous)) { + return { emit: true, kind: "escalated", severity }; + } + if (tierRank(next) < tierRank(previous)) { + return next === "ok" + ? { emit: true, kind: "recovered", severity: "info" } + : { emit: true, kind: "recovered", severity }; + } + if (next !== "ok" && (lastAlertAt === null || now - lastAlertAt >= renotifyMs)) { + return { emit: true, kind: "reminder", severity }; + } + return { emit: false }; +} diff --git a/src/stellar/relayer-monitor/relayer-monitor.config.ts b/src/stellar/relayer-monitor/relayer-monitor.config.ts new file mode 100644 index 0000000..b00d04b --- /dev/null +++ b/src/stellar/relayer-monitor/relayer-monitor.config.ts @@ -0,0 +1,100 @@ +import { Keypair, StrKey } from "@stellar/stellar-sdk"; +import { BalanceThresholds, parseXlmToStroops } from "./relayer-balance.policy"; + +export interface RelayerMonitorConfig { + enabled: boolean; + /** Why monitoring is off, when it is. */ + disabledReason?: string; + publicKey: string | null; + horizonUrl: string; + intervalMs: number; + feeWindowMs: number; + renotifyMs: number; + /** Consecutive failed checks before alerting that the balance is unknown. */ + failuresBeforeAlert: number; + thresholds: BalanceThresholds; +} + +const HORIZON_BY_NETWORK: Record = { + testnet: "https://horizon-testnet.stellar.org", + futurenet: "https://horizon-futurenet.stellar.org", + mainnet: "https://horizon.stellar.org", +}; + +function num(env: NodeJS.ProcessEnv, key: string, fallback: number, min = 0): number { + const raw = env[key]; + if (raw === undefined || raw.trim() === "") return fallback; + const value = Number(raw); + if (!Number.isFinite(value) || value < min) { + throw new Error(`${key} must be a number >= ${min} (got "${raw}")`); + } + return value; +} + +function xlm(env: NodeJS.ProcessEnv, key: string, fallback: string): bigint { + const raw = env[key]?.trim() || fallback; + const stroops = parseXlmToStroops(raw); + if (stroops < 0n) throw new Error(`${key} must not be negative`); + return stroops; +} + +/** + * The relayer's public key: RELAYER_PUBLIC_KEY if set (preferred — the + * monitor then never needs the secret), otherwise derived from the + * relayer secret the publisher already uses. + */ +function resolvePublicKey(env: NodeJS.ProcessEnv): string | null { + const explicit = env.RELAYER_PUBLIC_KEY?.trim(); + if (explicit) { + if (!StrKey.isValidEd25519PublicKey(explicit)) { + throw new Error("RELAYER_PUBLIC_KEY must be a valid Stellar public key (G...)"); + } + return explicit; + } + const secret = (env.ORACLE_RELAYER_SECRET ?? env.RELAYER_SECRET_KEY ?? "").trim(); + if (secret && StrKey.isValidEd25519SecretSeed(secret)) { + return Keypair.fromSecret(secret).publicKey(); + } + return null; +} + +export function loadRelayerMonitorConfig(env: NodeJS.ProcessEnv = process.env): RelayerMonitorConfig { + const network = (env.STELLAR_NETWORK ?? "testnet").trim().toLowerCase(); + const horizonUrl = env.STELLAR_HORIZON_URL?.trim() || HORIZON_BY_NETWORK[network] || HORIZON_BY_NETWORK.testnet; + + const thresholds: BalanceThresholds = { + warningStroops: xlm(env, "RELAYER_BALANCE_WARNING_XLM", "100"), + criticalStroops: xlm(env, "RELAYER_BALANCE_CRITICAL_XLM", "25"), + warningRunwayHours: num(env, "RELAYER_RUNWAY_WARNING_HOURS", 72), + criticalRunwayHours: num(env, "RELAYER_RUNWAY_CRITICAL_HOURS", 24), + warningRunwaySubmissions: num(env, "RELAYER_RUNWAY_WARNING_SUBMISSIONS", 1_000), + criticalRunwaySubmissions: num(env, "RELAYER_RUNWAY_CRITICAL_SUBMISSIONS", 200), + minSubmissionFeeStroops: xlm(env, "RELAYER_MIN_SUBMISSION_FEE_XLM", "0.01"), + }; + const pairs: Array<[string, number | bigint, number | bigint]> = [ + ["RELAYER_BALANCE", thresholds.criticalStroops, thresholds.warningStroops], + ["RELAYER_RUNWAY_*_HOURS", thresholds.criticalRunwayHours, thresholds.warningRunwayHours], + ["RELAYER_RUNWAY_*_SUBMISSIONS", thresholds.criticalRunwaySubmissions, thresholds.warningRunwaySubmissions], + ]; + for (const [name, critical, warning] of pairs) { + if (critical > warning) throw new Error(`${name}: the critical threshold must not exceed the warning threshold`); + } + + const publicKey = resolvePublicKey(env); + const optedOut = (env.RELAYER_MONITOR_ENABLED ?? "true").trim().toLowerCase() === "false"; + return { + enabled: !optedOut && publicKey !== null, + disabledReason: optedOut + ? "RELAYER_MONITOR_ENABLED=false" + : publicKey === null + ? "no relayer key configured (RELAYER_PUBLIC_KEY or ORACLE_RELAYER_SECRET)" + : undefined, + publicKey, + horizonUrl, + intervalMs: num(env, "RELAYER_MONITOR_INTERVAL_MS", 60_000, 1_000), + feeWindowMs: num(env, "RELAYER_FEE_WINDOW_HOURS", 24, 1) * 3_600_000, + renotifyMs: num(env, "RELAYER_ALERT_RENOTIFY_MINUTES", 60, 1) * 60_000, + failuresBeforeAlert: num(env, "RELAYER_MONITOR_FAILURES_BEFORE_ALERT", 3, 1), + thresholds, + }; +} diff --git a/src/stellar/relayer-monitor/relayer-monitor.module.ts b/src/stellar/relayer-monitor/relayer-monitor.module.ts new file mode 100644 index 0000000..67dbefd --- /dev/null +++ b/src/stellar/relayer-monitor/relayer-monitor.module.ts @@ -0,0 +1,26 @@ +import { Module } from "@nestjs/common"; +import { AlertingModule } from "../../alerting/alerting.module"; +import { AlertingService } from "../../alerting/alerting.service"; +import { MetricsService } from "../../metrics/metrics.service"; +import { PipelineHeartbeat } from "../../runtime/pipeline-heartbeat"; +import { HorizonRelayerClient } from "./horizon-relayer.client"; +import { RelayerBalanceMonitor } from "./relayer-balance-monitor.service"; +import { loadRelayerMonitorConfig } from "./relayer-monitor.config"; + +/** Relayer balance and fee-spend monitoring. Imported by the worker. */ +@Module({ + imports: [AlertingModule], + providers: [ + { + provide: RelayerBalanceMonitor, + inject: [AlertingService, { token: MetricsService, optional: true }, { token: PipelineHeartbeat, optional: true }], + useFactory: (alerting: AlertingService, metrics?: MetricsService, heartbeat?: PipelineHeartbeat) => { + const config = loadRelayerMonitorConfig(); + const client = config.publicKey ? new HorizonRelayerClient(config.horizonUrl, config.publicKey) : null; + return new RelayerBalanceMonitor(config, client, alerting, metrics, heartbeat); + }, + }, + ], + exports: [RelayerBalanceMonitor], +}) +export class RelayerMonitorModule {} diff --git a/src/stellar/relayer-monitor/relayer-monitor.spec.ts b/src/stellar/relayer-monitor/relayer-monitor.spec.ts new file mode 100644 index 0000000..be60403 --- /dev/null +++ b/src/stellar/relayer-monitor/relayer-monitor.spec.ts @@ -0,0 +1,439 @@ +import fc from "fast-check"; +import { Keypair } from "@stellar/stellar-sdk"; +import { AlertingService } from "../../alerting/alerting.service"; +import { PipelineHeartbeat } from "../../runtime/pipeline-heartbeat"; +import { FeeSpendTracker } from "./fee-spend-tracker"; +import { HorizonRelayerClient, RelayerAccountNotFoundError } from "./horizon-relayer.client"; +import { + BalanceThresholds, + FeeSpendSummary, + RelayerAccountSnapshot, + assessBalance, + decideAlert, + formatXlm, + minimumReserveStroops, + parseXlmToStroops, + spendableStroops, + tierRank, +} from "./relayer-balance.policy"; +import { RelayerBalanceMonitor, RelayerMetricsSink } from "./relayer-balance-monitor.service"; +import { RelayerMonitorConfig, loadRelayerMonitorConfig } from "./relayer-monitor.config"; + +const XLM = 10_000_000n; +const HOUR = 3_600_000; + +const thresholds: BalanceThresholds = { + warningStroops: 100n * XLM, + criticalStroops: 25n * XLM, + warningRunwayHours: 72, + criticalRunwayHours: 24, + warningRunwaySubmissions: 1_000, + criticalRunwaySubmissions: 200, + minSubmissionFeeStroops: XLM / 100n, +}; + +function account(balanceXlm: string, overrides: Partial = {}): RelayerAccountSnapshot { + return { + balanceStroops: parseXlmToStroops(balanceXlm), + sellingLiabilitiesStroops: 0n, + subentryCount: 0, + numSponsoring: 0, + numSponsored: 0, + ...overrides, + }; +} + +const NO_SPEND: FeeSpendSummary = { count: 0, totalStroops: 0n, avgStroops: 0, maxStroops: 0n, ratePerHourStroops: 0 }; + +describe("XLM amounts", () => { + it("parses Horizon balances exactly, without float rounding", () => { + expect(parseXlmToStroops("123.4567890")).toBe(1_234_567_890n); + expect(parseXlmToStroops("0.0000001")).toBe(1n); + expect(parseXlmToStroops("100")).toBe(1_000_000_000n); + expect(() => parseXlmToStroops("1.12345678")).toThrow(); + expect(() => parseXlmToStroops("1e3")).toThrow(); + }); + + it("round-trips through formatXlm", () => { + fc.assert( + fc.property(fc.bigInt({ min: 0n, max: 10n ** 17n }), (stroops) => { + return parseXlmToStroops(formatXlm(stroops).replace(" XLM", "")) === stroops; + }) + ); + }); +}); + +describe("reserve and spendable balance", () => { + it("applies the (2 + entries) × 0.5 XLM minimum reserve", () => { + expect(minimumReserveStroops(account("10"))).toBe(1n * XLM); + expect(minimumReserveStroops(account("10", { subentryCount: 3, numSponsoring: 1, numSponsored: 2 }))).toBe(2n * XLM); + }); + + it("subtracts reserve and selling liabilities and never goes negative", () => { + expect(spendableStroops(account("10", { sellingLiabilitiesStroops: 2n * XLM }))).toBe(7n * XLM); + expect(spendableStroops(account("0.5"))).toBe(0n); + }); +}); + +describe("assessBalance", () => { + it("is ok with a healthy balance and no history", () => { + const a = assessBalance(account("1000"), NO_SPEND, thresholds); + expect(a.tier).toBe("ok"); + expect(a.runwayHours).toBeNull(); + expect(a.reasons).toEqual([]); + }); + + it.each([ + ["warning", "90"], + ["critical", "20"], + ["exhausted", "1.005"], + ])("goes %s on absolute balance (balance %s XLM)", (tier, balance) => { + expect(assessBalance(account(balance), NO_SPEND, thresholds).tier).toBe(tier); + }); + + it("warns on time runway even when the balance looks large", () => { + // 1000 XLM spendable-ish, burning 20 XLM/h → ~50h left + const spend: FeeSpendSummary = { count: 10, totalStroops: 0n, avgStroops: 100, maxStroops: 100n, ratePerHourStroops: 20 * Number(XLM) }; + const a = assessBalance(account("1001"), spend, thresholds); + expect(a.tier).toBe("warning"); + expect(a.runwayHours).toBeCloseTo(50, 5); + expect(a.reasons[0]).toMatch(/50\.0h of fees left/); + }); + + it("goes critical on submission runway when per-publish fees are high", () => { + // 500 XLM spendable at 5 XLM per publish → 100 submissions left + const spend: FeeSpendSummary = { count: 3, totalStroops: 15n * XLM, avgStroops: 5 * Number(XLM), maxStroops: 5n * XLM, ratePerHourStroops: 1 }; + const a = assessBalance(account("501"), spend, thresholds); + expect(a.tier).toBe("critical"); + expect(a.runwaySubmissions).toBeCloseTo(100, 5); + }); + + it("is exhausted when the next fee, sized from the largest recent fee, cannot be covered", () => { + const spend: FeeSpendSummary = { count: 1, totalStroops: 3n * XLM, avgStroops: 3 * Number(XLM), maxStroops: 3n * XLM, ratePerHourStroops: 0 }; + const a = assessBalance(account("3.5"), spend, thresholds); + expect(a.nextFeeStroops).toBe(3n * XLM); + expect(a.tier).toBe("exhausted"); + }); + + it("never reports a better tier for a lower balance (property)", () => { + fc.assert( + fc.property( + fc.bigInt({ min: 0n, max: 10_000n * XLM }), + fc.bigInt({ min: 0n, max: 10_000n * XLM }), + fc.record({ + count: fc.integer({ min: 0, max: 1_000 }), + avg: fc.integer({ min: 0, max: 10_000_000 }), + max: fc.bigInt({ min: 0n, max: 100_000_000n }), + rate: fc.integer({ min: 0, max: 1_000_000_000 }), + }), + (b1, b2, s) => { + const spend: FeeSpendSummary = { + count: s.count, + totalStroops: 0n, + avgStroops: s.avg, + maxStroops: s.max, + ratePerHourStroops: s.rate, + }; + const [lo, hi] = b1 <= b2 ? [b1, b2] : [b2, b1]; + const tLo = assessBalance({ ...account("0"), balanceStroops: lo }, spend, thresholds).tier; + const tHi = assessBalance({ ...account("0"), balanceStroops: hi }, spend, thresholds).tier; + return tierRank(tLo) >= tierRank(tHi); + } + ) + ); + }); +}); + +describe("decideAlert", () => { + const RENOTIFY = HOUR; + + it("alerts on every escalation with matching severity", () => { + expect(decideAlert("ok", "warning", null, 0, RENOTIFY)).toEqual({ emit: true, kind: "escalated", severity: "warning" }); + expect(decideAlert("warning", "critical", 0, 1, RENOTIFY)).toEqual({ emit: true, kind: "escalated", severity: "critical" }); + expect(decideAlert("ok", "exhausted", null, 0, RENOTIFY)).toMatchObject({ severity: "critical" }); + }); + + it("announces recovery once", () => { + expect(decideAlert("critical", "ok", 0, 1, RENOTIFY)).toEqual({ emit: true, kind: "recovered", severity: "info" }); + expect(decideAlert("critical", "warning", 0, 1, RENOTIFY)).toEqual({ emit: true, kind: "recovered", severity: "warning" }); + expect(decideAlert("ok", "ok", null, 1, RENOTIFY)).toEqual({ emit: false }); + }); + + it("reminds while a problem persists, but not more often than the renotify interval", () => { + expect(decideAlert("critical", "critical", 0, RENOTIFY - 1, RENOTIFY)).toEqual({ emit: false }); + expect(decideAlert("critical", "critical", 0, RENOTIFY, RENOTIFY)).toEqual({ emit: true, kind: "reminder", severity: "critical" }); + }); +}); + +describe("FeeSpendTracker", () => { + it("dedupes by hash, counts failed transactions, and prunes outside the window", () => { + const t = new FeeSpendTracker(24 * HOUR, 0); + expect(t.record({ txHash: "a", feeChargedStroops: 100n, at: 0, successful: true })).toBe(true); + expect(t.record({ txHash: "a", feeChargedStroops: 100n, at: 0, successful: true })).toBe(false); + t.record({ txHash: "b", feeChargedStroops: 300n, at: HOUR, successful: false }); + + expect(t.summary(2 * HOUR)).toEqual({ count: 2, totalStroops: 400n, avgStroops: 200, maxStroops: 300n, ratePerHourStroops: 200 }); + expect(t.summary(24 * HOUR + 1).count).toBe(1); + }); + + it("does not extrapolate a startup burst into a huge hourly rate", () => { + const t = new FeeSpendTracker(24 * HOUR, 15 * 60_000); + t.record({ txHash: "a", feeChargedStroops: 1_000n, at: 0, successful: true }); + expect(t.summary(1_000).ratePerHourStroops).toBe(4_000); // 1000 per 15 min, not per second + }); +}); + +// ------------------------------------------------------------------ Horizon client + +const RELAYER = Keypair.random().publicKey(); +const BUMPER = Keypair.random().publicKey(); + +function horizonTx(n: number, overrides: Record = {}) { + return { + hash: `h${n}`, + paging_token: `pt${n}`, + fee_charged: "1000", + source_account: RELAYER, + fee_account: RELAYER, + successful: true, + created_at: new Date(1_800_000_000_000 + n * 60_000).toISOString(), + ...overrides, + }; +} + +function fakeHorizon(routes: Record) { + const calls: string[] = []; + const fetchFn = jest.fn(async (url: string) => { + calls.push(url); + const path = url.replace("https://horizon.test", ""); + const key = Object.keys(routes).find((k) => path.startsWith(k)); + const route = key ? routes[key] : { status: 404 }; + return { ok: (route.status ?? 200) < 400, status: route.status ?? 200, json: async () => route.body }; + }); + return { fetchFn, calls }; +} + +describe("HorizonRelayerClient", () => { + it("reads balance, liabilities and reserve inputs", async () => { + const { fetchFn } = fakeHorizon({ + [`/accounts/${RELAYER}`]: { + body: { + balances: [ + { asset_type: "credit_alphanum4", balance: "5.0000000" }, + { asset_type: "native", balance: "42.1234567", selling_liabilities: "1.0000000" }, + ], + subentry_count: 2, + num_sponsoring: 1, + }, + }, + }); + const client = new HorizonRelayerClient("https://horizon.test/", RELAYER, 1_000, fetchFn); + await expect(client.getAccount()).resolves.toEqual({ + balanceStroops: 421_234_567n, + sellingLiabilitiesStroops: 10_000_000n, + subentryCount: 2, + numSponsoring: 1, + numSponsored: 0, + }); + }); + + it("reports an unfunded account distinctly", async () => { + const { fetchFn } = fakeHorizon({}); + await expect(new HorizonRelayerClient("https://horizon.test", RELAYER, 1_000, fetchFn).getAccount()).rejects.toBeInstanceOf( + RelayerAccountNotFoundError + ); + }); + + it("backfills newest-first, returns oldest-first, and only counts fees the relayer paid", async () => { + const { fetchFn, calls } = fakeHorizon({ + [`/accounts/${RELAYER}/transactions`]: { + body: { _embedded: { records: [horizonTx(3), horizonTx(2, { fee_account: BUMPER }), horizonTx(1, { successful: false })] } }, + }, + }); + const page = await new HorizonRelayerClient("https://horizon.test", RELAYER, 1_000, fetchFn).listFeeSpend(null); + + expect(calls[0]).toContain("order=desc"); + expect(calls[0]).toContain("include_failed=true"); + expect(page.records.map((r) => [r.txHash, r.successful])).toEqual([ + ["h1", false], + ["h3", true], + ]); + expect(page.cursor).toBe("pt3"); + }); +}); + +// ------------------------------------------------------------------ config + +describe("loadRelayerMonitorConfig", () => { + it("prefers RELAYER_PUBLIC_KEY and picks Horizon from the network", () => { + const cfg = loadRelayerMonitorConfig({ RELAYER_PUBLIC_KEY: RELAYER, STELLAR_NETWORK: "mainnet" }); + expect(cfg).toMatchObject({ enabled: true, publicKey: RELAYER, horizonUrl: "https://horizon.stellar.org" }); + expect(cfg.thresholds.warningStroops).toBe(100n * XLM); + }); + + it("derives the public key from the relayer secret when needed", () => { + const kp = Keypair.random(); + expect(loadRelayerMonitorConfig({ ORACLE_RELAYER_SECRET: kp.secret() }).publicKey).toBe(kp.publicKey()); + }); + + it("is disabled, with a reason, when no key is configured or it is opted out", () => { + expect(loadRelayerMonitorConfig({})).toMatchObject({ enabled: false, disabledReason: expect.stringMatching(/no relayer key/) }); + expect(loadRelayerMonitorConfig({ RELAYER_PUBLIC_KEY: RELAYER, RELAYER_MONITOR_ENABLED: "false" }).enabled).toBe(false); + }); + + it.each([ + [{ RELAYER_BALANCE_WARNING_XLM: "10", RELAYER_BALANCE_CRITICAL_XLM: "20" }, /critical threshold/], + [{ RELAYER_MONITOR_INTERVAL_MS: "10" }, /RELAYER_MONITOR_INTERVAL_MS/], + [{ RELAYER_BALANCE_WARNING_XLM: "lots" }, /Invalid XLM amount/], + [{ RELAYER_PUBLIC_KEY: "GNOPE" }, /RELAYER_PUBLIC_KEY/], + ])("rejects bad config %#", (env, message) => { + expect(() => loadRelayerMonitorConfig({ RELAYER_PUBLIC_KEY: RELAYER, ...env })).toThrow(message); + }); +}); + +// ------------------------------------------------------------------ monitor + +describe("RelayerBalanceMonitor", () => { + const originalRole = process.env.PROCESS_ROLE; + afterEach(() => { + if (originalRole === undefined) delete process.env.PROCESS_ROLE; + else process.env.PROCESS_ROLE = originalRole; + }); + + const config: RelayerMonitorConfig = { + enabled: true, + publicKey: RELAYER, + horizonUrl: "https://horizon.test", + intervalMs: 60_000, + feeWindowMs: 24 * HOUR, + renotifyMs: HOUR, + failuresBeforeAlert: 2, + thresholds, + }; + + function setup() { + let now = 1_800_000_000_000; + let balance = "1000"; + let txs: Array> = []; + let failWith: Error | null = null; + const client = { + getAccount: jest.fn(async () => { + if (failWith) throw failWith; + return account(balance); + }), + listFeeSpend: jest.fn(async (cursor: string | null) => { + if (failWith) throw failWith; + const after = cursor ? txs.filter((t) => t.paging_token > cursor) : txs; + return { + records: after.map((t) => ({ txHash: t.hash, feeChargedStroops: BigInt(t.fee_charged), at: Date.parse(t.created_at), successful: t.successful })), + cursor: after.length ? after[after.length - 1].paging_token : cursor, + }; + }), + } as unknown as HorizonRelayerClient; + const alerting = new AlertingService(); + const metrics: RelayerMetricsSink = { recordRelayerBalance: jest.fn(), recordRelayerFee: jest.fn() }; + const heartbeat = new PipelineHeartbeat(HOUR, () => now); + const monitor = new RelayerBalanceMonitor(config, client, alerting, metrics, heartbeat, () => now); + return { + monitor, + alerting, + metrics, + heartbeat, + client, + setBalance: (b: string) => (balance = b), + addTx: (tx: ReturnType) => (txs = [...txs, tx]), + fail: (e: Error | null) => (failWith = e), + advance: (ms: number) => (now += ms), + titles: () => alerting.listRecent().map((a) => `${a.severity}:${a.title}`).reverse(), + }; + } + + it("raises tiered alerts as the balance drains, reminds, then reports recovery", async () => { + const t = setup(); + await t.monitor.check(); + expect(t.titles()).toEqual([]); + + t.setBalance("90"); + await t.monitor.check(); + t.setBalance("20"); + await t.monitor.check(); + t.advance(HOUR); + await t.monitor.check(); + t.setBalance("500"); + await t.monitor.check(); + + expect(t.titles()).toEqual([ + "warning:Relayer balance warning", + "critical:Relayer balance critical", + "critical:Relayer balance critical (still)", + "info:Relayer balance recovered to ok", + ]); + expect(t.monitor.getStatus().assessment?.tier).toBe("ok"); + }); + + it("tracks per-transaction fee spend, counting each transaction once", async () => { + const t = setup(); + t.addTx(horizonTx(1, { fee_charged: "50000" })); + await t.monitor.check(); + t.addTx(horizonTx(2, { fee_charged: "70000", successful: false })); + await t.monitor.check(); + await t.monitor.check(); + + expect(t.metrics.recordRelayerFee).toHaveBeenCalledTimes(2); + expect(t.metrics.recordRelayerFee).toHaveBeenLastCalledWith(70_000n, false); + expect(t.monitor.getStatus().feeSpend).toMatchObject({ count: 2, totalStroops: 120_000n, maxStroops: 70_000n }); + expect(t.metrics.recordRelayerBalance).toHaveBeenCalledTimes(3); + }); + + it("alerts when the balance has been unknown for several checks, and when it is back", async () => { + const t = setup(); + t.fail(new Error("Horizon down")); + await t.monitor.check(); + expect(t.titles()).toEqual([]); + await t.monitor.check(); + await t.monitor.check(); + t.fail(null); + await t.monitor.check(); + + expect(t.titles()).toEqual(["warning:Relayer balance unknown", "info:Relayer balance monitoring restored"]); + expect(t.monitor.getStatus().consecutiveFailures).toBe(0); + }); + + it("treats a missing account as exhausted immediately", async () => { + const t = setup(); + t.fail(new RelayerAccountNotFoundError(RELAYER)); + await t.monitor.check(); + await t.monitor.check(); + expect(t.titles()).toEqual(["critical:Relayer balance exhausted"]); + }); + + it("beats the pipeline heartbeat only on successful checks", async () => { + const t = setup(); + t.fail(new Error("down")); + await t.monitor.check(); + expect(t.heartbeat.status().lastBeats).toEqual({}); + t.fail(null); + await t.monitor.check(); + expect(Object.keys(t.heartbeat.status().lastBeats)).toEqual(["relayer-monitor"]); + }); + + it("does not start in an API-only process or without a key", async () => { + process.env.PROCESS_ROLE = "api"; + const t = setup(); + t.monitor.onModuleInit(); + expect(t.client.getAccount).not.toHaveBeenCalled(); + + process.env.PROCESS_ROLE = "worker"; + const off = new RelayerBalanceMonitor({ ...config, enabled: false, disabledReason: "test" }, null, new AlertingService()); + off.onModuleInit(); + await off.onModuleDestroy(); + }); + + it("starts polling in the worker and stops cleanly", async () => { + process.env.PROCESS_ROLE = "worker"; + const t = setup(); + t.monitor.onModuleInit(); + await t.monitor.onModuleDestroy(); + expect(t.client.getAccount).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/worker/worker.module.ts b/src/worker/worker.module.ts index f807579..bfd4032 100644 --- a/src/worker/worker.module.ts +++ b/src/worker/worker.module.ts @@ -13,11 +13,12 @@ import { OracleModule } from "../oracle/oracle.module"; import { ReadingBusModule } from "../oracle/reading-bus.module"; import { PolicyModule } from "../policy/policy.module"; import { StellarModule } from "../stellar/stellar.module"; +import { RelayerMonitorModule } from "../stellar/relayer-monitor/relayer-monitor.module"; import { RuntimeModule } from "../runtime/runtime.module"; /** * Root module for the worker tier: oracle polling, consensus evaluation, - * on-chain publishing and claim settlement. + * on-chain publishing, claim settlement and relayer balance monitoring. * * It shares the data layer with the API but leaves out everything that * serves requests — controllers are never bound (the worker is started as @@ -41,6 +42,7 @@ import { RuntimeModule } from "../runtime/runtime.module"; PolicyModule, OracleModule, ClaimModule, + RelayerMonitorModule, ], }) export class WorkerModule {}