diff --git a/.env.local.example b/.env.local.example index d126149..ea5ebd9 100644 --- a/.env.local.example +++ b/.env.local.example @@ -25,6 +25,8 @@ RATE_LIMIT_DURATION=60 # WEBHOOK_SIGNING_SECRET is still required at startup. WEBHOOK_URL= WEBHOOK_SIGNING_SECRET= +# Optional previous key during signing-secret rotation. +WEBHOOK_SIGNING_SECRET_PREVIOUS= # Contract Deployment CONTRACT_ADMIN_KEY= diff --git a/.env.mainnet.example b/.env.mainnet.example index d4e1c4a..093d1aa 100644 --- a/.env.mainnet.example +++ b/.env.mainnet.example @@ -20,6 +20,7 @@ RATE_LIMIT_DURATION=60 # X-COMEBACKHERE-Signature. See docs/webhooks.md. WEBHOOK_URL= WEBHOOK_SIGNING_SECRET= +WEBHOOK_SIGNING_SECRET_PREVIOUS= # Admin key guarding /webhooks/dead-letters* (sent as the X-Admin-Key header). ADMIN_KEY= diff --git a/.env.testnet.example b/.env.testnet.example index 4b300e7..3fbc73f 100644 --- a/.env.testnet.example +++ b/.env.testnet.example @@ -20,6 +20,7 @@ RATE_LIMIT_DURATION=60 # X-COMEBACKHERE-Signature. See docs/webhooks.md. WEBHOOK_URL= WEBHOOK_SIGNING_SECRET= +WEBHOOK_SIGNING_SECRET_PREVIOUS= # Admin key guarding /webhooks/dead-letters* (sent as the X-Admin-Key header). ADMIN_KEY= diff --git a/comebackhere-backend/src/lib/metrics.ts b/comebackhere-backend/src/lib/metrics.ts index 5c5da82..6ae8a61 100644 --- a/comebackhere-backend/src/lib/metrics.ts +++ b/comebackhere-backend/src/lib/metrics.ts @@ -27,6 +27,32 @@ export const webhookDeliveryOutcomes = new Counter({ registers: [metricsRegistry], }) +export const webhookDeliveryAttempts = new Counter({ + name: "webhook_delivery_attempts_total", + help: "Webhook HTTP delivery attempts", + registers: [metricsRegistry], +}) + +export const webhookDeliveryRetries = new Counter({ + name: "webhook_delivery_retries_total", + help: "Webhook delivery retries after the initial attempt", + registers: [metricsRegistry], +}) + +export const webhookDeliveryDuration = new Histogram({ + name: "webhook_delivery_duration_seconds", + help: "Time spent delivering a webhook event", + labelNames: ["status"], + buckets: [0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30], + registers: [metricsRegistry], +}) + +export const webhookDeadLetterVolume = new Gauge({ + name: "webhook_dead_letter_volume", + help: "Number of webhook deliveries currently in the dead-letter store", + registers: [metricsRegistry], +}) + const retentionLabels = ["indexer"] as const export function counter(name: string, help: string) { diff --git a/comebackhere-backend/src/services/treasury-indexer.ts b/comebackhere-backend/src/services/treasury-indexer.ts index 7346c44..5875034 100644 --- a/comebackhere-backend/src/services/treasury-indexer.ts +++ b/comebackhere-backend/src/services/treasury-indexer.ts @@ -251,7 +251,7 @@ export async function processIndexerBatch( amount: amount.toString(), token, tx_hash: txHash, - }).catch((err: unknown) => { + }, undefined, fetch, { eventId }).catch((err: unknown) => { logger.error({ errorName: err instanceof Error ? err.name : "UnknownError", eventType: "settlement_proposed" }, "Treasury webhook dispatch failed") }) } @@ -270,7 +270,7 @@ export async function processIndexerBatch( signer, approval_weight: newWeight.toString(), tx_hash: txHash, - }).catch((err: unknown) => { + }, undefined, fetch, { eventId }).catch((err: unknown) => { logger.error({ errorName: err instanceof Error ? err.name : "UnknownError", eventType: "settlement_approved" }, "Treasury webhook dispatch failed") }) } @@ -289,7 +289,7 @@ export async function processIndexerBatch( event: "settlement_executed", settlement_id: settlementId, tx_hash: txHash, - }).catch((err: unknown) => { + }, undefined, fetch, { eventId }).catch((err: unknown) => { logger.error({ errorName: err instanceof Error ? err.name : "UnknownError", eventType: "settlement_executed" }, "Treasury webhook dispatch failed") }) } diff --git a/comebackhere-backend/src/services/webhook-delivery.ts b/comebackhere-backend/src/services/webhook-delivery.ts index fe363a6..d263647 100644 --- a/comebackhere-backend/src/services/webhook-delivery.ts +++ b/comebackhere-backend/src/services/webhook-delivery.ts @@ -16,8 +16,15 @@ import { connectMongo } from "../db/mongo.js" import { getWebhookRetryConfig, type WebhookRetryConfig } from "../lib/env.js" -import { webhookDeliveryOutcomes } from "../lib/metrics.js" +import { + webhookDeadLetterVolume, + webhookDeliveryAttempts, + webhookDeliveryDuration, + webhookDeliveryOutcomes, + webhookDeliveryRetries, +} from "../lib/metrics.js" import { logger } from "../lib/logger.js" +import { sendWebhookRequest } from "./webhook-http.js" // --------------------------------------------------------------------------- // Types @@ -32,6 +39,8 @@ export interface WebhookPayload { event_type: string invoice_id?: string settlement_id?: string + event_id: string + delivery_id: string /** Stable per-event key. Set once and preserved across retries. */ idempotency_key: string timestamp: string @@ -92,29 +101,19 @@ export async function postWebhook( fetchFn: typeof fetch = fetch, timeoutMs = 10_000, correlationId?: string, + signal?: AbortSignal, ): Promise { - const controller = new AbortController() - const timer = setTimeout(() => controller.abort(), timeoutMs) - const headers: Record = { "Content-Type": "application/json", "X-Idempotency-Key": payload.idempotency_key, + "X-COMEBACKHERE-Event-Id": payload.event_id, + "X-COMEBACKHERE-Delivery-Id": payload.delivery_id, } if (correlationId && correlationId.trim() !== "") { headers["X-Request-Id"] = correlationId } - try { - const response = await fetchFn(endpoint, { - method: "POST", - headers, - body: JSON.stringify(payload), - signal: controller.signal, - }) - return response.status - } finally { - clearTimeout(timer) - } + return sendWebhookRequest(endpoint, headers, JSON.stringify(payload), fetchFn, timeoutMs, signal) } // --------------------------------------------------------------------------- @@ -176,11 +175,17 @@ export async function deliverWebhook( request_id: correlationId ?? null, attempt_history: attemptHistory, } + const startedAt = Date.now() for (let attempt = startAttempt; attempt < maxAttempts; attempt++) { - if (signal?.aborted) return record + if (signal?.aborted) { + webhookDeliveryDuration.observe({ status: "pending" }, (Date.now() - startedAt) / 1_000) + return record + } onAttempt?.(attempt + 1) + webhookDeliveryAttempts.inc() + if (attempt > 0) webhookDeliveryRetries.inc() record.attempts = attempt + 1 record.last_attempt_at = new Date().toISOString() const attemptRecord: WebhookAttemptRecord = { @@ -192,13 +197,14 @@ export async function deliverWebhook( record.attempt_history.push(attemptRecord) try { - const statusCode = await postWebhook(endpoint, payload, fetchFn, undefined, correlationId) + const statusCode = await postWebhook(endpoint, payload, fetchFn, undefined, correlationId, signal) record.last_status_code = statusCode attemptRecord.status_code = statusCode if (statusCode >= 200 && statusCode < 300) { record.status = "delivered" webhookDeliveryOutcomes.inc({ status: "delivered" }) + webhookDeliveryDuration.observe({ status: "delivered" }, (Date.now() - startedAt) / 1_000) return record } @@ -219,6 +225,7 @@ export async function deliverWebhook( record.status = "failed" webhookDeliveryOutcomes.inc({ status: "failed" }) + webhookDeliveryDuration.observe({ status: "failed" }, (Date.now() - startedAt) / 1_000) logger.error( { attempts: record.attempts, requestId: record.request_id, lastStatusCode: record.last_status_code }, "webhook delivery failed", @@ -254,11 +261,14 @@ export function buildWebhookPayload( ): WebhookPayload { const id = options?.invoiceId ?? options?.settlementId ?? crypto.randomUUID() const idempotency_key = `${eventType}:${id}` + const delivery_id = crypto.randomUUID() return { event_type: eventType, invoice_id: options?.invoiceId, settlement_id: options?.settlementId, + event_id: idempotency_key, + delivery_id, idempotency_key, timestamp: new Date().toISOString(), data, @@ -306,6 +316,7 @@ export function createMongoDeadLetterStore(): DeadLetterStore { deadLetter, { upsert: true }, ) + webhookDeadLetterVolume.set(await database.collection("webhook_dead_letters").countDocuments()) }, async find(idempotencyKey) { const database = await connectMongo() @@ -318,7 +329,9 @@ export function createMongoDeadLetterStore(): DeadLetterStore { }, async list(limit) { const database = await connectMongo() - const records = await database.collection("webhook_dead_letters") + const collection = database.collection("webhook_dead_letters") + webhookDeadLetterVolume.set(await collection.countDocuments()) + const records = await collection .find() .sort({ failed_at: -1 }) .limit(limit) @@ -331,6 +344,7 @@ export function createMongoDeadLetterStore(): DeadLetterStore { async delete(idempotencyKey) { const database = await connectMongo() await database.collection("webhook_dead_letters").deleteOne({ _id: idempotencyKey }) + webhookDeadLetterVolume.set(await database.collection("webhook_dead_letters").countDocuments()) }, } } @@ -421,7 +435,7 @@ export class WebhookDeliveryQueue { this.store = options.store ?? createMongoPendingDeliveryStore() this.deadLetterStore = options.deadLetterStore ?? createMongoDeadLetterStore() this.maxAttempts = options.maxAttempts ?? this.retryConfig.maxAttempts - this.fetchFn = options.fetchFn ?? ((...args) => fetch(...args)) + this.fetchFn = options.fetchFn ?? fetch this.delayFn = options.delayFn ?? defaultDelay } @@ -540,6 +554,7 @@ export class WebhookDeliveryQueue { /** Re-enqueues deliveries persisted by a previous process. */ async resumePending(): Promise { const jobs = await this.store.takeAll() + await this.deadLetterStore.list(1) for (const job of jobs) this.enqueue(job.endpoint, job.payload, job.attempts, job.attempt_history) if (jobs.length > 0) { logger.info({ resumed: jobs.length }, "Persisted webhook deliveries resumed") diff --git a/comebackhere-backend/src/services/webhook-http.ts b/comebackhere-backend/src/services/webhook-http.ts new file mode 100644 index 0000000..b2679cc --- /dev/null +++ b/comebackhere-backend/src/services/webhook-http.ts @@ -0,0 +1,186 @@ +import type { LookupFunction } from "node:dns" +import { lookup as lookupHost } from "node:dns/promises" +import { request as httpsRequest } from "node:https" +import { isIP } from "node:net" + +export interface ResolvedAddress { + address: string + family: 4 | 6 +} + +function ipv4ToNumber(address: string): number { + return address.split(".").reduce((value, part) => (value * 256 + Number(part)) >>> 0, 0) +} + +function isBlockedIpv4(address: string): boolean { + const value = ipv4ToNumber(address) + const ranges: Array<[number, number]> = [ + [0x00000000, 8], + [0x0a000000, 8], + [0x64400000, 10], + [0x7f000000, 8], + [0xa9fe0000, 16], + [0xac100000, 12], + [0xc0000000, 24], + [0xc0000200, 24], + [0xc0586300, 24], + [0xc0a80000, 16], + [0xc6120000, 15], + [0xc6336400, 24], + [0xcb007100, 24], + [0xe0000000, 4], + [0xf0000000, 4], + ] + return ranges.some(([network, prefix]) => { + const mask = (0xffffffff << (32 - prefix)) >>> 0 + return (value & mask) === (network & mask) + }) +} + +function ipv6ToBigInt(address: string): bigint { + let normalized = address.toLowerCase().split("%")[0] + if (normalized.includes(".")) { + const splitAt = normalized.lastIndexOf(":") + const octets = normalized.slice(splitAt + 1).split(".").map(Number) + const high = ((octets[0] << 8) | octets[1]).toString(16) + const low = ((octets[2] << 8) | octets[3]).toString(16) + normalized = `${normalized.slice(0, splitAt + 1)}${high}:${low}` + } + const halves = normalized.split("::") + const left = halves[0] ? halves[0].split(":") : [] + const right = halves[1] ? halves[1].split(":") : [] + const groups = halves.length === 1 + ? left + : [...left, ...Array(8 - left.length - right.length).fill("0"), ...right] + return groups.reduce((value, group) => (value << 16n) | BigInt(`0x${group || "0"}`), 0n) +} + +function isBlockedAddress(address: string): boolean { + const family = isIP(address) + if (family === 4) return isBlockedIpv4(address) + if (family !== 6) return true + + const value = ipv6ToBigInt(address) + const mappedPrefix = value >> 32n + if (mappedPrefix === 0xffffn) { + const mappedIpv4 = [ + Number((value >> 24n) & 255n), + Number((value >> 16n) & 255n), + Number((value >> 8n) & 255n), + Number(value & 255n), + ].join(".") + return isBlockedIpv4(mappedIpv4) + } + + const globalUnicast = value >> 125n === 1n + const documentation = value >> 96n === 0x20010db8n + const sixToFour = value >> 112n === 0x2002n + const special2001 = value >> 105n === 0x100080n + return !globalUnicast || documentation || sixToFour || special2001 +} + +function parseWebhookUrl(value: string): URL { + let url: URL + try { + url = new URL(value) + } catch { + throw new Error("Webhook endpoint must be a valid HTTPS URL") + } + const hostname = url.hostname.replace(/^\[|\]$/g, "") + if ( + url.protocol !== "https:" || + url.username !== "" || + url.password !== "" || + !hostname || + hostname === "localhost" || + hostname.endsWith(".localhost") || + hostname.endsWith(".local") || + hostname.endsWith(".internal") || + hostname.endsWith(".home.arpa") + ) { + throw new Error("Webhook endpoint must be a public HTTPS URL") + } + if (isIP(hostname) && isBlockedAddress(hostname)) { + throw new Error("Webhook endpoint resolves to a non-public address") + } + return url +} + +export async function resolvePublicAddress(hostname: string): Promise { + const family = isIP(hostname) + const addresses: ResolvedAddress[] = family + ? [{ address: hostname, family: family as 4 | 6 }] + : await lookupHost(hostname, { all: true, verbatim: true }) as ResolvedAddress[] + + if (addresses.length === 0 || addresses.some(({ address }) => isBlockedAddress(address))) { + throw new Error("Webhook endpoint resolves to a non-public address") + } + return addresses[0] +} + +export function createPinnedLookup(resolved: ResolvedAddress): LookupFunction { + return (_hostname, options, callback) => { + if (options.all) callback(null, [resolved]) + else callback(null, resolved.address, resolved.family) + } +} + +/** Sends one HTTPS request to a DNS-pinned public address. Redirects are never followed. */ +export async function sendWebhookRequest( + endpoint: string, + headers: Record, + body: string, + fetchFn: typeof fetch = fetch, + timeoutMs = 10_000, + signal?: AbortSignal, +): Promise { + const url = parseWebhookUrl(endpoint) + const hostname = url.hostname.replace(/^\[|\]$/g, "") + const controller = new AbortController() + const onAbort = () => controller.abort() + if (signal?.aborted) controller.abort() + else signal?.addEventListener("abort", onAbort, { once: true }) + const timer = setTimeout(() => controller.abort(), timeoutMs) + + try { + const resolved = await resolvePublicAddress(hostname) + if (controller.signal.aborted) throw new Error("Webhook request aborted or timed out") + + if (fetchFn !== fetch) { + const response = await fetchFn(endpoint, { + method: "POST", + headers, + body, + redirect: "manual", + signal: controller.signal, + }) + return response.status + } + + const lookup = createPinnedLookup(resolved) + + return await new Promise((resolve, reject) => { + const request = httpsRequest({ + hostname, + port: url.port ? Number(url.port) : 443, + path: `${url.pathname}${url.search}`, + method: "POST", + headers, + lookup, + servername: isIP(hostname) ? undefined : hostname, + agent: false, + }, (response) => { + response.resume() + resolve(response.statusCode ?? 0) + }) + const destroyOnAbort = () => request.destroy(new Error("Webhook request aborted or timed out")) + controller.signal.addEventListener("abort", destroyOnAbort, { once: true }) + request.once("error", reject) + request.once("close", () => controller.signal.removeEventListener("abort", destroyOnAbort)) + request.end(body) + }) + } finally { + clearTimeout(timer) + signal?.removeEventListener("abort", onAbort) + } +} \ No newline at end of file diff --git a/comebackhere-backend/src/services/webhooks.ts b/comebackhere-backend/src/services/webhooks.ts index 131f57a..0bbd421 100644 --- a/comebackhere-backend/src/services/webhooks.ts +++ b/comebackhere-backend/src/services/webhooks.ts @@ -24,14 +24,23 @@ import { createHmac, randomUUID, timingSafeEqual } from "crypto" import { connectMongo, type WebhookDeliveryHistoryRecord, type WebhookReplayRecord } from "../db/mongo.js" +import { + webhookDeliveryAttempts, + webhookDeliveryDuration, + webhookDeliveryOutcomes, +} from "../lib/metrics.js" +import { sendWebhookRequest } from "./webhook-http.js" /** The header name sent on every outbound webhook request. */ export const WEBHOOK_SIGNATURE_HEADER = "X-COMEBACKHERE-Signature" export const WEBHOOK_TIMESTAMP_HEADER = "X-COMEBACKHERE-Timestamp" export const WEBHOOK_LEGACY_SIGNATURE_HEADER = "X-COMEBACKHERE-Legacy-Signature" +export const WEBHOOK_PREVIOUS_SIGNATURE_HEADER = "X-COMEBACKHERE-Previous-Signature" export interface WebhookPayload { event: string + event_id?: string + delivery_id?: string [key: string]: unknown } @@ -49,6 +58,8 @@ export interface DispatchWebhookOptions { merchantAddress?: string correlationId?: string replayOf?: string + eventId?: string + deliveryId?: string } /** @@ -86,17 +97,21 @@ export function signPayload(secret: string, rawBody: string | Buffer | Uint8Arra * @returns `true` if the signature is valid, `false` otherwise. */ export function verifySignature( - secret: string, + secret: string | string[], rawBody: string | Buffer | Uint8Array, signature: string, ): boolean { try { - const expected = signPayload(secret, rawBody) - // Both buffers must have the same length for timingSafeEqual - const expectedBuf = Buffer.from(expected, "hex") + if (!/^[0-9a-f]{64}$/i.test(signature)) return false const actualBuf = Buffer.from(signature, "hex") - if (expectedBuf.length !== actualBuf.length) return false - return timingSafeEqual(expectedBuf, actualBuf) + const candidates = Array.isArray(secret) ? secret : [secret] + let valid = false + for (const candidate of candidates) { + const expectedBuf = Buffer.from(signPayload(candidate, rawBody), "hex") + const matches = expectedBuf.length === actualBuf.length && timingSafeEqual(expectedBuf, actualBuf) + valid = matches || valid + } + return valid } catch { return false } @@ -129,13 +144,21 @@ export async function dispatchWebhook( ) } - const rawBody = JSON.stringify(payload) + const eventId = options.eventId ?? payload.event_id ?? randomUUID() + const deliveryId = options.deliveryId ?? (options.replayOf ? randomUUID() : payload.delivery_id ?? eventId) + const deliveredPayload: WebhookPayload = { ...payload, event_id: eventId, delivery_id: deliveryId } + const rawBody = JSON.stringify(deliveredPayload) const timestamp = Math.floor(Date.now() / 1000).toString() const signature = signPayload(secret, `${timestamp}.${rawBody}`) const legacySignature = signPayload(secret, rawBody) - const deliveryId = randomUUID() + const previousSecret = process.env.WEBHOOK_SIGNING_SECRET_PREVIOUS + const previousSignature = previousSecret && previousSecret !== secret + ? signPayload(previousSecret, `${timestamp}.${rawBody}`) + : undefined + if (options.replayOf) webhookDeliveryRetries.inc() let status: number | null = null let failure: string | null = null + const startedAt = Date.now() try { const headers: Record = { @@ -143,19 +166,18 @@ export async function dispatchWebhook( [WEBHOOK_SIGNATURE_HEADER]: signature, [WEBHOOK_TIMESTAMP_HEADER]: timestamp, [WEBHOOK_LEGACY_SIGNATURE_HEADER]: legacySignature, + "X-COMEBACKHERE-Event-Id": eventId, + "X-COMEBACKHERE-Delivery-Id": deliveryId, } + if (previousSignature) headers[WEBHOOK_PREVIOUS_SIGNATURE_HEADER] = previousSignature if (options.correlationId) headers["X-Request-Id"] = options.correlationId - const response = await fetchImpl(url, { - method: "POST", - headers, - body: rawBody, - }) - status = response.status - if (!response.ok) failure = `HTTP ${response.status}` + webhookDeliveryAttempts.inc() + status = await sendWebhookRequest(url, headers, rawBody, fetchImpl) + if (status < 200 || status >= 300) failure = `HTTP ${status}` return { url, - status: response.status, - ok: response.ok, + status, + ok: failure === null, signature, legacySignature, timestamp, @@ -165,6 +187,9 @@ export async function dispatchWebhook( failure = err instanceof Error ? err.message : String(err) throw err } finally { + const delivered = status !== null && status >= 200 && status < 300 + webhookDeliveryOutcomes.inc({ status: delivered ? "delivered" : "failed" }) + webhookDeliveryDuration.observe({ status: delivered ? "delivered" : "failed" }, (Date.now() - startedAt) / 1_000) await persistWebhookAttempt({ deliveryId, merchantAddress: options.merchantAddress, @@ -213,7 +238,7 @@ async function persistWebhookAttempt(attempt: { delivery_id: attempt.deliveryId, merchant_address: attempt.merchantAddress, endpoint: attempt.endpoint, - payload: attempt.payload, + payload: deliveredPayload, status: replay.status, attempts: 1, last_status_code: attempt.status, diff --git a/comebackhere-backend/src/tests/graceful-shutdown.test.ts b/comebackhere-backend/src/tests/graceful-shutdown.test.ts index b384a19..102d31e 100644 --- a/comebackhere-backend/src/tests/graceful-shutdown.test.ts +++ b/comebackhere-backend/src/tests/graceful-shutdown.test.ts @@ -1,4 +1,9 @@ import { describe, it, expect, vi, beforeEach, afterEach } from "vitest" + +vi.mock("node:dns/promises", () => ({ + lookup: vi.fn().mockResolvedValue([{ address: "8.8.8.8", family: 4 }]), +})) + import { stopTreasuryIndexer } from "../services/treasury-indexer.js" import { stopIndexer } from "../indexer.js" import { closeMongo } from "../db/mongo.js" diff --git a/comebackhere-backend/src/tests/webhook-delivery.test.ts b/comebackhere-backend/src/tests/webhook-delivery.test.ts index 8059ddb..cae9ce4 100644 --- a/comebackhere-backend/src/tests/webhook-delivery.test.ts +++ b/comebackhere-backend/src/tests/webhook-delivery.test.ts @@ -1,4 +1,9 @@ import { describe, it, expect, vi } from "vitest" + +vi.mock("node:dns/promises", () => ({ + lookup: vi.fn().mockResolvedValue([{ address: "8.8.8.8", family: 4 }]), +})) + import { getWebhookRetryConfig } from "../lib/env.js" import { deliverWebhook, @@ -43,10 +48,13 @@ describe("webhook retry configuration", () => { }) function makePayload(overrides?: Partial): WebhookPayload { + const idempotencyKey = overrides?.idempotency_key ?? "invoice_paid:42" return { event_type: "invoice_paid", invoice_id: "42", - idempotency_key: "invoice_paid:42", + event_id: overrides?.event_id ?? idempotencyKey, + delivery_id: overrides?.delivery_id ?? "delivery-42", + idempotency_key: idempotencyKey, timestamp: new Date().toISOString(), data: { amount: 1_000_000 }, ...overrides, @@ -281,6 +289,23 @@ describe("deliverWebhook", () => { expect(capturedHeaders[1]["X-Idempotency-Key"]).toBe("invoice_paid:7") }) + it("preserves event and delivery IDs across retries", async () => { + const capturedHeaders: Record[] = [] + const fetchFn = vi.fn(async (_url: RequestInfo | URL, init?: RequestInit) => { + capturedHeaders.push((init?.headers ?? {}) as Record) + return { status: 500 } as Response + }) as unknown as typeof fetch + const payload = makePayload({ event_id: "chain-event-7", delivery_id: "delivery-7" }) + + await deliverWebhook("https://example.com/hook", payload, 2, fetchFn, noDelay) + + expect(capturedHeaders).toHaveLength(2) + expect(capturedHeaders.map((headers) => headers["X-COMEBACKHERE-Event-Id"])) + .toEqual(["chain-event-7", "chain-event-7"]) + expect(capturedHeaders.map((headers) => headers["X-COMEBACKHERE-Delivery-Id"])) + .toEqual(["delivery-7", "delivery-7"]) + }) + it("forwards X-Request-Id on every attempt when a correlationId is supplied", async () => { const capturedHeaders: Record[] = [] const fetchFn = vi.fn(async (_url: RequestInfo | URL, init?: RequestInit) => { diff --git a/comebackhere-backend/src/tests/webhooks.test.ts b/comebackhere-backend/src/tests/webhooks.test.ts index b962c61..2e08a27 100644 --- a/comebackhere-backend/src/tests/webhooks.test.ts +++ b/comebackhere-backend/src/tests/webhooks.test.ts @@ -21,8 +21,16 @@ import { verifySignature, dispatchWebhook, WEBHOOK_SIGNATURE_HEADER, + WEBHOOK_TIMESTAMP_HEADER, + WEBHOOK_PREVIOUS_SIGNATURE_HEADER, type WebhookPayload, } from "../services/webhooks.js" +import { createPinnedLookup, resolvePublicAddress, sendWebhookRequest } from "../services/webhook-http.js" +import { lookup as dnsLookup } from "node:dns/promises" + +vi.mock("node:dns/promises", () => ({ + lookup: vi.fn().mockResolvedValue([{ address: "8.8.8.8", family: 4 }]), +})) const TEST_SECRET = "super-secret-hmac-key-at-least-32-chars!" const TEST_URL = "https://merchant.example.com/webhooks" @@ -94,6 +102,15 @@ describe("verifySignature", () => { expect(verifySignature(TEST_SECRET, body, "not-a-valid-hex-signature!!")).toBe(false) }) + it("accepts signatures made with either key during a rotation overlap", () => { + const body = "timestamp.{\"event\":\"test\"}" + const oldSignature = signPayload("old-secret", body) + const newSignature = signPayload("new-secret", body) + + expect(verifySignature(["old-secret", "new-secret"], body, oldSignature)).toBe(true) + expect(verifySignature(["old-secret", "new-secret"], body, newSignature)).toBe(true) + }) + it("returns false when signature length does not match (timing-safe guard)", () => { const body = '{"event":"test"}' // Truncated hex — different byte length @@ -136,9 +153,11 @@ describe("verifySignature", () => { describe("dispatchWebhook", () => { let envBackup: string | undefined + let previousEnvBackup: string | undefined beforeEach(() => { envBackup = process.env.WEBHOOK_SIGNING_SECRET + previousEnvBackup = process.env.WEBHOOK_SIGNING_SECRET_PREVIOUS }) afterEach(() => { @@ -147,6 +166,11 @@ describe("dispatchWebhook", () => { } else { process.env.WEBHOOK_SIGNING_SECRET = envBackup } + if (previousEnvBackup === undefined) { + delete process.env.WEBHOOK_SIGNING_SECRET_PREVIOUS + } else { + process.env.WEBHOOK_SIGNING_SECRET_PREVIOUS = previousEnvBackup + } }) it("sends X-COMEBACKHERE-Signature header on the outbound request", async () => { @@ -173,10 +197,13 @@ describe("dispatchWebhook", () => { it("signature in the header verifies correctly against the body sent", async () => { let capturedBody = "" let capturedSignature = "" + let capturedTimestamp = "" const mockFetch = vi.fn().mockImplementation((_url: string, init: RequestInit) => { capturedBody = init?.body as string - capturedSignature = (init?.headers as Record)[WEBHOOK_SIGNATURE_HEADER] + const headers = init?.headers as Record + capturedSignature = headers[WEBHOOK_SIGNATURE_HEADER] + capturedTimestamp = headers[WEBHOOK_TIMESTAMP_HEADER] return Promise.resolve(new Response(null, { status: 200 })) }) @@ -188,14 +215,17 @@ describe("dispatchWebhook", () => { ) // The consumer should be able to verify the signature - expect(verifySignature(TEST_SECRET, capturedBody, capturedSignature)).toBe(true) + expect(verifySignature(TEST_SECRET, `${capturedTimestamp}.${capturedBody}`, capturedSignature)).toBe(true) }) it("a tampered body fails signature verification", async () => { let capturedSignature = "" + let capturedTimestamp = "" const mockFetch = vi.fn().mockImplementation((_url: string, init: RequestInit) => { - capturedSignature = (init?.headers as Record)[WEBHOOK_SIGNATURE_HEADER] + const headers = init?.headers as Record + capturedSignature = headers[WEBHOOK_SIGNATURE_HEADER] + capturedTimestamp = headers[WEBHOOK_TIMESTAMP_HEADER] return Promise.resolve(new Response(null, { status: 200 })) }) @@ -207,7 +237,7 @@ describe("dispatchWebhook", () => { ) const tampered = JSON.stringify({ event: "settlement_approved", settlement_id: 9999 }) - expect(verifySignature(TEST_SECRET, tampered, capturedSignature)).toBe(false) + expect(verifySignature(TEST_SECRET, `${capturedTimestamp}.${tampered}`, capturedSignature)).toBe(false) }) it("returns ok:true and status from the upstream response", async () => { @@ -281,6 +311,50 @@ describe("dispatchWebhook", () => { expect(capturedContentType).toBe("application/json") }) + + it("sends stable event and delivery identifiers in the payload and headers", async () => { + let capturedBody = "" + const capturedHeaders: Record = {} + const mockFetch = vi.fn().mockImplementation((_url: string, init: RequestInit) => { + capturedBody = init?.body as string + for (const [key, value] of Object.entries(init?.headers ?? {})) capturedHeaders[key] = value as string + return Promise.resolve(new Response(null, { status: 200 })) + }) + + const result = await dispatchWebhook( + TEST_URL, + { event: "settlement_executed", settlement_id: 1 }, + TEST_SECRET, + mockFetch as unknown as typeof fetch, + { eventId: "chain-event-123" }, + ) + + const sent = JSON.parse(capturedBody) as Record + expect(sent.event_id).toBe("chain-event-123") + expect(sent.delivery_id).toBe(result.deliveryId) + expect(capturedHeaders["X-COMEBACKHERE-Event-Id"]).toBe("chain-event-123") + expect(capturedHeaders["X-COMEBACKHERE-Delivery-Id"]).toBe(result.deliveryId) + }) + + it("sends a previous-key signature during the configured rotation overlap", async () => { + process.env.WEBHOOK_SIGNING_SECRET_PREVIOUS = "old-secret" + let capturedBody = "" + const capturedHeaders: Record = {} + const mockFetch = vi.fn().mockImplementation((_url: string, init: RequestInit) => { + capturedBody = init?.body as string + for (const [key, value] of Object.entries(init?.headers ?? {})) capturedHeaders[key] = value as string + return Promise.resolve(new Response(null, { status: 200 })) + }) + + await dispatchWebhook(TEST_URL, { event: "test" }, TEST_SECRET, mockFetch as unknown as typeof fetch) + + const timestamp = capturedHeaders[WEBHOOK_TIMESTAMP_HEADER] + expect(verifySignature( + "old-secret", + `${timestamp}.${capturedBody}`, + capturedHeaders[WEBHOOK_PREVIOUS_SIGNATURE_HEADER], + )).toBe(true) + }) }) // --------------------------------------------------------------------------- @@ -305,9 +379,16 @@ describe("dispatchWebhook — header contract", () => { return captured } - it("sends exactly Content-Type and the signature header", async () => { + it("sends the documented signature, timestamp, and identifier headers", async () => { const headers = await capture({ event: "settlement_executed", settlement_id: 1 }) - expect(Object.keys(headers).sort()).toEqual(["content-type", "x-comebackhere-signature"]) + expect(Object.keys(headers).sort()).toEqual([ + "content-type", + "x-comebackhere-delivery-id", + "x-comebackhere-event-id", + "x-comebackhere-legacy-signature", + "x-comebackhere-signature", + "x-comebackhere-timestamp", + ]) }) it("does not send X-Idempotency-Key", async () => { @@ -322,9 +403,7 @@ describe("dispatchWebhook — header contract", () => { expect(headers["x-request-id"]).toBeUndefined() }) - it("sends a bare digest with no timestamp envelope", async () => { - // The docs promise a bare 64-char lowercase hex digest, so a consumer can - // never reconstruct a "t=...,v1=..." style string from the header. + it("sends a bare digest and a separate timestamp header", async () => { const headers = await capture({ event: "settlement_executed", settlement_id: 1 }) expect(headers["x-comebackhere-signature"]).toMatch(/^[0-9a-f]{64}$/) }) @@ -356,9 +435,12 @@ describe("dispatchWebhook — header contract", () => { // bytes on the wire. let capturedBody = "" let capturedSignature = "" + let capturedTimestamp = "" const mockFetch = vi.fn().mockImplementation((_url: string, init: RequestInit) => { capturedBody = init?.body as string - capturedSignature = (init?.headers as Record)[WEBHOOK_SIGNATURE_HEADER] + const headers = init?.headers as Record + capturedSignature = headers[WEBHOOK_SIGNATURE_HEADER] + capturedTimestamp = headers[WEBHOOK_TIMESTAMP_HEADER] return Promise.resolve(new Response(null, { status: 200 })) }) @@ -369,6 +451,60 @@ describe("dispatchWebhook — header contract", () => { mockFetch as unknown as typeof fetch, ) - expect(verifySignature(TEST_SECRET, capturedBody, capturedSignature)).toBe(true) + expect(verifySignature(TEST_SECRET, `${capturedTimestamp}.${capturedBody}`, capturedSignature)).toBe(true) + }) +}) + +describe("webhook request destination safety", () => { + it("rejects private-address and non-HTTPS targets before sending", async () => { + const mockFetch = vi.fn() + + for (const host of ["127.0.0.1", "10.0.0.1", "169.254.169.254", "[::1]", "[fe80::1]"]) { + await expect(sendWebhookRequest(`https://${host}/hook`, {}, "{}", mockFetch as typeof fetch)) + .rejects.toThrow(/non-public/) + } + await expect(sendWebhookRequest("http://merchant.example/hook", {}, "{}", mockFetch as typeof fetch)) + .rejects.toThrow(/public HTTPS/) + expect(mockFetch).not.toHaveBeenCalled() + }) + + it("does not follow redirects", async () => { + const mockFetch = vi.fn().mockResolvedValue(new Response(null, { + status: 302, + headers: { Location: "https://127.0.0.1/metadata" }, + })) + + const status = await sendWebhookRequest(TEST_URL, {}, "{}", mockFetch as typeof fetch) + + expect(status).toBe(302) + expect(mockFetch).toHaveBeenCalledOnce() + expect(mockFetch.mock.calls[0][1]).toMatchObject({ redirect: "manual" }) + }) + + it("rejects every DNS answer if any answer is private", async () => { + vi.mocked(dnsLookup).mockImplementationOnce(async () => [ + { address: "1.1.1.1", family: 4 }, + { address: "127.0.0.1", family: 4 }, + ] as never) + + await expect(resolvePublicAddress("merchant.example")).rejects.toThrow(/non-public/) + }) + + it("rejects a hostname when DNS changes from public to private", async () => { + vi.mocked(dnsLookup) + .mockImplementationOnce(async () => [{ address: "1.1.1.1", family: 4 }] as never) + .mockImplementationOnce(async () => [{ address: "10.0.0.8", family: 4 }] as never) + + await expect(resolvePublicAddress("merchant.example")).resolves.toEqual({ address: "1.1.1.1", family: 4 }) + await expect(resolvePublicAddress("merchant.example")).rejects.toThrow(/non-public/) + }) + + it("pins the socket lookup to the address validated for this request", () => { + const lookup = createPinnedLookup({ address: "1.1.1.1", family: 4 }) + const callback = vi.fn() + + lookup("merchant.example", {}, callback) + + expect(callback).toHaveBeenCalledWith(null, "1.1.1.1", 4) }) }) diff --git a/docs/api-reference.md b/docs/api-reference.md index eced157..20cb8db 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -939,14 +939,19 @@ can verify payload authenticity before processing it. | `X-COMEBACKHERE-Signature` | HMAC-SHA256 of `timestamp.rawBody`, lowercase hex | | `X-COMEBACKHERE-Timestamp` | Unix timestamp in seconds | | `X-COMEBACKHERE-Legacy-Signature` | Temporary body-only HMAC for one release | +| `X-COMEBACKHERE-Previous-Signature` | Prior-key HMAC during secret rotation | +| `X-COMEBACKHERE-Event-Id` | Stable indexed chain event ID | +| `X-COMEBACKHERE-Delivery-Id` | Stable delivery ID | -The digest is computed over the **raw JSON request body** (exactly as sent over -the wire) using the `WEBHOOK_SIGNING_SECRET` environment variable as the key. It -is a bare 64-character lowercase hex digest — there is no `t=…,v1=…` envelope -and no timestamp, so the signature alone provides no replay window. Deduplicate -on your own key. +The digest is computed over the Unix timestamp, a period, and the **raw JSON +request body** (exactly as sent over the wire), using `WEBHOOK_SIGNING_SECRET` +as the key. The timestamp is sent separately in `X-COMEBACKHERE-Timestamp` and +must be within five minutes. During rotation, +`X-COMEBACKHERE-Previous-Signature` carries the same input signed with the prior +key. -> There is no `WEBHOOK_SECRET`: the backend reads `WEBHOOK_SIGNING_SECRET` only. +> There is no `WEBHOOK_SECRET`: the backend reads `WEBHOOK_SIGNING_SECRET` and +> the optional `WEBHOOK_SIGNING_SECRET_PREVIOUS` only. ### Verification (Node.js example) @@ -1014,6 +1019,7 @@ top level — there is no `data` envelope and no `event_type` key: | ----------------------- | ------------------------------------------------------------------ | | `WEBHOOK_URL` | Merchant endpoint that receives webhook POSTs | | `WEBHOOK_SIGNING_SECRET`| HMAC-SHA256 signing secret (minimum 32 characters recommended). **Required at startup.** | +| `WEBHOOK_SIGNING_SECRET_PREVIOUS` | Optional previous key during rotation; remove after receivers accept the current key. | Set both variables in your deployment environment. If `WEBHOOK_URL` is not set, webhook delivery is skipped silently (no error). If `WEBHOOK_SIGNING_SECRET` is diff --git a/docs/webhooks.md b/docs/webhooks.md index 353a01c..25c347d 100644 --- a/docs/webhooks.md +++ b/docs/webhooks.md @@ -4,12 +4,14 @@ COMEBACKHERE can send signed HTTP POST requests to your configured endpoint when settlement events occur on-chain. This page documents the events, their exact payload shapes, and how to verify the HMAC-SHA256 signature. -> **Security note:** Treat your `WEBHOOK_SIGNING_SECRET` with the same care as a -> private key. Anyone who holds it can forge valid webhook signatures. Rotate it -> immediately if it is ever exposed. +> **Security note:** Treat both signing secrets with the same care as a private +> key. Anyone who holds one can forge valid webhook signatures. Rotate a +> compromised secret immediately. -> **There is no `WEBHOOK_SECRET`.** The backend reads `WEBHOOK_SIGNING_SECRET` -> and nothing else — see [Configuration](#configuration). +> **There is no `WEBHOOK_SECRET`.** The backend requires +> `WEBHOOK_SIGNING_SECRET` and optionally reads +> `WEBHOOK_SIGNING_SECRET_PREVIOUS` during rotation — see +> [Configuration](#configuration). See also: [`## Webhooks` in the API Reference](./api-reference.md#webhooks) for the configuration environment variables. @@ -27,6 +29,7 @@ are **not** equivalent. Read this section before writing a receiver. | Signed | **Yes** — `X-COMEBACKHERE-Signature` | **No** | | `X-Idempotency-Key` | Not sent | Sent | | `X-Request-Id` | Not sent | Sent when a correlation id is supplied | +| Event and delivery IDs | Stable chain event ID and delivery ID in payload and headers | Stable `event_id` and `delivery_id` fields and headers | | Retries / dead letters | None — single attempt, failures are logged | Yes, with backoff and dead-letter persistence | | Used by | `treasury-indexer` for `settlement_proposed`, `settlement_approved`, `settlement_executed` | Nothing in production code; the queue is never enqueued to | @@ -45,6 +48,7 @@ responsible for deduplication** — see [Deduplicating deliveries](#deduplicatin | ------------------------ | ---------------------------------------------------------------------- | | `WEBHOOK_URL` | Your endpoint that receives `POST` requests from the backend. If unset, no webhooks are sent. | | `WEBHOOK_SIGNING_SECRET` | HMAC-SHA256 signing secret (minimum 32 characters recommended). **Required at startup** — the backend refuses to boot without it. | +| `WEBHOOK_SIGNING_SECRET_PREVIOUS` | Optional prior HMAC key. During rotation, the backend emits signatures for both keys until this variable is removed. | | `WEBHOOK_MAX_ATTEMPTS` | Maximum delivery attempts (default `5`) — retry queue only | | `WEBHOOK_BASE_DELAY_MS` | Initial retry delay in milliseconds (default `1000`) — retry queue only | | `WEBHOOK_MAX_DELAY_MS` | Maximum exponential backoff delay in milliseconds (default `60000`) — retry queue only | @@ -57,14 +61,37 @@ If `WEBHOOK_SIGNING_SECRET` is missing, `validateEnv()` in `src/lib/env.ts` fails the process at startup with a message naming the variable. It is never a runtime warning. +### Destination restrictions + +Webhook destinations must use HTTPS and resolve only to public IP addresses. +Loopback, private, link-local, reserved, and local-only targets are rejected. +The dispatcher pins the validated DNS answer to the connection, so a DNS change +between validation and connection cannot redirect a request to an internal +address. HTTP redirects are not followed; configure the final HTTPS destination +directly. + +### Signing-secret rotation + +1. Set `WEBHOOK_SIGNING_SECRET` to the new key and + `WEBHOOK_SIGNING_SECRET_PREVIOUS` to the current key. +2. Restart the backend. Requests carry the current signature in + `X-COMEBACKHERE-Signature` and an overlap signature in + `X-COMEBACKHERE-Previous-Signature`. +3. Update receivers to accept either key for the overlap window. Both signatures + cover the timestamp and exact raw body; the previous key is never returned by + the API. +4. Remove `WEBHOOK_SIGNING_SECRET_PREVIOUS` and restart after receivers have + switched to the new key. + --- ## Signature verification Every webhook sent by the live dispatch path includes an `X-COMEBACKHERE-Signature` header containing a lowercase hex-encoded -HMAC-SHA256 digest of the **raw request body** (the exact bytes sent over the -wire), keyed by your `WEBHOOK_SIGNING_SECRET`. +HMAC-SHA256 digest of `timestamp.rawBody`, keyed by `WEBHOOK_SIGNING_SECRET`. +During rotation, a second digest using the prior key is sent in +`X-COMEBACKHERE-Previous-Signature`. ### Signature format @@ -73,14 +100,14 @@ wire), keyed by your `WEBHOOK_SIGNING_SECRET`. | Header name | `X-COMEBACKHERE-Signature` | | Algorithm | HMAC-SHA256 | | Encoding | lowercase hex, 64 characters | -| Signed input | the exact request body bytes, UTF-8 | -| Envelope | **none** — a bare digest, not a `t=…,v1=…` string | +| Signed input | Unix timestamp, `.`, then exact request body bytes | +| Timestamp header | `X-COMEBACKHERE-Timestamp`, Unix seconds | +| Envelope | **none** — each signature is a bare digest, not a `t=…,v1=…` string | -There is no `t=` timestamp prefix and no version field. Consequently **there is -no signature timestamp and no replay window**: a captured request body and its -signature stay valid forever. Replay defence has to come from your own -deduplication (see -[Deduplicating deliveries](#deduplicating-deliveries)), not from the signature. +The timestamp is a separate header and is included in the signed input. Reject +requests more than five minutes from the current time to limit replay. Use the +stable `event_id` to deduplicate chain events and `delivery_id` to correlate an +individual send. ### Algorithm summary @@ -100,6 +127,10 @@ Headers sent on a **live dispatch** request: | Header | Value | | -------------------------- | ---------------------------------------- | | `X-COMEBACKHERE-Signature` | Lowercase hex-encoded HMAC-SHA256 digest | +| `X-COMEBACKHERE-Previous-Signature` | Prior-key digest during rotation | +| `X-COMEBACKHERE-Timestamp` | Unix timestamp in seconds | +| `X-COMEBACKHERE-Event-Id` | Stable identifier for the indexed chain event | +| `X-COMEBACKHERE-Delivery-Id` | Stable identifier for this delivery | | `Content-Type` | `application/json` | Additional headers sent by the **retry queue** path (not used today): @@ -107,6 +138,8 @@ Additional headers sent by the **retry queue** path (not used today): | Header | Value | | ---------------------- | ------------------------------------------------------------ | | `X-Idempotency-Key` | Stable per-event key (see [Deduplicating deliveries](#deduplicating-deliveries)) | +| `X-COMEBACKHERE-Event-Id` | Stable event identifier, preserved across retries | +| `X-COMEBACKHERE-Delivery-Id` | Stable delivery identifier, preserved across retries | | `X-Request-Id` | Correlation ID forwarded from the originating request, when available | If you need a `X-COMEBACKHERE-Signature` on every delivery, verify it @@ -183,34 +216,49 @@ import ( "crypto/hmac" "crypto/sha256" "encoding/hex" + "encoding/json" "io" + "math" "net/http" + "os" + "strconv" + "time" ) // VerifyWebhookSignature returns true if the signature header is a valid -// HMAC-SHA256 of the raw request body under the given secret. -func VerifyWebhookSignature(rawBody []byte, signature, secret string) bool { +// HMAC-SHA256 of timestamp + "." + rawBody under the given secret. +func VerifyWebhookSignature(rawBody []byte, signature, timestamp, secret string) bool { + seconds, err := strconv.ParseInt(timestamp, 10, 64) + if err != nil || math.Abs(float64(time.Now().Unix()-seconds)) > 300 { + return false + } + expected := hmac.New(sha256.New, []byte(secret)) + expected.Write([]byte(timestamp + ".")) expected.Write(rawBody) - expectedHex := hex.EncodeToString(expected.Sum(nil)) + actual, err := hex.DecodeString(signature) + if err != nil { + return false + } // Use constant-time comparison to prevent timing attacks - return hmac.Equal([]byte(expectedHex), []byte(signature)) + return hmac.Equal(expected.Sum(nil), actual) } // Example: verify webhook in an HTTP handler func HandleWebhook(w http.ResponseWriter, r *http.Request) { - // Read raw body before parsing JSON + // Read raw body before parsing JSON rawBody, err := io.ReadAll(r.Body) if err != nil { http.Error(w, "failed to read body", http.StatusBadRequest) return } - signature := r.Header.Get("X-COMEBACKHERE-Signature") + signature := r.Header.Get("X-COMEBACKHERE-Signature") + timestamp := r.Header.Get("X-COMEBACKHERE-Timestamp") secret := os.Getenv("WEBHOOK_SIGNING_SECRET") - if !VerifyWebhookSignature(rawBody, signature, secret) { + if !VerifyWebhookSignature(rawBody, signature, timestamp, secret) { http.Error(w, "invalid signature", http.StatusUnauthorized) return } @@ -236,32 +284,27 @@ func HandleWebhook(w http.ResponseWriter, r *http.Request) { ## Deduplicating deliveries -The live dispatch path sends **no** `X-Idempotency-Key` header and includes no -`idempotency_key` field in the body. There is no stable per-event key to dedupe -on out of the box. +The live dispatch path does not send `X-Idempotency-Key`. Instead, the payload +includes stable `event_id` and `delivery_id` fields, also sent in +`X-COMEBACKHERE-Event-Id` and `X-COMEBACKHERE-Delivery-Id` headers. `event_id` +is derived from the indexed chain event and remains stable when that event is +reprocessed; `delivery_id` remains stable across retries of that delivery. -Deduplicate on a tuple you build yourself from the body. For settlement events -`(event, settlement_id, tx_hash)` is stable across redeliveries of the same -chain event, and `settlement_approved` can legitimately repeat with different -signers, so include the signer when you need to distinguish approvals: - -``` -dedupe_key = "::" - # settlement_approved: ":::" +Deduplicate on `event_id` before applying side effects. If a receiver needs to +track retries or delivery attempts, use `delivery_id`. -Examples: - settlement_proposed:15:9f2c... - settlement_executed:15:41ab... - settlement_approved:15:GDR7...:41ab... +```typescript +if (await alreadyProcessed(payload.event_id)) return +await recordProcessed(payload.event_id) +await applyEvent(payload) ``` -Store the key and reject duplicates before applying side effects. This is the -only replay protection available, because the signature carries no timestamp. +Store the event ID and reject duplicates before applying side effects. Also +enforce the signature timestamp tolerance described above. -The retry queue path, when it is wired up, additionally sends an -`X-Idempotency-Key` header and an `idempotency_key` body field derived as -`:`. Do not build a receiver that depends on that -header today. +The dormant retry queue path uses an `X-Idempotency-Key` header and an +`idempotency_key` field. Live receivers should use `event_id` and `delivery_id` +as described above. --- @@ -280,6 +323,8 @@ arrive as **strings** and never lose precision. Account addresses and | Field | Type | Present on | Description | | --------------- | ------ | ----------------------------- | ------------------------------------ | | `event` | string | all | One of the event names below | +| `event_id` | string | all | Stable indexed chain event ID | +| `delivery_id` | string | all | Stable delivery ID | | `settlement_id` | number | all | Numeric settlement ID | | `tx_hash` | string | all | Stellar transaction hash | | `merchant_address` | string | `settlement_proposed` | Stellar public key of the merchant | @@ -408,13 +453,37 @@ replay one with `POST /webhooks/dead-letters/:id/replay`; both require the > These two routes are live today, but nothing populates the collection while the > retry queue is dormant. +### Delivery metrics and alerts + +The `/metrics` endpoint exposes `webhook_delivery_total` by final status, +`webhook_delivery_attempts_total`, `webhook_delivery_retries_total`, +`webhook_delivery_duration_seconds`, and `webhook_dead_letter_volume`. +Scrape the endpoint with Prometheus and tune alert windows and thresholds for +your traffic. For example: + +```yaml +groups: + - name: webhook-delivery + rules: + - alert: WebhookDeliveryFailureRateHigh + expr: rate(webhook_delivery_total{status="failed"}[10m]) > 0.1 + for: 5m + - alert: WebhookDeadLettersAccumulating + expr: webhook_dead_letter_volume > 10 + for: 10m + - alert: WebhookDeliveryLatencyHigh + expr: histogram_quantile(0.95, rate(webhook_delivery_duration_seconds_bucket[10m])) > 5 + for: 10m +``` + ### Delivery record fields The backend maintains an internal delivery record for every webhook event: | Field | Type | Description | | ------------------- | ---------------- | ------------------------------------------------- | -| `idempotency_key` | string | Stable key identifying this event | +| `event_id` | string | Stable indexed event identifier | +| `delivery_id` | string | Stable identifier for this delivery | | `endpoint` | string | Merchant URL the POST was sent to | | `status` | `delivered` \| `failed` \| `pending` | Final delivery outcome | | `attempts` | number | Total number of delivery attempts made | @@ -447,11 +516,11 @@ Your endpoint should: | -------------------------------------- | ---------------------------------------------------------------------------- | | Signature verification fails | Body was parsed before reading the raw bytes, or wrong `WEBHOOK_SIGNING_SECRET` | | `timingSafeEqual` throws | The signature header was not 64 hex characters; compare lengths first | -| Duplicate events processed | Dedupe key not checked; no idempotency key is sent on the live path | +| Duplicate events processed | Receiver does not persist and deduplicate the stable `event_id` | | Deliveries time out | Endpoint performs synchronous work before responding; move work off the request path | | No webhooks received | `WEBHOOK_URL` not set, or set to an unreachable address | | Occasional missing events | Expected: the live path is single-attempt with no retry | -| A replayed body still verifies | Expected: the signature has no timestamp; dedupe on your own key | +| A replayed body still verifies | Enforce the five-minute timestamp window and deduplicate on `event_id` | | Backend refuses to start | `WEBHOOK_SIGNING_SECRET` is unset — `validateEnv()` fails fast | --- diff --git a/scripts/check_webhook_docs_sync.sh b/scripts/check_webhook_docs_sync.sh index cbff61b..97eb65f 100755 --- a/scripts/check_webhook_docs_sync.sh +++ b/scripts/check_webhook_docs_sync.sh @@ -18,7 +18,7 @@ # operator to set one. # 4. WEBHOOK_SIGNING_SECRET is in the startup-validated set, so the docs do # not describe it as optional. -# 5. The docs do not claim the live dispatch path sends X-Idempotency-Key. +# 5. The docs match the live dispatch path's stable IDs and signature headers. # 6. The docs do not promise an envelope (event_type/idempotency_key/data) # that the live payload does not contain. # @@ -168,7 +168,7 @@ require_in "$IMPL" 'digest\("hex"\)' \ "webhooks.ts hex-encodes the digest" require_in "$IMPL" 'timingSafeEqual' \ "webhooks.ts compares signatures in constant time" -require_in "$IMPL" 'length !== actualBuf\.length' \ +require_in "$IMPL" 'expectedBuf\.length === actualBuf\.length' \ "webhooks.ts length-checks before timingSafeEqual" require_in "$DOC" 'HMAC-SHA256' "$DOC names the HMAC-SHA256 algorithm" @@ -177,13 +177,17 @@ require_in "$DOC" 'constant-time' "$DOC requires a constant-time comparison" require_in "$DOC" 'timingSafeEqual.*throws|timingSafeEqual` throws' \ "$DOC warns that timingSafeEqual throws on a length mismatch" -# The signature carries no timestamp, so the docs must not promise one. +# The signature includes a timestamp in its signed input, sent in a separate header. if grep -qE 't=[^ ]*,v1=|svix|Stripe-Signature' "$IMPL"; then info "$IMPL now builds a signature envelope -- update the Signature format table in $DOC" else - ok "webhooks.ts sends a bare digest with no timestamp envelope" - require_in "$DOC" 'no signature timestamp|no timestamp' \ - "$DOC states there is no signature timestamp" + ok "webhooks.ts sends bare digests with a separate timestamp header" + require_in "$IMPL" 'WEBHOOK_TIMESTAMP_HEADER' \ + "webhooks.ts includes the timestamp header" + require_in "$DOC" 'X-COMEBACKHERE-Timestamp' \ + "$DOC documents the timestamp header" + require_in "$DOC" 'five minutes|300' \ + "$DOC documents the timestamp tolerance" fi # ── 3. Secret variable name ────────────────────────────────────────────────── @@ -273,14 +277,8 @@ if printf '%s' "$SEND_BLOCK" | grep -q 'X-Idempotency-Key'; then ok "dispatchWebhook sends X-Idempotency-Key -- the docs may describe it as sent" else ok "dispatchWebhook does not send X-Idempotency-Key" - # The doc must not claim every outbound POST carries it. - if grep -qiE '(every|all) (outbound )?webhooks? .*includes an? .?X-Idempotency-Key' "$DOC"; then - fail "$DOC claims every outbound webhook carries X-Idempotency-Key, but the live path does not send it" - else - ok "$DOC does not claim the live path sends X-Idempotency-Key" - fi - require_in "$DOC" 'not sent|does not send|no.*idempotency key' \ - "$DOC states that the live path sends no idempotency key" + require_in "$DOC" 'stable `event_id`|event_id.*stable' \ + "$DOC documents stable live event IDs" fi # ── 6. Payload shape ─────────────────────────────────────────────────────────