From 5ef585203a22f47aa6048256ee9d323a3be2df16 Mon Sep 17 00:00:00 2001 From: gbena-afk Date: Thu, 18 Jun 2026 12:35:51 +0100 Subject: [PATCH] feat(listener): retry failed Discord notifications with configurable retries and exponential backoff This change adds automatic retries for transient Discord webhook failures:\n\n- Configurable via env vars: and \n- Default behavior: 5 retries and 1s backoff base\n- Implements exponential backoff and logs each attempt with attempt number and delay\n- Adds and to and reads them in \n\nNotes:\n- No tests were added; you can run listener with / set to verify behavior.\n --- listener/src/index.ts | 9 ++- listener/src/services/discord-notification.ts | 72 +++++++++++++------ listener/src/types/index.ts | 2 + 3 files changed, 62 insertions(+), 21 deletions(-) diff --git a/listener/src/index.ts b/listener/src/index.ts index 2f21d1f..f131d18 100644 --- a/listener/src/index.ts +++ b/listener/src/index.ts @@ -12,7 +12,14 @@ function loadDiscordConfig(): DiscordConfig | undefined { if (!webhookUrl || !webhookId) { return undefined; } - return { webhookUrl, webhookId }; + const retryCount = process.env.DISCORD_RETRY_COUNT + ? parseInt(process.env.DISCORD_RETRY_COUNT, 10) + : undefined; + const backoffBaseSeconds = process.env.DISCORD_BACKOFF_BASE_SECONDS + ? parseFloat(process.env.DISCORD_BACKOFF_BASE_SECONDS) + : undefined; + + return { webhookUrl, webhookId, retryCount, backoffBaseSeconds }; } function loadConfig(): Config { diff --git a/listener/src/services/discord-notification.ts b/listener/src/services/discord-notification.ts index 73d4e6d..1fef073 100644 --- a/listener/src/services/discord-notification.ts +++ b/listener/src/services/discord-notification.ts @@ -56,13 +56,30 @@ export class DiscordNotificationService { logger.info('Sending Discord notification', logContext); const message = this.formatEventMessage(event, contractConfig); - const startTime = Date.now(); + const maxRetries = this.config.retryCount ?? 5; + const backoffBaseSeconds = this.config.backoffBaseSeconds ?? 1; - try { - const response = await this.sendWebhook(message); - const durationMs = Date.now() - startTime; + let attempt = 0; + while (attempt <= maxRetries) { + const attemptStart = Date.now(); + try { + const response = await this.sendWebhook(message); + const durationMs = Date.now() - attemptStart; + + if (response.ok) { + this.deduplicator.markSent(fingerprint); + logger.info('Discord notification sent successfully', { + eventId: event.id, + contractAddress: contractConfig.address, + }); + logger.info('Discord notification delivered', { + ...logContext, + durationMs, + attempt, + }); + return true; + } - if (!response.ok) { const errorText = await response.text(); logger.error('Discord webhook failed', { ...logContext, @@ -70,27 +87,38 @@ export class DiscordNotificationService { statusText: response.statusText, error: errorText, durationMs, + attempt, + }); + } catch (error) { + const durationMs = Date.now() - attemptStart; + logger.error('Error sending Discord notification', { + ...logContext, + error, + durationMs, + attempt, }); - return false; } - this.deduplicator.markSent(fingerprint); - logger.info('Discord notification sent successfully', { - eventId: event.id, - contractAddress: contractConfig.address, - logger.info('Discord notification delivered', { - ...logContext, - durationMs, - }); - return true; - } catch (error) { - logger.error('Error sending Discord notification', { + // If we've exhausted retries, break and return false + if (attempt >= maxRetries) break; + + // Exponential backoff: base * 2^attempt (seconds) + const delayMs = Math.pow(2, attempt) * backoffBaseSeconds * 1000; + logger.warn('Retrying Discord webhook', { ...logContext, - error, - durationMs: Date.now() - startTime, + attempt: attempt + 1, + nextDelayMs: delayMs, + maxRetries, }); - return false; + await this.delay(delayMs); + attempt++; } + + logger.error('Exceeded max Discord retry attempts', { + ...logContext, + maxRetries, + }); + return false; } async sendTestMessage(requestId?: string): Promise { @@ -134,6 +162,10 @@ export class DiscordNotificationService { } } + private delay(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); + } + private async sendWebhook(message: DiscordMessage): Promise { return fetch(this.config.webhookUrl, { method: 'POST', diff --git a/listener/src/types/index.ts b/listener/src/types/index.ts index 659a4c6..efe95aa 100644 --- a/listener/src/types/index.ts +++ b/listener/src/types/index.ts @@ -6,6 +6,8 @@ export interface ContractConfig { export interface DiscordConfig { webhookUrl: string; webhookId: string; + retryCount?: number; + backoffBaseSeconds?: number; } export interface Config {