From 0f6f66a43144cc1ff035bff49ee70fcd958e3dc1 Mon Sep 17 00:00:00 2001 From: Ezedike-egwom Collins Date: Thu, 22 Oct 2026 22:21:59 +0100 Subject: [PATCH] fix(monitor): make alert webhook optional and log-only --- .github/workflows/tholos-monitor.yml | 8 ++ packages/tholos-monitor/.env.example | 5 +- packages/tholos-monitor/README.md | 8 +- packages/tholos-monitor/src/config.test.ts | 8 +- packages/tholos-monitor/src/config.ts | 24 +++--- packages/tholos-monitor/src/main.ts | 8 +- packages/tholos-monitor/src/poller.test.ts | 10 +++ packages/tholos-monitor/src/poller.ts | 92 ++++++++++++++-------- 8 files changed, 105 insertions(+), 58 deletions(-) diff --git a/.github/workflows/tholos-monitor.yml b/.github/workflows/tholos-monitor.yml index 194bc7a..c0dd2df 100644 --- a/.github/workflows/tholos-monitor.yml +++ b/.github/workflows/tholos-monitor.yml @@ -87,6 +87,14 @@ jobs: - name: Build run: pnpm build + - name: Warn when alerts are disabled + env: + ALERT_WEBHOOK_URL: ${{ secrets.ALERT_WEBHOOK_URL }} + run: | + if [ -z "$ALERT_WEBHOOK_URL" ]; then + echo "::warning::ALERT_WEBHOOK_URL is not set. The monitor will poll and log events but cannot send alerts." + fi + - name: Run monitor env: THOLOS_V1_CONTRACT_ID: ${{ vars.THOLOS_V1_CONTRACT_ID }} diff --git a/packages/tholos-monitor/.env.example b/packages/tholos-monitor/.env.example index 748a27b..5949f3e 100644 --- a/packages/tholos-monitor/.env.example +++ b/packages/tholos-monitor/.env.example @@ -6,8 +6,9 @@ THOLOS_V1_CONTRACT_ID= THOLOS_V2_CONTRACT_ID= -# Required: any endpoint that accepts a JSON POST (Slack/Discord incoming -# webhook, or your own). +# Optional: any endpoint that accepts a JSON POST (Slack/Discord incoming +# webhook, or your own). When unset, the monitor polls and logs events but +# does not send alerts. ALERT_WEBHOOK_URL= # Optional overrides — leave unset to use the defaults in README.md. diff --git a/packages/tholos-monitor/README.md b/packages/tholos-monitor/README.md index 293808c..35f0c08 100644 --- a/packages/tholos-monitor/README.md +++ b/packages/tholos-monitor/README.md @@ -93,7 +93,7 @@ since this is a plain Node service rather than a Vite app. | --- | --- | --- | --- | | `THOLOS_V1_CONTRACT_ID` | At least one of this or `THOLOS_V2_CONTRACT_ID` | — | v1 deployment to monitor, e.g. the [canonical testnet deployment](../../docs/src/DEPLOYMENT.md#canonical-testnet-deployment). | | `THOLOS_V2_CONTRACT_ID` | At least one of this or `THOLOS_V1_CONTRACT_ID` | — | v2 deployment to monitor. v2 has no canonical testnet deployment yet (see `docs/src/DEPLOYMENT_V2.md`) — use your own. | -| `ALERT_WEBHOOK_URL` | Yes | — | Any endpoint that accepts a JSON `POST` — a Slack/Discord incoming webhook, or your own. See "Alert payload" below for the body shape. | +| `ALERT_WEBHOOK_URL` | No | — | Any endpoint that accepts a JSON `POST`, such as a Slack/Discord incoming webhook or your own. See "Alert payload" below for the body shape. When unset, the monitor polls and logs events but does not send alerts. | | `SOROBAN_RPC_URL` | No | `https://soroban-testnet.stellar.org` | Same default as the demo app's config. | | `NETWORK_PASSPHRASE` | No | `Test SDF Network ; September 2015` | Currently informational (not sent to the RPC — `getEvents` is filtered by contract id, not network); kept alongside the RPC URL so a future mainnet config has an obvious place to set both together. | | `ALERT_MIN_SEVERITY` | No | `warning` | `info` \| `warning` \| `critical`. Events below this are still logged, just not sent to the webhook. | @@ -105,7 +105,9 @@ since this is a plain Node service rather than a Vite app. Invalid or missing required configuration fails fast at startup with a message naming the problem (`src/config.ts`), rather than surfacing later as -a confusing runtime error. +a confusing runtime error. `ALERT_WEBHOOK_URL` is not required: with no +webhook configured the run still completes, logs a warning, and records +every event to standard output instead of sending alerts. ## Running it @@ -131,7 +133,7 @@ and variables → Actions: | `THOLOS_V1_CONTRACT_ID` | Variable | At least one of this or `THOLOS_V2_CONTRACT_ID`. | | `THOLOS_V2_CONTRACT_ID` | Variable | At least one of this or `THOLOS_V1_CONTRACT_ID`. | | `SOROBAN_RPC_URL` | Variable (optional) | Only if not using the default testnet RPC. | -| `ALERT_WEBHOOK_URL` | **Secret** | Kept as a secret, not a variable, since it's effectively a bearer credential for posting to whatever it points at (a Slack/Discord incoming webhook, typically). | +| `ALERT_WEBHOOK_URL` | **Secret** (optional) | Kept as a secret, not a variable, since it's effectively a bearer credential for posting to whatever it points at (a Slack/Discord incoming webhook, typically). Without it the scheduled run polls and logs but sends no alerts. | A `pnpm-lock.yaml` for this package (generated by running `pnpm install` once locally and committing the result) is required for diff --git a/packages/tholos-monitor/src/config.test.ts b/packages/tholos-monitor/src/config.test.ts index cbc1ffa..cf46d3a 100644 --- a/packages/tholos-monitor/src/config.test.ts +++ b/packages/tholos-monitor/src/config.test.ts @@ -19,11 +19,9 @@ test("loadConfig: requires at least one contract id", () => { ); }); -test("loadConfig: requires ALERT_WEBHOOK_URL", () => { - assert.throws( - () => loadConfig({ THOLOS_V1_CONTRACT_ID: BASE_ENV.THOLOS_V1_CONTRACT_ID }), - ConfigError, - ); +test("loadConfig: allows ALERT_WEBHOOK_URL to be omitted", () => { + const config = loadConfig({ THOLOS_V1_CONTRACT_ID: BASE_ENV.THOLOS_V1_CONTRACT_ID }); + assert.equal(config.alertWebhookUrl, undefined); }); test("loadConfig: rejects a malformed webhook URL", () => { diff --git a/packages/tholos-monitor/src/config.ts b/packages/tholos-monitor/src/config.ts index 7bb207d..b1b0716 100644 --- a/packages/tholos-monitor/src/config.ts +++ b/packages/tholos-monitor/src/config.ts @@ -20,7 +20,9 @@ export interface Config { * `loadConfig`. Both is the common case (monitor both deployments from * one process). */ contracts: Partial>; - alertWebhookUrl: string; + /** Optional. When unset, the monitor polls and logs events but does not + * send alerts. */ + alertWebhookUrl?: string; stateFilePath: string; alertMinSeverity: Severity; /** Consecutive failed runs (this is a scheduled script re-invoked by a @@ -97,25 +99,23 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): Config { } const alertWebhookUrl = optionalEnv(env, "ALERT_WEBHOOK_URL"); - if (!alertWebhookUrl) { - throw new ConfigError("ALERT_WEBHOOK_URL is required."); - } - try { - void new URL(alertWebhookUrl); - } catch { - throw new ConfigError(`ALERT_WEBHOOK_URL="${alertWebhookUrl}" is not a valid URL.`); + if (alertWebhookUrl !== undefined) { + try { + void new URL(alertWebhookUrl); + } catch { + throw new ConfigError(`ALERT_WEBHOOK_URL="${alertWebhookUrl}" is not a valid URL.`); + } } const contracts: Partial> = {}; if (tholos) contracts.tholos = tholos; if (tholosV2) contracts["tholos-v2"] = tholosV2; - return { + const config: Config = { rpcUrl: optionalEnv(env, "SOROBAN_RPC_URL") ?? DEFAULT_RPC_URL, networkPassphrase: optionalEnv(env, "NETWORK_PASSPHRASE") ?? DEFAULT_NETWORK_PASSPHRASE, contracts, - alertWebhookUrl, stateFilePath: optionalEnv(env, "STATE_FILE_PATH") ?? DEFAULT_STATE_FILE_PATH, alertMinSeverity: parseSeverity( optionalEnv(env, "ALERT_MIN_SEVERITY"), @@ -142,4 +142,8 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): Config { DEFAULT_ALERT_MAX_CONCURRENCY, ), }; + if (alertWebhookUrl !== undefined) { + config.alertWebhookUrl = alertWebhookUrl; + } + return config; } diff --git a/packages/tholos-monitor/src/main.ts b/packages/tholos-monitor/src/main.ts index 5453164..2d5f25f 100644 --- a/packages/tholos-monitor/src/main.ts +++ b/packages/tholos-monitor/src/main.ts @@ -24,9 +24,11 @@ async function main(): Promise { throw err; } - console.log( - `[main] tholos-monitor run starting. Alert webhook: ${redactUrl(config.alertWebhookUrl)}`, - ); + const alertTarget = + config.alertWebhookUrl === undefined + ? "none configured (poll and log only)" + : redactUrl(config.alertWebhookUrl); + console.log(`[main] tholos-monitor run starting. Alert webhook: ${alertTarget}`); const hadFailures = await runOnce(config); if (hadFailures) { diff --git a/packages/tholos-monitor/src/poller.test.ts b/packages/tholos-monitor/src/poller.test.ts index 1eb2ae2..837178f 100644 --- a/packages/tholos-monitor/src/poller.test.ts +++ b/packages/tholos-monitor/src/poller.test.ts @@ -174,3 +174,13 @@ test("classifyLogAndAlert: only alerts on events meeting alertMinSeverity (still await webhook.close(); } }); + +test("classifyLogAndAlert: completes without sending when no webhook is configured", async () => { + await assert.doesNotReject( + classifyLogAndAlert([makeEvent()], { + alertMinSeverity: "warning", + requestTimeoutMs: 5000, + maxConcurrentAlerts: 5, + }), + ); +}); diff --git a/packages/tholos-monitor/src/poller.ts b/packages/tholos-monitor/src/poller.ts index e4283d3..d67ee60 100644 --- a/packages/tholos-monitor/src/poller.ts +++ b/packages/tholos-monitor/src/poller.ts @@ -15,6 +15,7 @@ import type { Deployment, DecodedEvent, DeploymentState, + LivenessAlert, MonitorState, Severity, } from "./types.js"; @@ -52,7 +53,8 @@ function logEvent(event: ClassifiedEvent): void { export interface ClassifyLogAndAlertOptions { alertMinSeverity: Severity; - alertWebhookUrl: string; + /** Optional. When unset, threshold events are logged but not sent. */ + alertWebhookUrl?: string; requestTimeoutMs: number; /** Caps how many `sendAlert` calls run at once (see `runWithConcurrencyLimit` * below for why this needs a real cap, not just "fire them all"). */ @@ -118,14 +120,40 @@ export async function classifyLogAndAlert( toAlert.push(classified); } } + const webhookUrl = options.alertWebhookUrl; + if (webhookUrl === undefined) { + if (toAlert.length > 0) { + console.warn( + `[alerts] ALERT_WEBHOOK_URL not set; ${toAlert.length} alert(s) logged but not sent.`, + ); + } + return; + } await runWithConcurrencyLimit(toAlert, options.maxConcurrentAlerts, (classified) => sendAlert(buildEventAlertPayload(classified), { - webhookUrl: options.alertWebhookUrl, + webhookUrl, timeoutMs: options.requestTimeoutMs, }), ); } +/** + * Sends a liveness alert when a webhook is configured; otherwise logs it so + * the reason it is not delivered is visible in the run output. + */ +async function sendLivenessAlert(config: Config, alert: LivenessAlert): Promise { + if (config.alertWebhookUrl === undefined) { + console.warn( + `[alerts] ALERT_WEBHOOK_URL not set; liveness alert "${alert.kind}" logged but not sent.`, + ); + return; + } + await sendAlert(buildLivenessAlertPayload(alert), { + webhookUrl: config.alertWebhookUrl, + timeoutMs: config.requestTimeoutMs, + }); +} + /** Builds a `DeploymentState`, omitting `cursor` entirely rather than * setting it to `undefined` when there isn't one — required under * `exactOptionalPropertyTypes` (an optional property may be absent, but not @@ -197,12 +225,15 @@ async function pollOneDeployment( // See classifyLogAndAlert's own doc comment for why this dispatches // alerts concurrently rather than one at a time. - await classifyLogAndAlert(result.events, { + const classifyOptions: ClassifyLogAndAlertOptions = { alertMinSeverity: config.alertMinSeverity, - alertWebhookUrl: config.alertWebhookUrl, requestTimeoutMs: config.requestTimeoutMs, maxConcurrentAlerts: config.alertMaxConcurrency, - }); + }; + if (config.alertWebhookUrl !== undefined) { + classifyOptions.alertWebhookUrl = config.alertWebhookUrl; + } + await classifyLogAndAlert(result.events, classifyOptions); if (result.hitPageCap) { console.warn( @@ -211,15 +242,12 @@ async function pollOneDeployment( } if (consecutiveFailuresBefore > 0 && alertedForFailureStreak) { - await sendAlert( - buildLivenessAlertPayload({ - deployment, - kind: "recovered", - severity: "info", - message: `${deployment}: polling recovered after ${consecutiveFailuresBefore} consecutive failed run(s).`, - }), - { webhookUrl: config.alertWebhookUrl, timeoutMs: config.requestTimeoutMs }, - ); + await sendLivenessAlert(config, { + deployment, + kind: "recovered", + severity: "info", + message: `${deployment}: polling recovered after ${consecutiveFailuresBefore} consecutive failed run(s).`, + }); } state.deployments[deployment] = buildDeploymentState(0, false, result.nextCursor); @@ -229,16 +257,13 @@ async function pollOneDeployment( if (err instanceof RetentionGapError) { console.error(`[monitor] ${deployment}: ${lastError}`); - await sendAlert( - buildLivenessAlertPayload({ - deployment, - kind: "retention_gap", - severity: "critical", - message: `${deployment}: ${lastError} Re-anchoring to the current chain tip minus the configured lookback on the next run; events in the gap are unrecoverable from this RPC.`, - lastError, - }), - { webhookUrl: config.alertWebhookUrl, timeoutMs: config.requestTimeoutMs }, - ); + await sendLivenessAlert(config, { + deployment, + kind: "retention_gap", + severity: "critical", + message: `${deployment}: ${lastError} Re-anchoring to the current chain tip minus the configured lookback on the next run; events in the gap are unrecoverable from this RPC.`, + lastError, + }); // Drop the stale cursor so the next run re-derives a fresh // startLedger, instead of repeating the same failing request every // run from here on. A retention gap isn't the same kind of failure @@ -263,17 +288,14 @@ async function pollOneDeployment( !alertedForFailureStreak ) { nowAlerted = true; - await sendAlert( - buildLivenessAlertPayload({ - deployment, - kind: "rpc_failure", - severity: "critical", - message: `${deployment}: ${consecutiveFailures} consecutive failed runs against ${config.rpcUrl}.`, - consecutiveFailures, - lastError, - }), - { webhookUrl: config.alertWebhookUrl, timeoutMs: config.requestTimeoutMs }, - ); + await sendLivenessAlert(config, { + deployment, + kind: "rpc_failure", + severity: "critical", + message: `${deployment}: ${consecutiveFailures} consecutive failed runs against ${config.rpcUrl}.`, + consecutiveFailures, + lastError, + }); } state.deployments[deployment] = buildDeploymentState(