Merge pull request #288 from techulus/feature/cron-failure-notifications

Add cron failure notifications

Author: arjunkomath

web/components/settings/email-settings.tsx

@@ -19,7 +19,8 @@ type AlertField =
 	| "serverOfflineAlert"
 	| "buildFailure"
 	| "deploymentFailure"
-	| "deploymentMovedAlert";
+	| "deploymentMovedAlert"
+	| "cronFailure";
 
 type AlertSetting = {
 	field: AlertField;
@@ -49,13 +50,19 @@ const ALERT_SETTINGS: AlertSetting[] = [
 		description:
 			"Receive a notification when offline replicas need manual recovery",
 	},
+	{
+		field: "cronFailure",
+		label: "Cron Failure Alert",
+		description: "Receive a notification when a cron run fails",
+	},
 ];
 
 type State = {
 	serverOfflineAlert: boolean;
 	buildFailure: boolean;
 	deploymentFailure: boolean;
 	deploymentMovedAlert: boolean;
+	cronFailure: boolean;
 	isSavingAlerts: boolean;
 };
 
@@ -70,6 +77,7 @@ function createInitialState(props: Props): State {
 		buildFailure: alertsConfig?.buildFailure ?? true,
 		deploymentFailure: alertsConfig?.deploymentFailure ?? true,
 		deploymentMovedAlert: alertsConfig?.deploymentMovedAlert ?? true,
+		cronFailure: alertsConfig?.cronFailure ?? true,
 		isSavingAlerts: false,
 	};
 }
@@ -99,6 +107,7 @@ export function EmailSettings({ initialAlertsConfig }: Props) {
 				buildFailure: state.buildFailure,
 				deploymentFailure: state.deploymentFailure,
 				deploymentMovedAlert: state.deploymentMovedAlert,
+				cronFailure: state.cronFailure,
 			});
 			toast.success("Alert settings saved");
 			router.refresh();
@@ -120,7 +129,8 @@ export function EmailSettings({ initialAlertsConfig }: Props) {
 		state.deploymentFailure !==
 			(initialAlertsConfig?.deploymentFailure ?? true) ||
 		state.deploymentMovedAlert !==
-			(initialAlertsConfig?.deploymentMovedAlert ?? true);
+			(initialAlertsConfig?.deploymentMovedAlert ?? true) ||
+		state.cronFailure !== (initialAlertsConfig?.cronFailure ?? true);
 
 	return (
 		<div className="space-y-6">

web/lib/email/index.ts

@@ -261,6 +261,60 @@ async function sendBuildFailureAlert(
 	});
 }
 
+type CronFailureAlertOptions = {
+	to: string;
+	serviceId: string;
+	path: string;
+	statusCode: number | null;
+	error: string | null;
+};
+
+async function sendCronFailureAlert(
+	options: CronFailureAlertOptions,
+): Promise<void> {
+	const [result] = await db
+		.select({
+			serviceName: services.name,
+			projectName: projects.name,
+			projectSlug: projects.slug,
+			envName: environments.name,
+		})
+		.from(services)
+		.innerJoin(projects, eq(projects.id, services.projectId))
+		.innerJoin(environments, eq(environments.id, services.environmentId))
+		.where(eq(services.id, options.serviceId));
+
+	if (!result) return;
+
+	const baseUrl = getAppBaseUrl();
+	const serviceUrl = baseUrl
+		? `${baseUrl}/dashboard/projects/${result.projectSlug}/${result.envName}/services/${options.serviceId}`
+		: undefined;
+	const details = [
+		{ label: "Service", value: result.serviceName },
+		{ label: "Project", value: result.projectName },
+		{ label: "Cron Path", value: options.path },
+		...(options.statusCode !== null
+			? [{ label: "HTTP Status", value: String(options.statusCode) }]
+			: []),
+		...(options.error ? [{ label: "Error", value: options.error }] : []),
+	];
+
+	await sendAlert({
+		to: options.to,
+		subject: `Cron Failed: ${result.serviceName}`,
+		template: Alert({
+			bannerText: "CRON FAILED",
+			heading: "Cron Failure Alert",
+			description: `The cron job "${options.path}" for service "${result.serviceName}" in project "${result.projectName}" has failed.`,
+			details,
+			buttonText: serviceUrl ? "View Service" : undefined,
+			buttonUrl: serviceUrl,
+			baseUrl,
+		}),
+	});
+}
+
 type DeploymentFailureAlertOptions = {
 	to: string;
 	serviceId: string;
@@ -400,6 +454,9 @@ export async function deliverNotificationEmail(
 		case "build.failed":
 			await sendBuildFailureAlert({ ...event, to });
 			return;
+		case "cron.failed":
+			await sendCronFailureAlert({ ...event, to });
+			return;
 		case "deployment.failed":
 			await sendDeploymentFailureAlert({ ...event, to });
 	}

web/lib/inngest/events/notification.ts

@@ -29,6 +29,14 @@ export type NotificationEvent =
 			serverId: string | null;
 			failedStage?: string;
 	  }
+	| {
+			kind: "cron.failed";
+			occurrenceId: string;
+			serviceId: string;
+			path: string;
+			statusCode: number | null;
+			error: string | null;
+	  }
 	| {
 			kind: "member.invited";
 			occurrenceId: string;

web/lib/notifications/index.ts

@@ -37,6 +37,8 @@ export async function notificationEventIsEnabled(event: NotificationEvent) {
 			return config?.buildFailure !== false;
 		case "deployment.failed":
 			return config?.deploymentFailure !== false;
+		case "cron.failed":
+			return config?.cronFailure !== false;
 	}
 }
 
@@ -81,6 +83,13 @@ export async function renderInAppNotification(event: NotificationEvent) {
 			href: `${serviceHref}/builds/${event.buildId}`,
 		};
 	}
+	if (event.kind === "cron.failed") {
+		return {
+			title: `Cron failed: ${context.serviceName}`,
+			body: `${event.path}: ${event.error ?? "Cron request failed"}`,
+			href: serviceHref,
+		};
+	}
 	return {
 		title: `Deployment failed: ${context.serviceName}`,
 		body: event.failedStage

web/lib/service-crons.ts

@@ -5,6 +5,7 @@ import { CronExpressionParser } from "cron-parser";
 import { db } from "@/db";
 import { secrets, serviceCrons, services } from "@/db/schema";
 import { decryptSecret } from "@/lib/crypto";
+import { notify } from "@/lib/notifications";
 import { isSafeCronPath, nextOccurrenceAfter } from "@/lib/public-api";
 import { ingestCronLog, type CronLog } from "@/lib/victoria-logs";
 
@@ -256,5 +257,20 @@ export async function executeServiceCron(
 		log_type: "cron",
 	};
 	await ingestCronLog(log);
+	if (status === "failed") {
+		notify({
+			kind: "cron.failed",
+			occurrenceId: cronEventId(cronId, scheduledFor),
+			serviceId: row.serviceId,
+			path: row.cron.path,
+			statusCode,
+			error,
+		}).catch((cause) => {
+			console.error(
+				"[service-cron] failed to enqueue cron failure notification:",
+				cause,
+			);
+		});
+	}
 	return { stale: false as const, status, statusCode, error };
 }

web/lib/settings-keys.ts

@@ -89,6 +89,7 @@ export const emailAlertsConfigSchema = z.object({
 	buildFailure: z.boolean(),
 	deploymentFailure: z.boolean(),
 	deploymentMovedAlert: z.boolean(),
+	cronFailure: z.boolean(),
 });
 
 export type EmailAlertsConfig = z.infer<typeof emailAlertsConfigSchema>;

web/tests/notifications.test.ts

@@ -99,12 +99,49 @@ describe("notification pipeline", () => {
 		).resolves.toBeNull();
 	});
 
+	it("renders cron failures with the service deep link", async () => {
+		mocks.select.mockReturnValueOnce({
+			from: vi.fn(() => ({
+				innerJoin: vi.fn(() => ({
+					innerJoin: vi.fn(() => ({
+						where: vi.fn(() =>
+							Promise.resolve([
+								{
+									serviceName: "API",
+									projectName: "Cloud",
+									projectSlug: "cloud",
+									environmentName: "production",
+								},
+							]),
+						),
+					})),
+				})),
+			})),
+		});
+
+		await expect(
+			renderInAppNotification({
+				kind: "cron.failed",
+				occurrenceId: "cron-1",
+				serviceId: "service-1",
+				path: "/jobs/nightly",
+				statusCode: 500,
+				error: "HTTP status 500",
+			}),
+		).resolves.toEqual({
+			title: "Cron failed: API",
+			body: "/jobs/nightly: HTTP status 500",
+			href: "/dashboard/projects/cloud/production/services/service-1",
+		});
+	});
+
 	it("maps every operational event to its alert toggle", async () => {
 		mocks.getAlertsConfig.mockResolvedValue({
 			serverOfflineAlert: false,
 			buildFailure: false,
 			deploymentFailure: false,
 			deploymentMovedAlert: false,
+			cronFailure: false,
 		});
 
 		await expect(
@@ -141,17 +178,29 @@ describe("notification pipeline", () => {
 				serverId: "server-1",
 			}),
 		).resolves.toBe(false);
+		await expect(
+			notificationEventIsEnabled({
+				kind: "cron.failed",
+				occurrenceId: "cron-1",
+				serviceId: "service-1",
+				path: "/jobs/nightly",
+				statusCode: 500,
+				error: "HTTP status 500",
+			}),
+		).resolves.toBe(false);
 	});
 
 	it("defaults missing alert settings to enabled", async () => {
 		mocks.getAlertsConfig.mockResolvedValue(null);
 
 		await expect(
 			notificationEventIsEnabled({
-				kind: "build.failed",
-				occurrenceId: "build-1",
+				kind: "cron.failed",
+				occurrenceId: "cron-1",
 				serviceId: "service-1",
-				buildId: "build-1",
+				path: "/jobs/nightly",
+				statusCode: null,
+				error: "Cron request failed",
 			}),
 		).resolves.toBe(true);
 	});
@@ -162,6 +211,7 @@ describe("notification pipeline", () => {
 			buildFailure: true,
 			deploymentFailure: true,
 			deploymentMovedAlert: true,
+			cronFailure: true,
 		});
 
 		await deliverInAppNotification({