fix: database health cron

src/db/migrations/0045_needy_martin_li.sql

@@ -0,0 +1 @@
+ALTER TYPE "public"."event_kind" ADD VALUE 'error_health_database';

src/db/migrations/0046_mysterious_menace.sql

@@ -0,0 +1 @@
+ALTER TABLE "databases" ADD COLUMN "health_error_count" integer;

src/db/migrations/meta/_journal.json

@@ -316,6 +316,20 @@
       "when": 1774648872657,
       "tag": "0044_steep_wiccan",
       "breakpoints": true
+    },
+    {
+      "idx": 45,
+      "version": "7",
+      "when": 1774696680244,
+      "tag": "0045_needy_martin_li",
+      "breakpoints": true
+    },
+    {
+      "idx": 46,
+      "version": "7",
+      "when": 1774706071024,
+      "tag": "0046_mysterious_menace",
+      "breakpoints": true
     }
   ]
 }

src/db/schema/07_database.ts

@@ -19,6 +19,7 @@ export const database = pgTable("databases", {
     backupPolicy: text("backup_policy"),
     isWaitingForBackup: boolean("is_waiting_for_backup").default(false).notNull(),
     backupToRestore: text("backup_to_restore"),
+    healthErrorCount: integer("health_error_count"),
     agentId: uuid("agent_id")
         .notNull()
         .references(() => agent.id, {onDelete: "cascade"}),

src/db/schema/10_alert-policy.ts

@@ -6,7 +6,7 @@ import {database} from "@/db/schema/07_database";
 import {createSelectSchema} from "drizzle-zod";
 import {z} from "zod";
 
-export const eventKindEnum = pgEnum('event_kind', ['error_backup', 'error_restore', 'success_restore', 'success_backup', 'weekly_report', 'error_health_agent']);
+export const eventKindEnum = pgEnum('event_kind', ['error_backup', 'error_restore', 'success_restore', 'success_backup', 'weekly_report', 'error_health_agent', 'error_health_database']);
 
 export const alertPolicy = pgTable('alert_policy', {
     id: uuid('id').defaultRandom().primaryKey(),

src/db/services/healthcheck.ts

@@ -1,10 +1,8 @@
 import {db} from "@/db";
 import * as drizzleDb from "@/db";
 import {and, eq, gte, isNotNull, lt} from "drizzle-orm";
-import {enforceRetention} from "@/lib/tasks/database";
 import {dispatchNotification} from "@/features/notifications/dispatch";
-import {EventKind, EventPayload} from "@/features/notifications/types";
-import {DatabaseWith} from "@/db/schema/07_database";
+import {EventPayload} from "@/features/notifications/types";
 
 export async function getHealthLast12hLogs({id}: { id: string }) {
     const now = new Date()
@@ -44,102 +42,6 @@ export async function deleteHealthLogsOlderThan12h() {
     return logsToDelete.length
 }
 
-
-//
-// export async function sendNotificationsHealthCheck(event: EventKind) {
-//
-//
-//
-//     const date = new Date();
-//     let level: "info" | "critical" = "info";
-//     let message = "";
-//     let error: string | null = null;
-//
-//     switch (event) {
-//         case "error_backup":
-//         case "error_restore":
-//             level = "critical";
-//             message = `An error occurred during ${event.includes("backup") ? "backup" : "restore"} on ${date.toISOString()}.`;
-//             error = "Check database connection or agent";
-//             break;
-//         case "success_backup":
-//         case "success_restore":
-//             level = "info";
-//             message = `${event.includes("backup") ? "Backup" : "Restore"} completed successfully at ${date.toISOString()}.`;
-//             break;
-//         case "weekly_report":
-//             level = "info";
-//             message = `Weekly report generated at ${date.toISOString()}.`;
-//             break;
-//     }
-//
-//
-//
-//
-//
-//
-//
-//
-//
-//
-//     const activePolicies = database.alertPolicies.filter(policy =>
-//         policy.enabled && policy.eventKinds.includes(event)
-//     );
-//
-//     const promises = activePolicies.map(alertPolicy => {
-//         const date = new Date();
-//         let level: "info" | "critical" = "info";
-//         let message = "";
-//         let error: string | null = null;
-//
-//         switch (event) {
-//             case "error_backup":
-//             case "error_restore":
-//                 level = "critical";
-//                 message = `An error occurred during ${event.includes("backup") ? "backup" : "restore"} on ${date.toISOString()}.`;
-//                 error = "Check database connection or agent";
-//                 break;
-//             case "success_backup":
-//             case "success_restore":
-//                 level = "info";
-//                 message = `${event.includes("backup") ? "Backup" : "Restore"} completed successfully at ${date.toISOString()}.`;
-//                 break;
-//             case "weekly_report":
-//                 level = "info";
-//                 message = `Weekly report generated at ${date.toISOString()}.`;
-//                 break;
-//         }
-//
-//         const titleMap: Record<EventKind, string> = {
-//             error_backup: `Backup Notification`,
-//             error_restore: `Restore Notification`,
-//             success_backup: `Backup Notification`,
-//             success_restore: `Restore Notification`,
-//             weekly_report: `Weekly Report Notification`,
-//         };
-//
-//         const payload: EventPayload = {
-//             title: titleMap[event],
-//             message,
-//             level: level,
-//             event: event,
-//             data: {
-//                 host: database.name,
-//                 id: database.id,
-//                 agentDatabaseId: database.agentDatabaseId,
-//                 error,
-//             },
-//         };
-//
-//         return dispatchNotification(payload, alertPolicy.id, undefined, undefined);
-//     });
-//
-//
-//     return Promise.all(promises);
-// }
-//
-
-
 export async function checkAgentsHealthError() {
     const agents = await db.query.agent.findMany({
         where: isNotNull(drizzleDb.schemas.agent.lastContact),
@@ -200,4 +102,86 @@ export async function checkAgentsHealthError() {
 
         }
     }
-}
+}
+
+
+
+export async function checkDatabasesHealthError() {
+
+    const databases = await db.query.database.findMany({
+        where: isNotNull(drizzleDb.schemas.database.lastContact),
+        with: {
+            agent: true,
+            alertPolicies: true
+        }
+    })
+
+    const now = new Date();
+
+    for (const database of databases) {
+        if (!database.lastContact) continue;
+
+        const lastContactDate = new Date(database.lastContact);
+        const diffMinutes = (now.getTime() - lastContactDate.getTime()) / 1000 / 60;
+
+        if (diffMinutes > 10) {
+            if ((database.healthErrorCount ?? 0) < 3) {
+
+                const newHealthErrorCount = (database.healthErrorCount ?? 0) + 1
+                await db.update(drizzleDb.schemas.database)
+                    .set({
+                        healthErrorCount: newHealthErrorCount,
+                    })
+                    .where(eq(drizzleDb.schemas.database.id, database.id));
+
+                const settings = await db.query.setting.findFirst({
+                    where: eq(drizzleDb.schemas.setting.name, "system"),
+                    with: { notificationChannel: true },
+                });
+
+                const defaultPolicy = settings?.notificationChannel
+                    ? [{
+                        id: null,
+                        notificationChannelId: settings.notificationChannel.id,
+                        enabled: settings.notificationChannel.enabled,
+                        eventKinds: ["error_health_database"]
+                    }]
+                    : [];
+
+                const policiesToUse = (database.alertPolicies && database.alertPolicies.length > 0)
+                    ? database.alertPolicies.filter(policy => policy.enabled && policy.eventKinds.includes("error_health_database"))
+                    : defaultPolicy;
+
+                if (!policiesToUse || policiesToUse.length === 0) {
+                    continue
+                }
+
+
+                const promises = policiesToUse.map(alertPolicy => {
+
+                    const payload: EventPayload = {
+                        title: "Database down",
+                        message: `Database ${database.name} is down, (notification number: ${newHealthErrorCount}/3)`,
+                        level: "critical",
+                        event: "error_health_database",
+                        data: {
+                            agent: database.name,
+                            id: database.id,
+                            error: "Database is down",
+                        },
+                    };
+
+                    console.log("[Database Healthcheck] :", payload);
+
+                    return dispatchNotification(payload, alertPolicy.id == null ? undefined : alertPolicy.id, alertPolicy.id ? undefined : alertPolicy.notificationChannelId, undefined);
+                });
+
+                await Promise.all(promises);
+
+            }
+
+        }
+    }
+}
+
+