fix: database health cron

app/(customer)/dashboard/(organization)/projects/[projectId]/database/[databaseId]/page.tsx

@@ -8,6 +8,7 @@ import {getOrganizationProjectDatabases} from "@/lib/services";
 import {getActiveMember, getOrganization} from "@/lib/auth/auth";
 import {BackupModalProvider} from "@/components/wrappers/dashboard/database/backup/backup-modal-context";
 import {DatabaseContent} from "@/components/wrappers/dashboard/projects/database/database-content";
+import {getHealthLast12hLogs} from "@/db/services/healthcheck";
 
 export default async function RoutePage(props: PageParams<{
     projectId: string;
@@ -83,6 +84,8 @@ export default async function RoutePage(props: PageParams<{
         notFound();
     }
 
+    const databaseHealthLogs = dbItem ? await getHealthLast12hLogs({ id: dbItem.id }) : []
+
 
     const successRate = totalBackups > 0 ? (successfulBackups / totalBackups) * 100 : null;
 
@@ -95,6 +98,7 @@ export default async function RoutePage(props: PageParams<{
                     activeMember={activeMember}
                     settings={settings}
                     database={dbItem}
+                    databaseHealthLogs={databaseHealthLogs}
                     isAlreadyRestore={isAlreadyRestore}
                     restorations={restorations}
                     backups={backups}

app/api/agent/[agentId]/status/helpers.ts

@@ -96,7 +96,7 @@ export async function handleDatabases(body: Body, agent: Agent, lastContact: Dat
                 .set(withUpdatedAt({
                     name: db.name,
                     agentId: agent.id,
-                    lastContact: db.pingStatus ? lastContact : null,
+                    lastContact: db.pingStatus ? lastContact : existingDatabase.lastContact,
                 }))
                 .where(eq(drizzleDb.schemas.database.id, existingDatabase.id))
                 .returning();

src/components/wrappers/dashboard/database/backup/actions/get-data.action.ts

@@ -7,6 +7,7 @@ import * as drizzleDb from "@/db";
 import {BackupWith, Restoration} from "@/db/schema/07_database";
 import {getOrganizationChannels} from "@/db/services/notification-channel";
 import {getOrganizationStorageChannels} from "@/db/services/storage-channel";
+import {getHealthLast12hLogs} from "@/db/services/healthcheck";
 
 export const getDatabaseDataAction = userAction
     .schema(
@@ -50,7 +51,9 @@ export const getDatabaseDataAction = userAction
         const successfulBackups = backups.filter(b => b.status === "success").length;
         const successRate = totalBackups > 0 ? (successfulBackups / totalBackups) * 100 : null;
 
+        // @ts-ignore
         let activeOrganizationChannels = [];
+        // @ts-ignore
         let activeOrganizationStorageChannels = [];
 
         if (database?.project?.organizationId) {
@@ -61,16 +64,20 @@ export const getDatabaseDataAction = userAction
             activeOrganizationStorageChannels = organizationStorageChannels.filter(channel => channel.enabled);
         }
 
+
         return {
             database,
             backups,
             restorations,
+            // @ts-ignore
             activeOrganizationChannels,
+            // @ts-ignore
             activeOrganizationStorageChannels,
             stats: {
                 totalBackups,
                 availableBackups,
                 successRate
-            }
+            },
+            health: database ? await getHealthLast12hLogs({ id: database.id }) : []
         };
     });

src/components/wrappers/dashboard/database/channels-policy/policy.schema.ts

@@ -3,7 +3,7 @@ import {z} from "zod";
 export const PolicySchema = z.object({
     channelId: z.string().min(1, "Please select channel"),
     eventKinds: z.array(z.enum([
-        'error_backup', 'error_restore', 'success_restore', 'success_backup', 'weekly_report'
+        'error_backup', 'error_restore', 'success_restore', 'success_backup', 'weekly_report', 'error_health_database'
     ]))
         .optional(),
     enabled: z.boolean().default(true),
@@ -17,16 +17,16 @@ export const PoliciesSchema = z.object({
 export type PoliciesType = z.infer<typeof PoliciesSchema>;
 export type PolicyType = z.infer<typeof PolicySchema>;
 
-
-export const EVENT_KIND_OPTIONS = [
-    {label: "Error Backup", value: "error_backup"},
-    {label: "Error Restore", value: "error_restore"},
-    {label: "Success Restore", value: "success_restore"},
-    {label: "Success Backup", value: "success_backup"},
-    // {label: "Weekly Report", value: "weekly_report"},
-];
-
 export const EVENT_KIND_BACKUP_ONLY_OPTIONS = [
     {label: "Error Backup", value: "error_backup"},
     {label: "Success Backup", value: "success_backup"},
+    {label: "Health Ping Fail", value: "error_health_database"},
 ];
+
+export const EVENT_KIND_OPTIONS = [
+    ...EVENT_KIND_BACKUP_ONLY_OPTIONS,
+    {label: "Error Restore", value: "error_restore"},
+    {label: "Success Restore", value: "success_restore"},
+    // {label: "Weekly Report", value: "weekly_report"},
+];
+

src/components/wrappers/dashboard/database/health/health-modal.tsx

@@ -0,0 +1,43 @@
+"use client"
+
+import {useState} from "react";
+import {Button} from "@/components/ui/button";
+import {DatabaseWith} from "@/db/schema/07_database";
+import {HeartPulse} from "lucide-react";
+import {
+    Sheet,
+    SheetContent,
+    SheetTitle,
+    SheetTrigger
+} from "@/components/ui/sheet";
+import {HealthcheckLog} from "@/db/schema/15_healthcheck-log";
+import {HealthCheckGraph} from "@/components/wrappers/dashboard/health/heath-grid";
+
+type HealthModalProps = {
+    database: DatabaseWith,
+    healthLogs: HealthcheckLog[]
+}
+
+export const HealthModal = ({database, healthLogs}: HealthModalProps) => {
+    const [open, setOpen] = useState(false);
+
+    return (
+        <Sheet open={open} onOpenChange={setOpen}>
+            <SheetTrigger asChild>
+                <Button variant="outline" onClick={() => setOpen(true)}>
+                    <HeartPulse/>
+                </Button>
+            </SheetTrigger>
+            <SheetContent side="bottom">
+                <div className="flex  pl-5 pt-4">
+                    <SheetTitle>
+                        Database Health Status
+                    </SheetTitle>
+                </div>
+                <div className="px-4 pb-5">
+                    <HealthCheckGraph logs={healthLogs} />
+                </div>
+            </SheetContent>
+        </Sheet>
+    )
+}

src/components/wrappers/dashboard/projects/database/database-content.tsx

@@ -20,6 +20,8 @@ import { ChannelPoliciesModal } from "@/components/wrappers/dashboard/database/c
 import { HardDrive, Megaphone } from "lucide-react";
 import { ImportModal } from "@/components/wrappers/dashboard/database/import/import-modal";
 import { BackupButton } from "@/components/wrappers/dashboard/backup/backup-button/backup-button";
+import {HealthModal} from "@/components/wrappers/dashboard/database/health/health-modal";
+import {HealthcheckLog} from "@/db/schema/15_healthcheck-log";
 
 export type DatabaseContentProps = {
   settings: Setting;
@@ -34,6 +36,7 @@ export type DatabaseContentProps = {
   organizationId: string;
   activeOrganizationChannels: any[];
   activeOrganizationStorageChannels: any[];
+  databaseHealthLogs: HealthcheckLog[]
 };
 
 export const DatabaseContent = (props: DatabaseContentProps) => {
@@ -64,6 +67,7 @@ export const DatabaseContent = (props: DatabaseContentProps) => {
         availableBackups: props.availableBackups,
         successRate: props.successRate,
       },
+      health: props.databaseHealthLogs
     },
     staleTime: 0,
     gcTime: 0,
@@ -118,10 +122,13 @@ export const DatabaseContent = (props: DatabaseContentProps) => {
                   organizationId={props.organizationId}
                 />
                 <ImportModal database={database} />
+                <HealthModal database={database} healthLogs={data?.health ?? []}/>
               </div>
+
+
               <div className="flex items-center gap-2">
                 <BackupButton
-                  disable={isAlreadyBackup}
+                  disable={isAlreadyBackup || !database.lastContact}
                   databaseId={database.id}
                 />
               </div>

src/db/migrations/0045_needy_martin_li.sql

@@ -0,0 +1 @@
+ALTER TYPE "public"."event_kind" ADD VALUE 'error_health_database';

src/db/migrations/0046_mysterious_menace.sql

@@ -0,0 +1 @@
+ALTER TABLE "databases" ADD COLUMN "health_error_count" integer;

src/db/migrations/meta/_journal.json

@@ -316,6 +316,20 @@
       "when": 1774648872657,
       "tag": "0044_steep_wiccan",
       "breakpoints": true
+    },
+    {
+      "idx": 45,
+      "version": "7",
+      "when": 1774696680244,
+      "tag": "0045_needy_martin_li",
+      "breakpoints": true
+    },
+    {
+      "idx": 46,
+      "version": "7",
+      "when": 1774706071024,
+      "tag": "0046_mysterious_menace",
+      "breakpoints": true
     }
   ]
 }

src/db/schema/07_database.ts

@@ -19,6 +19,7 @@ export const database = pgTable("databases", {
     backupPolicy: text("backup_policy"),
     isWaitingForBackup: boolean("is_waiting_for_backup").default(false).notNull(),
     backupToRestore: text("backup_to_restore"),
+    healthErrorCount: integer("health_error_count"),
     agentId: uuid("agent_id")
         .notNull()
         .references(() => agent.id, {onDelete: "cascade"}),

src/db/schema/10_alert-policy.ts

@@ -6,7 +6,7 @@ import {database} from "@/db/schema/07_database";
 import {createSelectSchema} from "drizzle-zod";
 import {z} from "zod";
 
-export const eventKindEnum = pgEnum('event_kind', ['error_backup', 'error_restore', 'success_restore', 'success_backup', 'weekly_report', 'error_health_agent']);
+export const eventKindEnum = pgEnum('event_kind', ['error_backup', 'error_restore', 'success_restore', 'success_backup', 'weekly_report', 'error_health_agent', 'error_health_database']);
 
 export const alertPolicy = pgTable('alert_policy', {
     id: uuid('id').defaultRandom().primaryKey(),

src/db/services/healthcheck.ts

@@ -1,10 +1,8 @@
 import {db} from "@/db";
 import * as drizzleDb from "@/db";
 import {and, eq, gte, isNotNull, lt} from "drizzle-orm";
-import {enforceRetention} from "@/lib/tasks/database";
 import {dispatchNotification} from "@/features/notifications/dispatch";
-import {EventKind, EventPayload} from "@/features/notifications/types";
+import {EventPayload} from "@/features/notifications/types";
-import {DatabaseWith} from "@/db/schema/07_database";
 
 export async function getHealthLast12hLogs({id}: { id: string }) {
     const now = new Date()
@@ -44,102 +42,6 @@ export async function deleteHealthLogsOlderThan12h() {
     return logsToDelete.length
 }
 
-
-//
-// export async function sendNotificationsHealthCheck(event: EventKind) {
-//
-//
-//
-//     const date = new Date();
-//     let level: "info" | "critical" = "info";
-//     let message = "";
-//     let error: string | null = null;
-//
-//     switch (event) {
-//         case "error_backup":
-//         case "error_restore":
-//             level = "critical";
-//             message = `An error occurred during ${event.includes("backup") ? "backup" : "restore"} on ${date.toISOString()}.`;
-//             error = "Check database connection or agent";
-//             break;
-//         case "success_backup":
-//         case "success_restore":
-//             level = "info";
-//             message = `${event.includes("backup") ? "Backup" : "Restore"} completed successfully at ${date.toISOString()}.`;
-//             break;
-//         case "weekly_report":
-//             level = "info";
-//             message = `Weekly report generated at ${date.toISOString()}.`;
-//             break;
-//     }
-//
-//
-//
-//
-//
-//
-//
-//
-//
-//
-//     const activePolicies = database.alertPolicies.filter(policy =>
-//         policy.enabled && policy.eventKinds.includes(event)
-//     );
-//
-//     const promises = activePolicies.map(alertPolicy => {
-//         const date = new Date();
-//         let level: "info" | "critical" = "info";
-//         let message = "";
-//         let error: string | null = null;
-//
-//         switch (event) {
-//             case "error_backup":
-//             case "error_restore":
-//                 level = "critical";
-//                 message = `An error occurred during ${event.includes("backup") ? "backup" : "restore"} on ${date.toISOString()}.`;
-//                 error = "Check database connection or agent";
-//                 break;
-//             case "success_backup":
-//             case "success_restore":
-//                 level = "info";
-//                 message = `${event.includes("backup") ? "Backup" : "Restore"} completed successfully at ${date.toISOString()}.`;
-//                 break;
-//             case "weekly_report":
-//                 level = "info";
-//                 message = `Weekly report generated at ${date.toISOString()}.`;
-//                 break;
-//         }
-//
-//         const titleMap: Record<EventKind, string> = {
-//             error_backup: `Backup Notification`,
-//             error_restore: `Restore Notification`,
-//             success_backup: `Backup Notification`,
-//             success_restore: `Restore Notification`,
-//             weekly_report: `Weekly Report Notification`,
-//         };
-//
-//         const payload: EventPayload = {
-//             title: titleMap[event],
-//             message,
-//             level: level,
-//             event: event,
-//             data: {
-//                 host: database.name,
-//                 id: database.id,
-//                 agentDatabaseId: database.agentDatabaseId,
-//                 error,
-//             },
-//         };
-//
-//         return dispatchNotification(payload, alertPolicy.id, undefined, undefined);
-//     });
-//
-//
-//     return Promise.all(promises);
-// }
-//
-
-
 export async function checkAgentsHealthError() {
     const agents = await db.query.agent.findMany({
         where: isNotNull(drizzleDb.schemas.agent.lastContact),
@@ -201,3 +103,85 @@ export async function checkAgentsHealthError() {
         }
     }
 }
+
+
+
+export async function checkDatabasesHealthError() {
+
+    const databases = await db.query.database.findMany({
+        where: isNotNull(drizzleDb.schemas.database.lastContact),
+        with: {
+            agent: true,
+            alertPolicies: true
+        }
+    })
+
+    const now = new Date();
+
+    for (const database of databases) {
+        if (!database.lastContact) continue;
+
+        const lastContactDate = new Date(database.lastContact);
+        const diffMinutes = (now.getTime() - lastContactDate.getTime()) / 1000 / 60;
+
+        if (diffMinutes > 10) {
+            if ((database.healthErrorCount ?? 0) < 3) {
+
+                const newHealthErrorCount = (database.healthErrorCount ?? 0) + 1
+                await db.update(drizzleDb.schemas.database)
+                    .set({
+                        healthErrorCount: newHealthErrorCount,
+                    })
+                    .where(eq(drizzleDb.schemas.database.id, database.id));
+
+                const settings = await db.query.setting.findFirst({
+                    where: eq(drizzleDb.schemas.setting.name, "system"),
+                    with: { notificationChannel: true },
+                });
+
+                const defaultPolicy = settings?.notificationChannel
+                    ? [{
+                        id: null,
+                        notificationChannelId: settings.notificationChannel.id,
+                        enabled: settings.notificationChannel.enabled,
+                        eventKinds: ["error_health_database"]
+                    }]
+                    : [];
+
+                const policiesToUse = (database.alertPolicies && database.alertPolicies.length > 0)
+                    ? database.alertPolicies.filter(policy => policy.enabled && policy.eventKinds.includes("error_health_database"))
+                    : defaultPolicy;
+
+                if (!policiesToUse || policiesToUse.length === 0) {
+                    continue
+                }
+
+
+                const promises = policiesToUse.map(alertPolicy => {
+
+                    const payload: EventPayload = {
+                        title: "Database down",
+                        message: `Database ${database.name} is down, (notification number: ${newHealthErrorCount}/3)`,
+                        level: "critical",
+                        event: "error_health_database",
+                        data: {
+                            agent: database.name,
+                            id: database.id,
+                            error: "Database is down",
+                        },
+                    };
+
+                    console.log("[Database Healthcheck] :", payload);
+
+                    return dispatchNotification(payload, alertPolicy.id == null ? undefined : alertPolicy.id, alertPolicy.id ? undefined : alertPolicy.notificationChannelId, undefined);
+                });
+
+                await Promise.all(promises);
+
+            }
+
+        }
+    }
+}
+
+

src/features/notifications/types.ts

@@ -18,4 +18,4 @@ export interface EventPayload {
     data?: Record<string, any>;
 }
 
-export type EventKind = ("error_backup" | "error_restore" | "success_restore" | "success_backup" | "weekly_report" | "error_health_agent")
+export type EventKind = ("error_backup" | "error_restore" | "success_restore" | "success_backup" | "weekly_report" | "error_health_agent" | "error_health_database")

src/lib/tasks/index.ts

@@ -2,7 +2,11 @@ import cron from "node-cron";
 import {retentionCleanTask} from "@/lib/tasks/database";
 import {env} from "@/env.mjs";
 import {backupCleanTask} from "@/lib/tasks/cleaning";
-import {checkAgentsHealthError, deleteHealthLogsOlderThan12h} from "@/db/services/healthcheck";
+import {
+    checkAgentsHealthError,
+    checkDatabasesHealthError,
+    deleteHealthLogsOlderThan12h
+} from "@/db/services/healthcheck";
 
 export const retentionJob = cron.schedule(env.RETENTION_CRON, async () => {
     try {
@@ -36,6 +40,7 @@ export const healthcheckAgentAndDatabaseJob = cron.schedule(env.HEALTHCHECK_CRON
     try {
         console.log("Healthcheck Job : Starting task");
         await checkAgentsHealthError();
+        await checkDatabasesHealthError()
     } catch (err) {
         console.error(`[CRON] Error:`, err);
     }