feat: added heath_error_count and the cron for health check agent errors

src/db/migrations/0043_peaceful_chat.sql

@@ -0,0 +1 @@
+ALTER TABLE "agents" ADD COLUMN "health_error_count" integer;

src/db/migrations/meta/_journal.json

@@ -302,6 +302,13 @@
       "when": 1774472168308,
       "tag": "0042_breezy_namora",
       "breakpoints": true
+    },
+    {
+      "idx": 43,
+      "version": "7",
+      "when": 1774643615673,
+      "tag": "0043_peaceful_chat",
+      "breakpoints": true
     }
   ]
 }

src/db/schema/08_agent.ts

@@ -1,4 +1,4 @@
-import {boolean, pgTable, text, timestamp, uuid} from "drizzle-orm/pg-core";
+import {boolean, pgTable, text, timestamp, uuid, integer} from "drizzle-orm/pg-core";
 import {createSelectSchema} from "drizzle-zod";
 import {z} from "zod";
 import {Database, database} from "@/db/schema/07_database";
@@ -10,6 +10,7 @@ export const agent = pgTable("agents", {
     slug: text("slug").notNull().unique(),
     version: text("version"),
     name: text("name").notNull().notNull(),
+    healthErrorCount: integer("health_error_count"),
     description: text("description").notNull(),
     isArchived: boolean("is_archived").default(false),
     lastContact: timestamp("last_contact"),

src/db/schema/10_alert-policy.ts

@@ -6,7 +6,7 @@ import {database} from "@/db/schema/07_database";
 import {createSelectSchema} from "drizzle-zod";
 import {z} from "zod";
 
-export const eventKindEnum = pgEnum('event_kind', ['error_backup', 'error_restore', 'success_restore', 'success_backup', 'weekly_report']);
+export const eventKindEnum = pgEnum('event_kind', ['error_backup', 'error_restore', 'success_restore', 'success_backup', 'weekly_report', 'error_health_agent']);
 
 export const alertPolicy = pgTable('alert_policy', {
     id: uuid('id').defaultRandom().primaryKey(),

src/db/services/healthcheck.ts

@@ -45,7 +45,6 @@ export async function deleteHealthLogsOlderThan12h() {
 }
 
 
-
 //
 // export async function sendNotificationsHealthCheck(event: EventKind) {
 //
@@ -139,57 +138,66 @@ export async function deleteHealthLogsOlderThan12h() {
 //     return Promise.all(promises);
 // }
 //
-//
-//
-// export async function checkAgentsHealthError() {
-//     const agents = await db.query.agent.findMany({
-//         where: isNotNull(drizzleDb.schemas.agent.lastContact),
-//     });
-//
-//     const settings = await db.query.setting.findFirst({
-//         where: (fields, { eq }) => eq(fields.name, "system"),
-//     });
-//
-//     if (!settings) {
-//         throw new Error("System settings not found");
-//     }
-//
-//     const now = new Date();
-//
-//     for (const agent of agents) {
-//         if (!agent.lastContact) continue;
-//
-//         const lastContactDate = new Date(agent.lastContact);
-//         const diffMinutes = (now.getTime() - lastContactDate.getTime()) / 1000 / 60;
-//
-//         if (diffMinutes > 10) {
-//             if ((agent.health_error_count ?? 0) < 3) {
-//                 await db.update(drizzleDb.schemas.agent)
-//                     .set({
-//                         health_error_count: (agent.health_error_count ?? 0) + 1,
-//                     })
-//                     .where(drizzleDb.schemas.agent.id.eq(agent.id));
-//
-//                 const payload: EventPayload = {
-//                     title: "Agent down",
-//                     message: `Agent ${agent.name} is down`,
-//                     level: "critical",
-//                     event: "error_health_agent",
-//                     data: {
-//                         agent: agent.name,
-//                         id: agent.id,
-//                         error: "Agent is down",
-//                     },
-//                 };
-//
-//                 await dispatchNotification(
-//                     payload,
-//                     undefined,
-//                     settings.defaultNotificationChannelId,
-//                     undefined
-//                 );
-//             }
-//
-//         }
-//     }
-// }
+
+
+export async function checkAgentsHealthError() {
+    const agents = await db.query.agent.findMany({
+        where: isNotNull(drizzleDb.schemas.agent.lastContact),
+    });
+
+    const settings = await db.query.setting.findFirst({
+        where: (fields, {eq}) => eq(fields.name, "system"),
+    });
+
+    if (!settings) {
+        throw new Error("System settings not found");
+    }
+
+    if (!settings.defaultNotificationChannelId) {
+        console.error("No default notification channel id found.");
+        return
+    }
+
+    const now = new Date();
+
+    for (const agent of agents) {
+        if (!agent.lastContact) continue;
+
+        const lastContactDate = new Date(agent.lastContact);
+        const diffMinutes = (now.getTime() - lastContactDate.getTime()) / 1000 / 60;
+
+        if (diffMinutes > 10) {
+            if ((agent.healthErrorCount ?? 0) < 3) {
+
+                const newHealthErrorCount = (agent.healthErrorCount ?? 0) + 1
+                await db.update(drizzleDb.schemas.agent)
+                    .set({
+                        healthErrorCount: newHealthErrorCount,
+                    })
+                    .where(eq(drizzleDb.schemas.agent.id, agent.id));
+
+                const payload: EventPayload = {
+                    title: "Agent down",
+                    message: `Agent ${agent.name} is down, (notification number: ${newHealthErrorCount}/3)`,
+                    level: "critical",
+                    event: "error_health_agent",
+                    data: {
+                        agent: agent.name,
+                        id: agent.id,
+                        error: "Agent is down",
+                    },
+                };
+
+                console.log("[Agent Healthcheck] :", payload);
+
+                await dispatchNotification(
+                    payload,
+                    undefined,
+                    settings.defaultNotificationChannelId,
+                    undefined
+                );
+            }
+
+        }
+    }
+}