feat: added heath_error_count and the cron for health check agent errors

app/api/agent/[agentId]/status/route.ts

@@ -62,7 +62,8 @@ export async function POST(
             .update(drizzleDb.schemas.agent)
             .set(withUpdatedAt({
                 version: body.version,
-                lastContact: lastContact
+                lastContact: lastContact,
+                healthErrorCount: null
             }))
             .where(eq(drizzleDb.schemas.agent.id, agentId));
 

src/components/wrappers/dashboard/agent/agent-card/agent-card.tsx

@@ -52,6 +52,11 @@ export const AgentCard = (props: agentCardProps) => {
                                 <Badge variant="secondary" className="h-5 px-1.5 text-[10px] font-bold tracking-wider">
                                     v{agent.version}
                                 </Badge>
+                                {agent.healthErrorCount && (
+                                    <Badge variant="destructive" className="h-5 px-1.5 text-[10px] font-bold tracking-wider">
+                                        down
+                                    </Badge>
+                                )}
                                 {isUpdateAvailable && (
                                     <Tooltip>
                                         <TooltipTrigger asChild>

src/components/wrappers/dashboard/agent/agent-content.tsx

@@ -50,7 +50,8 @@ export const AgentContentPage = ({edgeKey, agent: initialAgent}: AgentContentPag
             <div className="flex flex-col sm:flex-row sm:justify-between gap-6 ">
                 <Card className="w-full sm:w-auto flex-1  transition-all  border-border/50 bg-card ">
                     <CardHeader className="flex flex-row items-center justify-between space-y-0 pb-2">
-                        <CardTitle className="text-sm font-semibold uppercase tracking-wider text-muted-foreground">Databases</CardTitle>
+                        <CardTitle
+                            className="text-sm font-semibold uppercase tracking-wider text-muted-foreground">Databases</CardTitle>
                         <Server className="h-4 w-4 text-muted-foreground opacity-50"/>
                     </CardHeader>
                     <CardContent>
@@ -61,26 +62,32 @@ export const AgentContentPage = ({edgeKey, agent: initialAgent}: AgentContentPag
 
                 <Card className="w-full sm:w-auto flex-1  transition-all  border-border/50 bg-card ">
                     <CardHeader className="flex flex-row items-center justify-between space-y-0 pb-2">
-                        <CardTitle className="text-sm font-semibold uppercase tracking-wider text-muted-foreground">Last contact</CardTitle>
+                        <CardTitle className="text-sm font-semibold uppercase tracking-wider text-muted-foreground">Last
+                            contact</CardTitle>
                         <Server className="h-4 w-4 text-muted-foreground opacity-50"/>
                     </CardHeader>
                     <CardContent>
-                        <div className="text-3xl font-bold tracking-tight">{formatDateLastContact(agent.lastContact)}</div>
+                        <div
+                            className="text-3xl font-bold tracking-tight">{formatDateLastContact(agent.lastContact)}</div>
                         <p className="text-xs text-muted-foreground mt-1">Status heartbeat</p>
                     </CardContent>
                 </Card>
             </div>
 
+            {agent.lastContact && (
                 <HealthCheckGraph logs={agentHealthLogs}/>
+            )}
 
             <div className="space-y-6">
                 <Accordion type="single" collapsible defaultValue={!agent.lastContact ? "registration" : undefined}>
-                    <AccordionItem value="registration" className="border rounded-xl px-6 bg-card shadow-sm overflow-hidden transition-all duration-300">
+                    <AccordionItem value="registration"
+                                   className="border rounded-xl px-6 bg-card shadow-sm overflow-hidden transition-all duration-300">
                         <AccordionTrigger className="hover:no-underline py-4 group">
                             <div className="flex items-center gap-3">
                                 <span className="text-xl font-bold tracking-tight">Registration & Setup</span>
                                 {!agent.lastContact && (
-                                    <Badge variant="outline" className="bg-orange-500/10 text-orange-600 border-orange-500/20 animate-pulse">
+                                    <Badge variant="outline"
+                                           className="bg-orange-500/10 text-orange-600 border-orange-500/20 animate-pulse">
                                         Action Required
                                     </Badge>
                                 )}
@@ -106,7 +113,7 @@ export const AgentContentPage = ({edgeKey, agent: initialAgent}: AgentContentPag
                             </p>
                         </div>
                     </div>
-                <Separator className="opacity-50" />
+                    <Separator className="opacity-50"/>
                     <CardsWithPagination
                         cardsPerPage={4}
                         numberOfColumns={2}

src/db/migrations/0043_peaceful_chat.sql

@@ -0,0 +1 @@
+ALTER TABLE "agents" ADD COLUMN "health_error_count" integer;

src/db/migrations/meta/_journal.json

@@ -302,6 +302,13 @@
       "when": 1774472168308,
       "tag": "0042_breezy_namora",
       "breakpoints": true
+    },
+    {
+      "idx": 43,
+      "version": "7",
+      "when": 1774643615673,
+      "tag": "0043_peaceful_chat",
+      "breakpoints": true
     }
   ]
 }

src/db/schema/08_agent.ts

@@ -1,4 +1,4 @@
-import {boolean, pgTable, text, timestamp, uuid} from "drizzle-orm/pg-core";
+import {boolean, pgTable, text, timestamp, uuid, integer} from "drizzle-orm/pg-core";
 import {createSelectSchema} from "drizzle-zod";
 import {z} from "zod";
 import {Database, database} from "@/db/schema/07_database";
@@ -10,6 +10,7 @@ export const agent = pgTable("agents", {
     slug: text("slug").notNull().unique(),
     version: text("version"),
     name: text("name").notNull().notNull(),
+    healthErrorCount: integer("health_error_count"),
     description: text("description").notNull(),
     isArchived: boolean("is_archived").default(false),
     lastContact: timestamp("last_contact"),

src/db/schema/10_alert-policy.ts

@@ -6,7 +6,7 @@ import {database} from "@/db/schema/07_database";
 import {createSelectSchema} from "drizzle-zod";
 import {z} from "zod";
 
-export const eventKindEnum = pgEnum('event_kind', ['error_backup', 'error_restore', 'success_restore', 'success_backup', 'weekly_report']);
+export const eventKindEnum = pgEnum('event_kind', ['error_backup', 'error_restore', 'success_restore', 'success_backup', 'weekly_report', 'error_health_agent']);
 
 export const alertPolicy = pgTable('alert_policy', {
     id: uuid('id').defaultRandom().primaryKey(),

src/db/services/healthcheck.ts

@@ -45,7 +45,6 @@ export async function deleteHealthLogsOlderThan12h() {
 }
 
 
-
 //
 // export async function sendNotificationsHealthCheck(event: EventKind) {
 //
@@ -139,57 +138,66 @@ export async function deleteHealthLogsOlderThan12h() {
 //     return Promise.all(promises);
 // }
 //
-//
+
-//
+
-// export async function checkAgentsHealthError() {
+export async function checkAgentsHealthError() {
-//     const agents = await db.query.agent.findMany({
+    const agents = await db.query.agent.findMany({
-//         where: isNotNull(drizzleDb.schemas.agent.lastContact),
+        where: isNotNull(drizzleDb.schemas.agent.lastContact),
-//     });
+    });
-//
+
-//     const settings = await db.query.setting.findFirst({
+    const settings = await db.query.setting.findFirst({
-//         where: (fields, { eq }) => eq(fields.name, "system"),
+        where: (fields, {eq}) => eq(fields.name, "system"),
-//     });
+    });
-//
+
-//     if (!settings) {
+    if (!settings) {
-//         throw new Error("System settings not found");
+        throw new Error("System settings not found");
-//     }
+    }
-//
+
-//     const now = new Date();
+    if (!settings.defaultNotificationChannelId) {
-//
+        console.error("No default notification channel id found.");
-//     for (const agent of agents) {
+        return
-//         if (!agent.lastContact) continue;
+    }
-//
+
-//         const lastContactDate = new Date(agent.lastContact);
+    const now = new Date();
-//         const diffMinutes = (now.getTime() - lastContactDate.getTime()) / 1000 / 60;
+
-//
+    for (const agent of agents) {
-//         if (diffMinutes > 10) {
+        if (!agent.lastContact) continue;
-//             if ((agent.health_error_count ?? 0) < 3) {
+
-//                 await db.update(drizzleDb.schemas.agent)
+        const lastContactDate = new Date(agent.lastContact);
-//                     .set({
+        const diffMinutes = (now.getTime() - lastContactDate.getTime()) / 1000 / 60;
-//                         health_error_count: (agent.health_error_count ?? 0) + 1,
+
-//                     })
+        if (diffMinutes > 10) {
-//                     .where(drizzleDb.schemas.agent.id.eq(agent.id));
+            if ((agent.healthErrorCount ?? 0) < 3) {
-//
+
-//                 const payload: EventPayload = {
+                const newHealthErrorCount = (agent.healthErrorCount ?? 0) + 1
-//                     title: "Agent down",
+                await db.update(drizzleDb.schemas.agent)
-//                     message: `Agent ${agent.name} is down`,
+                    .set({
-//                     level: "critical",
+                        healthErrorCount: newHealthErrorCount,
-//                     event: "error_health_agent",
+                    })
-//                     data: {
+                    .where(eq(drizzleDb.schemas.agent.id, agent.id));
-//                         agent: agent.name,
+
-//                         id: agent.id,
+                const payload: EventPayload = {
-//                         error: "Agent is down",
+                    title: "Agent down",
-//                     },
+                    message: `Agent ${agent.name} is down, (notification number: ${newHealthErrorCount}/3)`,
-//                 };
+                    level: "critical",
-//
+                    event: "error_health_agent",
-//                 await dispatchNotification(
+                    data: {
-//                     payload,
+                        agent: agent.name,
-//                     undefined,
+                        id: agent.id,
-//                     settings.defaultNotificationChannelId,
+                        error: "Agent is down",
-//                     undefined
+                    },
-//                 );
+                };
-//             }
+
-//
+                console.log("[Agent Healthcheck] :", payload);
-//         }
+
-//     }
+                await dispatchNotification(
-// }
+                    payload,
+                    undefined,
+                    settings.defaultNotificationChannelId,
+                    undefined
+                );
+            }
+
+        }
+    }
+}

src/env.mjs

@@ -45,6 +45,12 @@ export const env = createEnv({
         process.env.NODE_ENV === "production" ? "0 * * * *" : "* * * * *",
       ),
 
+    HEALTHCHECK_CRON: z
+        .string()
+        .default(
+            process.env.NODE_ENV === "production" ? "0 * * * *" : "* * * * *",
+        ),
+
 
 
     AUTH_OIDC_ID: z.string().optional().default("oidc"),

src/features/notifications/helpers.ts

@@ -6,27 +6,6 @@ import {eq} from "drizzle-orm";
 import * as drizzleDb from "@/db";
 
 export async function sendNotificationsBackupRestore(database: DatabaseWith, event: EventKind) {
-    // if (!database.alertPolicies || database.alertPolicies.length === 0) {
-    //     return [];
-    // }
-    //
-    // const activePolicies = database.alertPolicies.filter(policy =>
-    //     policy.enabled && policy.eventKinds.includes(event)
-    // );
-    //
-    // const settings = await db.query.setting.findFirst({
-    //     where: eq(drizzleDb.schemas.setting.name, "system"),
-    //     with: {notificationChannel: true},
-    // });
-    //
-    // const defaultPolicy = settings?.notificationChannel
-    //     ? [{
-    //         id: null,
-    //         notificationChannelId: settings.notificationChannel.id,
-    //         enabled: settings.notificationChannel.enabled,
-    //     }]
-    //     : [];
-
 
     const settings = await db.query.setting.findFirst({
         where: eq(drizzleDb.schemas.setting.name, "system"),
@@ -38,6 +17,7 @@ export async function sendNotificationsBackupRestore(database: DatabaseWith, eve
             id: null,
             notificationChannelId: settings.notificationChannel.id,
             enabled: settings.notificationChannel.enabled,
+            eventKinds: ["error_backup" , "error_restore"]
         }]
         : [];
 
@@ -79,6 +59,7 @@ export async function sendNotificationsBackupRestore(database: DatabaseWith, eve
             success_backup: `Backup Notification`,
             success_restore: `Restore Notification`,
             weekly_report: `Weekly Report Notification`,
+            error_health_agent: "Health Agent Notification",
         };
 
         const payload: EventPayload = {

src/features/notifications/types.ts

@@ -18,4 +18,4 @@ export interface EventPayload {
     data?: Record<string, any>;
 }
 
-export type EventKind = ("error_backup" | "error_restore" | "success_restore" | "success_backup" | "weekly_report")
+export type EventKind = ("error_backup" | "error_restore" | "success_restore" | "success_backup" | "weekly_report" | "error_health_agent")

src/lib/tasks/index.ts

@@ -2,7 +2,7 @@ import cron from "node-cron";
 import {retentionCleanTask} from "@/lib/tasks/database";
 import {env} from "@/env.mjs";
 import {backupCleanTask} from "@/lib/tasks/cleaning";
-import {deleteHealthLogsOlderThan12h} from "@/db/services/healthcheck";
+import {checkAgentsHealthError, deleteHealthLogsOlderThan12h} from "@/db/services/healthcheck";
 
 export const retentionJob = cron.schedule(env.RETENTION_CRON, async () => {
     try {
@@ -30,3 +30,13 @@ export const cleaningHealthcheckLogsJob = cron.schedule(env.CLEANING_HEALTHCHECK
         console.error(`[CRON] Error:`, err);
     }
 });
+
+
+export const healthcheckAgentAndDatabaseJob = cron.schedule(env.HEALTHCHECK_CRON, async () => {
+    try {
+        console.log("Healthcheck Job : Starting task");
+        await checkAgentsHealthError();
+    } catch (err) {
+        console.error(`[CRON] Error:`, err);
+    }
+});

src/utils/init.ts

@@ -2,7 +2,7 @@ import { env } from "@/env.mjs";
 import { db, makeMigration } from "@/db";
 import { eq } from "drizzle-orm";
 import * as drizzleDb from "@/db";
-import {cleaningHealthcheckLogsJob, cleaningJob, retentionJob} from "@/lib/tasks";
+import {cleaningHealthcheckLogsJob, cleaningJob, healthcheckAgentAndDatabaseJob, retentionJob} from "@/lib/tasks";
 import { generateRSAKeys, getOrCreateMasterKey } from "@/utils/rsa-keys";
 import { StorageProviderKind } from "@/features/storages/types";
 
@@ -18,6 +18,7 @@ export async function init() {
   await setupCronJobs();
   await setupCleaningJobs();
   await setupCleaningHealthLogsJobs();
+  await setupHealthCheckJobs();
 
   if (
     (env.AUTH_GOOGLE_ID && env.AUTH_GOOGLE_SECRET) ||
@@ -46,6 +47,11 @@ async function setupCleaningHealthLogsJobs() {
   cleaningHealthcheckLogsJob.start();
   console.log("==== Cleaning Healthcheck Logs job started ====");
 }
+async function setupHealthCheckJobs() {
+  console.log("==== Setting up Healthcheck Jobs ====");
+  healthcheckAgentAndDatabaseJob.start();
+  console.log("==== Cleaning Healthcheck job started ====");
+}
 
 async function createSettingsIfNotExist() {
   await db.transaction(async (tx) => {