External/portabase
1
0
Fork 0
mirror of https://github.com/Portabase/portabase.git synced 2026-07-14 11:16:13 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat: added heath_error_count and the cron for health check agent errors

Browse Source
This commit is contained in:
Charles GTE
2026-03-27 22:13:32 +01:00
parent bded273f94
commit 595ccadfb3
14 changed files with 2674 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/components/wrappers/dashboard/agent/agent-card/agent-card.tsx
+5
View File
@@ -52,6 +52,11 @@ export const AgentCard = (props: agentCardProps) => {
<Badge variant="secondary" className="h-5 px-1.5 text-[10px] font-bold tracking-wider">
v{agent.version}
</Badge>
{agent.healthErrorCount && (
<Badge variant="destructive" className="h-5 px-1.5 text-[10px] font-bold tracking-wider">
down
</Badge>
)}
{isUpdateAvailable && (
<Tooltip>
<TooltipTrigger asChild>
src/components/wrappers/dashboard/agent/agent-content.tsx
+28 -21
View File
@@ -50,7 +50,8 @@ export const AgentContentPage = ({edgeKey, agent: initialAgent}: AgentContentPag
<div className="flex flex-col sm:flex-row sm:justify-between gap-6 ">
<Card className="w-full sm:w-auto flex-1 transition-all border-border/50 bg-card ">
<CardHeader className="flex flex-row items-center justify-between space-y-0 pb-2">
<CardTitle className="text-sm font-semibold uppercase tracking-wider text-muted-foreground">Databases</CardTitle>
<CardTitle
className="text-sm font-semibold uppercase tracking-wider text-muted-foreground">Databases</CardTitle>
<Server className="h-4 w-4 text-muted-foreground opacity-50"/>
</CardHeader>
<CardContent>
@@ -61,26 +62,32 @@ export const AgentContentPage = ({edgeKey, agent: initialAgent}: AgentContentPag
<Card className="w-full sm:w-auto flex-1 transition-all border-border/50 bg-card ">
<CardHeader className="flex flex-row items-center justify-between space-y-0 pb-2">
<CardTitle className="text-sm font-semibold uppercase tracking-wider text-muted-foreground">Last contact</CardTitle>
<CardTitle className="text-sm font-semibold uppercase tracking-wider text-muted-foreground">Last
contact</CardTitle>
<Server className="h-4 w-4 text-muted-foreground opacity-50"/>
</CardHeader>
<CardContent>
<div className="text-3xl font-bold tracking-tight">{formatDateLastContact(agent.lastContact)}</div>
<div
className="text-3xl font-bold tracking-tight">{formatDateLastContact(agent.lastContact)}</div>
<p className="text-xs text-muted-foreground mt-1">Status heartbeat</p>
</CardContent>
</Card>
</div>
<HealthCheckGraph logs={agentHealthLogs}/>
{agent.lastContact && (
<HealthCheckGraph logs={agentHealthLogs}/>
)}
<div className="space-y-6">
<Accordion type="single" collapsible defaultValue={!agent.lastContact ? "registration" : undefined}>
<AccordionItem value="registration" className="border rounded-xl px-6 bg-card shadow-sm overflow-hidden transition-all duration-300">
<AccordionItem value="registration"
className="border rounded-xl px-6 bg-card shadow-sm overflow-hidden transition-all duration-300">
<AccordionTrigger className="hover:no-underline py-4 group">
<div className="flex items-center gap-3">
<span className="text-xl font-bold tracking-tight">Registration & Setup</span>
{!agent.lastContact && (
<Badge variant="outline" className="bg-orange-500/10 text-orange-600 border-orange-500/20 animate-pulse">
<Badge variant="outline"
className="bg-orange-500/10 text-orange-600 border-orange-500/20 animate-pulse">
Action Required
</Badge>
)}
@@ -97,23 +104,23 @@ export const AgentContentPage = ({edgeKey, agent: initialAgent}: AgentContentPag
</div>
{agent.databases.length > 0 && (
<div className="space-y-6">
<div className="flex items-center justify-between px-1">
<div className="space-y-1">
<h2 className="text-2xl font-bold tracking-tight">Managed Databases</h2>
<p className="text-sm text-muted-foreground">
Resources currently connected to this agent.
</p>
<div className="space-y-6">
<div className="flex items-center justify-between px-1">
<div className="space-y-1">
<h2 className="text-2xl font-bold tracking-tight">Managed Databases</h2>
<p className="text-sm text-muted-foreground">
Resources currently connected to this agent.
</p>
</div>
</div>
<Separator className="opacity-50"/>
<CardsWithPagination
cardsPerPage={4}
numberOfColumns={2}
data={agent.databases}
cardItem={AgentDatabaseCard}
/>
</div>
<Separator className="opacity-50" />
<CardsWithPagination
cardsPerPage={4}
numberOfColumns={2}
data={agent.databases}
cardItem={AgentDatabaseCard}
/>
</div>
)}
</div>
)
src/db/migrations/0043_peaceful_chat.sql
+1
View File
@@ -0,0 +1 @@
ALTER TABLE "agents" ADD COLUMN "health_error_count" integer;
src/db/migrations/meta/0043_snapshot.json
+2538
View File
File diff suppressed because it is too large Load Diff
src/db/migrations/meta/_journal.json
+7
View File
@@ -302,6 +302,13 @@
"when": 1774472168308,
"tag": "0042_breezy_namora",
"breakpoints": true
},
{
"idx": 43,
"version": "7",
"when": 1774643615673,
"tag": "0043_peaceful_chat",
"breakpoints": true
}
]
}
src/db/schema/08_agent.ts
+2 -1
View File
@@ -1,4 +1,4 @@
import {boolean, pgTable, text, timestamp, uuid} from "drizzle-orm/pg-core";
import {boolean, pgTable, text, timestamp, uuid, integer} from "drizzle-orm/pg-core";
import {createSelectSchema} from "drizzle-zod";
import {z} from "zod";
import {Database, database} from "@/db/schema/07_database";
@@ -10,6 +10,7 @@ export const agent = pgTable("agents", {
slug: text("slug").notNull().unique(),
version: text("version"),
name: text("name").notNull().notNull(),
healthErrorCount: integer("health_error_count"),
description: text("description").notNull(),
isArchived: boolean("is_archived").default(false),
lastContact: timestamp("last_contact"),
src/db/schema/10_alert-policy.ts
+1 -1
View File
@@ -6,7 +6,7 @@ import {database} from "@/db/schema/07_database";
import {createSelectSchema} from "drizzle-zod";
import {z} from "zod";
export const eventKindEnum = pgEnum('event_kind', ['error_backup', 'error_restore', 'success_restore', 'success_backup', 'weekly_report']);
export const eventKindEnum = pgEnum('event_kind', ['error_backup', 'error_restore', 'success_restore', 'success_backup', 'weekly_report', 'error_health_agent']);
export const alertPolicy = pgTable('alert_policy', {
id: uuid('id').defaultRandom().primaryKey(),
src/db/services/healthcheck.ts
+63 -55
View File
@@ -45,7 +45,6 @@ export async function deleteHealthLogsOlderThan12h() {
}
//
// export async function sendNotificationsHealthCheck(event: EventKind) {
//
@@ -139,57 +138,66 @@ export async function deleteHealthLogsOlderThan12h() {
// return Promise.all(promises);
// }
//
//
//
// export async function checkAgentsHealthError() {
// const agents = await db.query.agent.findMany({
// where: isNotNull(drizzleDb.schemas.agent.lastContact),
// });
//
// const settings = await db.query.setting.findFirst({
// where: (fields, { eq }) => eq(fields.name, "system"),
// });
//
// if (!settings) {
// throw new Error("System settings not found");
// }
//
// const now = new Date();
//
// for (const agent of agents) {
// if (!agent.lastContact) continue;
//
// const lastContactDate = new Date(agent.lastContact);
// const diffMinutes = (now.getTime() - lastContactDate.getTime()) / 1000 / 60;
//
// if (diffMinutes > 10) {
// if ((agent.health_error_count ?? 0) < 3) {
// await db.update(drizzleDb.schemas.agent)
// .set({
// health_error_count: (agent.health_error_count ?? 0) + 1,
// })
// .where(drizzleDb.schemas.agent.id.eq(agent.id));
//
// const payload: EventPayload = {
// title: "Agent down",
// message: `Agent ${agent.name} is down`,
// level: "critical",
// event: "error_health_agent",
// data: {
// agent: agent.name,
// id: agent.id,
// error: "Agent is down",
// },
// };
//
// await dispatchNotification(
// payload,
// undefined,
// settings.defaultNotificationChannelId,
// undefined
// );
// }
//
// }
// }
// }
export async function checkAgentsHealthError() {
const agents = await db.query.agent.findMany({
where: isNotNull(drizzleDb.schemas.agent.lastContact),
});
const settings = await db.query.setting.findFirst({
where: (fields, {eq}) => eq(fields.name, "system"),
});
if (!settings) {
throw new Error("System settings not found");
}
if (!settings.defaultNotificationChannelId) {
console.error("No default notification channel id found.");
return
}
const now = new Date();
for (const agent of agents) {
if (!agent.lastContact) continue;
const lastContactDate = new Date(agent.lastContact);
const diffMinutes = (now.getTime() - lastContactDate.getTime()) / 1000 / 60;
if (diffMinutes > 10) {
if ((agent.healthErrorCount ?? 0) < 3) {
const newHealthErrorCount = (agent.healthErrorCount ?? 0) + 1
await db.update(drizzleDb.schemas.agent)
.set({
healthErrorCount: newHealthErrorCount,
})
.where(eq(drizzleDb.schemas.agent.id, agent.id));
const payload: EventPayload = {
title: "Agent down",
message: `Agent ${agent.name} is down, (notification number: ${newHealthErrorCount}/3)`,
level: "critical",
event: "error_health_agent",
data: {
agent: agent.name,
id: agent.id,
error: "Agent is down",
},
};
console.log("[Agent Healthcheck] :", payload);
await dispatchNotification(
payload,
undefined,
settings.defaultNotificationChannelId,
undefined
);
}
}
}
}
src/env.mjs
+6
View File
@@ -45,6 +45,12 @@ export const env = createEnv({
process.env.NODE_ENV === "production" ? "0 * * * *" : "* * * * *",
),
HEALTHCHECK_CRON: z
.string()
.default(
process.env.NODE_ENV === "production" ? "0 * * * *" : "* * * * *",
),
AUTH_OIDC_ID: z.string().optional().default("oidc"),
src/features/notifications/helpers.ts
+2 -21
View File
@@ -6,27 +6,6 @@ import {eq} from "drizzle-orm";
import * as drizzleDb from "@/db";
export async function sendNotificationsBackupRestore(database: DatabaseWith, event: EventKind) {
// if (!database.alertPolicies || database.alertPolicies.length === 0) {
// return [];
// }
//
// const activePolicies = database.alertPolicies.filter(policy =>
// policy.enabled && policy.eventKinds.includes(event)
// );
//
// const settings = await db.query.setting.findFirst({
// where: eq(drizzleDb.schemas.setting.name, "system"),
// with: {notificationChannel: true},
// });
//
// const defaultPolicy = settings?.notificationChannel
// ? [{
// id: null,
// notificationChannelId: settings.notificationChannel.id,
// enabled: settings.notificationChannel.enabled,
// }]
// : [];
const settings = await db.query.setting.findFirst({
where: eq(drizzleDb.schemas.setting.name, "system"),
@@ -38,6 +17,7 @@ export async function sendNotificationsBackupRestore(database: DatabaseWith, eve
id: null,
notificationChannelId: settings.notificationChannel.id,
enabled: settings.notificationChannel.enabled,
eventKinds: ["error_backup" , "error_restore"]
}]
: [];
@@ -79,6 +59,7 @@ export async function sendNotificationsBackupRestore(database: DatabaseWith, eve
success_backup: `Backup Notification`,
success_restore: `Restore Notification`,
weekly_report: `Weekly Report Notification`,
error_health_agent: "Health Agent Notification",
};
const payload: EventPayload = {
src/features/notifications/types.ts
+1 -1
View File
@@ -18,4 +18,4 @@ export interface EventPayload {
data?: Record<string, any>;
}
export type EventKind = ("error_backup" | "error_restore" | "success_restore" | "success_backup" | "weekly_report")
export type EventKind = ("error_backup" | "error_restore" | "success_restore" | "success_backup" | "weekly_report" | "error_health_agent")
src/lib/tasks/index.ts
+11 -1
View File
@@ -2,7 +2,7 @@ import cron from "node-cron";
import {retentionCleanTask} from "@/lib/tasks/database";
import {env} from "@/env.mjs";
import {backupCleanTask} from "@/lib/tasks/cleaning";
import {deleteHealthLogsOlderThan12h} from "@/db/services/healthcheck";
import {checkAgentsHealthError, deleteHealthLogsOlderThan12h} from "@/db/services/healthcheck";
export const retentionJob = cron.schedule(env.RETENTION_CRON, async () => {
try {
@@ -29,4 +29,14 @@ export const cleaningHealthcheckLogsJob = cron.schedule(env.CLEANING_HEALTHCHECK
} catch (err) {
console.error(`[CRON] Error:`, err);
}
});
export const healthcheckAgentAndDatabaseJob = cron.schedule(env.HEALTHCHECK_CRON, async () => {
try {
console.log("Healthcheck Job : Starting task");
await checkAgentsHealthError();
} catch (err) {
console.error(`[CRON] Error:`, err);
}
});
src/utils/init.ts
+7 -1
View File
@@ -2,7 +2,7 @@ import { env } from "@/env.mjs";
import { db, makeMigration } from "@/db";
import { eq } from "drizzle-orm";
import * as drizzleDb from "@/db";
import {cleaningHealthcheckLogsJob, cleaningJob, retentionJob} from "@/lib/tasks";
import {cleaningHealthcheckLogsJob, cleaningJob, healthcheckAgentAndDatabaseJob, retentionJob} from "@/lib/tasks";
import { generateRSAKeys, getOrCreateMasterKey } from "@/utils/rsa-keys";
import { StorageProviderKind } from "@/features/storages/types";
@@ -18,6 +18,7 @@ export async function init() {
await setupCronJobs();
await setupCleaningJobs();
await setupCleaningHealthLogsJobs();
await setupHealthCheckJobs();
if (
(env.AUTH_GOOGLE_ID && env.AUTH_GOOGLE_SECRET) ||
@@ -46,6 +47,11 @@ async function setupCleaningHealthLogsJobs() {
cleaningHealthcheckLogsJob.start();
console.log("==== Cleaning Healthcheck Logs job started ====");
}
async function setupHealthCheckJobs() {
console.log("==== Setting up Healthcheck Jobs ====");
healthcheckAgentAndDatabaseJob.start();
console.log("==== Cleaning Healthcheck job started ====");
}
async function createSettingsIfNotExist() {
await db.transaction(async (tx) => {