External/portabase
1
0
Fork 0
mirror of https://github.com/Portabase/portabase.git synced 2026-07-14 11:16:13 +02:00
Code Issues Packages Projects Releases Wiki Activity

fix: database health cron

Browse Source
This commit is contained in:
Charles GTE
2026-03-28 15:10:46 +01:00
parent 7aecd911c0
commit 0262a1c176
16 changed files with 5270 additions and 117 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/(customer)/dashboard/(organization)/projects/[projectId]/database/[databaseId]/page.tsx
+4
View File
@@ -8,6 +8,7 @@ import {getOrganizationProjectDatabases} from "@/lib/services";
import {getActiveMember, getOrganization} from "@/lib/auth/auth";
import {BackupModalProvider} from "@/components/wrappers/dashboard/database/backup/backup-modal-context";
import {DatabaseContent} from "@/components/wrappers/dashboard/projects/database/database-content";
import {getHealthLast12hLogs} from "@/db/services/healthcheck";
export default async function RoutePage(props: PageParams<{
projectId: string;
@@ -83,6 +84,8 @@ export default async function RoutePage(props: PageParams<{
notFound();
}
const databaseHealthLogs = dbItem ? await getHealthLast12hLogs({ id: dbItem.id }) : []
const successRate = totalBackups > 0 ? (successfulBackups / totalBackups) * 100 : null;
@@ -95,6 +98,7 @@ export default async function RoutePage(props: PageParams<{
activeMember={activeMember}
settings={settings}
database={dbItem}
databaseHealthLogs={databaseHealthLogs}
isAlreadyRestore={isAlreadyRestore}
restorations={restorations}
backups={backups}
app/api/agent/[agentId]/status/helpers.ts
+1 -1
View File
@@ -96,7 +96,7 @@ export async function handleDatabases(body: Body, agent: Agent, lastContact: Dat
.set(withUpdatedAt({
name: db.name,
agentId: agent.id,
lastContact: db.pingStatus ? lastContact : null,
lastContact: db.pingStatus ? lastContact : existingDatabase.lastContact,
}))
.where(eq(drizzleDb.schemas.database.id, existingDatabase.id))
.returning();
src/components/wrappers/dashboard/database/backup/actions/get-data.action.ts
+8 -1
View File
@@ -7,6 +7,7 @@ import * as drizzleDb from "@/db";
import {BackupWith, Restoration} from "@/db/schema/07_database";
import {getOrganizationChannels} from "@/db/services/notification-channel";
import {getOrganizationStorageChannels} from "@/db/services/storage-channel";
import {getHealthLast12hLogs} from "@/db/services/healthcheck";
export const getDatabaseDataAction = userAction
.schema(
@@ -50,7 +51,9 @@ export const getDatabaseDataAction = userAction
const successfulBackups = backups.filter(b => b.status === "success").length;
const successRate = totalBackups > 0 ? (successfulBackups / totalBackups) * 100 : null;
// @ts-ignore
let activeOrganizationChannels = [];
// @ts-ignore
let activeOrganizationStorageChannels = [];
if (database?.project?.organizationId) {
@@ -61,16 +64,20 @@ export const getDatabaseDataAction = userAction
activeOrganizationStorageChannels = organizationStorageChannels.filter(channel => channel.enabled);
}
return {
database,
backups,
restorations,
// @ts-ignore
activeOrganizationChannels,
// @ts-ignore
activeOrganizationStorageChannels,
stats: {
totalBackups,
availableBackups,
successRate
}
},
health: database ? await getHealthLast12hLogs({ id: database.id }) : []
};
});
src/components/wrappers/dashboard/database/channels-policy/policy.schema.ts
+11 -11
View File
@@ -3,7 +3,7 @@ import {z} from "zod";
export const PolicySchema = z.object({
channelId: z.string().min(1, "Please select channel"),
eventKinds: z.array(z.enum([
'error_backup', 'error_restore', 'success_restore', 'success_backup', 'weekly_report'
'error_backup', 'error_restore', 'success_restore', 'success_backup', 'weekly_report', 'error_health_database'
]))
.optional(),
enabled: z.boolean().default(true),
@@ -17,16 +17,16 @@ export const PoliciesSchema = z.object({
export type PoliciesType = z.infer<typeof PoliciesSchema>;
export type PolicyType = z.infer<typeof PolicySchema>;
export const EVENT_KIND_OPTIONS = [
{label: "Error Backup", value: "error_backup"},
{label: "Error Restore", value: "error_restore"},
{label: "Success Restore", value: "success_restore"},
{label: "Success Backup", value: "success_backup"},
// {label: "Weekly Report", value: "weekly_report"},
];
export const EVENT_KIND_BACKUP_ONLY_OPTIONS = [
{label: "Error Backup", value: "error_backup"},
{label: "Success Backup", value: "success_backup"},
];
{label: "Health Ping Fail", value: "error_health_database"},
];
export const EVENT_KIND_OPTIONS = [
...EVENT_KIND_BACKUP_ONLY_OPTIONS,
{label: "Error Restore", value: "error_restore"},
{label: "Success Restore", value: "success_restore"},
// {label: "Weekly Report", value: "weekly_report"},
];
src/components/wrappers/dashboard/database/health/health-modal.tsx
+43
View File
@@ -0,0 +1,43 @@
"use client"
import {useState} from "react";
import {Button} from "@/components/ui/button";
import {DatabaseWith} from "@/db/schema/07_database";
import {HeartPulse} from "lucide-react";
import {
Sheet,
SheetContent,
SheetTitle,
SheetTrigger
} from "@/components/ui/sheet";
import {HealthcheckLog} from "@/db/schema/15_healthcheck-log";
import {HealthCheckGraph} from "@/components/wrappers/dashboard/health/heath-grid";
type HealthModalProps = {
database: DatabaseWith,
healthLogs: HealthcheckLog[]
}
export const HealthModal = ({database, healthLogs}: HealthModalProps) => {
const [open, setOpen] = useState(false);
return (
<Sheet open={open} onOpenChange={setOpen}>
<SheetTrigger asChild>
<Button variant="outline" onClick={() => setOpen(true)}>
<HeartPulse/>
</Button>
</SheetTrigger>
<SheetContent side="bottom">
<div className="flex pl-5 pt-4">
<SheetTitle>
Database Health Status
</SheetTitle>
</div>
<div className="px-4 pb-5">
<HealthCheckGraph logs={healthLogs} />
</div>
</SheetContent>
</Sheet>
)
}
src/components/wrappers/dashboard/projects/database/database-content.tsx
+8 -1
View File
@@ -20,6 +20,8 @@ import { ChannelPoliciesModal } from "@/components/wrappers/dashboard/database/c
import { HardDrive, Megaphone } from "lucide-react";
import { ImportModal } from "@/components/wrappers/dashboard/database/import/import-modal";
import { BackupButton } from "@/components/wrappers/dashboard/backup/backup-button/backup-button";
import {HealthModal} from "@/components/wrappers/dashboard/database/health/health-modal";
import {HealthcheckLog} from "@/db/schema/15_healthcheck-log";
export type DatabaseContentProps = {
settings: Setting;
@@ -34,6 +36,7 @@ export type DatabaseContentProps = {
organizationId: string;
activeOrganizationChannels: any[];
activeOrganizationStorageChannels: any[];
databaseHealthLogs: HealthcheckLog[]
};
export const DatabaseContent = (props: DatabaseContentProps) => {
@@ -64,6 +67,7 @@ export const DatabaseContent = (props: DatabaseContentProps) => {
availableBackups: props.availableBackups,
successRate: props.successRate,
},
health: props.databaseHealthLogs
},
staleTime: 0,
gcTime: 0,
@@ -118,10 +122,13 @@ export const DatabaseContent = (props: DatabaseContentProps) => {
organizationId={props.organizationId}
/>
<ImportModal database={database} />
<HealthModal database={database} healthLogs={data?.health ?? []}/>
</div>
<div className="flex items-center gap-2">
<BackupButton
disable={isAlreadyBackup}
disable={isAlreadyBackup || !database.lastContact}
databaseId={database.id}
/>
</div>
src/db/migrations/0045_needy_martin_li.sql
+1
View File
@@ -0,0 +1 @@
ALTER TYPE "public"."event_kind" ADD VALUE 'error_health_database';
src/db/migrations/0046_mysterious_menace.sql
+1
View File
@@ -0,0 +1 @@
ALTER TABLE "databases" ADD COLUMN "health_error_count" integer;
src/db/migrations/meta/0045_snapshot.json
+2540
View File
File diff suppressed because it is too large Load Diff
src/db/migrations/meta/0046_snapshot.json
+2546
View File
File diff suppressed because it is too large Load Diff
src/db/migrations/meta/_journal.json
+14
View File
@@ -316,6 +316,20 @@
"when": 1774648872657,
"tag": "0044_steep_wiccan",
"breakpoints": true
},
{
"idx": 45,
"version": "7",
"when": 1774696680244,
"tag": "0045_needy_martin_li",
"breakpoints": true
},
{
"idx": 46,
"version": "7",
"when": 1774706071024,
"tag": "0046_mysterious_menace",
"breakpoints": true
}
]
}
src/db/schema/07_database.ts
+1
View File
@@ -19,6 +19,7 @@ export const database = pgTable("databases", {
backupPolicy: text("backup_policy"),
isWaitingForBackup: boolean("is_waiting_for_backup").default(false).notNull(),
backupToRestore: text("backup_to_restore"),
healthErrorCount: integer("health_error_count"),
agentId: uuid("agent_id")
.notNull()
.references(() => agent.id, {onDelete: "cascade"}),
src/db/schema/10_alert-policy.ts
+1 -1
View File
@@ -6,7 +6,7 @@ import {database} from "@/db/schema/07_database";
import {createSelectSchema} from "drizzle-zod";
import {z} from "zod";
export const eventKindEnum = pgEnum('event_kind', ['error_backup', 'error_restore', 'success_restore', 'success_backup', 'weekly_report', 'error_health_agent']);
export const eventKindEnum = pgEnum('event_kind', ['error_backup', 'error_restore', 'success_restore', 'success_backup', 'weekly_report', 'error_health_agent', 'error_health_database']);
export const alertPolicy = pgTable('alert_policy', {
id: uuid('id').defaultRandom().primaryKey(),
src/db/services/healthcheck.ts
+84 -100
View File
@@ -1,10 +1,8 @@
import {db} from "@/db";
import * as drizzleDb from "@/db";
import {and, eq, gte, isNotNull, lt} from "drizzle-orm";
import {enforceRetention} from "@/lib/tasks/database";
import {dispatchNotification} from "@/features/notifications/dispatch";
import {EventKind, EventPayload} from "@/features/notifications/types";
import {DatabaseWith} from "@/db/schema/07_database";
import {EventPayload} from "@/features/notifications/types";
export async function getHealthLast12hLogs({id}: { id: string }) {
const now = new Date()
@@ -44,102 +42,6 @@ export async function deleteHealthLogsOlderThan12h() {
return logsToDelete.length
}
//
// export async function sendNotificationsHealthCheck(event: EventKind) {
//
//
//
// const date = new Date();
// let level: "info" | "critical" = "info";
// let message = "";
// let error: string | null = null;
//
// switch (event) {
// case "error_backup":
// case "error_restore":
// level = "critical";
// message = `An error occurred during ${event.includes("backup") ? "backup" : "restore"} on ${date.toISOString()}.`;
// error = "Check database connection or agent";
// break;
// case "success_backup":
// case "success_restore":
// level = "info";
// message = `${event.includes("backup") ? "Backup" : "Restore"} completed successfully at ${date.toISOString()}.`;
// break;
// case "weekly_report":
// level = "info";
// message = `Weekly report generated at ${date.toISOString()}.`;
// break;
// }
//
//
//
//
//
//
//
//
//
//
// const activePolicies = database.alertPolicies.filter(policy =>
// policy.enabled && policy.eventKinds.includes(event)
// );
//
// const promises = activePolicies.map(alertPolicy => {
// const date = new Date();
// let level: "info" | "critical" = "info";
// let message = "";
// let error: string | null = null;
//
// switch (event) {
// case "error_backup":
// case "error_restore":
// level = "critical";
// message = `An error occurred during ${event.includes("backup") ? "backup" : "restore"} on ${date.toISOString()}.`;
// error = "Check database connection or agent";
// break;
// case "success_backup":
// case "success_restore":
// level = "info";
// message = `${event.includes("backup") ? "Backup" : "Restore"} completed successfully at ${date.toISOString()}.`;
// break;
// case "weekly_report":
// level = "info";
// message = `Weekly report generated at ${date.toISOString()}.`;
// break;
// }
//
// const titleMap: Record<EventKind, string> = {
// error_backup: `Backup Notification`,
// error_restore: `Restore Notification`,
// success_backup: `Backup Notification`,
// success_restore: `Restore Notification`,
// weekly_report: `Weekly Report Notification`,
// };
//
// const payload: EventPayload = {
// title: titleMap[event],
// message,
// level: level,
// event: event,
// data: {
// host: database.name,
// id: database.id,
// agentDatabaseId: database.agentDatabaseId,
// error,
// },
// };
//
// return dispatchNotification(payload, alertPolicy.id, undefined, undefined);
// });
//
//
// return Promise.all(promises);
// }
//
export async function checkAgentsHealthError() {
const agents = await db.query.agent.findMany({
where: isNotNull(drizzleDb.schemas.agent.lastContact),
@@ -200,4 +102,86 @@ export async function checkAgentsHealthError() {
}
}
}
}
export async function checkDatabasesHealthError() {
const databases = await db.query.database.findMany({
where: isNotNull(drizzleDb.schemas.database.lastContact),
with: {
agent: true,
alertPolicies: true
}
})
const now = new Date();
for (const database of databases) {
if (!database.lastContact) continue;
const lastContactDate = new Date(database.lastContact);
const diffMinutes = (now.getTime() - lastContactDate.getTime()) / 1000 / 60;
if (diffMinutes > 10) {
if ((database.healthErrorCount ?? 0) < 3) {
const newHealthErrorCount = (database.healthErrorCount ?? 0) + 1
await db.update(drizzleDb.schemas.database)
.set({
healthErrorCount: newHealthErrorCount,
})
.where(eq(drizzleDb.schemas.database.id, database.id));
const settings = await db.query.setting.findFirst({
where: eq(drizzleDb.schemas.setting.name, "system"),
with: { notificationChannel: true },
});
const defaultPolicy = settings?.notificationChannel
? [{
id: null,
notificationChannelId: settings.notificationChannel.id,
enabled: settings.notificationChannel.enabled,
eventKinds: ["error_health_database"]
}]
: [];
const policiesToUse = (database.alertPolicies && database.alertPolicies.length > 0)
? database.alertPolicies.filter(policy => policy.enabled && policy.eventKinds.includes("error_health_database"))
: defaultPolicy;
if (!policiesToUse || policiesToUse.length === 0) {
continue
}
const promises = policiesToUse.map(alertPolicy => {
const payload: EventPayload = {
title: "Database down",
message: `Database ${database.name} is down, (notification number: ${newHealthErrorCount}/3)`,
level: "critical",
event: "error_health_database",
data: {
agent: database.name,
id: database.id,
error: "Database is down",
},
};
console.log("[Database Healthcheck] :", payload);
return dispatchNotification(payload, alertPolicy.id == null ? undefined : alertPolicy.id, alertPolicy.id ? undefined : alertPolicy.notificationChannelId, undefined);
});
await Promise.all(promises);
}
}
}
}
src/features/notifications/types.ts
+1 -1
View File
@@ -18,4 +18,4 @@ export interface EventPayload {
data?: Record<string, any>;
}
export type EventKind = ("error_backup" | "error_restore" | "success_restore" | "success_backup" | "weekly_report" | "error_health_agent")
export type EventKind = ("error_backup" | "error_restore" | "success_restore" | "success_backup" | "weekly_report" | "error_health_agent" | "error_health_database")
src/lib/tasks/index.ts
+6 -1
View File
@@ -2,7 +2,11 @@ import cron from "node-cron";
import {retentionCleanTask} from "@/lib/tasks/database";
import {env} from "@/env.mjs";
import {backupCleanTask} from "@/lib/tasks/cleaning";
import {checkAgentsHealthError, deleteHealthLogsOlderThan12h} from "@/db/services/healthcheck";
import {
checkAgentsHealthError,
checkDatabasesHealthError,
deleteHealthLogsOlderThan12h
} from "@/db/services/healthcheck";
export const retentionJob = cron.schedule(env.RETENTION_CRON, async () => {
try {
@@ -36,6 +40,7 @@ export const healthcheckAgentAndDatabaseJob = cron.schedule(env.HEALTHCHECK_CRON
try {
console.log("Healthcheck Job : Starting task");
await checkAgentsHealthError();
await checkDatabasesHealthError()
} catch (err) {
console.error(`[CRON] Error:`, err);
}