fix: stale-ongoing-backups (#352)

* fix: add STALE_BACKUP_THRESHOLD_HOURS env (#351)

* fix: fail stale ongoing backups and notify (#351)

* fix: prefer waiting backup over stale ongoing in dispatch (#351)

* fix: prefer waiting over ongoing by status priority in dispatch (#351)

---------

Co-authored-by: charles-gauthereau <charles.gauthereau@soluce-technologies.com>

src/env.mjs

@@ -64,6 +64,7 @@ export const env = createEnv({
                 process.env.NODE_ENV === "production" ? "0 * * * *" : "* * * * *",
             ),
 
+        STALE_BACKUP_THRESHOLD_HOURS: z.coerce.number().default(6),
 
         AUTH_OIDC_ID: z.string().optional().default("oidc"),
         AUTH_OIDC_TITLE: z.string().optional(),
@@ -147,6 +148,7 @@ export const env = createEnv({
 
         RETENTION_CRON: process.env.RETENTION_CRON,
         CLEANING_HEALTHCHECK_LOGS_CRON: process.env.CLEANING_HEALTHCHECK_LOGS_CRON,
+        STALE_BACKUP_THRESHOLD_HOURS: process.env.STALE_BACKUP_THRESHOLD_HOURS,
 
         AUTH_OIDC_ID: process.env.AUTH_OIDC_ID,
         AUTH_OIDC_TITLE: process.env.AUTH_OIDC_TITLE,

src/lib/tasks/cleaning/index.ts

@@ -1,8 +1,10 @@
 import {db} from "@/db";
-import {and, eq, isNotNull, isNull} from "drizzle-orm";
+import {and, eq, isNotNull, isNull, lt} from "drizzle-orm";
 import * as drizzleDb from "@/db";
 import {withUpdatedAt} from "@/db/utils";
 import {logger} from "@/lib/logger";
+import {env} from "@/env.mjs";
+import {sendNotificationsBackupRestore} from "@/features/notifications/utils/notifications.helpers";
 
 const log = logger.child({module: "tasks/cleaning"});
 
@@ -23,6 +25,36 @@ export const backupCleanTask = async () => {
                 .where(eq(drizzleDb.schemas.backup.id, backup.id));
         }
 
+        const staleCutoff = new Date(Date.now() - env.STALE_BACKUP_THRESHOLD_HOURS * 60 * 60 * 1000);
+
+        const staleOngoingBackups = await db.query.backup.findMany({
+            where: and(
+                isNull(drizzleDb.schemas.backup.deletedAt),
+                eq(drizzleDb.schemas.backup.status, "ongoing"),
+                lt(drizzleDb.schemas.backup.createdAt, staleCutoff)
+            )
+        });
+        log.debug(`Stale ongoing backups to fail: ${staleOngoingBackups.length}`);
+
+        for (const backup of staleOngoingBackups) {
+            await db.update(drizzleDb.schemas.backup).set(withUpdatedAt({
+                status: "failed",
+            }))
+                .where(eq(drizzleDb.schemas.backup.id, backup.id));
+
+            try {
+                const database = await db.query.database.findFirst({
+                    where: eq(drizzleDb.schemas.database.id, backup.databaseId),
+                    with: {alertPolicies: true},
+                });
+                if (database) {
+                    await sendNotificationsBackupRestore(database, "error_backup");
+                }
+            } catch (notifyError) {
+                log.error({name: "backupCleanTask", error: notifyError}, "Stale backup notification failed");
+            }
+        }
+
         const failedBackups = await db.query.backup.findMany({
             where: and(
                 isNull(drizzleDb.schemas.backup.deletedAt),