fix: log job when storage file missing for restoration

app/api/agent/[agentId]/status/route.ts

@@ -1,8 +1,8 @@
 import { NextResponse } from "next/server";
-import { handleDatabases } from "./helpers";
+import { handleDatabases } from "@/features/agents/utils/status/status.helpers";
+import { Body } from "@/features/agents/types";
 import * as drizzleDb from "@/db";
 import { db } from "@/db";
-import { EDbmsSchema } from "@/db/schema/types";
 import { and, eq } from "drizzle-orm";
 import { withUpdatedAt } from "@/db/utils";
 import { logger } from "@/lib/logger";
@@ -10,18 +10,6 @@ import { isUUID } from "@/utils/text";
 
 const log = logger.child({ module: "api/agent/status/route" });
 
-export type databaseAgent = {
-  name: string;
-  dbms: EDbmsSchema;
-  generatedId: string;
-  pingStatus: boolean;
-};
-
-export type Body = {
-  version: string;
-  databases: databaseAgent[];
-};
-
 export async function POST(
   request: Request,
   { params }: { params: Promise<{ agentId: string }> },

src/features/agents/types/index.ts

@@ -0,0 +1,13 @@
+import {EDbmsSchema} from "@/db/schema/types";
+
+export type databaseAgent = {
+    name: string;
+    dbms: EDbmsSchema;
+    generatedId: string;
+    pingStatus: boolean;
+};
+
+export type Body = {
+    version: string;
+    databases: databaseAgent[];
+};

src/features/agents/utils/status/restoration.helpers.ts

@@ -0,0 +1,37 @@
+import * as drizzleDb from "@/db";
+import {db as dbClient} from "@/db";
+import {eq} from "drizzle-orm";
+import {logger} from "@/lib/logger";
+import {sendNotificationsBackupRestore} from "@/features/notifications/utils/notifications.helpers";
+
+const log = logger.child({module: "api/agent/status/restoration"});
+
+export async function handleFailedRestoration(restorationId: string, databaseId: string, reason: string): Promise<void> {
+    try {
+        await dbClient
+            .insert(drizzleDb.schemas.jobLog)
+            .values({
+                backupId: null,
+                restorationId,
+                loggedAt: new Date(),
+                entryType: "log",
+                level: "error",
+                message: reason,
+                command: null,
+                output: null,
+                exitCode: null,
+                durationMs: null,
+            });
+
+        const databaseWithPolicies = await dbClient.query.database.findFirst({
+            where: eq(drizzleDb.schemas.database.id, databaseId),
+            with: {alertPolicies: true},
+        });
+
+        if (databaseWithPolicies) {
+            await sendNotificationsBackupRestore(databaseWithPolicies, "error_restore");
+        }
+    } catch (err) {
+        log.error({error: err, name: "handleFailedRestoration"}, "Failed to record restoration failure job log / notification");
+    }
+}

src/features/agents/utils/status/status.helpers.ts

@@ -1,9 +1,9 @@
 import {NextResponse} from "next/server";
-import {Body} from "./route";
+import {Body} from "@/features/agents/types";
 import {Agent} from "@/db/schema/08_agent";
 import {DatabaseWith} from "@/db/schema/07_database";
 import * as drizzleDb from "@/db";
-import {db, db as dbClient} from "@/db";
+import {db as dbClient} from "@/db";
 import {and, eq, inArray, desc, sql} from "drizzle-orm";
 import {dbmsEnumSchema, EDbmsSchema} from "@/db/schema/types";
 import {withUpdatedAt} from "@/db/utils";
@@ -13,7 +13,9 @@ import {isUUID} from "@/utils/text";
 import {StorageInput} from "@/features/storages/types";
 import {dispatchStorage} from "@/features/storages/utils/storages.dispatch";
 import {getMasterServerKeyContent} from "@/features/agents/actions/keys.action";
-import {encryptStorages, isAgentVersionAtLeast, MIN_AGENT_VERSION_STORAGE_ENC} from "@/utils/status-crypto";
+import {getDatabaseStorageChannels, PingDatabaseStorageChannels} from "./storage-channels.helpers";
+import {applyStorageEncryption} from "./storage-encryption.helpers";
+import {handleFailedRestoration} from "./restoration.helpers";
 
 const log = logger.child({module: "api/agent/status/helpers"});
 
@@ -202,6 +204,9 @@ export async function handleDatabases(body: Body, agent: Agent, lastContact: Dat
 
                         const errorMessage = "Failed to get backup URL";
                         log.error({error: errorMessage, name: "handleDatabases"}, "Restoration failed");
+
+                        await handleFailedRestoration(restoration.id, databaseUpdated.id, errorMessage);
+
                         continue;
                     }
                 } catch (err) {
@@ -210,6 +215,7 @@ export async function handleDatabases(body: Body, agent: Agent, lastContact: Dat
                         .update(drizzleDb.schemas.restoration)
                         .set(withUpdatedAt({status: "failed"}))
                         .where(eq(drizzleDb.schemas.restoration.id, restoration.id));
+                    await handleFailedRestoration(restoration.id, databaseUpdated.id, "Restoration crashed unexpectedly");
                     continue;
                 }
 
@@ -226,101 +232,3 @@ export async function handleDatabases(body: Body, agent: Agent, lastContact: Dat
     }
     return databasesResponse;
 }
-
-
-type PingDatabaseStorageChannels = {
-    id: string;
-    config: any
-    provider: string
-}
-
-async function getDatabaseStorageChannels(databaseId: string): Promise<PingDatabaseStorageChannels[]> {
-
-    const database = await db.query.database.findFirst({
-        where: eq(drizzleDb.schemas.database.id, databaseId),
-        with: {
-            project: true,
-            retentionPolicy: true,
-            alertPolicies: true,
-            storagePolicies: true
-        }
-    });
-
-    if (!database) {
-        return []
-    }
-
-    const settings = await db.query.setting.findFirst({
-        where: eq(drizzleDb.schemas.setting.name, "system"),
-        with: {storageChannel: true},
-    });
-
-    const defaultStorageChannel: PingDatabaseStorageChannels[] = settings?.storageChannel
-        ? [{
-            id: settings.storageChannel.id,
-            provider: settings.storageChannel.provider,
-            config: settings.storageChannel.config,
-        }]
-        : [];
-
-
-    const enabledDatabaseStorageChannels = await Promise.all(
-        (database.storagePolicies ?? [])
-            .filter(p => p.enabled)
-            .map(async policy => {
-                const storageChannel = await db.query.storageChannel.findFirst({
-                    where: eq(drizzleDb.schemas.storageChannel.id, policy.storageChannelId),
-                });
-
-                if (!storageChannel) return null;
-
-                return {
-                    id: storageChannel.id,
-                    config: storageChannel.config,
-                    provider: storageChannel.provider,
-                } as PingDatabaseStorageChannels;
-            })
-    );
-
-    const filteredChannels: PingDatabaseStorageChannels[] = enabledDatabaseStorageChannels.filter(
-        (c): c is PingDatabaseStorageChannels => c !== null
-    );
-
-    return filteredChannels.length > 0 ? filteredChannels : defaultStorageChannel;
-}
-
-function applyStorageEncryption(
-    entry: Record<string, any>,
-    version: string | undefined,
-    masterKey: Buffer | null,
-    agentId: string,
-): void {
-    if (!masterKey) return;
-    if (!Array.isArray(entry.storages) || entry.storages.length === 0) return;
-    if (!isAgentVersionAtLeast(version, MIN_AGENT_VERSION_STORAGE_ENC)) {
-        log.warn(
-            {
-                name: "applyStorageEncryption",
-                agentId,
-                agentVersion: version ?? "unknown",
-                requiredVersion: MIN_AGENT_VERSION_STORAGE_ENC,
-            },
-            `\n============================================================\n` +
-            `  ⚠️  OUTDATED AGENT — STORAGE CREDENTIALS SENT UNENCRYPTED\n` +
-            `  Agent ${agentId} reports v${version ?? "unknown"} (< required v${MIN_AGENT_VERSION_STORAGE_ENC}).\n` +
-            `  Update this agent to v${MIN_AGENT_VERSION_STORAGE_ENC}+ to encrypt storage credentials in transit.\n` +
-            `============================================================`,
-        );
-        return;
-    }
-
-    try {
-        const ciphertext = encryptStorages(entry.storages, masterKey);
-        entry.storages_ciphertext = ciphertext;
-        entry.storages_encrypted = true;
-        entry.storages = [];
-    } catch (err) {
-        log.error({error: err, name: "applyStorageEncryption"}, "Storage encryption failed; sending plaintext");
-    }
-}
-

src/features/agents/utils/status/storage-channels.helpers.ts

@@ -0,0 +1,64 @@
+import * as drizzleDb from "@/db";
+import {db} from "@/db";
+import {eq} from "drizzle-orm";
+
+export type PingDatabaseStorageChannels = {
+    id: string;
+    config: any
+    provider: string
+}
+
+export async function getDatabaseStorageChannels(databaseId: string): Promise<PingDatabaseStorageChannels[]> {
+
+    const database = await db.query.database.findFirst({
+        where: eq(drizzleDb.schemas.database.id, databaseId),
+        with: {
+            project: true,
+            retentionPolicy: true,
+            alertPolicies: true,
+            storagePolicies: true
+        }
+    });
+
+    if (!database) {
+        return []
+    }
+
+    const settings = await db.query.setting.findFirst({
+        where: eq(drizzleDb.schemas.setting.name, "system"),
+        with: {storageChannel: true},
+    });
+
+    const defaultStorageChannel: PingDatabaseStorageChannels[] = settings?.storageChannel
+        ? [{
+            id: settings.storageChannel.id,
+            provider: settings.storageChannel.provider,
+            config: settings.storageChannel.config,
+        }]
+        : [];
+
+
+    const enabledDatabaseStorageChannels = await Promise.all(
+        (database.storagePolicies ?? [])
+            .filter(p => p.enabled)
+            .map(async policy => {
+                const storageChannel = await db.query.storageChannel.findFirst({
+                    where: eq(drizzleDb.schemas.storageChannel.id, policy.storageChannelId),
+                });
+
+                if (!storageChannel) return null;
+
+                return {
+                    id: storageChannel.id,
+                    config: storageChannel.config,
+                    provider: storageChannel.provider,
+                } as PingDatabaseStorageChannels;
+            })
+    );
+
+    const filteredChannels: PingDatabaseStorageChannels[] = enabledDatabaseStorageChannels.filter(
+        (c): c is PingDatabaseStorageChannels => c !== null
+    );
+
+    return filteredChannels.length > 0 ? filteredChannels : defaultStorageChannel;
+}

src/features/agents/utils/status/storage-encryption.helpers.ts

@@ -0,0 +1,39 @@
+import {logger} from "@/lib/logger";
+import {encryptStorages, isAgentVersionAtLeast, MIN_AGENT_VERSION_STORAGE_ENC} from "@/utils/status-crypto";
+
+const log = logger.child({module: "api/agent/status/storage-encryption"});
+
+export function applyStorageEncryption(
+    entry: Record<string, any>,
+    version: string | undefined,
+    masterKey: Buffer | null,
+    agentId: string,
+): void {
+    if (!masterKey) return;
+    if (!Array.isArray(entry.storages) || entry.storages.length === 0) return;
+    if (!isAgentVersionAtLeast(version, MIN_AGENT_VERSION_STORAGE_ENC)) {
+        log.warn(
+            {
+                name: "applyStorageEncryption",
+                agentId,
+                agentVersion: version ?? "unknown",
+                requiredVersion: MIN_AGENT_VERSION_STORAGE_ENC,
+            },
+            `\n============================================================\n` +
+            `  ⚠️  OUTDATED AGENT — STORAGE CREDENTIALS SENT UNENCRYPTED\n` +
+            `  Agent ${agentId} reports v${version ?? "unknown"} (< required v${MIN_AGENT_VERSION_STORAGE_ENC}).\n` +
+            `  Update this agent to v${MIN_AGENT_VERSION_STORAGE_ENC}+ to encrypt storage credentials in transit.\n` +
+            `============================================================`,
+        );
+        return;
+    }
+
+    try {
+        const ciphertext = encryptStorages(entry.storages, masterKey);
+        entry.storages_ciphertext = ciphertext;
+        entry.storages_encrypted = true;
+        entry.storages = [];
+    } catch (err) {
+        log.error({error: err, name: "applyStorageEncryption"}, "Storage encryption failed; sending plaintext");
+    }
+}