diff --git a/app/api/agent/[agentId]/backup/helpers.ts b/app/api/agent/[agentId]/backup/helpers.ts index 3da9b06a..7c915ef2 100644 --- a/app/api/agent/[agentId]/backup/helpers.ts +++ b/app/api/agent/[agentId]/backup/helpers.ts @@ -1,53 +1,52 @@ -import fs from "node:fs"; -import forge from "node-forge"; +import { NextResponse } from "next/server"; +import { eq } from "drizzle-orm"; +import { db } from "@/db"; +import * as drizzleDb from "@/db"; + +export function withAgentCheck(handler: Function) { + return async (request: Request, context: { params: Promise<{ agentId: string }> }) => { + try { + const agentId = (await context.params).agentId; + + const agent = await db.query.agent.findFirst({ + where: eq(drizzleDb.schemas.agent.id, agentId), + }); + + if (!agent) { + return NextResponse.json( + { error: "Agent not found" }, + { status: 404 } + ); + } + + return handler(request, { ...context, agent }); + } catch (err) { + console.error("Error in agent middleware:", err); + return NextResponse.json( + { error: "Internal server error" }, + { status: 500 } + ); + } + }; +} -export async function decryptedDump(file: File, aesKeyHex: string, ivHex: string, fileExtension: string): Promise { - const privateKeyPem = fs.readFileSync("private/keys/server_private.pem", "utf8"); - const privateKey = forge.pki.privateKeyFromPem(privateKeyPem); - - // Decrypt AES key with RSA-OAEP - const encryptedAesKey = forge.util.hexToBytes(aesKeyHex); - const aesKey = privateKey.decrypt(encryptedAesKey, "RSA-OAEP", { - md: forge.md.sha256.create(), - mgf1: {md: forge.md.sha256.create()}, +export async function getDatabaseOrThrow(generatedId: string) { + const database = await db.query.database.findFirst({ + where: eq(drizzleDb.schemas.database.agentDatabaseId, generatedId), + with: { + project: true, + alertPolicies: true, + storagePolicies: true + } }); - // Read encrypted file content - const encryptedBuffer = Buffer.from(await file.arrayBuffer()); - const iv = forge.util.hexToBytes(ivHex); - - // AES decryption - const decipher = forge.cipher.createDecipher("AES-CBC", aesKey); - decipher.start({iv}); - decipher.update(forge.util.createBuffer(encryptedBuffer.toString("binary"))); - const success = decipher.finish(); - - if (!success) { - throw new Error("Decryption failed"); + if (!database) { + throw NextResponse.json( + { error: "Database associated with generatedId not found" }, + { status: 404 } + ); } - const decryptedBytes = decipher.output.getBytes(); - const decryptedBuffer = Buffer.from(decryptedBytes, "binary"); - - // Return a File so you can use file.arrayBuffer() later - return new File( - [decryptedBuffer], - file.name.replace(/\.enc$/, fileExtension), - {type: "application/octet-stream"} - ); -} - - -export function getFileExtension(dbType: string) { - switch (dbType) { - case "postgresql": - return ".dump"; - case "mysql": - return ".sql"; - default: - return ".dump"; - } -} - - + return database; +} \ No newline at end of file diff --git a/app/api/agent/[agentId]/backup/old/helpers.ts b/app/api/agent/[agentId]/backup/old/helpers.ts new file mode 100644 index 00000000..3da9b06a --- /dev/null +++ b/app/api/agent/[agentId]/backup/old/helpers.ts @@ -0,0 +1,53 @@ +import fs from "node:fs"; +import forge from "node-forge"; + + +export async function decryptedDump(file: File, aesKeyHex: string, ivHex: string, fileExtension: string): Promise { + const privateKeyPem = fs.readFileSync("private/keys/server_private.pem", "utf8"); + const privateKey = forge.pki.privateKeyFromPem(privateKeyPem); + + // Decrypt AES key with RSA-OAEP + const encryptedAesKey = forge.util.hexToBytes(aesKeyHex); + const aesKey = privateKey.decrypt(encryptedAesKey, "RSA-OAEP", { + md: forge.md.sha256.create(), + mgf1: {md: forge.md.sha256.create()}, + }); + + // Read encrypted file content + const encryptedBuffer = Buffer.from(await file.arrayBuffer()); + const iv = forge.util.hexToBytes(ivHex); + + // AES decryption + const decipher = forge.cipher.createDecipher("AES-CBC", aesKey); + decipher.start({iv}); + decipher.update(forge.util.createBuffer(encryptedBuffer.toString("binary"))); + const success = decipher.finish(); + + if (!success) { + throw new Error("Decryption failed"); + } + + const decryptedBytes = decipher.output.getBytes(); + const decryptedBuffer = Buffer.from(decryptedBytes, "binary"); + + // Return a File so you can use file.arrayBuffer() later + return new File( + [decryptedBuffer], + file.name.replace(/\.enc$/, fileExtension), + {type: "application/octet-stream"} + ); +} + + +export function getFileExtension(dbType: string) { + switch (dbType) { + case "postgresql": + return ".dump"; + case "mysql": + return ".sql"; + default: + return ".dump"; + } +} + + diff --git a/app/api/agent/[agentId]/backup/old/route.ts b/app/api/agent/[agentId]/backup/old/route.ts new file mode 100644 index 00000000..ef9cb150 --- /dev/null +++ b/app/api/agent/[agentId]/backup/old/route.ts @@ -0,0 +1,169 @@ +import {NextResponse} from "next/server"; +import {isUuidv4} from "@/utils/verify-uuid"; +import {v4 as uuidv4} from "uuid"; +import * as drizzleDb from "@/db"; +import {db} from "@/db"; +import {Backup} from "@/db/schema/07_database"; +import {and, eq} from "drizzle-orm"; +import {withUpdatedAt} from "@/db/utils"; +import {decryptedDump, getFileExtension} from "./helpers"; +import {sendNotificationsBackupRestore} from "@/features/notifications/helpers"; +import {storeBackupFiles} from "@/features/storages/helpers"; +import {eventEmitter} from "@/features/shared/event"; + +export async function POST( + request: Request, + {params}: { params: Promise<{ agentId: string }> } +) { + try { + const contentType = request.headers.get("Content-Type"); + + if (!contentType || !contentType.includes("multipart/form-data")) { + return NextResponse.json( + {error: "Unsupported or missing Content-Type"}, + {status: 400} + ); + } + + eventEmitter.emit('modification', {update: true}); + + const agentId = (await params).agentId; + const formData = await request.formData(); + const aesKeyHex = formData.get("aes_key") as string; + const ivHex = formData.get("iv") as string; + const generatedId = formData.get("generatedId") as string | null; + const method = formData.get("method") as string | null; + + + if (!generatedId || !isUuidv4(generatedId)) { + return NextResponse.json( + {error: "generatedId is not a valid UUID"}, + {status: 400} + ); + } + + const agent = await db.query.agent.findFirst({ + where: eq(drizzleDb.schemas.agent.id, agentId), + }); + + if (!agent) { + return NextResponse.json( + {error: "Agent not found"}, + {status: 404} + ); + } + + const database = await db.query.database.findFirst({ + where: eq(drizzleDb.schemas.database.agentDatabaseId, generatedId), + with: { + project: true, + alertPolicies: true, + storagePolicies: true + } + }); + + if (!database) { + return NextResponse.json( + {error: "Database associated with generatedId not found"}, + {status: 404} + ); + } + + let backup: Backup | null | undefined = null; + + if (method === "automatic") { + [backup] = await db + .insert(drizzleDb.schemas.backup) + .values({ + status: 'ongoing', + databaseId: database.id, + }) + .returning(); + + + if (!backup) { + return NextResponse.json( + {error: "Unable to create an automatic backup"}, + {status: 500} + ); + } + } else { + backup = await db.query.backup.findFirst({ + where: and( + eq(drizzleDb.schemas.backup.status, 'ongoing'), + eq(drizzleDb.schemas.backup.databaseId, database.id), + ), + }); + + + if (!backup) { + return NextResponse.json( + {error: "Unable to find the corresponding backup"}, + {status: 404} + ); + } + } + + const status = formData.get("status") as string | null; + + if (status === "success") { + const file = formData.get("file") as File | null; + const extension = formData.get("extension") as string | null; + if (!aesKeyHex || !ivHex) { + return NextResponse.json({error: "Missing fields"}, {status: 400}); + } + + + if (!file) { + return NextResponse.json( + {error: "File is required for successful backup"}, + {status: 400} + ); + } + const fileSizeBytes = file.size; + // const fileExtension = '.' + (file.name.split('.').pop()?.toLowerCase() || ''); + const fileExtension = extension ? extension : getFileExtension(database.dbms) + const decryptedFile = await decryptedDump(file, aesKeyHex, ivHex, fileExtension); + const uuid = uuidv4(); + const fileName = `${uuid}${fileExtension}`; + const buffer = Buffer.from(await decryptedFile.arrayBuffer()); + + + const storageResults = await storeBackupFiles(backup, database, buffer, fileName) + eventEmitter.emit('modification', {update: true}); + + await sendNotificationsBackupRestore(database, "success_backup"); + + return NextResponse.json( + { + message: "Backup successfully uploaded", + }, + {status: 200} + ); + } else { + + await db + .update(drizzleDb.schemas.backup) + .set(withUpdatedAt({status: 'failed'})) + .where(eq(drizzleDb.schemas.backup.id, backup.id)); + + eventEmitter.emit('modification', {update: true}); + + await sendNotificationsBackupRestore(database, "error_backup"); + + + return NextResponse.json( + { + message: "Backup successfully updated with status failed", + }, + {status: 200} + ); + } + } catch (error) { + console.error("Error in POST handler:", error); + return NextResponse.json( + {error: "Internal server error"}, + {status: 500} + ); + } +} diff --git a/app/api/agent/[agentId]/backup/route.ts b/app/api/agent/[agentId]/backup/route.ts index ef9cb150..315bb567 100644 --- a/app/api/agent/[agentId]/backup/route.ts +++ b/app/api/agent/[agentId]/backup/route.ts @@ -1,73 +1,34 @@ import {NextResponse} from "next/server"; -import {isUuidv4} from "@/utils/verify-uuid"; -import {v4 as uuidv4} from "uuid"; -import * as drizzleDb from "@/db"; -import {db} from "@/db"; -import {Backup} from "@/db/schema/07_database"; import {and, eq} from "drizzle-orm"; +import * as drizzleDb from "@/db"; +import {db as dbClient, db} from "@/db"; +import {getDatabaseOrThrow, withAgentCheck} from "./helpers"; +import {Backup} from "@/db/schema/07_database"; import {withUpdatedAt} from "@/db/utils"; -import {decryptedDump, getFileExtension} from "./helpers"; -import {sendNotificationsBackupRestore} from "@/features/notifications/helpers"; -import {storeBackupFiles} from "@/features/storages/helpers"; import {eventEmitter} from "@/features/shared/event"; +import {sendNotificationsBackupRestore} from "@/features/notifications/helpers"; +import {EventKind} from "@/features/notifications/types"; -export async function POST( - request: Request, - {params}: { params: Promise<{ agentId: string }> } -) { +export type BodyPost = { + method: "manual" | "automatic" + generatedId: string +} + +export type BodyPatch = { + backupId: string + status: "success" | "failed" + size: number + generatedId: string +} + +export const POST = withAgentCheck(async (request: Request, {params, agent}: { + params: Promise<{ agentId: string }>, + agent: any +}) => { try { - const contentType = request.headers.get("Content-Type"); - - if (!contentType || !contentType.includes("multipart/form-data")) { - return NextResponse.json( - {error: "Unsupported or missing Content-Type"}, - {status: 400} - ); - } - - eventEmitter.emit('modification', {update: true}); - - const agentId = (await params).agentId; - const formData = await request.formData(); - const aesKeyHex = formData.get("aes_key") as string; - const ivHex = formData.get("iv") as string; - const generatedId = formData.get("generatedId") as string | null; - const method = formData.get("method") as string | null; - - - if (!generatedId || !isUuidv4(generatedId)) { - return NextResponse.json( - {error: "generatedId is not a valid UUID"}, - {status: 400} - ); - } - - const agent = await db.query.agent.findFirst({ - where: eq(drizzleDb.schemas.agent.id, agentId), - }); - - if (!agent) { - return NextResponse.json( - {error: "Agent not found"}, - {status: 404} - ); - } - - const database = await db.query.database.findFirst({ - where: eq(drizzleDb.schemas.database.agentDatabaseId, generatedId), - with: { - project: true, - alertPolicies: true, - storagePolicies: true - } - }); - - if (!database) { - return NextResponse.json( - {error: "Database associated with generatedId not found"}, - {status: 404} - ); - } + const body: BodyPost = await request.json(); + const method = body.method + const database = await getDatabaseOrThrow(body.generatedId); let backup: Backup | null | undefined = null; @@ -79,8 +40,6 @@ export async function POST( databaseId: database.id, }) .returning(); - - if (!backup) { return NextResponse.json( {error: "Unable to create an automatic backup"}, @@ -104,66 +63,78 @@ export async function POST( } } - const status = formData.get("status") as string | null; - if (status === "success") { - const file = formData.get("file") as File | null; - const extension = formData.get("extension") as string | null; - if (!aesKeyHex || !ivHex) { - return NextResponse.json({error: "Missing fields"}, {status: 400}); - } + eventEmitter.emit('modification', {update: true}); - - if (!file) { - return NextResponse.json( - {error: "File is required for successful backup"}, - {status: 400} - ); - } - const fileSizeBytes = file.size; - // const fileExtension = '.' + (file.name.split('.').pop()?.toLowerCase() || ''); - const fileExtension = extension ? extension : getFileExtension(database.dbms) - const decryptedFile = await decryptedDump(file, aesKeyHex, ivHex, fileExtension); - const uuid = uuidv4(); - const fileName = `${uuid}${fileExtension}`; - const buffer = Buffer.from(await decryptedFile.arrayBuffer()); - - - const storageResults = await storeBackupFiles(backup, database, buffer, fileName) - eventEmitter.emit('modification', {update: true}); - - await sendNotificationsBackupRestore(database, "success_backup"); - - return NextResponse.json( - { - message: "Backup successfully uploaded", - }, - {status: 200} - ); - } else { - - await db - .update(drizzleDb.schemas.backup) - .set(withUpdatedAt({status: 'failed'})) - .where(eq(drizzleDb.schemas.backup.id, backup.id)); - - eventEmitter.emit('modification', {update: true}); - - await sendNotificationsBackupRestore(database, "error_backup"); - - - return NextResponse.json( - { - message: "Backup successfully updated with status failed", - }, - {status: 200} - ); - } + return NextResponse.json( + { + message: "Init backup success", + backup: backup, + }, + {status: 200} + ); } catch (error) { - console.error("Error in POST handler:", error); + console.error("Error in POST for INIT backup:", error); return NextResponse.json( {error: "Internal server error"}, {status: 500} ); } -} +}); + +export const PATCH = withAgentCheck(async (request: Request, {params, agent}: { + params: Promise<{ agentId: string }>, + agent: any +}) => { + try { + const body: BodyPatch = await request.json(); + + console.log(body); + const status = body.status + const backupId = body.backupId + const backupSize = body.size + + const database = await getDatabaseOrThrow(body.generatedId); + + const backup = await db.query.backup.findFirst({ + where: eq(drizzleDb.schemas.backup.id, backupId), + }); + + if (!backup) { + return NextResponse.json( + {error: "No backup found"}, + {status: 500} + ); + } + + const [backupUpdated] = await dbClient + .update(drizzleDb.schemas.backup) + .set(withUpdatedAt({ + status: status, + fileSize: backupSize + })) + .where(eq(drizzleDb.schemas.backup.id, backup.id)) + .returning(); + + + eventEmitter.emit('modification', {update: true}); + await sendNotificationsBackupRestore(database, `${status}_backup` as EventKind); + + return NextResponse.json( + { + message: "Backup successfully updated", + backup: backupUpdated, + }, + {status: 200} + ); + } catch (error) { + + + console.error("Error in PATCH backup:", error); + return NextResponse.json( + {error: "Internal server error"}, + {status: 500} + ); + } +}); + diff --git a/app/api/agent/[agentId]/backup/upload/init/route.ts b/app/api/agent/[agentId]/backup/upload/init/route.ts new file mode 100644 index 00000000..3d0f7661 --- /dev/null +++ b/app/api/agent/[agentId]/backup/upload/init/route.ts @@ -0,0 +1,76 @@ +import {NextResponse} from "next/server"; +import {and, eq} from "drizzle-orm"; +import * as drizzleDb from "@/db"; +import {db} from "@/db"; +import {getDatabaseOrThrow, withAgentCheck} from "../../helpers"; +import {isUuidv4} from "@/utils/verify-uuid"; +import {eventEmitter} from "@/features/shared/event"; + +export type Body = { + generatedId: string + storageChannelId: string + backupId: string +} +export const POST = withAgentCheck(async (request: Request, {params, agent}: { + params: Promise<{ agentId: string }>, + agent: any +}) => { + try { + const body: Body = await request.json(); + + console.log("body", body); + + const generatedId = body.generatedId; + const storageChannelId = body.storageChannelId; + const backupId = body.backupId; + + if (!generatedId || !isUuidv4(generatedId)) { + return NextResponse.json( + {error: "generatedId is not a valid UUID"}, + {status: 400} + ); + } + + const database = await getDatabaseOrThrow(generatedId); + + const backup = await db.query.backup.findFirst({ + where: and( + eq(drizzleDb.schemas.backup.id, backupId), + eq(drizzleDb.schemas.backup.databaseId, database.id), + ), + }); + + if (!backup) { + return NextResponse.json( + {error: "Unable to find the corresponding backup"}, + {status: 404} + ); + } + + const [backupStorage] = await db + .insert(drizzleDb.schemas.backupStorage) + .values({ + backupId: backup.id, + storageChannelId: storageChannelId, + status: "pending", + }) + .returning(); + + eventEmitter.emit('modification', {update: true}); + + return NextResponse.json( + { + message: "Backup storage successfully created", + backupStorage: backupStorage + }, + {status: 200} + ); + } catch (error) { + console.error("Error in POST for INIT backup:", error); + return NextResponse.json( + {error: "Internal server error"}, + {status: 500} + ); + } +}); + diff --git a/app/api/agent/[agentId]/backup/upload/status/route.ts b/app/api/agent/[agentId]/backup/upload/status/route.ts new file mode 100644 index 00000000..0b0031c5 --- /dev/null +++ b/app/api/agent/[agentId]/backup/upload/status/route.ts @@ -0,0 +1,93 @@ +import {NextResponse} from "next/server"; +import {and, eq} from "drizzle-orm"; +import * as drizzleDb from "@/db"; +import {db as dbClient, db} from "@/db"; +import {withUpdatedAt} from "@/db/utils"; +import {getDatabaseOrThrow, withAgentCheck} from "../../helpers"; +import {eventEmitter} from "@/features/shared/event"; + +export type Body = { + generatedId: string + status: "success" | "failed" + backupStorageId: string + path: string + size: number + backupId: string +} +export const PATCH = withAgentCheck(async (request: Request, {params, agent}: { + params: Promise<{ agentId: string }>, + agent: any +}) => { + try { + const body: Body = await request.json(); + const generatedId = body.generatedId; + const status = body.status; + const filePath = body.path; + const fileSize = body.size; + const backupStorageId = body.backupStorageId; + const backupId = body.backupId; + + + console.log("body", body); + + const database = await getDatabaseOrThrow(generatedId); + + const backup = await db.query.backup.findFirst({ + where: and( + eq(drizzleDb.schemas.backup.id, backupId), + eq(drizzleDb.schemas.backup.databaseId, database.id), + ), + with: { + storages: true + } + }); + + if (!backup) { + return NextResponse.json( + {error: "Unable to find the corresponding backup"}, + {status: 404} + ); + } + + const [backupStorage] = await dbClient + .update(drizzleDb.schemas.backupStorage) + .set(withUpdatedAt({ + status: status, + path: filePath, + size: fileSize + })) + .where(eq(drizzleDb.schemas.backupStorage.id, backupStorageId)) + .returning(); + + + if (backup.storages.length > 0) { + const hasSuccessfulStorage = backup.storages.some( + (storage) => storage.status === "success" + ); + + if (hasSuccessfulStorage && backup.status !== "success") { + await db + .update(drizzleDb.schemas.backup) + .set({status: "success"}) + .where(eq(drizzleDb.schemas.backup.id, backup.id)); + } + } + + eventEmitter.emit('modification', {update: true}); + + + return NextResponse.json({ + message: "Backup status successfully updated", + backupStorage: backupStorage + }, + {status: 200} + ); + } catch (error) { + console.error("Error in POST for INIT backup:", error); + return NextResponse.json( + {error: "Internal server error"}, + {status: 500} + ); + } +}); + diff --git a/app/api/agent/[agentId]/status/helpers.ts b/app/api/agent/[agentId]/status/helpers.ts index 06685ba0..ccf11841 100644 --- a/app/api/agent/[agentId]/status/helpers.ts +++ b/app/api/agent/[agentId]/status/helpers.ts @@ -2,9 +2,9 @@ import {NextResponse} from "next/server"; import {Body} from "./route"; import {isUuidv4} from "@/utils/verify-uuid"; import {Agent} from "@/db/schema/08_agent"; -import {Database} from "@/db/schema/07_database"; +import {Database, DatabaseWith} from "@/db/schema/07_database"; import * as drizzleDb from "@/db"; -import {db as dbClient} from "@/db"; +import {db, db as dbClient} from "@/db"; import {and, eq, inArray} from "drizzle-orm"; import {dbmsEnumSchema, EDbmsSchema} from "@/db/schema/types"; import {withUpdatedAt} from "@/db/utils"; @@ -14,9 +14,11 @@ import {dispatchStorage} from "@/features/storages/dispatch"; export async function handleDatabases(body: Body, agent: Agent, lastContact: Date) { const databasesResponse = []; - const formatDatabase = (database: Database, backupAction: boolean, restoreAction: boolean, UrlBackup: string) => ({ + const formatDatabase = (database: DatabaseWith, backupAction: boolean, restoreAction: boolean, UrlBackup: string | null, storages: PingDatabaseStorageChannels[], urlMeta: string | null) => ({ generatedId: database.agentDatabaseId, dbms: database.dbms, + storages: storages, + encrypt: false, data: { backup: { action: backupAction, @@ -25,6 +27,7 @@ export async function handleDatabases(body: Body, agent: Agent, lastContact: Dat restore: { action: restoreAction, file: UrlBackup, + metaFile: urlMeta }, }, }); @@ -32,12 +35,16 @@ export async function handleDatabases(body: Body, agent: Agent, lastContact: Dat for (const db of body.databases) { const existingDatabase = await dbClient.query.database.findFirst({ - where: eq(drizzleDb.schemas.database.agentDatabaseId, db.generatedId) + where: eq(drizzleDb.schemas.database.agentDatabaseId, db.generatedId), + with: { + project: true + } }); let backupAction: boolean = false let restoreAction: boolean = false - let urlBackup: string = "" + let urlBackup: string | null = null; + let urlMeta: string | null = null if (!existingDatabase) { if (!isUuidv4(db.generatedId)) { @@ -63,8 +70,11 @@ export async function handleDatabases(body: Body, agent: Agent, lastContact: Dat }) .returning(); + if (databaseCreated) { - databasesResponse.push(formatDatabase(databaseCreated, backupAction, restoreAction, urlBackup)); + const storages = await getDatabaseStorageChannels(databaseCreated.id) + + databasesResponse.push(formatDatabase(databaseCreated, backupAction, restoreAction, urlBackup, storages, null)); } } else { @@ -78,7 +88,6 @@ export async function handleDatabases(body: Body, agent: Agent, lastContact: Dat .where(eq(drizzleDb.schemas.database.id, existingDatabase.id)) .returning(); - const activeBackup = await dbClient.query.backup.findFirst({ where: and( eq(drizzleDb.schemas.backup.databaseId, databaseUpdated.id), @@ -86,7 +95,6 @@ export async function handleDatabases(body: Body, agent: Agent, lastContact: Dat ) }) - const restoration = await dbClient.query.restoration.findFirst({ where: and(eq(drizzleDb.schemas.restoration.databaseId, databaseUpdated.id), eq(drizzleDb.schemas.restoration.status, "waiting")), with: { @@ -94,7 +102,6 @@ export async function handleDatabases(body: Body, agent: Agent, lastContact: Dat } }) - if (activeBackup && activeBackup.status == "waiting") { backupAction = true @@ -107,13 +114,11 @@ export async function handleDatabases(body: Body, agent: Agent, lastContact: Dat if (restoration) { restoreAction = true - if (!restoration.backupStorage || restoration.backupStorage.status != "success" || !restoration.backupStorage.path) { restoreAction = false continue; } - const input: StorageInput = { action: "get", data: { @@ -126,12 +131,26 @@ export async function handleDatabases(body: Body, agent: Agent, lastContact: Dat } }; + const inputMeta: StorageInput = { + action: "get", + data: { + path: `${restoration.backupStorage.path}.meta`, + signedUrl: true, + }, + metadata: { + storageId: restoration.backupStorage.storageChannelId, + fileKind: "backups" + } + }; + try { const result = await dispatchStorage(input, undefined, restoration.backupStorage.storageChannelId); + const resultMeta = await dispatchStorage(inputMeta, undefined, restoration.backupStorage.storageChannelId); if (result.success) { - urlBackup = result.url ?? ""; + urlBackup = result.url ?? null; + urlMeta = resultMeta.url ?? null } else { await dbClient .update(drizzleDb.schemas.restoration) @@ -151,53 +170,78 @@ export async function handleDatabases(body: Body, agent: Agent, lastContact: Dat continue; } - - // const fileName = backupToRestore?.file - // - // let data: SafeActionResult, object> | undefined - // - // try { - // - // if (settings.storage == "local") { - // data = await getFileUrlPresignedLocal({fileName: fileName!}) - // } else if (settings.storage == "s3") { - // - // data = await getFileUrlPreSignedS3Action(`backups/${backupToRestore?.database.project?.slug}/${fileName}`); - // } - // - // if (data?.data?.success) { - // urlBackup = data.data.value ?? ""; - // } else { - // await dbClient - // .update(drizzleDb.schemas.restoration) - // .set({status: "failed"}) - // .where(eq(drizzleDb.schemas.restoration.id, restoration.id)); - // - // // @ts-ignore - // const errorMessage = data?.data?.actionError?.message || "Failed to get presigned URL"; - // console.error("Restoration failed: ", errorMessage); - // - // continue; - // } - // } catch (err) { - // console.error("Restoration crashed unexpectedly:", err); - // await dbClient - // .update(drizzleDb.schemas.restoration) - // .set({status: "failed"}) - // .where(eq(drizzleDb.schemas.restoration.id, restoration.id)); - // - // continue; - // } await dbClient .update(drizzleDb.schemas.restoration) .set({status: "ongoing"}) .where(eq(drizzleDb.schemas.restoration.id, restoration.id)); } - databasesResponse.push(formatDatabase(databaseUpdated, backupAction, restoreAction, urlBackup)); + const storages = await getDatabaseStorageChannels(databaseUpdated.id) + databasesResponse.push(formatDatabase(databaseUpdated, backupAction, restoreAction, urlBackup, storages, urlMeta)); } } + return databasesResponse; } + +type PingDatabaseStorageChannels = { + id: string; + config: any + provider: string +} + +async function getDatabaseStorageChannels(databaseId: string): Promise { + + const database = await db.query.database.findFirst({ + where: eq(drizzleDb.schemas.database.id, databaseId), + with: { + project: true, + retentionPolicy: true, + alertPolicies: true, + storagePolicies: true + } + }); + + if (!database) { + return [] + } + + const settings = await db.query.setting.findFirst({ + where: eq(drizzleDb.schemas.setting.name, "system"), + with: {storageChannel: true}, + }); + + const defaultStorageChannel: PingDatabaseStorageChannels[] = settings?.storageChannel + ? [{ + id: settings.storageChannel.id, + provider: settings.storageChannel.provider, + config: settings.storageChannel.config, + }] + : []; + + + const enabledDatabaseStorageChannels = await Promise.all( + (database.storagePolicies ?? []) + .filter(p => p.enabled) + .map(async policy => { + const storageChannel = await db.query.storageChannel.findFirst({ + where: eq(drizzleDb.schemas.storageChannel.id, policy.storageChannelId), + }); + + if (!storageChannel) return null; + + return { + id: storageChannel.id, + config: storageChannel.config, + provider: storageChannel.provider, + } as PingDatabaseStorageChannels; + }) + ); + + const filteredChannels: PingDatabaseStorageChannels[] = enabledDatabaseStorageChannels.filter( + (c): c is PingDatabaseStorageChannels => c !== null + ); + + return filteredChannels.length > 0 ? filteredChannels : defaultStorageChannel; +} + diff --git a/app/api/agent/[agentId]/status/route.ts b/app/api/agent/[agentId]/status/route.ts index 30fbf2aa..d733b0a0 100644 --- a/app/api/agent/[agentId]/status/route.ts +++ b/app/api/agent/[agentId]/status/route.ts @@ -26,12 +26,14 @@ export async function POST( ) { try { const agentId = (await params).agentId + console.log(agentId) const body: Body = await request.json(); const lastContact = new Date(); let message: string if (!isUuidv4(agentId)) { message = "agentId is not a valid uuid" + console.error(message) return NextResponse.json( {error: "agentId is not a valid uuid"}, {status: 500} @@ -66,6 +68,7 @@ export async function POST( databases: databasesResponse } + return Response.json(response) } catch (error) { console.error('Error in POST handler:', error); diff --git a/app/api/tus/hooks/route.ts b/app/api/tus/hooks/route.ts index 997fd711..4cacc0a9 100644 --- a/app/api/tus/hooks/route.ts +++ b/app/api/tus/hooks/route.ts @@ -3,120 +3,75 @@ import fs from "fs"; import path from "path"; -// -// export async function POST(request: Request) { -// try { -// const body = await request.json(); -// console.log("tusd hook payload:", body); -// -// return NextResponse.json({ -// "Upload-Defer-Length": true, -// "Upload-Metadata": "name somefilename", -// }); -// } catch (error) { -// console.error("Error in POST handler:", error); -// return NextResponse.json( -// { error: "Internal server error" }, -// { status: 500 } -// ); -// } -// } - -// export async function POST(request: Request) { -// try { -// const body = await request.json(); -// console.log("TUSD hook payload:", body); -// -// // Pre-create hook: must return Upload-Defer-Length or Upload-Length -// if (body.event === "pre-create") { -// return NextResponse.json({ -// "Upload-Defer-Length": true, // defer the upload length -// "Upload-Metadata": "name somefilename" // optional metadata -// }); -// } -// -// // Post-receive hook: handle finished uploads -// if (body.event === "post-receive") { -// console.log(`Upload finished: ${body.id}, size: ${body.upload_length}`); -// // process the file, AES encryption, move, etc. -// } -// -// return NextResponse.json({ ok: true }); // safe fallback for other events -// } catch (error) { -// console.error("Hook error:", error); -// return NextResponse.json({ error: "Internal server error" }, { status: 500 }); -// } -// } - export async function POST(request: Request) { try { const body = await request.json(); - console.log("TUSD hook payload:", body); const event = body.Event const headers = event.HTTPRequest.Header + const uploadLength = headers["X-File-Size"]?.[0]; + const uploadOffset = headers["Upload-Offset"]?.[0]; + const status = headers["X-Status"]?.[0]; - // if (body.event === "pre-create") { - // return NextResponse.json({ - // "Upload-Defer-Length": true - // }); - // } - console.log("TUSD hook payload:", body.Type); - console.log("TUSD hook payload:", headers); + console.log(`Upload ID : ${event.Upload.ID} (${uploadOffset}/${uploadLength})`); - if ( - body.Type === "post-receive" && - event.Upload.SizeIsDeferred === false && - event.Upload.Offset === event.Upload.Size - ) { - const id = event.Upload.ID; - // const extensionFile = headers["X-Extension"]?.[0] ?? ".bin"; - const extensionFile = ".enc"; - const filePath = path.join(process.cwd(), "/private/uploads/backups/"); - fs.renameSync( - `${filePath}${id}`, - `${filePath}${id}${extensionFile}` - ); + if (status === "success") { + if ( + body.Type === "post-receive" && + event.Upload.SizeIsDeferred === false && + event.Upload.Offset === event.Upload.Size + ) { + const id = event.Upload.ID; + const fileName = headers["X-File-Name"]?.[0]; + const filePath = headers["X-File-Path"]?.[0]; + if (!filePath) { + return NextResponse.json({error: "Missing X-File-Path"}, {status: 500}); + } + + + const uploadDir = path.join(process.cwd(), "/private/uploads/"); + + const oldFilePath = path.join(uploadDir, "tmp", id); + const newFilePath = path.join(uploadDir, filePath); + + fs.mkdirSync(path.dirname(newFilePath), {recursive: true}); + + let retries = 10; + while (!fs.existsSync(oldFilePath)) { + if (retries-- === 0) { + return NextResponse.json({error: `Upload file not found: ${oldFilePath}`}, {status: 500}); + } + await new Promise(r => setTimeout(r, 200)); + } + + fs.renameSync(oldFilePath, newFilePath); + + const infoFilePath = `${oldFilePath}.info`; + if (fs.existsSync(infoFilePath)) { + fs.unlinkSync(infoFilePath); + } + + const metadataHeaderB64 = headers["Upload-Metadata"]?.[0]; + + if (metadataHeaderB64) { + const metadataHeader = Buffer.from(metadataHeaderB64, "base64").toString("utf-8"); + if (metadataHeader) { + const tomlContent = metadataHeader + .split(",") + .map((pair) => { + const [key, value] = pair.split(" "); + const escapedValue = value.replace(/"/g, '\\"'); + return `${key} = "${escapedValue}"`; + }) + .join("\n"); + const metaFilePath = `${newFilePath}.meta`; + fs.writeFileSync(metaFilePath, tomlContent, "utf-8"); + } + } + } } - - - - // if (body.Type === "post-receive"){ - // // const id = event.Upload.ID - // // const extensionFile = headers["X-Extension"]?.[0] ?? ".bin"; - // // const extensionFile = ".enc"; - // // const filePath = path.join(process.cwd(), "/private/uploads/backups/"); - // // - // // fs.renameSync( - // // `${filePath}${id}`, - // // `${filePath}${id}${extensionFile}` - // // ); - // } - - // Post-receive hook: handle finished uploads - if (body.event === "post-receive") { - - - - console.log(`Upload finished: ${body.id}, size: ${body.upload_length}`); - // process the file, AES encryption, move, etc. - - const id = body.id; - const ext = body.upload.metaData?.extension ?? ".bin"; - console.log(body) - console.log("Upload finished: ", id); - console.log("Ext: ", ext); - - // fs.renameSync( - // `/data/uploads/backups/${id}`, - // `/data/uploads/backups/${id}${ext}` - // ); - - } - - - return NextResponse.json({}); // no Upload-Defer-Length + return NextResponse.json({}); } catch (error) { console.error("Hook error:", error); return NextResponse.json({error: "Internal server error"}, {status: 500}); diff --git a/docker-compose.yml b/docker-compose.yml index f6e66795..02ba5acd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -22,17 +22,13 @@ services: ports: - "1080:8080" command: > - -upload-dir /data/uploads/backups + -upload-dir /data/uploads/tmp -hooks-http http://localhost:8887/api/tus/hooks -max-size 21474836480 - - # -max-size 54975581388 - # -max-size 54975581388 - extra_hosts: - "localhost:host-gateway" volumes: - - ./private/uploads/backups:/data/uploads/backups + - ./private/uploads/tmp:/data/uploads/tmp volumes: postgres-data: diff --git a/proxy.ts b/proxy.ts index 0541851d..a7dfd6e7 100644 --- a/proxy.ts +++ b/proxy.ts @@ -53,6 +53,8 @@ function checkRouteExists(pathname: string) { const routePatterns = [ /^\/api\/agent\/[^/]+\/status\/?$/, /^\/api\/agent\/[^/]+\/backup\/?$/, + /^\/api\/agent\/[^/]+\/backup\/upload\/init\/?$/, + /^\/api\/agent\/[^/]+\/backup\/upload\/status\/?$/, /^\/api\/agent\/[^/]+\/restore\/?$/, /^\/api\/files\/images\/[^/]+\/?$/, /^\/api\/files\/backups\/?$/, diff --git a/src/components/wrappers/dashboard/admin/channels/channel/channel-form/providers/storages/forms/s3.form.tsx b/src/components/wrappers/dashboard/admin/channels/channel/channel-form/providers/storages/forms/s3.form.tsx index 9efef758..db72647f 100644 --- a/src/components/wrappers/dashboard/admin/channels/channel/channel-form/providers/storages/forms/s3.form.tsx +++ b/src/components/wrappers/dashboard/admin/channels/channel/channel-form/providers/storages/forms/s3.form.tsx @@ -82,7 +82,7 @@ export const StorageS3Form = ({form}: StorageS3FormProps) => { ( Use SSL diff --git a/src/components/wrappers/dashboard/database/import/upload-backup.action.ts b/src/components/wrappers/dashboard/database/import/upload-backup.action.ts index 51503a39..403b09c0 100644 --- a/src/components/wrappers/dashboard/database/import/upload-backup.action.ts +++ b/src/components/wrappers/dashboard/database/import/upload-backup.action.ts @@ -4,13 +4,11 @@ import {ServerActionResult} from "@/types/action-type"; import {db} from "@/db"; import * as drizzleDb from "@/db"; import {Backup} from "@/db/schema/07_database"; -import {getFileExtension} from "../../../../../../app/api/agent/[agentId]/backup/helpers"; import {v4 as uuidv4} from "uuid"; import {eq} from "drizzle-orm"; -import {uploadLocalPrivate, uploadS3Private} from "@/features/upload/private/upload.action"; import {z} from "zod"; -import {env} from "@/env.mjs"; import {storeBackupFiles} from "@/features/storages/helpers"; +import {getFileExtension} from "@/features/api/upload/helpers/file"; export const uploadBackupAction = userAction diff --git a/src/db/schema/13_storage-policy.ts b/src/db/schema/13_storage-policy.ts index d2fc0f55..0c2c4ab9 100644 --- a/src/db/schema/13_storage-policy.ts +++ b/src/db/schema/13_storage-policy.ts @@ -1,10 +1,13 @@ import {boolean, pgTable, uuid} from "drizzle-orm/pg-core"; import {timestamps} from "@/db/schema/00_common"; import {relations} from "drizzle-orm"; -import {database} from "@/db/schema/07_database"; +import {Backup, Database, database, Restoration, RetentionPolicy} from "@/db/schema/07_database"; import {createSelectSchema} from "drizzle-zod"; import {z} from "zod"; -import {storageChannel} from "@/db/schema/12_storage-channel"; +import {StorageChannel, storageChannel} from "@/db/schema/12_storage-channel"; +import {Agent} from "@/db/schema/08_agent"; +import {Project} from "@/db/schema/06_project"; +import {AlertPolicy} from "@/db/schema/10_alert-policy"; export const storagePolicy = pgTable('storage_policy', { id: uuid('id').defaultRandom().primaryKey(), @@ -31,3 +34,8 @@ export const storagePolicyRelations = relations(storagePolicy, ({one}) => ({ export const storagePolicySchema = createSelectSchema(storagePolicy); export type StoragePolicy = z.infer; + + +export type StoragePolicyWith = StoragePolicy & { + storageChannel: StorageChannel; +}; diff --git a/src/features/storages/providers/s3.ts b/src/features/storages/providers/s3.ts index 2ea04b9b..ab521b7d 100644 --- a/src/features/storages/providers/s3.ts +++ b/src/features/storages/providers/s3.ts @@ -8,7 +8,7 @@ type S3Config = { secretKey: string; bucketName: string; port?: number; - useSSL?: boolean; + ssl?: boolean; }; async function getS3Client(config: S3Config) { @@ -17,7 +17,7 @@ async function getS3Client(config: S3Config) { accessKey: config.accessKey, secretKey: config.secretKey, port: config.port ?? 443, - useSSL: config.useSSL ?? true, + useSSL: config.ssl ?? true, }); }