fix: new storage architecture

This commit is contained in:
charlesgauthereau
2026-02-08 20:42:42 +01:00
parent cdd01e695c
commit c328620f3d
15 changed files with 705 additions and 338 deletions
+46 -47
View File
@@ -1,53 +1,52 @@
import fs from "node:fs";
import forge from "node-forge";
import { NextResponse } from "next/server";
import { eq } from "drizzle-orm";
import { db } from "@/db";
import * as drizzleDb from "@/db";
export function withAgentCheck(handler: Function) {
return async (request: Request, context: { params: Promise<{ agentId: string }> }) => {
try {
const agentId = (await context.params).agentId;
const agent = await db.query.agent.findFirst({
where: eq(drizzleDb.schemas.agent.id, agentId),
});
if (!agent) {
return NextResponse.json(
{ error: "Agent not found" },
{ status: 404 }
);
}
return handler(request, { ...context, agent });
} catch (err) {
console.error("Error in agent middleware:", err);
return NextResponse.json(
{ error: "Internal server error" },
{ status: 500 }
);
}
};
}
export async function decryptedDump(file: File, aesKeyHex: string, ivHex: string, fileExtension: string): Promise<File> {
const privateKeyPem = fs.readFileSync("private/keys/server_private.pem", "utf8");
const privateKey = forge.pki.privateKeyFromPem(privateKeyPem);
// Decrypt AES key with RSA-OAEP
const encryptedAesKey = forge.util.hexToBytes(aesKeyHex);
const aesKey = privateKey.decrypt(encryptedAesKey, "RSA-OAEP", {
md: forge.md.sha256.create(),
mgf1: {md: forge.md.sha256.create()},
export async function getDatabaseOrThrow(generatedId: string) {
const database = await db.query.database.findFirst({
where: eq(drizzleDb.schemas.database.agentDatabaseId, generatedId),
with: {
project: true,
alertPolicies: true,
storagePolicies: true
}
});
// Read encrypted file content
const encryptedBuffer = Buffer.from(await file.arrayBuffer());
const iv = forge.util.hexToBytes(ivHex);
// AES decryption
const decipher = forge.cipher.createDecipher("AES-CBC", aesKey);
decipher.start({iv});
decipher.update(forge.util.createBuffer(encryptedBuffer.toString("binary")));
const success = decipher.finish();
if (!success) {
throw new Error("Decryption failed");
if (!database) {
throw NextResponse.json(
{ error: "Database associated with generatedId not found" },
{ status: 404 }
);
}
const decryptedBytes = decipher.output.getBytes();
const decryptedBuffer = Buffer.from(decryptedBytes, "binary");
// Return a File so you can use file.arrayBuffer() later
return new File(
[decryptedBuffer],
file.name.replace(/\.enc$/, fileExtension),
{type: "application/octet-stream"}
);
}
export function getFileExtension(dbType: string) {
switch (dbType) {
case "postgresql":
return ".dump";
case "mysql":
return ".sql";
default:
return ".dump";
}
}
return database;
}
@@ -0,0 +1,53 @@
import fs from "node:fs";
import forge from "node-forge";
export async function decryptedDump(file: File, aesKeyHex: string, ivHex: string, fileExtension: string): Promise<File> {
const privateKeyPem = fs.readFileSync("private/keys/server_private.pem", "utf8");
const privateKey = forge.pki.privateKeyFromPem(privateKeyPem);
// Decrypt AES key with RSA-OAEP
const encryptedAesKey = forge.util.hexToBytes(aesKeyHex);
const aesKey = privateKey.decrypt(encryptedAesKey, "RSA-OAEP", {
md: forge.md.sha256.create(),
mgf1: {md: forge.md.sha256.create()},
});
// Read encrypted file content
const encryptedBuffer = Buffer.from(await file.arrayBuffer());
const iv = forge.util.hexToBytes(ivHex);
// AES decryption
const decipher = forge.cipher.createDecipher("AES-CBC", aesKey);
decipher.start({iv});
decipher.update(forge.util.createBuffer(encryptedBuffer.toString("binary")));
const success = decipher.finish();
if (!success) {
throw new Error("Decryption failed");
}
const decryptedBytes = decipher.output.getBytes();
const decryptedBuffer = Buffer.from(decryptedBytes, "binary");
// Return a File so you can use file.arrayBuffer() later
return new File(
[decryptedBuffer],
file.name.replace(/\.enc$/, fileExtension),
{type: "application/octet-stream"}
);
}
export function getFileExtension(dbType: string) {
switch (dbType) {
case "postgresql":
return ".dump";
case "mysql":
return ".sql";
default:
return ".dump";
}
}
+169
View File
@@ -0,0 +1,169 @@
import {NextResponse} from "next/server";
import {isUuidv4} from "@/utils/verify-uuid";
import {v4 as uuidv4} from "uuid";
import * as drizzleDb from "@/db";
import {db} from "@/db";
import {Backup} from "@/db/schema/07_database";
import {and, eq} from "drizzle-orm";
import {withUpdatedAt} from "@/db/utils";
import {decryptedDump, getFileExtension} from "./helpers";
import {sendNotificationsBackupRestore} from "@/features/notifications/helpers";
import {storeBackupFiles} from "@/features/storages/helpers";
import {eventEmitter} from "@/features/shared/event";
export async function POST(
request: Request,
{params}: { params: Promise<{ agentId: string }> }
) {
try {
const contentType = request.headers.get("Content-Type");
if (!contentType || !contentType.includes("multipart/form-data")) {
return NextResponse.json(
{error: "Unsupported or missing Content-Type"},
{status: 400}
);
}
eventEmitter.emit('modification', {update: true});
const agentId = (await params).agentId;
const formData = await request.formData();
const aesKeyHex = formData.get("aes_key") as string;
const ivHex = formData.get("iv") as string;
const generatedId = formData.get("generatedId") as string | null;
const method = formData.get("method") as string | null;
if (!generatedId || !isUuidv4(generatedId)) {
return NextResponse.json(
{error: "generatedId is not a valid UUID"},
{status: 400}
);
}
const agent = await db.query.agent.findFirst({
where: eq(drizzleDb.schemas.agent.id, agentId),
});
if (!agent) {
return NextResponse.json(
{error: "Agent not found"},
{status: 404}
);
}
const database = await db.query.database.findFirst({
where: eq(drizzleDb.schemas.database.agentDatabaseId, generatedId),
with: {
project: true,
alertPolicies: true,
storagePolicies: true
}
});
if (!database) {
return NextResponse.json(
{error: "Database associated with generatedId not found"},
{status: 404}
);
}
let backup: Backup | null | undefined = null;
if (method === "automatic") {
[backup] = await db
.insert(drizzleDb.schemas.backup)
.values({
status: 'ongoing',
databaseId: database.id,
})
.returning();
if (!backup) {
return NextResponse.json(
{error: "Unable to create an automatic backup"},
{status: 500}
);
}
} else {
backup = await db.query.backup.findFirst({
where: and(
eq(drizzleDb.schemas.backup.status, 'ongoing'),
eq(drizzleDb.schemas.backup.databaseId, database.id),
),
});
if (!backup) {
return NextResponse.json(
{error: "Unable to find the corresponding backup"},
{status: 404}
);
}
}
const status = formData.get("status") as string | null;
if (status === "success") {
const file = formData.get("file") as File | null;
const extension = formData.get("extension") as string | null;
if (!aesKeyHex || !ivHex) {
return NextResponse.json({error: "Missing fields"}, {status: 400});
}
if (!file) {
return NextResponse.json(
{error: "File is required for successful backup"},
{status: 400}
);
}
const fileSizeBytes = file.size;
// const fileExtension = '.' + (file.name.split('.').pop()?.toLowerCase() || '');
const fileExtension = extension ? extension : getFileExtension(database.dbms)
const decryptedFile = await decryptedDump(file, aesKeyHex, ivHex, fileExtension);
const uuid = uuidv4();
const fileName = `${uuid}${fileExtension}`;
const buffer = Buffer.from(await decryptedFile.arrayBuffer());
const storageResults = await storeBackupFiles(backup, database, buffer, fileName)
eventEmitter.emit('modification', {update: true});
await sendNotificationsBackupRestore(database, "success_backup");
return NextResponse.json(
{
message: "Backup successfully uploaded",
},
{status: 200}
);
} else {
await db
.update(drizzleDb.schemas.backup)
.set(withUpdatedAt({status: 'failed'}))
.where(eq(drizzleDb.schemas.backup.id, backup.id));
eventEmitter.emit('modification', {update: true});
await sendNotificationsBackupRestore(database, "error_backup");
return NextResponse.json(
{
message: "Backup successfully updated with status failed",
},
{status: 200}
);
}
} catch (error) {
console.error("Error in POST handler:", error);
return NextResponse.json(
{error: "Internal server error"},
{status: 500}
);
}
}
+92 -121
View File
@@ -1,73 +1,34 @@
import {NextResponse} from "next/server";
import {isUuidv4} from "@/utils/verify-uuid";
import {v4 as uuidv4} from "uuid";
import * as drizzleDb from "@/db";
import {db} from "@/db";
import {Backup} from "@/db/schema/07_database";
import {and, eq} from "drizzle-orm";
import * as drizzleDb from "@/db";
import {db as dbClient, db} from "@/db";
import {getDatabaseOrThrow, withAgentCheck} from "./helpers";
import {Backup} from "@/db/schema/07_database";
import {withUpdatedAt} from "@/db/utils";
import {decryptedDump, getFileExtension} from "./helpers";
import {sendNotificationsBackupRestore} from "@/features/notifications/helpers";
import {storeBackupFiles} from "@/features/storages/helpers";
import {eventEmitter} from "@/features/shared/event";
import {sendNotificationsBackupRestore} from "@/features/notifications/helpers";
import {EventKind} from "@/features/notifications/types";
export async function POST(
request: Request,
{params}: { params: Promise<{ agentId: string }> }
) {
export type BodyPost = {
method: "manual" | "automatic"
generatedId: string
}
export type BodyPatch = {
backupId: string
status: "success" | "failed"
size: number
generatedId: string
}
export const POST = withAgentCheck(async (request: Request, {params, agent}: {
params: Promise<{ agentId: string }>,
agent: any
}) => {
try {
const contentType = request.headers.get("Content-Type");
if (!contentType || !contentType.includes("multipart/form-data")) {
return NextResponse.json(
{error: "Unsupported or missing Content-Type"},
{status: 400}
);
}
eventEmitter.emit('modification', {update: true});
const agentId = (await params).agentId;
const formData = await request.formData();
const aesKeyHex = formData.get("aes_key") as string;
const ivHex = formData.get("iv") as string;
const generatedId = formData.get("generatedId") as string | null;
const method = formData.get("method") as string | null;
if (!generatedId || !isUuidv4(generatedId)) {
return NextResponse.json(
{error: "generatedId is not a valid UUID"},
{status: 400}
);
}
const agent = await db.query.agent.findFirst({
where: eq(drizzleDb.schemas.agent.id, agentId),
});
if (!agent) {
return NextResponse.json(
{error: "Agent not found"},
{status: 404}
);
}
const database = await db.query.database.findFirst({
where: eq(drizzleDb.schemas.database.agentDatabaseId, generatedId),
with: {
project: true,
alertPolicies: true,
storagePolicies: true
}
});
if (!database) {
return NextResponse.json(
{error: "Database associated with generatedId not found"},
{status: 404}
);
}
const body: BodyPost = await request.json();
const method = body.method
const database = await getDatabaseOrThrow(body.generatedId);
let backup: Backup | null | undefined = null;
@@ -79,8 +40,6 @@ export async function POST(
databaseId: database.id,
})
.returning();
if (!backup) {
return NextResponse.json(
{error: "Unable to create an automatic backup"},
@@ -104,66 +63,78 @@ export async function POST(
}
}
const status = formData.get("status") as string | null;
if (status === "success") {
const file = formData.get("file") as File | null;
const extension = formData.get("extension") as string | null;
if (!aesKeyHex || !ivHex) {
return NextResponse.json({error: "Missing fields"}, {status: 400});
}
eventEmitter.emit('modification', {update: true});
if (!file) {
return NextResponse.json(
{error: "File is required for successful backup"},
{status: 400}
);
}
const fileSizeBytes = file.size;
// const fileExtension = '.' + (file.name.split('.').pop()?.toLowerCase() || '');
const fileExtension = extension ? extension : getFileExtension(database.dbms)
const decryptedFile = await decryptedDump(file, aesKeyHex, ivHex, fileExtension);
const uuid = uuidv4();
const fileName = `${uuid}${fileExtension}`;
const buffer = Buffer.from(await decryptedFile.arrayBuffer());
const storageResults = await storeBackupFiles(backup, database, buffer, fileName)
eventEmitter.emit('modification', {update: true});
await sendNotificationsBackupRestore(database, "success_backup");
return NextResponse.json(
{
message: "Backup successfully uploaded",
},
{status: 200}
);
} else {
await db
.update(drizzleDb.schemas.backup)
.set(withUpdatedAt({status: 'failed'}))
.where(eq(drizzleDb.schemas.backup.id, backup.id));
eventEmitter.emit('modification', {update: true});
await sendNotificationsBackupRestore(database, "error_backup");
return NextResponse.json(
{
message: "Backup successfully updated with status failed",
},
{status: 200}
);
}
return NextResponse.json(
{
message: "Init backup success",
backup: backup,
},
{status: 200}
);
} catch (error) {
console.error("Error in POST handler:", error);
console.error("Error in POST for INIT backup:", error);
return NextResponse.json(
{error: "Internal server error"},
{status: 500}
);
}
}
});
export const PATCH = withAgentCheck(async (request: Request, {params, agent}: {
params: Promise<{ agentId: string }>,
agent: any
}) => {
try {
const body: BodyPatch = await request.json();
console.log(body);
const status = body.status
const backupId = body.backupId
const backupSize = body.size
const database = await getDatabaseOrThrow(body.generatedId);
const backup = await db.query.backup.findFirst({
where: eq(drizzleDb.schemas.backup.id, backupId),
});
if (!backup) {
return NextResponse.json(
{error: "No backup found"},
{status: 500}
);
}
const [backupUpdated] = await dbClient
.update(drizzleDb.schemas.backup)
.set(withUpdatedAt({
status: status,
fileSize: backupSize
}))
.where(eq(drizzleDb.schemas.backup.id, backup.id))
.returning();
eventEmitter.emit('modification', {update: true});
await sendNotificationsBackupRestore(database, `${status}_backup` as EventKind);
return NextResponse.json(
{
message: "Backup successfully updated",
backup: backupUpdated,
},
{status: 200}
);
} catch (error) {
console.error("Error in PATCH backup:", error);
return NextResponse.json(
{error: "Internal server error"},
{status: 500}
);
}
});
@@ -0,0 +1,76 @@
import {NextResponse} from "next/server";
import {and, eq} from "drizzle-orm";
import * as drizzleDb from "@/db";
import {db} from "@/db";
import {getDatabaseOrThrow, withAgentCheck} from "../../helpers";
import {isUuidv4} from "@/utils/verify-uuid";
import {eventEmitter} from "@/features/shared/event";
export type Body = {
generatedId: string
storageChannelId: string
backupId: string
}
export const POST = withAgentCheck(async (request: Request, {params, agent}: {
params: Promise<{ agentId: string }>,
agent: any
}) => {
try {
const body: Body = await request.json();
console.log("body", body);
const generatedId = body.generatedId;
const storageChannelId = body.storageChannelId;
const backupId = body.backupId;
if (!generatedId || !isUuidv4(generatedId)) {
return NextResponse.json(
{error: "generatedId is not a valid UUID"},
{status: 400}
);
}
const database = await getDatabaseOrThrow(generatedId);
const backup = await db.query.backup.findFirst({
where: and(
eq(drizzleDb.schemas.backup.id, backupId),
eq(drizzleDb.schemas.backup.databaseId, database.id),
),
});
if (!backup) {
return NextResponse.json(
{error: "Unable to find the corresponding backup"},
{status: 404}
);
}
const [backupStorage] = await db
.insert(drizzleDb.schemas.backupStorage)
.values({
backupId: backup.id,
storageChannelId: storageChannelId,
status: "pending",
})
.returning();
eventEmitter.emit('modification', {update: true});
return NextResponse.json(
{
message: "Backup storage successfully created",
backupStorage: backupStorage
},
{status: 200}
);
} catch (error) {
console.error("Error in POST for INIT backup:", error);
return NextResponse.json(
{error: "Internal server error"},
{status: 500}
);
}
});
@@ -0,0 +1,93 @@
import {NextResponse} from "next/server";
import {and, eq} from "drizzle-orm";
import * as drizzleDb from "@/db";
import {db as dbClient, db} from "@/db";
import {withUpdatedAt} from "@/db/utils";
import {getDatabaseOrThrow, withAgentCheck} from "../../helpers";
import {eventEmitter} from "@/features/shared/event";
export type Body = {
generatedId: string
status: "success" | "failed"
backupStorageId: string
path: string
size: number
backupId: string
}
export const PATCH = withAgentCheck(async (request: Request, {params, agent}: {
params: Promise<{ agentId: string }>,
agent: any
}) => {
try {
const body: Body = await request.json();
const generatedId = body.generatedId;
const status = body.status;
const filePath = body.path;
const fileSize = body.size;
const backupStorageId = body.backupStorageId;
const backupId = body.backupId;
console.log("body", body);
const database = await getDatabaseOrThrow(generatedId);
const backup = await db.query.backup.findFirst({
where: and(
eq(drizzleDb.schemas.backup.id, backupId),
eq(drizzleDb.schemas.backup.databaseId, database.id),
),
with: {
storages: true
}
});
if (!backup) {
return NextResponse.json(
{error: "Unable to find the corresponding backup"},
{status: 404}
);
}
const [backupStorage] = await dbClient
.update(drizzleDb.schemas.backupStorage)
.set(withUpdatedAt({
status: status,
path: filePath,
size: fileSize
}))
.where(eq(drizzleDb.schemas.backupStorage.id, backupStorageId))
.returning();
if (backup.storages.length > 0) {
const hasSuccessfulStorage = backup.storages.some(
(storage) => storage.status === "success"
);
if (hasSuccessfulStorage && backup.status !== "success") {
await db
.update(drizzleDb.schemas.backup)
.set({status: "success"})
.where(eq(drizzleDb.schemas.backup.id, backup.id));
}
}
eventEmitter.emit('modification', {update: true});
return NextResponse.json({
message: "Backup status successfully updated",
backupStorage: backupStorage
},
{status: 200}
);
} catch (error) {
console.error("Error in POST for INIT backup:", error);
return NextResponse.json(
{error: "Internal server error"},
{status: 500}
);
}
});