External/portabase
1
0
Fork 0
mirror of https://github.com/Portabase/portabase.git synced 2026-07-14 11:16:13 +02:00
Code Issues Packages Projects Releases Wiki Activity

fix: new storage architecture

Browse Source
This commit is contained in:
charlesgauthereau
2026-02-08 20:42:42 +01:00
parent cdd01e695c
commit c328620f3d
15 changed files with 705 additions and 338 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/api/agent/[agentId]/backup/helpers.ts
+46 -47
View File
@@ -1,53 +1,52 @@
import fs from "node:fs";
import forge from "node-forge";
import { NextResponse } from "next/server";
import { eq } from "drizzle-orm";
import { db } from "@/db";
import * as drizzleDb from "@/db";
export function withAgentCheck(handler: Function) {
return async (request: Request, context: { params: Promise<{ agentId: string }> }) => {
try {
const agentId = (await context.params).agentId;
const agent = await db.query.agent.findFirst({
where: eq(drizzleDb.schemas.agent.id, agentId),
});
if (!agent) {
return NextResponse.json(
{ error: "Agent not found" },
{ status: 404 }
);
}
return handler(request, { ...context, agent });
} catch (err) {
console.error("Error in agent middleware:", err);
return NextResponse.json(
{ error: "Internal server error" },
{ status: 500 }
);
}
};
}
export async function decryptedDump(file: File, aesKeyHex: string, ivHex: string, fileExtension: string): Promise<File> {
const privateKeyPem = fs.readFileSync("private/keys/server_private.pem", "utf8");
const privateKey = forge.pki.privateKeyFromPem(privateKeyPem);
// Decrypt AES key with RSA-OAEP
const encryptedAesKey = forge.util.hexToBytes(aesKeyHex);
const aesKey = privateKey.decrypt(encryptedAesKey, "RSA-OAEP", {
md: forge.md.sha256.create(),
mgf1: {md: forge.md.sha256.create()},
export async function getDatabaseOrThrow(generatedId: string) {
const database = await db.query.database.findFirst({
where: eq(drizzleDb.schemas.database.agentDatabaseId, generatedId),
with: {
project: true,
alertPolicies: true,
storagePolicies: true
}
});
// Read encrypted file content
const encryptedBuffer = Buffer.from(await file.arrayBuffer());
const iv = forge.util.hexToBytes(ivHex);
// AES decryption
const decipher = forge.cipher.createDecipher("AES-CBC", aesKey);
decipher.start({iv});
decipher.update(forge.util.createBuffer(encryptedBuffer.toString("binary")));
const success = decipher.finish();
if (!success) {
throw new Error("Decryption failed");
if (!database) {
throw NextResponse.json(
{ error: "Database associated with generatedId not found" },
{ status: 404 }
);
}
const decryptedBytes = decipher.output.getBytes();
const decryptedBuffer = Buffer.from(decryptedBytes, "binary");
// Return a File so you can use file.arrayBuffer() later
return new File(
[decryptedBuffer],
file.name.replace(/\.enc$/, fileExtension),
{type: "application/octet-stream"}
);
}
export function getFileExtension(dbType: string) {
switch (dbType) {
case "postgresql":
return ".dump";
case "mysql":
return ".sql";
default:
return ".dump";
}
}
return database;
}
app/api/agent/[agentId]/backup/old/helpers.ts
+53
View File
@@ -0,0 +1,53 @@
import fs from "node:fs";
import forge from "node-forge";
export async function decryptedDump(file: File, aesKeyHex: string, ivHex: string, fileExtension: string): Promise<File> {
const privateKeyPem = fs.readFileSync("private/keys/server_private.pem", "utf8");
const privateKey = forge.pki.privateKeyFromPem(privateKeyPem);
// Decrypt AES key with RSA-OAEP
const encryptedAesKey = forge.util.hexToBytes(aesKeyHex);
const aesKey = privateKey.decrypt(encryptedAesKey, "RSA-OAEP", {
md: forge.md.sha256.create(),
mgf1: {md: forge.md.sha256.create()},
});
// Read encrypted file content
const encryptedBuffer = Buffer.from(await file.arrayBuffer());
const iv = forge.util.hexToBytes(ivHex);
// AES decryption
const decipher = forge.cipher.createDecipher("AES-CBC", aesKey);
decipher.start({iv});
decipher.update(forge.util.createBuffer(encryptedBuffer.toString("binary")));
const success = decipher.finish();
if (!success) {
throw new Error("Decryption failed");
}
const decryptedBytes = decipher.output.getBytes();
const decryptedBuffer = Buffer.from(decryptedBytes, "binary");
// Return a File so you can use file.arrayBuffer() later
return new File(
[decryptedBuffer],
file.name.replace(/\.enc$/, fileExtension),
{type: "application/octet-stream"}
);
}
export function getFileExtension(dbType: string) {
switch (dbType) {
case "postgresql":
return ".dump";
case "mysql":
return ".sql";
default:
return ".dump";
}
}
app/api/agent/[agentId]/backup/old/route.ts
+169
View File
@@ -0,0 +1,169 @@
import {NextResponse} from "next/server";
import {isUuidv4} from "@/utils/verify-uuid";
import {v4 as uuidv4} from "uuid";
import * as drizzleDb from "@/db";
import {db} from "@/db";
import {Backup} from "@/db/schema/07_database";
import {and, eq} from "drizzle-orm";
import {withUpdatedAt} from "@/db/utils";
import {decryptedDump, getFileExtension} from "./helpers";
import {sendNotificationsBackupRestore} from "@/features/notifications/helpers";
import {storeBackupFiles} from "@/features/storages/helpers";
import {eventEmitter} from "@/features/shared/event";
export async function POST(
request: Request,
{params}: { params: Promise<{ agentId: string }> }
) {
try {
const contentType = request.headers.get("Content-Type");
if (!contentType || !contentType.includes("multipart/form-data")) {
return NextResponse.json(
{error: "Unsupported or missing Content-Type"},
{status: 400}
);
}
eventEmitter.emit('modification', {update: true});
const agentId = (await params).agentId;
const formData = await request.formData();
const aesKeyHex = formData.get("aes_key") as string;
const ivHex = formData.get("iv") as string;
const generatedId = formData.get("generatedId") as string | null;
const method = formData.get("method") as string | null;
if (!generatedId || !isUuidv4(generatedId)) {
return NextResponse.json(
{error: "generatedId is not a valid UUID"},
{status: 400}
);
}
const agent = await db.query.agent.findFirst({
where: eq(drizzleDb.schemas.agent.id, agentId),
});
if (!agent) {
return NextResponse.json(
{error: "Agent not found"},
{status: 404}
);
}
const database = await db.query.database.findFirst({
where: eq(drizzleDb.schemas.database.agentDatabaseId, generatedId),
with: {
project: true,
alertPolicies: true,
storagePolicies: true
}
});
if (!database) {
return NextResponse.json(
{error: "Database associated with generatedId not found"},
{status: 404}
);
}
let backup: Backup | null | undefined = null;
if (method === "automatic") {
[backup] = await db
.insert(drizzleDb.schemas.backup)
.values({
status: 'ongoing',
databaseId: database.id,
})
.returning();
if (!backup) {
return NextResponse.json(
{error: "Unable to create an automatic backup"},
{status: 500}
);
}
} else {
backup = await db.query.backup.findFirst({
where: and(
eq(drizzleDb.schemas.backup.status, 'ongoing'),
eq(drizzleDb.schemas.backup.databaseId, database.id),
),
});
if (!backup) {
return NextResponse.json(
{error: "Unable to find the corresponding backup"},
{status: 404}
);
}
}
const status = formData.get("status") as string | null;
if (status === "success") {
const file = formData.get("file") as File | null;
const extension = formData.get("extension") as string | null;
if (!aesKeyHex || !ivHex) {
return NextResponse.json({error: "Missing fields"}, {status: 400});
}
if (!file) {
return NextResponse.json(
{error: "File is required for successful backup"},
{status: 400}
);
}
const fileSizeBytes = file.size;
// const fileExtension = '.' + (file.name.split('.').pop()?.toLowerCase() || '');
const fileExtension = extension ? extension : getFileExtension(database.dbms)
const decryptedFile = await decryptedDump(file, aesKeyHex, ivHex, fileExtension);
const uuid = uuidv4();
const fileName = `${uuid}${fileExtension}`;
const buffer = Buffer.from(await decryptedFile.arrayBuffer());
const storageResults = await storeBackupFiles(backup, database, buffer, fileName)
eventEmitter.emit('modification', {update: true});
await sendNotificationsBackupRestore(database, "success_backup");
return NextResponse.json(
{
message: "Backup successfully uploaded",
},
{status: 200}
);
} else {
await db
.update(drizzleDb.schemas.backup)
.set(withUpdatedAt({status: 'failed'}))
.where(eq(drizzleDb.schemas.backup.id, backup.id));
eventEmitter.emit('modification', {update: true});
await sendNotificationsBackupRestore(database, "error_backup");
return NextResponse.json(
{
message: "Backup successfully updated with status failed",
},
{status: 200}
);
}
} catch (error) {
console.error("Error in POST handler:", error);
return NextResponse.json(
{error: "Internal server error"},
{status: 500}
);
}
}
app/api/agent/[agentId]/backup/route.ts
+92 -121
View File
@@ -1,73 +1,34 @@
import {NextResponse} from "next/server";
import {isUuidv4} from "@/utils/verify-uuid";
import {v4 as uuidv4} from "uuid";
import * as drizzleDb from "@/db";
import {db} from "@/db";
import {Backup} from "@/db/schema/07_database";
import {and, eq} from "drizzle-orm";
import * as drizzleDb from "@/db";
import {db as dbClient, db} from "@/db";
import {getDatabaseOrThrow, withAgentCheck} from "./helpers";
import {Backup} from "@/db/schema/07_database";
import {withUpdatedAt} from "@/db/utils";
import {decryptedDump, getFileExtension} from "./helpers";
import {sendNotificationsBackupRestore} from "@/features/notifications/helpers";
import {storeBackupFiles} from "@/features/storages/helpers";
import {eventEmitter} from "@/features/shared/event";
import {sendNotificationsBackupRestore} from "@/features/notifications/helpers";
import {EventKind} from "@/features/notifications/types";
export async function POST(
request: Request,
{params}: { params: Promise<{ agentId: string }> }
) {
export type BodyPost = {
method: "manual" | "automatic"
generatedId: string
}
export type BodyPatch = {
backupId: string
status: "success" | "failed"
size: number
generatedId: string
}
export const POST = withAgentCheck(async (request: Request, {params, agent}: {
params: Promise<{ agentId: string }>,
agent: any
}) => {
try {
const contentType = request.headers.get("Content-Type");
if (!contentType || !contentType.includes("multipart/form-data")) {
return NextResponse.json(
{error: "Unsupported or missing Content-Type"},
{status: 400}
);
}
eventEmitter.emit('modification', {update: true});
const agentId = (await params).agentId;
const formData = await request.formData();
const aesKeyHex = formData.get("aes_key") as string;
const ivHex = formData.get("iv") as string;
const generatedId = formData.get("generatedId") as string | null;
const method = formData.get("method") as string | null;
if (!generatedId || !isUuidv4(generatedId)) {
return NextResponse.json(
{error: "generatedId is not a valid UUID"},
{status: 400}
);
}
const agent = await db.query.agent.findFirst({
where: eq(drizzleDb.schemas.agent.id, agentId),
});
if (!agent) {
return NextResponse.json(
{error: "Agent not found"},
{status: 404}
);
}
const database = await db.query.database.findFirst({
where: eq(drizzleDb.schemas.database.agentDatabaseId, generatedId),
with: {
project: true,
alertPolicies: true,
storagePolicies: true
}
});
if (!database) {
return NextResponse.json(
{error: "Database associated with generatedId not found"},
{status: 404}
);
}
const body: BodyPost = await request.json();
const method = body.method
const database = await getDatabaseOrThrow(body.generatedId);
let backup: Backup | null | undefined = null;
@@ -79,8 +40,6 @@ export async function POST(
databaseId: database.id,
})
.returning();
if (!backup) {
return NextResponse.json(
{error: "Unable to create an automatic backup"},
@@ -104,66 +63,78 @@ export async function POST(
}
}
const status = formData.get("status") as string | null;
if (status === "success") {
const file = formData.get("file") as File | null;
const extension = formData.get("extension") as string | null;
if (!aesKeyHex || !ivHex) {
return NextResponse.json({error: "Missing fields"}, {status: 400});
}
eventEmitter.emit('modification', {update: true});
if (!file) {
return NextResponse.json(
{error: "File is required for successful backup"},
{status: 400}
);
}
const fileSizeBytes = file.size;
// const fileExtension = '.' + (file.name.split('.').pop()?.toLowerCase() || '');
const fileExtension = extension ? extension : getFileExtension(database.dbms)
const decryptedFile = await decryptedDump(file, aesKeyHex, ivHex, fileExtension);
const uuid = uuidv4();
const fileName = `${uuid}${fileExtension}`;
const buffer = Buffer.from(await decryptedFile.arrayBuffer());
const storageResults = await storeBackupFiles(backup, database, buffer, fileName)
eventEmitter.emit('modification', {update: true});
await sendNotificationsBackupRestore(database, "success_backup");
return NextResponse.json(
{
message: "Backup successfully uploaded",
},
{status: 200}
);
} else {
await db
.update(drizzleDb.schemas.backup)
.set(withUpdatedAt({status: 'failed'}))
.where(eq(drizzleDb.schemas.backup.id, backup.id));
eventEmitter.emit('modification', {update: true});
await sendNotificationsBackupRestore(database, "error_backup");
return NextResponse.json(
{
message: "Backup successfully updated with status failed",
},
{status: 200}
);
}
return NextResponse.json(
{
message: "Init backup success",
backup: backup,
},
{status: 200}
);
} catch (error) {
console.error("Error in POST handler:", error);
console.error("Error in POST for INIT backup:", error);
return NextResponse.json(
{error: "Internal server error"},
{status: 500}
);
}
}
});
export const PATCH = withAgentCheck(async (request: Request, {params, agent}: {
params: Promise<{ agentId: string }>,
agent: any
}) => {
try {
const body: BodyPatch = await request.json();
console.log(body);
const status = body.status
const backupId = body.backupId
const backupSize = body.size
const database = await getDatabaseOrThrow(body.generatedId);
const backup = await db.query.backup.findFirst({
where: eq(drizzleDb.schemas.backup.id, backupId),
});
if (!backup) {
return NextResponse.json(
{error: "No backup found"},
{status: 500}
);
}
const [backupUpdated] = await dbClient
.update(drizzleDb.schemas.backup)
.set(withUpdatedAt({
status: status,
fileSize: backupSize
}))
.where(eq(drizzleDb.schemas.backup.id, backup.id))
.returning();
eventEmitter.emit('modification', {update: true});
await sendNotificationsBackupRestore(database, `${status}_backup` as EventKind);
return NextResponse.json(
{
message: "Backup successfully updated",
backup: backupUpdated,
},
{status: 200}
);
} catch (error) {
console.error("Error in PATCH backup:", error);
return NextResponse.json(
{error: "Internal server error"},
{status: 500}
);
}
});
app/api/agent/[agentId]/backup/upload/init/route.ts
+76
View File
@@ -0,0 +1,76 @@
import {NextResponse} from "next/server";
import {and, eq} from "drizzle-orm";
import * as drizzleDb from "@/db";
import {db} from "@/db";
import {getDatabaseOrThrow, withAgentCheck} from "../../helpers";
import {isUuidv4} from "@/utils/verify-uuid";
import {eventEmitter} from "@/features/shared/event";
export type Body = {
generatedId: string
storageChannelId: string
backupId: string
}
export const POST = withAgentCheck(async (request: Request, {params, agent}: {
params: Promise<{ agentId: string }>,
agent: any
}) => {
try {
const body: Body = await request.json();
console.log("body", body);
const generatedId = body.generatedId;
const storageChannelId = body.storageChannelId;
const backupId = body.backupId;
if (!generatedId || !isUuidv4(generatedId)) {
return NextResponse.json(
{error: "generatedId is not a valid UUID"},
{status: 400}
);
}
const database = await getDatabaseOrThrow(generatedId);
const backup = await db.query.backup.findFirst({
where: and(
eq(drizzleDb.schemas.backup.id, backupId),
eq(drizzleDb.schemas.backup.databaseId, database.id),
),
});
if (!backup) {
return NextResponse.json(
{error: "Unable to find the corresponding backup"},
{status: 404}
);
}
const [backupStorage] = await db
.insert(drizzleDb.schemas.backupStorage)
.values({
backupId: backup.id,
storageChannelId: storageChannelId,
status: "pending",
})
.returning();
eventEmitter.emit('modification', {update: true});
return NextResponse.json(
{
message: "Backup storage successfully created",
backupStorage: backupStorage
},
{status: 200}
);
} catch (error) {
console.error("Error in POST for INIT backup:", error);
return NextResponse.json(
{error: "Internal server error"},
{status: 500}
);
}
});
app/api/agent/[agentId]/backup/upload/status/route.ts
+93
View File
@@ -0,0 +1,93 @@
import {NextResponse} from "next/server";
import {and, eq} from "drizzle-orm";
import * as drizzleDb from "@/db";
import {db as dbClient, db} from "@/db";
import {withUpdatedAt} from "@/db/utils";
import {getDatabaseOrThrow, withAgentCheck} from "../../helpers";
import {eventEmitter} from "@/features/shared/event";
export type Body = {
generatedId: string
status: "success" | "failed"
backupStorageId: string
path: string
size: number
backupId: string
}
export const PATCH = withAgentCheck(async (request: Request, {params, agent}: {
params: Promise<{ agentId: string }>,
agent: any
}) => {
try {
const body: Body = await request.json();
const generatedId = body.generatedId;
const status = body.status;
const filePath = body.path;
const fileSize = body.size;
const backupStorageId = body.backupStorageId;
const backupId = body.backupId;
console.log("body", body);
const database = await getDatabaseOrThrow(generatedId);
const backup = await db.query.backup.findFirst({
where: and(
eq(drizzleDb.schemas.backup.id, backupId),
eq(drizzleDb.schemas.backup.databaseId, database.id),
),
with: {
storages: true
}
});
if (!backup) {
return NextResponse.json(
{error: "Unable to find the corresponding backup"},
{status: 404}
);
}
const [backupStorage] = await dbClient
.update(drizzleDb.schemas.backupStorage)
.set(withUpdatedAt({
status: status,
path: filePath,
size: fileSize
}))
.where(eq(drizzleDb.schemas.backupStorage.id, backupStorageId))
.returning();
if (backup.storages.length > 0) {
const hasSuccessfulStorage = backup.storages.some(
(storage) => storage.status === "success"
);
if (hasSuccessfulStorage && backup.status !== "success") {
await db
.update(drizzleDb.schemas.backup)
.set({status: "success"})
.where(eq(drizzleDb.schemas.backup.id, backup.id));
}
}
eventEmitter.emit('modification', {update: true});
return NextResponse.json({
message: "Backup status successfully updated",
backupStorage: backupStorage
},
{status: 200}
);
} catch (error) {
console.error("Error in POST for INIT backup:", error);
return NextResponse.json(
{error: "Internal server error"},
{status: 500}
);
}
});
app/api/agent/[agentId]/status/helpers.ts
+96 -52
View File
@@ -2,9 +2,9 @@ import {NextResponse} from "next/server";
import {Body} from "./route";
import {isUuidv4} from "@/utils/verify-uuid";
import {Agent} from "@/db/schema/08_agent";
import {Database} from "@/db/schema/07_database";
import {Database, DatabaseWith} from "@/db/schema/07_database";
import * as drizzleDb from "@/db";
import {db as dbClient} from "@/db";
import {db, db as dbClient} from "@/db";
import {and, eq, inArray} from "drizzle-orm";
import {dbmsEnumSchema, EDbmsSchema} from "@/db/schema/types";
import {withUpdatedAt} from "@/db/utils";
@@ -14,9 +14,11 @@ import {dispatchStorage} from "@/features/storages/dispatch";
export async function handleDatabases(body: Body, agent: Agent, lastContact: Date) {
const databasesResponse = [];
const formatDatabase = (database: Database, backupAction: boolean, restoreAction: boolean, UrlBackup: string) => ({
const formatDatabase = (database: DatabaseWith, backupAction: boolean, restoreAction: boolean, UrlBackup: string | null, storages: PingDatabaseStorageChannels[], urlMeta: string | null) => ({
generatedId: database.agentDatabaseId,
dbms: database.dbms,
storages: storages,
encrypt: false,
data: {
backup: {
action: backupAction,
@@ -25,6 +27,7 @@ export async function handleDatabases(body: Body, agent: Agent, lastContact: Dat
restore: {
action: restoreAction,
file: UrlBackup,
metaFile: urlMeta
},
},
});
@@ -32,12 +35,16 @@ export async function handleDatabases(body: Body, agent: Agent, lastContact: Dat
for (const db of body.databases) {
const existingDatabase = await dbClient.query.database.findFirst({
where: eq(drizzleDb.schemas.database.agentDatabaseId, db.generatedId)
where: eq(drizzleDb.schemas.database.agentDatabaseId, db.generatedId),
with: {
project: true
}
});
let backupAction: boolean = false
let restoreAction: boolean = false
let urlBackup: string = ""
let urlBackup: string | null = null;
let urlMeta: string | null = null
if (!existingDatabase) {
if (!isUuidv4(db.generatedId)) {
@@ -63,8 +70,11 @@ export async function handleDatabases(body: Body, agent: Agent, lastContact: Dat
})
.returning();
if (databaseCreated) {
databasesResponse.push(formatDatabase(databaseCreated, backupAction, restoreAction, urlBackup));
const storages = await getDatabaseStorageChannels(databaseCreated.id)
databasesResponse.push(formatDatabase(databaseCreated, backupAction, restoreAction, urlBackup, storages, null));
}
} else {
@@ -78,7 +88,6 @@ export async function handleDatabases(body: Body, agent: Agent, lastContact: Dat
.where(eq(drizzleDb.schemas.database.id, existingDatabase.id))
.returning();
const activeBackup = await dbClient.query.backup.findFirst({
where: and(
eq(drizzleDb.schemas.backup.databaseId, databaseUpdated.id),
@@ -86,7 +95,6 @@ export async function handleDatabases(body: Body, agent: Agent, lastContact: Dat
)
})
const restoration = await dbClient.query.restoration.findFirst({
where: and(eq(drizzleDb.schemas.restoration.databaseId, databaseUpdated.id), eq(drizzleDb.schemas.restoration.status, "waiting")),
with: {
@@ -94,7 +102,6 @@ export async function handleDatabases(body: Body, agent: Agent, lastContact: Dat
}
})
if (activeBackup && activeBackup.status == "waiting") {
backupAction = true
@@ -107,13 +114,11 @@ export async function handleDatabases(body: Body, agent: Agent, lastContact: Dat
if (restoration) {
restoreAction = true
if (!restoration.backupStorage || restoration.backupStorage.status != "success" || !restoration.backupStorage.path) {
restoreAction = false
continue;
}
const input: StorageInput = {
action: "get",
data: {
@@ -126,12 +131,26 @@ export async function handleDatabases(body: Body, agent: Agent, lastContact: Dat
}
};
const inputMeta: StorageInput = {
action: "get",
data: {
path: `${restoration.backupStorage.path}.meta`,
signedUrl: true,
},
metadata: {
storageId: restoration.backupStorage.storageChannelId,
fileKind: "backups"
}
};
try {
const result = await dispatchStorage(input, undefined, restoration.backupStorage.storageChannelId);
const resultMeta = await dispatchStorage(inputMeta, undefined, restoration.backupStorage.storageChannelId);
if (result.success) {
urlBackup = result.url ?? "";
urlBackup = result.url ?? null;
urlMeta = resultMeta.url ?? null
} else {
await dbClient
.update(drizzleDb.schemas.restoration)
@@ -151,53 +170,78 @@ export async function handleDatabases(body: Body, agent: Agent, lastContact: Dat
continue;
}
// const fileName = backupToRestore?.file
//
// let data: SafeActionResult<string, ZodString, readonly [], {
// _errors?: string[] | undefined;
// }, readonly [], ServerActionResult<string>, object> | undefined
//
// try {
//
// if (settings.storage == "local") {
// data = await getFileUrlPresignedLocal({fileName: fileName!})
// } else if (settings.storage == "s3") {
//
// data = await getFileUrlPreSignedS3Action(`backups/${backupToRestore?.database.project?.slug}/${fileName}`);
// }
//
// if (data?.data?.success) {
// urlBackup = data.data.value ?? "";
// } else {
// await dbClient
// .update(drizzleDb.schemas.restoration)
// .set({status: "failed"})
// .where(eq(drizzleDb.schemas.restoration.id, restoration.id));
//
// // @ts-ignore
// const errorMessage = data?.data?.actionError?.message || "Failed to get presigned URL";
// console.error("Restoration failed: ", errorMessage);
//
// continue;
// }
// } catch (err) {
// console.error("Restoration crashed unexpectedly:", err);
// await dbClient
// .update(drizzleDb.schemas.restoration)
// .set({status: "failed"})
// .where(eq(drizzleDb.schemas.restoration.id, restoration.id));
//
// continue;
// }
await dbClient
.update(drizzleDb.schemas.restoration)
.set({status: "ongoing"})
.where(eq(drizzleDb.schemas.restoration.id, restoration.id));
}
databasesResponse.push(formatDatabase(databaseUpdated, backupAction, restoreAction, urlBackup));
const storages = await getDatabaseStorageChannels(databaseUpdated.id)
databasesResponse.push(formatDatabase(databaseUpdated, backupAction, restoreAction, urlBackup, storages, urlMeta));
}
}
return databasesResponse;
}
type PingDatabaseStorageChannels = {
id: string;
config: any
provider: string
}
async function getDatabaseStorageChannels(databaseId: string): Promise<PingDatabaseStorageChannels[]> {
const database = await db.query.database.findFirst({
where: eq(drizzleDb.schemas.database.id, databaseId),
with: {
project: true,
retentionPolicy: true,
alertPolicies: true,
storagePolicies: true
}
});
if (!database) {
return []
}
const settings = await db.query.setting.findFirst({
where: eq(drizzleDb.schemas.setting.name, "system"),
with: {storageChannel: true},
});
const defaultStorageChannel: PingDatabaseStorageChannels[] = settings?.storageChannel
? [{
id: settings.storageChannel.id,
provider: settings.storageChannel.provider,
config: settings.storageChannel.config,
}]
: [];
const enabledDatabaseStorageChannels = await Promise.all(
(database.storagePolicies ?? [])
.filter(p => p.enabled)
.map(async policy => {
const storageChannel = await db.query.storageChannel.findFirst({
where: eq(drizzleDb.schemas.storageChannel.id, policy.storageChannelId),
});
if (!storageChannel) return null;
return {
id: storageChannel.id,
config: storageChannel.config,
provider: storageChannel.provider,
} as PingDatabaseStorageChannels;
})
);
const filteredChannels: PingDatabaseStorageChannels[] = enabledDatabaseStorageChannels.filter(
(c): c is PingDatabaseStorageChannels => c !== null
);
return filteredChannels.length > 0 ? filteredChannels : defaultStorageChannel;
}
app/api/agent/[agentId]/status/route.ts
+3
View File
@@ -26,12 +26,14 @@ export async function POST(
) {
try {
const agentId = (await params).agentId
console.log(agentId)
const body: Body = await request.json();
const lastContact = new Date();
let message: string
if (!isUuidv4(agentId)) {
message = "agentId is not a valid uuid"
console.error(message)
return NextResponse.json(
{error: "agentId is not a valid uuid"},
{status: 500}
@@ -66,6 +68,7 @@ export async function POST(
databases: databasesResponse
}
return Response.json(response)
} catch (error) {
console.error('Error in POST handler:', error);
app/api/tus/hooks/route.ts
+59 -104
View File
@@ -3,120 +3,75 @@ import fs from "fs";
import path from "path";
//
// export async function POST(request: Request) {
// try {
// const body = await request.json();
// console.log("tusd hook payload:", body);
//
// return NextResponse.json({
// "Upload-Defer-Length": true,
// "Upload-Metadata": "name somefilename",
// });
// } catch (error) {
// console.error("Error in POST handler:", error);
// return NextResponse.json(
// { error: "Internal server error" },
// { status: 500 }
// );
// }
// }
// export async function POST(request: Request) {
// try {
// const body = await request.json();
// console.log("TUSD hook payload:", body);
//
// // Pre-create hook: must return Upload-Defer-Length or Upload-Length
// if (body.event === "pre-create") {
// return NextResponse.json({
// "Upload-Defer-Length": true, // defer the upload length
// "Upload-Metadata": "name somefilename" // optional metadata
// });
// }
//
// // Post-receive hook: handle finished uploads
// if (body.event === "post-receive") {
// console.log(`Upload finished: ${body.id}, size: ${body.upload_length}`);
// // process the file, AES encryption, move, etc.
// }
//
// return NextResponse.json({ ok: true }); // safe fallback for other events
// } catch (error) {
// console.error("Hook error:", error);
// return NextResponse.json({ error: "Internal server error" }, { status: 500 });
// }
// }
export async function POST(request: Request) {
try {
const body = await request.json();
console.log("TUSD hook payload:", body);
const event = body.Event
const headers = event.HTTPRequest.Header
const uploadLength = headers["X-File-Size"]?.[0];
const uploadOffset = headers["Upload-Offset"]?.[0];
const status = headers["X-Status"]?.[0];
// if (body.event === "pre-create") {
// return NextResponse.json({
// "Upload-Defer-Length": true
// });
// }
console.log("TUSD hook payload:", body.Type);
console.log("TUSD hook payload:", headers);
console.log(`Upload ID : ${event.Upload.ID} (${uploadOffset}/${uploadLength})`);
if (
body.Type === "post-receive" &&
event.Upload.SizeIsDeferred === false &&
event.Upload.Offset === event.Upload.Size
) {
const id = event.Upload.ID;
// const extensionFile = headers["X-Extension"]?.[0] ?? ".bin";
const extensionFile = ".enc";
const filePath = path.join(process.cwd(), "/private/uploads/backups/");
fs.renameSync(
`${filePath}${id}`,
`${filePath}${id}${extensionFile}`
);
if (status === "success") {
if (
body.Type === "post-receive" &&
event.Upload.SizeIsDeferred === false &&
event.Upload.Offset === event.Upload.Size
) {
const id = event.Upload.ID;
const fileName = headers["X-File-Name"]?.[0];
const filePath = headers["X-File-Path"]?.[0];
if (!filePath) {
return NextResponse.json({error: "Missing X-File-Path"}, {status: 500});
}
const uploadDir = path.join(process.cwd(), "/private/uploads/");
const oldFilePath = path.join(uploadDir, "tmp", id);
const newFilePath = path.join(uploadDir, filePath);
fs.mkdirSync(path.dirname(newFilePath), {recursive: true});
let retries = 10;
while (!fs.existsSync(oldFilePath)) {
if (retries-- === 0) {
return NextResponse.json({error: `Upload file not found: ${oldFilePath}`}, {status: 500});
}
await new Promise(r => setTimeout(r, 200));
}
fs.renameSync(oldFilePath, newFilePath);
const infoFilePath = `${oldFilePath}.info`;
if (fs.existsSync(infoFilePath)) {
fs.unlinkSync(infoFilePath);
}
const metadataHeaderB64 = headers["Upload-Metadata"]?.[0];
if (metadataHeaderB64) {
const metadataHeader = Buffer.from(metadataHeaderB64, "base64").toString("utf-8");
if (metadataHeader) {
const tomlContent = metadataHeader
.split(",")
.map((pair) => {
const [key, value] = pair.split(" ");
const escapedValue = value.replace(/"/g, '\\"');
return `${key} = "${escapedValue}"`;
})
.join("\n");
const metaFilePath = `${newFilePath}.meta`;
fs.writeFileSync(metaFilePath, tomlContent, "utf-8");
}
}
}
}
// if (body.Type === "post-receive"){
// // const id = event.Upload.ID
// // const extensionFile = headers["X-Extension"]?.[0] ?? ".bin";
// // const extensionFile = ".enc";
// // const filePath = path.join(process.cwd(), "/private/uploads/backups/");
// //
// // fs.renameSync(
// // `${filePath}${id}`,
// // `${filePath}${id}${extensionFile}`
// // );
// }
// Post-receive hook: handle finished uploads
if (body.event === "post-receive") {
console.log(`Upload finished: ${body.id}, size: ${body.upload_length}`);
// process the file, AES encryption, move, etc.
const id = body.id;
const ext = body.upload.metaData?.extension ?? ".bin";
console.log(body)
console.log("Upload finished: ", id);
console.log("Ext: ", ext);
// fs.renameSync(
// `/data/uploads/backups/${id}`,
// `/data/uploads/backups/${id}${ext}`
// );
}
return NextResponse.json({}); // no Upload-Defer-Length
return NextResponse.json({});
} catch (error) {
console.error("Hook error:", error);
return NextResponse.json({error: "Internal server error"}, {status: 500});
docker-compose.yml
+2 -6
View File
@@ -22,17 +22,13 @@ services:
ports:
- "1080:8080"
command: >
-upload-dir /data/uploads/backups
-upload-dir /data/uploads/tmp
-hooks-http http://localhost:8887/api/tus/hooks
-max-size 21474836480
# -max-size 54975581388
# -max-size 54975581388
extra_hosts:
- "localhost:host-gateway"
volumes:
- ./private/uploads/backups:/data/uploads/backups
- ./private/uploads/tmp:/data/uploads/tmp
volumes:
postgres-data:
proxy.ts
+2
View File
@@ -53,6 +53,8 @@ function checkRouteExists(pathname: string) {
const routePatterns = [
/^\/api\/agent\/[^/]+\/status\/?$/,
/^\/api\/agent\/[^/]+\/backup\/?$/,
/^\/api\/agent\/[^/]+\/backup\/upload\/init\/?$/,
/^\/api\/agent\/[^/]+\/backup\/upload\/status\/?$/,
/^\/api\/agent\/[^/]+\/restore\/?$/,
/^\/api\/files\/images\/[^/]+\/?$/,
/^\/api\/files\/backups\/?$/,
src/components/wrappers/dashboard/admin/channels/channel/channel-form/providers/storages/forms/s3.form.tsx
+1 -1
View File
@@ -82,7 +82,7 @@ export const StorageS3Form = ({form}: StorageS3FormProps) => {
<FormField
control={form.control}
name="config.useSSL"
name="config.ssl"
render={({ field }) => (
<FormItem>
<FormLabel>Use SSL</FormLabel>
src/components/wrappers/dashboard/database/import/upload-backup.action.ts
+1 -3
View File
@@ -4,13 +4,11 @@ import {ServerActionResult} from "@/types/action-type";
import {db} from "@/db";
import * as drizzleDb from "@/db";
import {Backup} from "@/db/schema/07_database";
import {getFileExtension} from "../../../../../../app/api/agent/[agentId]/backup/helpers";
import {v4 as uuidv4} from "uuid";
import {eq} from "drizzle-orm";
import {uploadLocalPrivate, uploadS3Private} from "@/features/upload/private/upload.action";
import {z} from "zod";
import {env} from "@/env.mjs";
import {storeBackupFiles} from "@/features/storages/helpers";
import {getFileExtension} from "@/features/api/upload/helpers/file";
export const uploadBackupAction = userAction
src/db/schema/13_storage-policy.ts
+10 -2
View File
@@ -1,10 +1,13 @@
import {boolean, pgTable, uuid} from "drizzle-orm/pg-core";
import {timestamps} from "@/db/schema/00_common";
import {relations} from "drizzle-orm";
import {database} from "@/db/schema/07_database";
import {Backup, Database, database, Restoration, RetentionPolicy} from "@/db/schema/07_database";
import {createSelectSchema} from "drizzle-zod";
import {z} from "zod";
import {storageChannel} from "@/db/schema/12_storage-channel";
import {StorageChannel, storageChannel} from "@/db/schema/12_storage-channel";
import {Agent} from "@/db/schema/08_agent";
import {Project} from "@/db/schema/06_project";
import {AlertPolicy} from "@/db/schema/10_alert-policy";
export const storagePolicy = pgTable('storage_policy', {
id: uuid('id').defaultRandom().primaryKey(),
@@ -31,3 +34,8 @@ export const storagePolicyRelations = relations(storagePolicy, ({one}) => ({
export const storagePolicySchema = createSelectSchema(storagePolicy);
export type StoragePolicy = z.infer<typeof storagePolicySchema>;
export type StoragePolicyWith = StoragePolicy & {
storageChannel: StorageChannel;
};
src/features/storages/providers/s3.ts
+2 -2
View File
@@ -8,7 +8,7 @@ type S3Config = {
secretKey: string;
bucketName: string;
port?: number;
useSSL?: boolean;
ssl?: boolean;
};
async function getS3Client(config: S3Config) {
@@ -17,7 +17,7 @@ async function getS3Client(config: S3Config) {
accessKey: config.accessKey,
secretKey: config.secretKey,
port: config.port ?? 443,
useSSL: config.useSSL ?? true,
useSSL: config.ssl ?? true,
});
}