External/portabase
1
0
Fork 0
mirror of https://github.com/Portabase/portabase.git synced 2026-07-14 11:16:13 +02:00
Code Issues Packages Projects Releases Wiki Activity

Working on the s3 save dump feature.

Browse Source
This commit is contained in:
charlesgauthereau
2025-08-02 20:13:29 +02:00
parent d13e4b7d40
commit 449c9c2cfe
11 changed files with 213 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/(customer)/dashboard/(organization)/projects/[projectId]/database/[databaseId]/page.tsx
+11 -1
View File
@@ -36,6 +36,9 @@ export default async function RoutePage(props: PageParams<{
const dbItem = await db.query.database.findFirst({
where: and(inArray(drizzleDb.schemas.backup.id, databasesProject.ids ?? []), eq(drizzleDb.schemas.database.id, databaseId), eq(drizzleDb.schemas.database.projectId, projectId)),
with: {
project: true
}
});
if (!dbItem) {
@@ -71,6 +74,12 @@ export default async function RoutePage(props: PageParams<{
.then((rows) => rows.length),
]);
const [settings] = await db.select().from(drizzleDb.schemas.setting).where(eq(drizzleDb.schemas.setting.name, "system")).limit(1);
if (!settings) {
notFound();
}
const successRate = totalBackups > 0 ? (successfulBackups / totalBackups) * 100 : null;
return (
@@ -88,7 +97,8 @@ export default async function RoutePage(props: PageParams<{
<PageDescription className="mt-5 sm:mt-0">{dbItem.description}</PageDescription>
<PageContent className="flex flex-col w-full h-full">
<DatabaseKpi successRate={successRate} database={dbItem} totalBackups={totalBackups}/>
<DatabaseTabs database={dbItem} isAlreadyRestore={isAlreadyRestore} backups={backups}
<DatabaseTabs settings={settings} database={dbItem} isAlreadyRestore={isAlreadyRestore}
backups={backups}
restorations={restorations}/>
</PageContent>
</Page>
app/api/agent/[agentId]/backup/route.ts
+18 -2
View File
@@ -1,12 +1,13 @@
import {NextResponse} from "next/server";
import {isUuidv4} from "@/utils/verify-uuid";
import {uploadLocalPrivate} from "@/features/upload/private/upload.action";
import {uploadLocalPrivate, uploadS3Private} from "@/features/upload/private/upload.action";
import {v4 as uuidv4} from "uuid";
import {eventEmitter} from "../../../events/route";
import {db} from "@/db";
import {Backup} from "@/db/schema/06_database";
import {and, eq} from "drizzle-orm";
import * as drizzleDb from "@/db";
import {env} from "@/env.mjs";
export async function POST(
request: Request,
@@ -48,6 +49,9 @@ export async function POST(
const database = await db.query.database.findFirst({
where: eq(drizzleDb.schemas.database.agentDatabaseId, generatedId),
with: {
project: true
}
});
if (!database) {
@@ -108,7 +112,19 @@ export async function POST(
const fileName = `${uuid}.dump`;
const buffer = Buffer.from(await file.arrayBuffer());
const {success, message, filePath} = await uploadLocalPrivate(fileName, buffer);
const [settings] = await db.select().from(drizzleDb.schemas.setting).where(eq(drizzleDb.schemas.setting.name, "system")).limit(1);
if (!settings) {
throw new Error("System settings not found.");
}
let success: boolean, message: string, filePath: string;
const result =
settings.storage === "local"
? await uploadLocalPrivate(fileName, buffer)
: await uploadS3Private(`${database.project?.slug}/${fileName}`, buffer, env.S3_BUCKET_NAME!);
({success, message, filePath} = result);
if (!success) {
return NextResponse.json(
app/api/agent/[agentId]/status/route.ts
+1 -1
View File
@@ -18,7 +18,7 @@ export type Body = {
databases: databaseAgent[]
}
// Function to test the get file url presigned local
export async function GET(request: Request) {
const url = await getFileUrlPresignedLocal("d4a7fa35-2506-4d01-a612-a8ef2e2cc1c5.dump")
return Response.json({
portabase.config.ts
+2
View File
@@ -13,6 +13,7 @@ export const PORTABASE_DEFAULT_SETTINGS = {
"https://code.iconify.com",
"https://cdn.iconify.design",
"https://api.iconify.design",
],
STYLE_SRC: [
"'self'",
@@ -31,6 +32,7 @@ export const PORTABASE_DEFAULT_SETTINGS = {
"https://cdn.iconify.design",
"https://code.iconify.com",
"https://api.iconify.design",
"http://localhost:9000",
],
FONT_SRC: [
"'self'",
src/components/wrappers/dashboard/profile/avatar/avatar-with-upload.tsx
+2 -1
View File
@@ -7,6 +7,7 @@ import { useMutation } from "@tanstack/react-query";
import { updateImageUserAction } from "@/components/wrappers/dashboard/profile/avatar/avatar.action";
import { useRouter } from "next/navigation";
import { User } from "@/db/schema/01_user";
import {ChangeEvent} from "react";
export type AvatarWithUploadProps = {
user: User;
@@ -43,7 +44,7 @@ export const AvatarWithUpload = (props: AvatarWithUploadProps) => {
},
});
const handleImageUpload = async (event: React.ChangeEvent<HTMLInputElement>) => {
const handleImageUpload = async (event: ChangeEvent<HTMLInputElement>) => {
const file = event.target.files?.[0];
if (!file) return;
if (!file.type.includes("image")) {
src/components/wrappers/dashboard/projects/database/database-tabs.tsx
+5 -3
View File
@@ -7,13 +7,15 @@ import { eventUpdate } from "@/types/events";
import { backupColumns } from "@/features/dashboard/backup/columns";
import { restoreColumns } from "@/features/dashboard/restore/columns";
import { DataTable } from "@/components/wrappers/common/table/data-table";
import {Backup, Database, Restoration} from "@/db/schema/06_database";
import {Backup, Database, DatabaseWith, Restoration} from "@/db/schema/06_database";
import {Setting} from "@/db/schema/00_setting";
export type DatabaseTabsProps = {
settings: Setting
backups: Backup[];
restorations: Restoration[];
isAlreadyRestore: boolean;
database: Database;
database: DatabaseWith;
};
export const DatabaseTabs = (props: DatabaseTabsProps) => {
@@ -41,7 +43,7 @@ export const DatabaseTabs = (props: DatabaseTabsProps) => {
</TabsList>
<TabsContent className="h-full justify-between" value="backup">
<DataTable columns={backupColumns(props.isAlreadyRestore)} data={props.backups} enablePagination />
<DataTable columns={backupColumns(props.isAlreadyRestore, props.settings, props.database)} data={props.backups} enablePagination />
</TabsContent>
<TabsContent className="h-full justify-between" value="restore">
<DataTable columns={restoreColumns(props.isAlreadyRestore)} data={props.restorations} enablePagination />
src/db/schema/06_database.ts
+24 -23
View File
@@ -1,10 +1,10 @@
import { pgTable, text, boolean, timestamp, uuid, integer, pgEnum, uniqueIndex } from "drizzle-orm/pg-core";
import { Agent, agent } from "./07_agent";
import { Project, project } from "./05_project";
import { relations } from "drizzle-orm";
import { dbmsEnum, statusEnum } from "./types";
import { createSelectSchema } from "drizzle-zod";
import { z } from "zod";
import {pgTable, text, boolean, timestamp, uuid, integer, pgEnum, uniqueIndex} from "drizzle-orm/pg-core";
import {Agent, agent} from "./07_agent";
import {Project, project} from "./05_project";
import {relations} from "drizzle-orm";
import {dbmsEnum, statusEnum} from "./types";
import {createSelectSchema} from "drizzle-zod";
import {z} from "zod";
export const database = pgTable("databases", {
id: uuid("id").primaryKey().defaultRandom(),
@@ -19,7 +19,7 @@ export const database = pgTable("databases", {
updatedAt: timestamp("updated_at"),
agentId: uuid("agent_id")
.notNull()
.references(() => agent.id, { onDelete: "cascade" }),
.references(() => agent.id, {onDelete: "cascade"}),
lastContact: timestamp("last_contact"),
projectId: uuid("project_id")
@@ -36,7 +36,7 @@ export const backup = pgTable(
updatedAt: timestamp("updated_at"),
databaseId: uuid("database_id")
.notNull()
.references(() => database.id, { onDelete: "cascade" }),
.references(() => database.id, {onDelete: "cascade"}),
},
// (table) => [uniqueIndex("database_id_status_unique").on(table.databaseId, table.status)]
);
@@ -48,25 +48,25 @@ export const restoration = pgTable("restorations", {
updatedAt: timestamp("updated_at"),
backupId: uuid("backup_id")
.notNull()
.references(() => backup.id, { onDelete: "cascade" }),
databaseId: uuid("database_id").references(() => database.id, { onDelete: "cascade" }),
.references(() => backup.id, {onDelete: "cascade"}),
databaseId: uuid("database_id").references(() => database.id, {onDelete: "cascade"}),
});
export const databaseRelations = relations(database, ({ one, many }) => ({
agent: one(agent, { fields: [database.agentId], references: [agent.id] }),
project: one(project, { fields: [database.projectId], references: [project.id] }),
export const databaseRelations = relations(database, ({one, many}) => ({
agent: one(agent, {fields: [database.agentId], references: [agent.id]}),
project: one(project, {fields: [database.projectId], references: [project.id]}),
backups: many(backup),
restorations: many(restoration),
}));
export const backupRelations = relations(backup, ({ one, many }) => ({
database: one(database, { fields: [backup.databaseId], references: [database.id] }),
export const backupRelations = relations(backup, ({one, many}) => ({
database: one(database, {fields: [backup.databaseId], references: [database.id]}),
restorations: many(restoration),
}));
export const restorationRelations = relations(restoration, ({ one }) => ({
backup: one(backup, { fields: [restoration.backupId], references: [backup.id] }),
database: one(database, { fields: [restoration.databaseId], references: [database.id] }),
export const restorationRelations = relations(restoration, ({one}) => ({
backup: one(backup, {fields: [restoration.backupId], references: [backup.id]}),
database: one(database, {fields: [restoration.databaseId], references: [database.id]}),
}));
export const databaseSchema = createSelectSchema(database);
@@ -79,8 +79,9 @@ export const restorationSchema = createSelectSchema(restoration);
export type Restoration = z.infer<typeof restorationSchema>;
export type DatabaseWith = Database & {
agent: Agent;
project: Project;
backups: Backup[];
restorations: Restoration[];
agent?: Agent | null;
project?: Project | null;
backups?: Backup[] | null;
restorations?: Restoration[] | null;
};
src/features/dashboard/backup/columns.tsx
+21 -4
View File
@@ -12,18 +12,23 @@ import {
import {Button} from "@/components/ui/button";
import {Download, MoreHorizontal, Trash2} from "lucide-react";
import {ReloadIcon} from "@radix-ui/react-icons";
import {getFileUrlPresignedLocal} from "@/features/upload/private/upload.action";
import {
getFileUrlPresignedLocal,
getFileUrlPresignedS3,
getFileUrlPreSignedS3Action
} from "@/features/upload/private/upload.action";
import {useMutation} from "@tanstack/react-query";
import {createRestorationAction, deleteBackupAction} from "@/features/dashboard/restore/restore.action";
import {toast} from "sonner";
import {useRouter} from "next/navigation";
import {StatusBadge} from "@/components/wrappers/common/status-badge";
import {Backup} from "@/db/schema/06_database";
import {Backup, DatabaseWith} from "@/db/schema/06_database";
import {formatFrenchDate} from "@/utils/date-formatting";
import {TooltipCustom} from "@/components/wrappers/common/tooltip-custom";
import {Setting} from "@/db/schema/00_setting";
export function backupColumns(isAlreadyRestore: boolean): ColumnDef<Backup>[] {
export function backupColumns(isAlreadyRestore: boolean, settings: Setting, database: DatabaseWith): ColumnDef<Backup>[] {
return [
{
@@ -98,7 +103,19 @@ export function backupColumns(isAlreadyRestore: boolean): ColumnDef<Backup>[] {
};
const handleDownload = async (fileName: string) => {
const url = await getFileUrlPresignedLocal(fileName);
let url: string = "";
if (settings.storage == "local") {
url = await getFileUrlPresignedLocal(fileName);
} else if (settings.storage == "s3") {
const data = await getFileUrlPreSignedS3Action(`backups/${database.project?.slug}/${fileName}`)
if (data?.data?.success) {
url = data.data.value ?? "";
} else {
// @ts-ignore
const errorMessage = data?.data?.actionError?.message || "Failed to get file!";
toast.error(errorMessage);
}
}
window.open(url, "_self");
};
src/features/upload/private/upload.action.ts
+78 -5
View File
@@ -1,16 +1,25 @@
"use server";
import { mkdir, writeFile } from "fs/promises";
import {mkdir, writeFile} from "fs/promises";
import path from "path";
import * as fs from "node:fs";
import { getServerUrl } from "@/utils/get-server-url";
import {getServerUrl} from "@/utils/get-server-url";
import {createPresignedUrlToDownload, saveFileInBucket} from "@/utils/s3-file-management";
import crypto from "crypto";
import {env} from "@/env.mjs";
import {userAction} from "@/safe-actions";
import {z} from "zod";
import {ServerActionResult} from "@/types/action-type";
import {Backup} from "@/db/schema/06_database";
import {db} from "@/db";
import * as drizzleDb from "@/db";
const privateLocalDir = "private/uploads/files/";
const privateS3Dir = "backups/";
export async function uploadLocalPrivate(fileName: string, buffer: any) {
try {
await mkdir(path.join(process.cwd(), privateLocalDir), { recursive: true });
await mkdir(path.join(process.cwd(), privateLocalDir), {recursive: true});
await writeFile(path.join(process.cwd(), privateLocalDir, fileName), buffer);
return {
@@ -24,10 +33,31 @@ export async function uploadLocalPrivate(fileName: string, buffer: any) {
}
}
export async function uploadS3Private(fileName: string, buffer: any, bucketName: string) {
try {
await saveFileInBucket({
bucketName,
fileName: `${privateS3Dir}${fileName}`,
file: buffer,
});
return {
success: true,
filePath: `${privateS3Dir}${fileName}`,
message: "File uploaded successfully",
};
} catch (error) {
console.error("Error occurred:", error);
throw new Error("An error occurred while importing the private file");
}
}
export async function getFileUrlPresignedLocal(fileName: string) {
try {
const filePath = path.join(privateLocalDir, fileName);
await mkdir(path.join(process.cwd(), privateLocalDir), { recursive: true });
await mkdir(path.join(process.cwd(), privateLocalDir), {recursive: true});
if (!fs.existsSync(filePath)) {
console.error("File not found at:", filePath);
@@ -43,3 +73,46 @@ export async function getFileUrlPresignedLocal(fileName: string) {
throw error;
}
}
export async function getFileUrlPresignedS3(fileName: string) {
try {
return await createPresignedUrlToDownload({
bucketName: env.S3_BUCKET_NAME!,
fileName: fileName,
});
} catch (error) {
throw error;
}
}
export const getFileUrlPreSignedS3Action = userAction.schema(z.string()).action(async ({parsedInput}): Promise<ServerActionResult<string>> => {
try {
const url = await createPresignedUrlToDownload({
bucketName: env.S3_BUCKET_NAME!,
fileName: parsedInput,
});
return {
success: true,
value: url,
actionSuccess: {
message: "Successfully get url",
messageParams: {fileName: parsedInput},
},
};
} catch (error) {
console.error("Error creating backup:", error);
return {
success: false,
actionError: {
message: "Failed to create url pre signed s3.",
status: 500,
cause: error instanceof Error ? error.message : "Unknown error",
messageParams: {fileName: parsedInput},
},
};
}
});
src/features/upload/public/upload.action.ts
+7 -5
View File
@@ -30,11 +30,13 @@ export const uploadImageAction = userAction.schema(z.instanceof(FormData)).actio
let result: void | UploadedObjectInfo;
const bucketName = "public-image-bucket";
if (settings.storage === "local") {
result = await uploadLocal(fileName, buffer);
} else if (settings.storage === "s3") {
result = await uploadS3Compatible(bucketName, fileName, buffer);
}
// TODO : Do not delete
// if (settings.storage === "local") {
// result = await uploadLocal(fileName, buffer);
// } else if (settings.storage === "s3") {
// result = await uploadS3Compatible(bucketName, fileName, buffer);
// }
result = await uploadLocal(fileName, buffer);
const url = getUrl(fileName, settings, bucketName);
console.log(url);
src/utils/s3-file-management.ts
+44 -10
View File
@@ -23,16 +23,23 @@ async function getS3Client() {
secretKey: settings.s3SecretAccessKey ?? "",
};
const s3Client =
env.NODE_ENV === "production"
? new Minio.Client({
...baseConfig,
})
: new Minio.Client({
...baseConfig,
port: Number(env.S3_PORT ?? 0),
useSSL: env.S3_USE_SSL === "true",
});
// const s3Client =
// env.NODE_ENV === "production"
// ? new Minio.Client({
// ...baseConfig,
// useSSL: env.S3_USE_SSL === "true",
// })
// : new Minio.Client({
// ...baseConfig,
// port: Number(env.S3_PORT ?? 0),
// useSSL: env.S3_USE_SSL === "true",
// });
const s3Client = new Minio.Client({
...baseConfig,
port: Number(env.S3_PORT ?? 0),
useSSL: env.S3_USE_SSL === "true",
})
return s3Client;
}
@@ -168,3 +175,30 @@ export async function createPublicBucket({bucketName}: { bucketName: string }) {
console.error("Error creating bucket:", error);
}
}
/**
* Generate a presigned URL for downloading a file from a private S3 bucket
* @param bucketName name of the bucket
* @param fileName name of the file
* @param expiry expiry time in seconds (default 1 hour)
* @returns presigned download URL
*/
export async function createPresignedUrlToDownload({
bucketName,
fileName,
expiry = 60 * 60, // 1 hour
}: {
bucketName: string;
fileName: string;
expiry?: number;
}) {
const s3Client = await getS3Client();
// Optionally: ensure file exists
const fileExists = await checkFileExistsInBucket({ bucketName, fileName });
if (!fileExists) {
throw new Error("File does not exist in the bucket.");
}
return await s3Client.presignedGetObject(bucketName, fileName, expiry);
}