External/portabase
1
0
Fork 0
mirror of https://github.com/Portabase/portabase.git synced 2026-07-14 11:16:13 +02:00
Code Issues Packages Projects Releases Wiki Activity

chore: working on backup api for new storage system.

Browse Source
This commit is contained in:
charlesgauthereau
2026-01-15 21:27:36 +01:00
parent dd1f791b3e
commit 670f6a1974
16 changed files with 437 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/(customer)/dashboard/(admin)/storages/channels/page.tsx
+6 -2
View File
@@ -3,7 +3,7 @@ import {Page, PageActions, PageContent, PageHeader, PageTitle} from "@/features/
import {Metadata} from "next";
import {ChannelsSection} from "@/components/wrappers/dashboard/admin/channels/channels-section";
import {db} from "@/db";
import {desc, isNotNull, isNull, not} from "drizzle-orm";
import {desc, eq, isNotNull, isNull, not} from "drizzle-orm";
import * as drizzleDb from "@/db";
import {StorageChannelWith} from "@/db/schema/12_storage-channel";
import {ChannelAddEditModal} from "@/components/wrappers/dashboard/admin/channels/channel/channel-add-edit-modal";
@@ -29,6 +29,10 @@ export default async function RoutePage(props: PageParams<{}>) {
},
});
const settings = await db.query.setting.findFirst({
where: eq(drizzleDb.schemas.setting.name, "system"),
});
return (
<Page>
@@ -39,7 +43,7 @@ export default async function RoutePage(props: PageParams<{}>) {
</PageActions>
</PageHeader>
<PageContent>
<ChannelsSection kind={"storage"} organizations={organizations} channels={storageChannels}/>
<ChannelsSection defaultStorageChannelId={settings?.defaultStorageChannelId} kind={"storage"} organizations={organizations} channels={storageChannels}/>
</PageContent>
</Page>
);
app/api/agent/[agentId]/backup/helpers.ts
-3
View File
@@ -1,8 +1,5 @@
import fs from "node:fs";
import forge from "node-forge";
import {EventPayload} from "@/features/notifications/types";
import {dispatchNotification} from "@/features/notifications/dispatch";
import {Database, DatabaseWith} from "@/db/schema/07_database";
export async function decryptedDump(file: File, aesKeyHex: string, ivHex: string, fileExtension: string): Promise<File> {
app/api/agent/[agentId]/backup/route.ts
+34 -25
View File
@@ -11,6 +11,7 @@ import {env} from "@/env.mjs";
import {withUpdatedAt} from "@/db/utils";
import {decryptedDump, getFileExtension} from "./helpers";
import {sendNotificationsBackupRestore} from "@/features/notifications/helpers";
import {storeBackupFiles} from "@/features/storages/helpers";
export async function POST(
request: Request,
@@ -25,6 +26,7 @@ export async function POST(
{status: 400}
);
}
eventEmitter.emit('modification', {update: true});
const agentId = (await params).agentId;
@@ -56,7 +58,8 @@ export async function POST(
where: eq(drizzleDb.schemas.database.agentDatabaseId, generatedId),
with: {
project: true,
alertPolicies: true
alertPolicies: true,
storagePolicies: true
}
});
@@ -125,35 +128,41 @@ export async function POST(
const uuid = uuidv4();
const fileName = `${uuid}${fileExtension}`;
const buffer = Buffer.from(await decryptedFile.arrayBuffer());
const [settings] = await db.select().from(drizzleDb.schemas.setting).where(eq(drizzleDb.schemas.setting.name, "system")).limit(1);
if (!settings) {
throw new Error("System settings not found.");
}
let success: boolean, message: string, filePath: string;
const result =
settings.storage === "local"
? await uploadLocalPrivate(fileName, buffer)
: await uploadS3Private(`${database.project?.slug}/${fileName}`, buffer, env.S3_BUCKET_NAME!);
({success, message, filePath} = result);
// const [settings] = await db.select().from(drizzleDb.schemas.setting).where(eq(drizzleDb.schemas.setting.name, "system")).limit(1);
// if (!settings) {
// throw new Error("System settings not found.");
// }
if (!success) {
return NextResponse.json(
{error: message},
{status: 500}
);
}
// let success: boolean, message: string, filePath: string;
//
// const result =
// settings.storage === "local"
// ? await uploadLocalPrivate(fileName, buffer)
// : await uploadS3Private(`${database.project?.slug}/${fileName}`, buffer, env.S3_BUCKET_NAME!);
//
// ({success, message, filePath} = result);
//
// if (!success) {
// return NextResponse.json(
// {error: message},
// {status: 500}
// );
// }
await db
.update(drizzleDb.schemas.backup)
.set(withUpdatedAt({
file: fileName,
fileSize: fileSizeBytes,
status: 'success',
}))
.where(eq(drizzleDb.schemas.backup.id, backup.id));
await storeBackupFiles(backup, database, buffer, fileName)
//
// await db
// .update(drizzleDb.schemas.backup)
// .set(withUpdatedAt({
// file: fileName,
// fileSize: fileSizeBytes,
// status: 'success',
// }))
// .where(eq(drizzleDb.schemas.backup.id, backup.id));
eventEmitter.emit('modification', {update: true});
src/components/wrappers/dashboard/admin/channels/channel/channel-add-edit-modal.tsx
+55 -35
View File
@@ -42,6 +42,7 @@ export const ChannelAddEditModal = ({
}: ChannelAddModalProps) => {
const isMobile = useIsMobile();
const [openInternal, setOpen] = useState(open);
const isLocalSystem = channel?.provider == "local";
const isCreate = !Boolean(channel);
@@ -82,46 +83,65 @@ export const ChannelAddEditModal = ({
</DialogDescription>
</DialogHeader>
<div>
{adminView ?
<Tabs className="flex flex-col flex-1" defaultValue="configuration">
<TabsList className="grid w-full grid-cols-2">
<TabsTrigger value="configuration">Configuration</TabsTrigger>
<TabsTrigger value="organizations">Organizations</TabsTrigger>
</TabsList>
<TabsContent className="h-full justify-between" value="configuration">
<ChannelForm
kind={kind}
adminView={adminView}
defaultValues={channel}
organization={organization}
onSuccessAction={() => {
onOpenChangeAction?.(false)
setOpen(false);
}}
/>
</TabsContent>
<TabsContent className="h-full justify-between" value="organizations">
<>
{!isLocalSystem ? (
<>
{adminView ?
<Tabs className="flex flex-col flex-1" defaultValue="configuration">
<TabsList className="grid w-full grid-cols-2">
<TabsTrigger value="configuration">Configuration</TabsTrigger>
<TabsTrigger value="organizations">Organizations</TabsTrigger>
</TabsList>
<TabsContent className="h-full justify-between" value="configuration">
<ChannelForm
kind={kind}
adminView={adminView}
defaultValues={channel}
organization={organization}
onSuccessAction={() => {
onOpenChangeAction?.(false)
setOpen(false);
}}
/>
</TabsContent>
<TabsContent className="h-full justify-between" value="organizations">
<ChannelOrganisationForm
defaultValues={channel}
kind={kind}
organizations={organizations}
/>
</TabsContent>
</Tabs>
:
<>
<ChannelForm
kind={kind}
adminView={adminView}
defaultValues={channel}
organization={organization}
onSuccessAction={() => {
onOpenChangeAction?.(false)
setOpen(false);
}}
/>
</>
}
</>
)
:
<>
<ChannelOrganisationForm
defaultValues={channel}
kind={kind}
organizations={organizations}
/>
</TabsContent>
</Tabs>
:
<>
<ChannelForm
kind={kind}
adminView={adminView}
defaultValues={channel}
organization={organization}
onSuccessAction={() => {
onOpenChangeAction?.(false)
setOpen(false);
}}
/>
</>
}
</>
}
</>
</div>
</DialogContent>
</Dialog>
src/components/wrappers/dashboard/admin/channels/channel/channel-card/button-edit-channel.tsx
+7 -4
View File
@@ -33,6 +33,7 @@ export const EditChannelButton = ({
}: EditChannelButtonProps) => {
const router = useRouter();
const [isAddModalOpen, setIsAddModalOpen] = useState(false);
const isLocalSystem = channel.provider == "local";
const mutation = useMutation({
@@ -73,10 +74,12 @@ export const EditChannelButton = ({
return (
<>
<Switch checked={channel.enabled} onCheckedChange={async () => {
await mutation.mutateAsync(!channel.enabled)
}}
/>
{!isLocalSystem && (
<Switch checked={channel.enabled} onCheckedChange={async () => {
await mutation.mutateAsync(!channel.enabled)
}}/>
)}
<ChannelAddEditModal
kind={kind}
organizations={organizations}
src/components/wrappers/dashboard/admin/channels/channel/channel-card/channel-card.tsx
+18 -7
View File
@@ -21,13 +21,17 @@ export type ChannelCardProps = {
organizations?: OrganizationWithMembers[];
adminView?: boolean;
kind?: ChannelKind;
defaultStorageChannelId?: string | null | undefined
};
export const ChannelCard = (props: ChannelCardProps) => {
const {data, organization, kind, adminView} = props;
const {data, organization, kind, adminView, defaultStorageChannelId} = props;
const isMobile = useIsMobile()
const isDefaultSystemStorage = defaultStorageChannelId === data.id;
const isOwned = data.organizationId ? true : !organization;
const isLocalSystem = data.provider == "local";
@@ -47,11 +51,16 @@ export const ChannelCard = (props: ChannelCardProps) => {
<Badge variant="secondary" className="text-xs font-mono">
{data.provider}
</Badge>
{isDefaultSystemStorage && (
<Badge variant="secondary" className="text-xs font-mono">
default
</Badge>
)}
</div>
</div>
{kind && (
<div className="flex items-center gap-2">
{(isOwned && !isLocalSystem) && (
{(isOwned) && (
<>
<EditChannelButton
organizations={props.organizations}
@@ -60,11 +69,13 @@ export const ChannelCard = (props: ChannelCardProps) => {
channel={data}
kind={kind}
/>
<DeleteChannelButton
kind={kind}
organizationId={organization?.id}
channelId={data.id}
/>
{!isLocalSystem && (
<DeleteChannelButton
kind={kind}
organizationId={organization?.id}
channelId={data.id}
/>
)}
</>
)}
</div>
src/components/wrappers/dashboard/admin/channels/channel/channel-form/providers/storages/forms/s3.form.tsx
+33 -1
View File
@@ -3,6 +3,7 @@ import {FormControl, FormField, FormItem, FormLabel, FormMessage} from "@/compon
import {Input} from "@/components/ui/input";
import {Separator} from "@/components/ui/separator";
import {PasswordInput} from "@/components/ui/password-input";
import {Switch} from "@/components/ui/switch";
type StorageS3FormProps = {
@@ -59,12 +60,43 @@ export const StorageS3Form = ({form}: StorageS3FormProps) => {
<FormItem>
<FormLabel>Bucket name</FormLabel>
<FormControl>
<PasswordInput {...field} placeholder="portabase-dev"/>
<Input {...field} placeholder="portabase-dev"/>
</FormControl>
<FormMessage/>
</FormItem>
)}
/>
<FormField
control={form.control}
name="config.port"
render={({ field }) => (
<FormItem>
<FormLabel>Port</FormLabel>
<FormControl>
<Input {...field} type="number" placeholder="443" />
</FormControl>
<FormMessage />
</FormItem>
)}
/>
<FormField
control={form.control}
name="config.useSSL"
render={({ field }) => (
<FormItem>
<FormLabel>Use SSL</FormLabel>
<FormControl>
<Switch
checked={field.value}
onCheckedChange={field.onChange}
/>
</FormControl>
<FormMessage />
</FormItem>
)}
/>
</>
)
}
src/components/wrappers/dashboard/admin/channels/channels-section.tsx
+7 -4
View File
@@ -10,15 +10,17 @@ import {ChannelAddEditModal} from "@/components/wrappers/dashboard/admin/channel
import {ChannelKind, getChannelTextBasedOnKind} from "@/components/wrappers/dashboard/admin/channels/helpers/common";
type ChannelsSectionProps = {
channels: NotificationChannelWith[] | StorageChannelWith[]
organizations: OrganizationWithMembers[]
kind: ChannelKind;
channels: NotificationChannelWith[] | StorageChannelWith[],
organizations: OrganizationWithMembers[],
kind: ChannelKind,
defaultStorageChannelId?: string | null | undefined
}
export const ChannelsSection = ({
organizations,
channels,
kind
kind,
defaultStorageChannelId
}: ChannelsSectionProps) => {
const [isAddModalOpen, setIsAddModalOpen] = useState(false);
@@ -41,6 +43,7 @@ export const ChannelsSection = ({
adminView={true}
organizations={organizations}
kind={kind}
defaultStorageChannelId={defaultStorageChannelId}
/>
</div>
) : (
src/db/index.ts
+3 -1
View File
@@ -14,6 +14,7 @@ import * as alertPolicy from "./schema/10_alert-policy";
import * as notificationLog from "./schema/11_notification-log";
import * as storageChannel from "./schema/12_storage-channel";
import * as storagePolicy from "@/db/schema/13_storage-policy";
import * as backupStorage from "@/db/schema/14_storage-backup";
import {Pool} from "pg";
@@ -44,7 +45,8 @@ export const schemas = {
...alertPolicy,
...notificationLog,
...storageChannel,
...storagePolicy
...storagePolicy,
...backupStorage
};
export const db = drizzle({
src/db/schema/01_setting.ts
+11
View File
@@ -4,6 +4,12 @@ import {createSelectSchema} from "drizzle-zod";
import {z} from "zod";
import {timestamps} from "@/db/schema/00_common";
import {storageChannel} from "@/db/schema/12_storage-channel";
import {relations} from "drizzle-orm";
import {agent} from "@/db/schema/08_agent";
import {project} from "@/db/schema/06_project";
import {alertPolicy} from "@/db/schema/10_alert-policy";
import {storagePolicy} from "@/db/schema/13_storage-policy";
import {backup, database, restoration, retentionPolicy} from "@/db/schema/07_database";
export const setting = pgTable("settings", {
id: uuid("id").primaryKey().defaultRandom(),
@@ -23,5 +29,10 @@ export const setting = pgTable("settings", {
...timestamps
});
export const settingRelations = relations(setting, ({one, many}) => ({
storageChannel: one(storageChannel, {fields: [setting.defaultStorageChannelId], references: [storageChannel.id]}),
}));
export const settingSchema = createSelectSchema(setting);
export type Setting = z.infer<typeof settingSchema>;
src/features/storages/dispatch.ts
+60 -8
View File
@@ -5,6 +5,8 @@ import * as drizzleDb from '@/db';
import {db} from '@/db';
import type {StorageInput, StorageProviderKind, StorageResult,} from './types';
import {dispatchViaProvider} from "@/features/storages/providers";
import {StorageChannel} from "@/db/schema/12_storage-channel";
import {Json} from "drizzle-zod";
export async function dispatchStorage(
input: StorageInput,
@@ -13,34 +15,84 @@ export async function dispatchStorage(
organizationId?: string
): Promise<StorageResult> {
try {
if (!channelId) {
let channel: StorageChannel | null = null;
if (policyId) {
const policyDb = await db.query.storagePolicy.findFirst({
where: eq(drizzleDb.schemas.storagePolicy.id, policyId),
with: {
storageChannel: true
},
});
if (!policyDb || !policyDb.storageChannel) {
return {
success: false,
provider: null,
error: "Policy or associated channel not found",
};
}
if (!policyDb.enabled || !policyDb.storageChannel.enabled) {
return {
success: false,
provider: policyDb.storageChannel.provider as any,
error: "Policy or channel is disabled",
};
}
channel = {
...policyDb.storageChannel,
config: policyDb.storageChannel.config as Json,
};
}
if (channelId) {
const fetchedChannel = await db.query.storageChannel.findFirst({
where: eq(drizzleDb.schemas.storageChannel.id, channelId),
});
if (!fetchedChannel) {
return {
success: false,
provider: null,
error: "Channel not found",
};
}
channel = {
...fetchedChannel,
config: fetchedChannel.config as Json,
};
}
if (!channel) {
return {
success: false,
provider: null,
error: 'No storage channel provided',
error: "No valid channel to dispatch on storage",
};
}
const channel = await db.query.storageChannel.findFirst({
where: eq(drizzleDb.schemas.storageChannel.id, channelId),
});
if (!channel || !channel.enabled) {
if (!channel.enabled) {
return {
success: false,
provider: channel?.provider as StorageProviderKind,
error: 'Storage channel not found or disabled',
provider: null,
error: "Channel not active",
};
}
return await dispatchViaProvider(
channel.provider as StorageProviderKind,
channel.config,
input
);
} catch (err: any) {
return {
success: false,
src/features/storages/helpers.ts
+101 -5
View File
@@ -1,10 +1,106 @@
import {DatabaseWith} from "@/db/schema/07_database";
import {StorageChannel} from "@/db/schema/12_storage-channel";
import {sendNotificationsBackupRestore} from "@/features/notifications/helpers";
import { Backup, DatabaseWith } from "@/db/schema/07_database";
import { dispatchStorage } from "@/features/storages/dispatch";
import type { StorageInput, StorageResult } from "@/features/storages/types";
import * as drizzleDb from "@/db";
import { withUpdatedAt } from "@/db/utils";
import { eq } from "drizzle-orm";
import { db } from "@/db";
import { createHash } from "crypto";
function computeChecksum(buffer: Buffer): string {
return createHash("sha256").update(buffer).digest("hex");
}
export async function storeBackupFiles(
backup: Backup,
database: DatabaseWith,
file: Buffer,
fileName: string
): Promise<StorageResult[]> {
const settings = await db.query.setting.findFirst({
where: eq(drizzleDb.schemas.setting.name, "system"),
with: { storageChannel: true },
});
const defaultPolicy = settings?.storageChannel
? [{
id: null,
storageChannelId: settings.storageChannel.id,
enabled: true
}]
: [];
console.log(database.storagePolicies);
const policies = (database.storagePolicies?.filter(p => p.enabled) || defaultPolicy);
console.log("Policies", policies);
export async function storeFileBackup(database: DatabaseWith, file: Buffer) {
if (!policies.length) {
return [];
}
const path = `${database.project?.slug}/${fileName}`;
const size = file.length;
const checksum = computeChecksum(file);
const results = await Promise.all(
policies.map(async policy => {
const [backupStorage] = await db
.insert(drizzleDb.schemas.backupStorage)
.values({
backupId: backup.id,
storageChannelId: policy.storageChannelId,
status: "pending",
path,
size,
checksum,
})
.returning();
}
const input: StorageInput = {
action: "upload",
data: { path, file },
};
// const result = await dispatchStorage(input, policy.id);
let result: StorageResult;
try {
if (policy.id){
result = await dispatchStorage(input, policy.id);
}else{
result = await dispatchStorage(input, undefined, policy.storageChannelId);
}
} catch (err) {
console.error(err);
result = {
success: false,
provider: null,
error: err instanceof Error ? err.message : "Unknown error"
};
}
await db
.update(drizzleDb.schemas.backupStorage)
.set(withUpdatedAt({ status: result.success ? "success" : "failed" }))
.where(eq(drizzleDb.schemas.backupStorage.id, backupStorage.id));
return result;
})
);
console.log(results);
const backupStatus = results.some(r => r.success) ? "success" : "failed";
await db
.update(drizzleDb.schemas.backup)
.set(withUpdatedAt({ status: backupStatus }))
.where(eq(drizzleDb.schemas.backup.id, backup.id));
return results;
}
src/features/storages/providers/index.ts
+6 -5
View File
@@ -5,6 +5,7 @@ import type {
} from '../types';
import {uploadLocal, getLocal, deleteLocal} from './local';
import {deleteS3, getS3, uploadS3} from "@/features/storages/providers/s3";
type ProviderHandler = {
upload: (config: any, input: StorageInput & { action: 'upload' }) => Promise<StorageResult>;
@@ -18,11 +19,11 @@ const handlers: Record<StorageProviderKind, ProviderHandler> = {
get: getLocal,
delete: deleteLocal,
},
// s3: {
// upload: uploadS3,
// get: getS3,
// delete: deleteS3,
// },
s3: {
upload: uploadS3,
get: getS3,
delete: deleteS3,
},
// gcs: null as any,
// azure: null as any,
};
src/features/storages/providers/local.ts
+3 -1
View File
@@ -14,7 +14,9 @@ export async function uploadLocal(
const base = config.baseDir || BASE_DIR;
const fullPath = path.join(process.cwd(), base, input.data.path);
await mkdir(fullPath, {recursive: true});
const dir = path.dirname(fullPath);
await mkdir(dir, { recursive: true });
await writeFile(fullPath, input.data.file);
return {
src/features/storages/providers/s3.ts
+92
View File
@@ -0,0 +1,92 @@
import * as Minio from "minio";
import {StorageDeleteInput, StorageGetInput, StorageResult, StorageUploadInput} from "../types";
type S3Config = {
endPointUrl: string;
accessKey: string;
secretKey: string;
bucketName: string;
port?: number;
useSSL?: boolean;
};
async function getS3Client(config: S3Config) {
return new Minio.Client({
endPoint: config.endPointUrl,
accessKey: config.accessKey,
secretKey: config.secretKey,
port: config.port ?? 443,
useSSL: config.useSSL ?? true,
});
}
const BASE_DIR = "backups/";
async function ensureBucket(config: S3Config) {
const client = await getS3Client(config);
const exists = await client.bucketExists(config.bucketName);
if (!exists) await client.makeBucket(config.bucketName);
}
export async function uploadS3(config: S3Config, input: { data: StorageUploadInput }): Promise<StorageResult> {
const client = await getS3Client(config);
await ensureBucket(config);
const key = `${BASE_DIR}${input.data.path}`;
try {
await client.statObject(config.bucketName, key);
return {success: false, provider: "s3", error: "File already exists"};
} catch {
// continue if not found
}
await client.putObject(config.bucketName, key, input.data.file as Buffer);
return {
success: true,
provider: "s3",
};
}
export async function getS3(config: S3Config, input: { data: StorageGetInput }): Promise<StorageResult> {
const client = await getS3Client(config);
// const key = input.data.path;
const key = `${BASE_DIR}${input.data.path}`;
try {
await client.statObject(config.bucketName, key);
} catch {
return {success: false, provider: "s3", error: "File not found"};
}
const presignedUrl = await client.presignedGetObject(config.bucketName, key, 60);
const fileStream = await client.getObject(config.bucketName, key);
const chunks: Buffer[] = [];
for await (const chunk of fileStream) chunks.push(chunk as Buffer);
const buffer = Buffer.concat(chunks);
return {
success: true,
provider: "s3",
file: buffer,
url: presignedUrl,
};
}
export async function deleteS3(config: S3Config, input: { data: StorageDeleteInput }): Promise<StorageResult> {
const client = await getS3Client(config);
// const key = input.data.path;
const key = `${BASE_DIR}${input.data.path}`;
try {
await client.removeObject(config.bucketName, key);
return {success: true, provider: "s3"};
} catch (err: any) {
return {success: false, provider: "s3", error: err.message};
}
}
src/features/storages/types.ts
+1 -1
View File
@@ -1,6 +1,6 @@
export type StorageProviderKind =
| 'local'
// | 's3'
| 's3'
;
export type StorageAction =