diff --git a/app/(customer)/dashboard/(admin)/storages/channels/page.tsx b/app/(customer)/dashboard/(admin)/storages/channels/page.tsx index f3d0a5f8..e265ae74 100644 --- a/app/(customer)/dashboard/(admin)/storages/channels/page.tsx +++ b/app/(customer)/dashboard/(admin)/storages/channels/page.tsx @@ -3,7 +3,7 @@ import {Page, PageActions, PageContent, PageHeader, PageTitle} from "@/features/ import {Metadata} from "next"; import {ChannelsSection} from "@/components/wrappers/dashboard/admin/channels/channels-section"; import {db} from "@/db"; -import {desc, isNotNull, isNull, not} from "drizzle-orm"; +import {desc, eq, isNotNull, isNull, not} from "drizzle-orm"; import * as drizzleDb from "@/db"; import {StorageChannelWith} from "@/db/schema/12_storage-channel"; import {ChannelAddEditModal} from "@/components/wrappers/dashboard/admin/channels/channel/channel-add-edit-modal"; @@ -29,6 +29,10 @@ export default async function RoutePage(props: PageParams<{}>) { }, }); + const settings = await db.query.setting.findFirst({ + where: eq(drizzleDb.schemas.setting.name, "system"), + }); + return ( @@ -39,7 +43,7 @@ export default async function RoutePage(props: PageParams<{}>) { - + ); diff --git a/app/api/agent/[agentId]/backup/helpers.ts b/app/api/agent/[agentId]/backup/helpers.ts index 0df35e4f..3da9b06a 100644 --- a/app/api/agent/[agentId]/backup/helpers.ts +++ b/app/api/agent/[agentId]/backup/helpers.ts @@ -1,8 +1,5 @@ import fs from "node:fs"; import forge from "node-forge"; -import {EventPayload} from "@/features/notifications/types"; -import {dispatchNotification} from "@/features/notifications/dispatch"; -import {Database, DatabaseWith} from "@/db/schema/07_database"; export async function decryptedDump(file: File, aesKeyHex: string, ivHex: string, fileExtension: string): Promise { diff --git a/app/api/agent/[agentId]/backup/route.ts b/app/api/agent/[agentId]/backup/route.ts index 5c468585..8ebe25d6 100644 --- a/app/api/agent/[agentId]/backup/route.ts +++ b/app/api/agent/[agentId]/backup/route.ts @@ -11,6 +11,7 @@ import {env} from "@/env.mjs"; import {withUpdatedAt} from "@/db/utils"; import {decryptedDump, getFileExtension} from "./helpers"; import {sendNotificationsBackupRestore} from "@/features/notifications/helpers"; +import {storeBackupFiles} from "@/features/storages/helpers"; export async function POST( request: Request, @@ -25,6 +26,7 @@ export async function POST( {status: 400} ); } + eventEmitter.emit('modification', {update: true}); const agentId = (await params).agentId; @@ -56,7 +58,8 @@ export async function POST( where: eq(drizzleDb.schemas.database.agentDatabaseId, generatedId), with: { project: true, - alertPolicies: true + alertPolicies: true, + storagePolicies: true } }); @@ -125,35 +128,41 @@ export async function POST( const uuid = uuidv4(); const fileName = `${uuid}${fileExtension}`; const buffer = Buffer.from(await decryptedFile.arrayBuffer()); - const [settings] = await db.select().from(drizzleDb.schemas.setting).where(eq(drizzleDb.schemas.setting.name, "system")).limit(1); - if (!settings) { - throw new Error("System settings not found."); - } - let success: boolean, message: string, filePath: string; - const result = - settings.storage === "local" - ? await uploadLocalPrivate(fileName, buffer) - : await uploadS3Private(`${database.project?.slug}/${fileName}`, buffer, env.S3_BUCKET_NAME!); - ({success, message, filePath} = result); + // const [settings] = await db.select().from(drizzleDb.schemas.setting).where(eq(drizzleDb.schemas.setting.name, "system")).limit(1); + // if (!settings) { + // throw new Error("System settings not found."); + // } - if (!success) { - return NextResponse.json( - {error: message}, - {status: 500} - ); - } + // let success: boolean, message: string, filePath: string; + // + // const result = + // settings.storage === "local" + // ? await uploadLocalPrivate(fileName, buffer) + // : await uploadS3Private(`${database.project?.slug}/${fileName}`, buffer, env.S3_BUCKET_NAME!); + // + // ({success, message, filePath} = result); + // + // if (!success) { + // return NextResponse.json( + // {error: message}, + // {status: 500} + // ); + // } - await db - .update(drizzleDb.schemas.backup) - .set(withUpdatedAt({ - file: fileName, - fileSize: fileSizeBytes, - status: 'success', - })) - .where(eq(drizzleDb.schemas.backup.id, backup.id)); + await storeBackupFiles(backup, database, buffer, fileName) + + // + // await db + // .update(drizzleDb.schemas.backup) + // .set(withUpdatedAt({ + // file: fileName, + // fileSize: fileSizeBytes, + // status: 'success', + // })) + // .where(eq(drizzleDb.schemas.backup.id, backup.id)); eventEmitter.emit('modification', {update: true}); diff --git a/src/components/wrappers/dashboard/admin/channels/channel/channel-add-edit-modal.tsx b/src/components/wrappers/dashboard/admin/channels/channel/channel-add-edit-modal.tsx index 3506e52a..723fba26 100644 --- a/src/components/wrappers/dashboard/admin/channels/channel/channel-add-edit-modal.tsx +++ b/src/components/wrappers/dashboard/admin/channels/channel/channel-add-edit-modal.tsx @@ -42,6 +42,7 @@ export const ChannelAddEditModal = ({ }: ChannelAddModalProps) => { const isMobile = useIsMobile(); const [openInternal, setOpen] = useState(open); + const isLocalSystem = channel?.provider == "local"; const isCreate = !Boolean(channel); @@ -82,46 +83,65 @@ export const ChannelAddEditModal = ({
- {adminView ? - - - Configuration - Organizations - - - { - onOpenChangeAction?.(false) - setOpen(false); - }} - /> - - + <> + {!isLocalSystem ? ( + <> + {adminView ? + + + Configuration + Organizations + + + { + onOpenChangeAction?.(false) + setOpen(false); + }} + /> + + + + + + : + <> + { + onOpenChangeAction?.(false) + setOpen(false); + }} + /> + + } + + ) + + : + + <> - - - : - <> - { - onOpenChangeAction?.(false) - setOpen(false); - }} - /> - - } + + } + + +
diff --git a/src/components/wrappers/dashboard/admin/channels/channel/channel-card/button-edit-channel.tsx b/src/components/wrappers/dashboard/admin/channels/channel/channel-card/button-edit-channel.tsx index 4aff284d..612241a0 100644 --- a/src/components/wrappers/dashboard/admin/channels/channel/channel-card/button-edit-channel.tsx +++ b/src/components/wrappers/dashboard/admin/channels/channel/channel-card/button-edit-channel.tsx @@ -33,6 +33,7 @@ export const EditChannelButton = ({ }: EditChannelButtonProps) => { const router = useRouter(); const [isAddModalOpen, setIsAddModalOpen] = useState(false); + const isLocalSystem = channel.provider == "local"; const mutation = useMutation({ @@ -73,10 +74,12 @@ export const EditChannelButton = ({ return ( <> - { - await mutation.mutateAsync(!channel.enabled) - }} - /> + + {!isLocalSystem && ( + { + await mutation.mutateAsync(!channel.enabled) + }}/> + )} { - const {data, organization, kind, adminView} = props; + const {data, organization, kind, adminView, defaultStorageChannelId} = props; const isMobile = useIsMobile() + const isDefaultSystemStorage = defaultStorageChannelId === data.id; + const isOwned = data.organizationId ? true : !organization; const isLocalSystem = data.provider == "local"; @@ -47,11 +51,16 @@ export const ChannelCard = (props: ChannelCardProps) => { {data.provider} + {isDefaultSystemStorage && ( + + default + + )} {kind && (
- {(isOwned && !isLocalSystem) && ( + {(isOwned) && ( <> { channel={data} kind={kind} /> - + {!isLocalSystem && ( + + )} )}
diff --git a/src/components/wrappers/dashboard/admin/channels/channel/channel-form/providers/storages/forms/s3.form.tsx b/src/components/wrappers/dashboard/admin/channels/channel/channel-form/providers/storages/forms/s3.form.tsx index 01406b30..9efef758 100644 --- a/src/components/wrappers/dashboard/admin/channels/channel/channel-form/providers/storages/forms/s3.form.tsx +++ b/src/components/wrappers/dashboard/admin/channels/channel/channel-form/providers/storages/forms/s3.form.tsx @@ -3,6 +3,7 @@ import {FormControl, FormField, FormItem, FormLabel, FormMessage} from "@/compon import {Input} from "@/components/ui/input"; import {Separator} from "@/components/ui/separator"; import {PasswordInput} from "@/components/ui/password-input"; +import {Switch} from "@/components/ui/switch"; type StorageS3FormProps = { @@ -59,12 +60,43 @@ export const StorageS3Form = ({form}: StorageS3FormProps) => { Bucket name - + )} /> + ( + + Port + + + + + + )} + /> + + ( + + Use SSL + + + + + + )} + /> + ) } diff --git a/src/components/wrappers/dashboard/admin/channels/channels-section.tsx b/src/components/wrappers/dashboard/admin/channels/channels-section.tsx index cf1adfb2..4e632d7e 100644 --- a/src/components/wrappers/dashboard/admin/channels/channels-section.tsx +++ b/src/components/wrappers/dashboard/admin/channels/channels-section.tsx @@ -10,15 +10,17 @@ import {ChannelAddEditModal} from "@/components/wrappers/dashboard/admin/channel import {ChannelKind, getChannelTextBasedOnKind} from "@/components/wrappers/dashboard/admin/channels/helpers/common"; type ChannelsSectionProps = { - channels: NotificationChannelWith[] | StorageChannelWith[] - organizations: OrganizationWithMembers[] - kind: ChannelKind; + channels: NotificationChannelWith[] | StorageChannelWith[], + organizations: OrganizationWithMembers[], + kind: ChannelKind, + defaultStorageChannelId?: string | null | undefined } export const ChannelsSection = ({ organizations, channels, - kind + kind, + defaultStorageChannelId }: ChannelsSectionProps) => { const [isAddModalOpen, setIsAddModalOpen] = useState(false); @@ -41,6 +43,7 @@ export const ChannelsSection = ({ adminView={true} organizations={organizations} kind={kind} + defaultStorageChannelId={defaultStorageChannelId} /> ) : ( diff --git a/src/db/index.ts b/src/db/index.ts index acc6b41b..ec52874b 100644 --- a/src/db/index.ts +++ b/src/db/index.ts @@ -14,6 +14,7 @@ import * as alertPolicy from "./schema/10_alert-policy"; import * as notificationLog from "./schema/11_notification-log"; import * as storageChannel from "./schema/12_storage-channel"; import * as storagePolicy from "@/db/schema/13_storage-policy"; +import * as backupStorage from "@/db/schema/14_storage-backup"; import {Pool} from "pg"; @@ -44,7 +45,8 @@ export const schemas = { ...alertPolicy, ...notificationLog, ...storageChannel, - ...storagePolicy + ...storagePolicy, + ...backupStorage }; export const db = drizzle({ diff --git a/src/db/schema/01_setting.ts b/src/db/schema/01_setting.ts index 030d2a98..94d14bb0 100644 --- a/src/db/schema/01_setting.ts +++ b/src/db/schema/01_setting.ts @@ -4,6 +4,12 @@ import {createSelectSchema} from "drizzle-zod"; import {z} from "zod"; import {timestamps} from "@/db/schema/00_common"; import {storageChannel} from "@/db/schema/12_storage-channel"; +import {relations} from "drizzle-orm"; +import {agent} from "@/db/schema/08_agent"; +import {project} from "@/db/schema/06_project"; +import {alertPolicy} from "@/db/schema/10_alert-policy"; +import {storagePolicy} from "@/db/schema/13_storage-policy"; +import {backup, database, restoration, retentionPolicy} from "@/db/schema/07_database"; export const setting = pgTable("settings", { id: uuid("id").primaryKey().defaultRandom(), @@ -23,5 +29,10 @@ export const setting = pgTable("settings", { ...timestamps }); +export const settingRelations = relations(setting, ({one, many}) => ({ + storageChannel: one(storageChannel, {fields: [setting.defaultStorageChannelId], references: [storageChannel.id]}), +})); + + export const settingSchema = createSelectSchema(setting); export type Setting = z.infer; diff --git a/src/features/storages/dispatch.ts b/src/features/storages/dispatch.ts index 5e58d665..943fa1d9 100644 --- a/src/features/storages/dispatch.ts +++ b/src/features/storages/dispatch.ts @@ -5,6 +5,8 @@ import * as drizzleDb from '@/db'; import {db} from '@/db'; import type {StorageInput, StorageProviderKind, StorageResult,} from './types'; import {dispatchViaProvider} from "@/features/storages/providers"; +import {StorageChannel} from "@/db/schema/12_storage-channel"; +import {Json} from "drizzle-zod"; export async function dispatchStorage( input: StorageInput, @@ -13,34 +15,84 @@ export async function dispatchStorage( organizationId?: string ): Promise { try { - if (!channelId) { + + let channel: StorageChannel | null = null; + + if (policyId) { + const policyDb = await db.query.storagePolicy.findFirst({ + where: eq(drizzleDb.schemas.storagePolicy.id, policyId), + with: { + storageChannel: true + }, + }); + + if (!policyDb || !policyDb.storageChannel) { + return { + success: false, + provider: null, + error: "Policy or associated channel not found", + }; + } + + if (!policyDb.enabled || !policyDb.storageChannel.enabled) { + return { + success: false, + provider: policyDb.storageChannel.provider as any, + error: "Policy or channel is disabled", + }; + } + + channel = { + ...policyDb.storageChannel, + config: policyDb.storageChannel.config as Json, + }; + } + if (channelId) { + const fetchedChannel = await db.query.storageChannel.findFirst({ + where: eq(drizzleDb.schemas.storageChannel.id, channelId), + }); + if (!fetchedChannel) { + return { + success: false, + provider: null, + error: "Channel not found", + }; + } + + channel = { + ...fetchedChannel, + config: fetchedChannel.config as Json, + }; + } + + if (!channel) { return { success: false, provider: null, - error: 'No storage channel provided', + error: "No valid channel to dispatch on storage", }; } - const channel = await db.query.storageChannel.findFirst({ - where: eq(drizzleDb.schemas.storageChannel.id, channelId), - }); - if (!channel || !channel.enabled) { + if (!channel.enabled) { return { success: false, - provider: channel?.provider as StorageProviderKind, - error: 'Storage channel not found or disabled', + provider: null, + error: "Channel not active", }; } + return await dispatchViaProvider( channel.provider as StorageProviderKind, channel.config, input ); + + } catch (err: any) { return { success: false, diff --git a/src/features/storages/helpers.ts b/src/features/storages/helpers.ts index e6d03b6f..8479de79 100644 --- a/src/features/storages/helpers.ts +++ b/src/features/storages/helpers.ts @@ -1,10 +1,106 @@ -import {DatabaseWith} from "@/db/schema/07_database"; -import {StorageChannel} from "@/db/schema/12_storage-channel"; -import {sendNotificationsBackupRestore} from "@/features/notifications/helpers"; +import { Backup, DatabaseWith } from "@/db/schema/07_database"; +import { dispatchStorage } from "@/features/storages/dispatch"; +import type { StorageInput, StorageResult } from "@/features/storages/types"; +import * as drizzleDb from "@/db"; +import { withUpdatedAt } from "@/db/utils"; +import { eq } from "drizzle-orm"; +import { db } from "@/db"; +import { createHash } from "crypto"; + +function computeChecksum(buffer: Buffer): string { + return createHash("sha256").update(buffer).digest("hex"); +} + +export async function storeBackupFiles( + backup: Backup, + database: DatabaseWith, + file: Buffer, + fileName: string +): Promise { + + const settings = await db.query.setting.findFirst({ + where: eq(drizzleDb.schemas.setting.name, "system"), + with: { storageChannel: true }, + }); + + const defaultPolicy = settings?.storageChannel + ? [{ + id: null, + storageChannelId: settings.storageChannel.id, + enabled: true + }] + : []; + + console.log(database.storagePolicies); + + const policies = (database.storagePolicies?.filter(p => p.enabled) || defaultPolicy); + + console.log("Policies", policies); -export async function storeFileBackup(database: DatabaseWith, file: Buffer) { + if (!policies.length) { + return []; + } + const path = `${database.project?.slug}/${fileName}`; + const size = file.length; + const checksum = computeChecksum(file); + const results = await Promise.all( + policies.map(async policy => { + const [backupStorage] = await db + .insert(drizzleDb.schemas.backupStorage) + .values({ + backupId: backup.id, + storageChannelId: policy.storageChannelId, + status: "pending", + path, + size, + checksum, + }) + .returning(); -} \ No newline at end of file + const input: StorageInput = { + action: "upload", + data: { path, file }, + }; + + // const result = await dispatchStorage(input, policy.id); + let result: StorageResult; + + try { + if (policy.id){ + result = await dispatchStorage(input, policy.id); + }else{ + result = await dispatchStorage(input, undefined, policy.storageChannelId); + } + + } catch (err) { + console.error(err); + result = { + success: false, + provider: null, + error: err instanceof Error ? err.message : "Unknown error" + }; + } + + await db + .update(drizzleDb.schemas.backupStorage) + .set(withUpdatedAt({ status: result.success ? "success" : "failed" })) + .where(eq(drizzleDb.schemas.backupStorage.id, backupStorage.id)); + + return result; + }) + ); + + console.log(results); + + const backupStatus = results.some(r => r.success) ? "success" : "failed"; + + await db + .update(drizzleDb.schemas.backup) + .set(withUpdatedAt({ status: backupStatus })) + .where(eq(drizzleDb.schemas.backup.id, backup.id)); + + return results; +} diff --git a/src/features/storages/providers/index.ts b/src/features/storages/providers/index.ts index f30a130c..5950f7bc 100644 --- a/src/features/storages/providers/index.ts +++ b/src/features/storages/providers/index.ts @@ -5,6 +5,7 @@ import type { } from '../types'; import {uploadLocal, getLocal, deleteLocal} from './local'; +import {deleteS3, getS3, uploadS3} from "@/features/storages/providers/s3"; type ProviderHandler = { upload: (config: any, input: StorageInput & { action: 'upload' }) => Promise; @@ -18,11 +19,11 @@ const handlers: Record = { get: getLocal, delete: deleteLocal, }, - // s3: { - // upload: uploadS3, - // get: getS3, - // delete: deleteS3, - // }, + s3: { + upload: uploadS3, + get: getS3, + delete: deleteS3, + }, // gcs: null as any, // azure: null as any, }; diff --git a/src/features/storages/providers/local.ts b/src/features/storages/providers/local.ts index 2357c7e2..05f466bd 100644 --- a/src/features/storages/providers/local.ts +++ b/src/features/storages/providers/local.ts @@ -14,7 +14,9 @@ export async function uploadLocal( const base = config.baseDir || BASE_DIR; const fullPath = path.join(process.cwd(), base, input.data.path); - await mkdir(fullPath, {recursive: true}); + const dir = path.dirname(fullPath); + + await mkdir(dir, { recursive: true }); await writeFile(fullPath, input.data.file); return { diff --git a/src/features/storages/providers/s3.ts b/src/features/storages/providers/s3.ts new file mode 100644 index 00000000..774b9e91 --- /dev/null +++ b/src/features/storages/providers/s3.ts @@ -0,0 +1,92 @@ +import * as Minio from "minio"; +import {StorageDeleteInput, StorageGetInput, StorageResult, StorageUploadInput} from "../types"; + +type S3Config = { + endPointUrl: string; + accessKey: string; + secretKey: string; + bucketName: string; + port?: number; + useSSL?: boolean; +}; + +async function getS3Client(config: S3Config) { + return new Minio.Client({ + endPoint: config.endPointUrl, + accessKey: config.accessKey, + secretKey: config.secretKey, + port: config.port ?? 443, + useSSL: config.useSSL ?? true, + }); +} + +const BASE_DIR = "backups/"; + + +async function ensureBucket(config: S3Config) { + const client = await getS3Client(config); + const exists = await client.bucketExists(config.bucketName); + if (!exists) await client.makeBucket(config.bucketName); +} + +export async function uploadS3(config: S3Config, input: { data: StorageUploadInput }): Promise { + const client = await getS3Client(config); + await ensureBucket(config); + + const key = `${BASE_DIR}${input.data.path}`; + + try { + await client.statObject(config.bucketName, key); + return {success: false, provider: "s3", error: "File already exists"}; + } catch { + // continue if not found + } + + await client.putObject(config.bucketName, key, input.data.file as Buffer); + + return { + success: true, + provider: "s3", + }; +} + +export async function getS3(config: S3Config, input: { data: StorageGetInput }): Promise { + const client = await getS3Client(config); + + // const key = input.data.path; + const key = `${BASE_DIR}${input.data.path}`; + + + try { + await client.statObject(config.bucketName, key); + } catch { + return {success: false, provider: "s3", error: "File not found"}; + } + + const presignedUrl = await client.presignedGetObject(config.bucketName, key, 60); + + const fileStream = await client.getObject(config.bucketName, key); + const chunks: Buffer[] = []; + for await (const chunk of fileStream) chunks.push(chunk as Buffer); + const buffer = Buffer.concat(chunks); + + return { + success: true, + provider: "s3", + file: buffer, + url: presignedUrl, + }; +} + +export async function deleteS3(config: S3Config, input: { data: StorageDeleteInput }): Promise { + const client = await getS3Client(config); + // const key = input.data.path; + const key = `${BASE_DIR}${input.data.path}`; + + try { + await client.removeObject(config.bucketName, key); + return {success: true, provider: "s3"}; + } catch (err: any) { + return {success: false, provider: "s3", error: err.message}; + } +} diff --git a/src/features/storages/types.ts b/src/features/storages/types.ts index c0c5ec81..0bf5385e 100644 --- a/src/features/storages/types.ts +++ b/src/features/storages/types.ts @@ -1,6 +1,6 @@ export type StorageProviderKind = | 'local' -// | 's3' + | 's3' ; export type StorageAction =