chore: working on backup api for new storage system.

src/features/storages/dispatch.ts

@@ -5,6 +5,8 @@ import * as drizzleDb from '@/db';
 import {db} from '@/db';
 import type {StorageInput, StorageProviderKind, StorageResult,} from './types';
 import {dispatchViaProvider} from "@/features/storages/providers";
+import {StorageChannel} from "@/db/schema/12_storage-channel";
+import {Json} from "drizzle-zod";
 
 export async function dispatchStorage(
     input: StorageInput,
@@ -13,34 +15,84 @@ export async function dispatchStorage(
     organizationId?: string
 ): Promise<StorageResult> {
     try {
-        if (!channelId) {
+
+        let channel: StorageChannel | null = null;
+
+        if (policyId) {
+            const policyDb = await db.query.storagePolicy.findFirst({
+                where: eq(drizzleDb.schemas.storagePolicy.id, policyId),
+                with: {
+                    storageChannel: true
+                },
+            });
+
+            if (!policyDb || !policyDb.storageChannel) {
+                return {
+                    success: false,
+                    provider: null,
+                    error: "Policy or associated channel not found",
+                };
+            }
+
+            if (!policyDb.enabled || !policyDb.storageChannel.enabled) {
+                return {
+                    success: false,
+                    provider: policyDb.storageChannel.provider as any,
+                    error: "Policy or channel is disabled",
+                };
+            }
+
+            channel = {
+                ...policyDb.storageChannel,
+                config: policyDb.storageChannel.config as Json,
+            };
+        }
 
 
+        if (channelId) {
+            const fetchedChannel = await db.query.storageChannel.findFirst({
+                where: eq(drizzleDb.schemas.storageChannel.id, channelId),
+            });
 
+            if (!fetchedChannel) {
+                return {
+                    success: false,
+                    provider: null,
+                    error: "Channel not found",
+                };
+            }
+
+            channel = {
+                ...fetchedChannel,
+                config: fetchedChannel.config as Json,
+            };
+        }
+
+        if (!channel) {
             return {
                 success: false,
                 provider: null,
-                error: 'No storage channel provided',
+                error: "No valid channel to dispatch on storage",
             };
         }
 
-        const channel = await db.query.storageChannel.findFirst({
-            where: eq(drizzleDb.schemas.storageChannel.id, channelId),
-        });
 
-        if (!channel || !channel.enabled) {
+        if (!channel.enabled) {
             return {
                 success: false,
-                provider: channel?.provider as StorageProviderKind,
-                error: 'Storage channel not found or disabled',
+                provider: null,
+                error: "Channel not active",
             };
         }
 
+
         return await dispatchViaProvider(
             channel.provider as StorageProviderKind,
             channel.config,
             input
         );
+
+
     } catch (err: any) {
         return {
             success: false,

src/features/storages/helpers.ts

@@ -1,10 +1,106 @@
-import {DatabaseWith} from "@/db/schema/07_database";
-import {StorageChannel} from "@/db/schema/12_storage-channel";
-import {sendNotificationsBackupRestore} from "@/features/notifications/helpers";
+import { Backup, DatabaseWith } from "@/db/schema/07_database";
+import { dispatchStorage } from "@/features/storages/dispatch";
+import type { StorageInput, StorageResult } from "@/features/storages/types";
+import * as drizzleDb from "@/db";
+import { withUpdatedAt } from "@/db/utils";
+import { eq } from "drizzle-orm";
+import { db } from "@/db";
+import { createHash } from "crypto";
+
+function computeChecksum(buffer: Buffer): string {
+    return createHash("sha256").update(buffer).digest("hex");
+}
+
+export async function storeBackupFiles(
+    backup: Backup,
+    database: DatabaseWith,
+    file: Buffer,
+    fileName: string
+): Promise<StorageResult[]> {
+
+    const settings = await db.query.setting.findFirst({
+        where: eq(drizzleDb.schemas.setting.name, "system"),
+        with: { storageChannel: true },
+    });
+
+    const defaultPolicy = settings?.storageChannel
+        ? [{
+            id: null,
+            storageChannelId: settings.storageChannel.id,
+            enabled: true
+        }]
+        : [];
+
+    console.log(database.storagePolicies);
+
+    const policies = (database.storagePolicies?.filter(p => p.enabled) || defaultPolicy);
+
+    console.log("Policies", policies);
 
 
-export async function storeFileBackup(database: DatabaseWith, file: Buffer)  {
+    if (!policies.length) {
+        return [];
+    }
 
+    const path = `${database.project?.slug}/${fileName}`;
+    const size = file.length;
+    const checksum = computeChecksum(file);
 
+    const results = await Promise.all(
+        policies.map(async policy => {
+            const [backupStorage] = await db
+                .insert(drizzleDb.schemas.backupStorage)
+                .values({
+                    backupId: backup.id,
+                    storageChannelId: policy.storageChannelId,
+                    status: "pending",
+                    path,
+                    size,
+                    checksum,
+                })
+                .returning();
 
-}
+            const input: StorageInput = {
+                action: "upload",
+                data: { path, file },
+            };
+
+            // const result = await dispatchStorage(input, policy.id);
+            let result: StorageResult;
+
+            try {
+                if (policy.id){
+                    result = await dispatchStorage(input, policy.id);
+                }else{
+                    result = await dispatchStorage(input, undefined, policy.storageChannelId);
+                }
+
+            } catch (err) {
+                console.error(err);
+                result = {
+                    success: false,
+                    provider: null,
+                    error: err instanceof Error ? err.message : "Unknown error"
+                };
+            }
+
+            await db
+                .update(drizzleDb.schemas.backupStorage)
+                .set(withUpdatedAt({ status: result.success ? "success" : "failed" }))
+                .where(eq(drizzleDb.schemas.backupStorage.id, backupStorage.id));
+
+            return result;
+        })
+    );
+
+    console.log(results);
+
+    const backupStatus = results.some(r => r.success) ? "success" : "failed";
+
+    await db
+        .update(drizzleDb.schemas.backup)
+        .set(withUpdatedAt({ status: backupStatus }))
+        .where(eq(drizzleDb.schemas.backup.id, backup.id));
+
+    return results;
+}

src/features/storages/providers/index.ts

@@ -5,6 +5,7 @@ import type {
 } from '../types';
 
 import {uploadLocal, getLocal, deleteLocal} from './local';
+import {deleteS3, getS3, uploadS3} from "@/features/storages/providers/s3";
 
 type ProviderHandler = {
     upload: (config: any, input: StorageInput & { action: 'upload' }) => Promise<StorageResult>;
@@ -18,11 +19,11 @@ const handlers: Record<StorageProviderKind, ProviderHandler> = {
         get: getLocal,
         delete: deleteLocal,
     },
-    // s3: {
-    //     upload: uploadS3,
-    //     get: getS3,
-    //     delete: deleteS3,
-    // },
+    s3: {
+        upload: uploadS3,
+        get: getS3,
+        delete: deleteS3,
+    },
     // gcs: null as any,
     // azure: null as any,
 };

src/features/storages/providers/local.ts

@@ -14,7 +14,9 @@ export async function uploadLocal(
     const base = config.baseDir || BASE_DIR;
     const fullPath = path.join(process.cwd(), base, input.data.path);
 
-    await mkdir(fullPath, {recursive: true});
+    const dir = path.dirname(fullPath);
+
+    await mkdir(dir, { recursive: true });
     await writeFile(fullPath, input.data.file);
 
     return {

src/features/storages/providers/s3.ts

@@ -0,0 +1,92 @@
+import * as Minio from "minio";
+import {StorageDeleteInput, StorageGetInput, StorageResult, StorageUploadInput} from "../types";
+
+type S3Config = {
+    endPointUrl: string;
+    accessKey: string;
+    secretKey: string;
+    bucketName: string;
+    port?: number;
+    useSSL?: boolean;
+};
+
+async function getS3Client(config: S3Config) {
+    return new Minio.Client({
+        endPoint: config.endPointUrl,
+        accessKey: config.accessKey,
+        secretKey: config.secretKey,
+        port: config.port ?? 443,
+        useSSL: config.useSSL ?? true,
+    });
+}
+
+const BASE_DIR = "backups/";
+
+
+async function ensureBucket(config: S3Config) {
+    const client = await getS3Client(config);
+    const exists = await client.bucketExists(config.bucketName);
+    if (!exists) await client.makeBucket(config.bucketName);
+}
+
+export async function uploadS3(config: S3Config, input: { data: StorageUploadInput }): Promise<StorageResult> {
+    const client = await getS3Client(config);
+    await ensureBucket(config);
+
+    const key = `${BASE_DIR}${input.data.path}`;
+
+    try {
+        await client.statObject(config.bucketName, key);
+        return {success: false, provider: "s3", error: "File already exists"};
+    } catch {
+        // continue if not found
+    }
+
+    await client.putObject(config.bucketName, key, input.data.file as Buffer);
+
+    return {
+        success: true,
+        provider: "s3",
+    };
+}
+
+export async function getS3(config: S3Config, input: { data: StorageGetInput }): Promise<StorageResult> {
+    const client = await getS3Client(config);
+
+    // const key = input.data.path;
+    const key = `${BASE_DIR}${input.data.path}`;
+
+
+    try {
+        await client.statObject(config.bucketName, key);
+    } catch {
+        return {success: false, provider: "s3", error: "File not found"};
+    }
+
+    const presignedUrl = await client.presignedGetObject(config.bucketName, key, 60);
+
+    const fileStream = await client.getObject(config.bucketName, key);
+    const chunks: Buffer[] = [];
+    for await (const chunk of fileStream) chunks.push(chunk as Buffer);
+    const buffer = Buffer.concat(chunks);
+
+    return {
+        success: true,
+        provider: "s3",
+        file: buffer,
+        url: presignedUrl,
+    };
+}
+
+export async function deleteS3(config: S3Config, input: { data: StorageDeleteInput }): Promise<StorageResult> {
+    const client = await getS3Client(config);
+    // const key = input.data.path;
+    const key = `${BASE_DIR}${input.data.path}`;
+
+    try {
+        await client.removeObject(config.bucketName, key);
+        return {success: true, provider: "s3"};
+    } catch (err: any) {
+        return {success: false, provider: "s3", error: err.message};
+    }
+}

src/features/storages/types.ts

@@ -1,6 +1,6 @@
 export type StorageProviderKind =
     | 'local'
-// | 's3'
+    | 's3'
     ;
 
 export type StorageAction =