feat: Working on backend storage providers.

2026-07-14 11:16:13 +02:00 · 2026-01-13 22:22:01 +01:00
parent 67a89a12ac
commit d8e9e2dac3
15 changed files with 2506 additions and 108 deletions
@@ -0,0 +1,70 @@
+import {UseFormReturn} from "react-hook-form";
+import {FormControl, FormField, FormItem, FormLabel, FormMessage} from "@/components/ui/form";
+import {Input} from "@/components/ui/input";
+import {Separator} from "@/components/ui/separator";
+import {PasswordInput} from "@/components/ui/password-input";
+
+
+type StorageS3FormProps = {
+    form: UseFormReturn<any, any, any>
+}
+
+export const StorageS3Form = ({form}: StorageS3FormProps) => {
+    return (
+        <>
+            <Separator className="my-1"/>
+            <FormField
+                control={form.control}
+                name="config.endPointUrl"
+                render={({field}) => (
+                    <FormItem>
+                        <FormLabel>Endpoind URL</FormLabel>
+                        <FormControl>
+                            <Input {...field} placeholder="s3.exemple.com"/>
+                        </FormControl>
+                        <FormMessage/>
+                    </FormItem>
+                )}
+            />
+            <FormField
+                control={form.control}
+                name="config.accessKey"
+                render={({field}) => (
+                    <FormItem>
+                        <FormLabel>Access Key</FormLabel>
+                        <FormControl>
+                            <Input {...field} placeholder="A1b2C3d4E5f6G7h"/>
+                        </FormControl>
+                        <FormMessage/>
+                    </FormItem>
+                )}
+            />
+            <FormField
+                control={form.control}
+                name="config.secretKey"
+                render={({field}) => (
+                    <FormItem>
+                        <FormLabel>Secret Key</FormLabel>
+                        <FormControl>
+                            <PasswordInput {...field} placeholder="A1b2C3d4E5f6G7h"/>
+                        </FormControl>
+                        <FormMessage/>
+                    </FormItem>
+                )}
+            />
+            <FormField
+                control={form.control}
+                name="config.bucketName"
+                render={({field}) => (
+                    <FormItem>
+                        <FormLabel>Bucket name</FormLabel>
+                        <FormControl>
+                            <PasswordInput {...field} placeholder="portabase-dev"/>
+                        </FormControl>
+                        <FormMessage/>
+                    </FormItem>
+                )}
+            />
+        </>
+    )
+}
@@ -27,6 +27,9 @@ import {NotificationChannelWith} from "@/db/schema/09_notification-channel";
 import {StorageChannelWith} from "@/db/schema/12_storage-channel";
 import {ForwardRefExoticComponent, JSX, RefAttributes, SVGProps} from "react";
 import {LucideProps} from "lucide-react";
+import {
+    StorageS3Form
+} from "@/components/wrappers/dashboard/admin/channels/channel/channel-form/providers/storages/forms/s3.form";

 export type ChannelKind = "notification" | "storage";

@@ -80,6 +83,8 @@ export const renderChannelForm = (provider: string | undefined, form: UseFormRet
            return <NotifierNtfyForm form={form}/>;
        case "webhook":
            return <NotifierWebhookForm form={form}/>;
+        case "s3":
+            return <StorageS3Form form={form}/>
        case "local":
            return <></>
        default:
@@ -1,6 +1,17 @@
 import {Server} from "lucide-react";
+import type {SVGProps} from "react";

 export const storageTypes = [
    {value: "local", label: "Local", icon: Server},
-    {value: "s3", label: "s3", icon: Server},
+    {value: "s3", label: "s3", icon: S3Icon},
 ]
+
+export function S3Icon(props: SVGProps<SVGSVGElement>) {
+    return (
+        <svg xmlns="http://www.w3.org/2000/svg" width={256} height={199}  {...props} viewBox="0 0 24 24">
+            <path fill="currentColor"
+                  d="M13.207.006a2.16 2.16 0 0 0-1.62.582a2.15 2.15 0 0 0-.095 3.035l3.408 3.55a3.042 3.042 0 0 1-.663 4.688l-.463.239V7.285a15.42 15.42 0 0 0-8.018 10.487v.017l6.549-3.328v7.621L13.779 24V13.682l.897-.463a4.443 4.443 0 0 0 1.22-7.03l-3.37-3.525a.75.75 0 0 1 .037-1.055a.75.75 0 0 1 1.056.038l.467.486l-.006.006l4.07 4.244a.057.057 0 0 0 .082 0a.06.06 0 0 0 0-.07l-3.14-5.143l-.149.143l.149-.145C14.494.393 13.829.054 13.207.006m-.902 9.865v2.994l-4.152 2.149a14 14 0 0 1 2.767-3.928a14 14 0 0 1 1.385-1.215"/>
+        </svg>
+    );
+}
+
@@ -21,10 +21,10 @@ import {
 } from "@/components/wrappers/dashboard/database/alert-policy/alert-policy.action";
 import {useRouter} from "next/navigation";
 import {Switch} from "@/components/ui/switch";
-import {getNotificationChannelIcon} from "@/components/wrappers/dashboard/admin/notifications/helpers";
 import {Card} from "@/components/ui/card";
 import Link from "next/link";
 import {useIsMobile} from "@/hooks/use-mobile";
+import {getChannelIcon} from "@/components/wrappers/dashboard/admin/channels/helpers/common";

 type AlertPolicyFormProps = {
    onSuccess?: () => void;
@@ -239,7 +239,7 @@ export const AlertPolicyForm = ({database, notificationChannels, organizationId,
                                                                                {selectedChannel && (
                                                                                    <div className="flex items-center gap-2 min-w-0 w-full">
                                                                                        <div className="flex items-center justify-center h-4 w-4 shrink-0">
-                                                                                            {getNotificationChannelIcon(selectedChannel.provider)}
+                                                                                            {getChannelIcon(selectedChannel.provider)}
                                                                                        </div>
                                                                                        <span className="truncate font-medium text-sm min-w-0">
                                                                                          {selectedChannel.name}
@@ -257,7 +257,7 @@ export const AlertPolicyForm = ({database, notificationChannels, organizationId,
                                                                            <SelectItem key={channel.id.toString()} value={channel.id.toString()}>
                                                                                <div className="flex items-center gap-2 w-full min-w-0">
                                                                                    <div className="text-muted-foreground scale-90 shrink-0">
-                                                                                        {getNotificationChannelIcon(channel.provider)}
+                                                                                        {getChannelIcon(channel.provider)}
                                                                                    </div>
                                                                                    <span className="font-medium truncate min-w-0">{channel.name}</span>
                                                                                    <span className="text-xs text-muted-foreground ml-2 capitalize shrink-0">
@@ -0,0 +1,12 @@
+CREATE TABLE "storage_policy" (
+	"id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
+	"storage_channel_id" uuid NOT NULL,
+	"enabled" boolean DEFAULT true NOT NULL,
+	"database_id" uuid NOT NULL,
+	"updated_at" timestamp,
+	"created_at" timestamp DEFAULT now() NOT NULL,
+	"deleted_at" timestamp
+);
+--> statement-breakpoint
+ALTER TABLE "storage_policy" ADD CONSTRAINT "storage_policy_storage_channel_id_storage_channel_id_fk" FOREIGN KEY ("storage_channel_id") REFERENCES "public"."storage_channel"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
+ALTER TABLE "storage_policy" ADD CONSTRAINT "storage_policy_database_id_databases_id_fk" FOREIGN KEY ("database_id") REFERENCES "public"."databases"("id") ON DELETE cascade ON UPDATE no action;
@@ -148,6 +148,13 @@
      "when": 1768248015695,
      "tag": "0020_thankful_sunspot",
      "breakpoints": true
+    },
+    {
+      "idx": 21,
+      "version": "7",
+      "when": 1768337434929,
+      "tag": "0021_soft_blockbuster",
+      "breakpoints": true
    }
  ]
 }
@@ -1,4 +1,4 @@
-import {pgTable, text, boolean, timestamp, uuid, integer, pgEnum, uniqueIndex} from "drizzle-orm/pg-core";
+import {pgTable, text, boolean, timestamp, uuid, integer, pgEnum} from "drizzle-orm/pg-core";
 import {Agent, agent} from "./08_agent";
 import {Project, project} from "./06_project";
 import {relations} from "drizzle-orm";
@@ -6,11 +6,8 @@ import {dbmsEnum, statusEnum} from "./types";
 import {createSelectSchema} from "drizzle-zod";
 import {z} from "zod";
 import {timestamps} from "@/db/schema/00_common";
-import {member} from "@/db/schema/04_member";
-import {invitation} from "@/db/schema/05_invitation";
-import {organizationNotificationChannel} from "@/db/schema/09_notification-channel";
-import {organization} from "@/db/schema/03_organization";
 import {AlertPolicy, alertPolicy} from "@/db/schema/10_alert-policy";
+import {StoragePolicy, storagePolicy} from "@/db/schema/13_storage-policy";

 export const database = pgTable("databases", {
    id: uuid("id").primaryKey().defaultRandom(),
@@ -84,6 +81,7 @@ export const databaseRelations = relations(database, ({one, many}) => ({
    backups: many(backup),
    restorations: many(restoration),
    alertPolicies: many(alertPolicy),
+    storagePolicies: many(storagePolicy),
 }));

 export const backupRelations = relations(backup, ({one, many}) => ({
@@ -124,5 +122,6 @@ export type DatabaseWith = Database & {
    restorations?: Restoration[] | null;
    retentionPolicy?: RetentionPolicy | null;
    alertPolicies?: AlertPolicy[] | null;
+    storagePolicies?: StoragePolicy[] | null;
 };

@@ -0,0 +1,33 @@
+import {boolean, pgTable, uuid} from "drizzle-orm/pg-core";
+import {timestamps} from "@/db/schema/00_common";
+import {relations} from "drizzle-orm";
+import {database} from "@/db/schema/07_database";
+import {createSelectSchema} from "drizzle-zod";
+import {z} from "zod";
+import {storageChannel} from "@/db/schema/12_storage-channel";
+
+export const storagePolicy = pgTable('storage_policy', {
+    id: uuid('id').defaultRandom().primaryKey(),
+    storageChannelId: uuid('storage_channel_id')
+        .notNull()
+        .references(() => storageChannel.id, {onDelete: 'cascade'}),
+    enabled: boolean('enabled').default(true).notNull(),
+    databaseId: uuid('database_id')
+        .notNull()
+        .references(() => database.id, {onDelete: 'cascade'}),
+    ...timestamps
+});
+
+export const storagePolicyRelations = relations(storagePolicy, ({one}) => ({
+    storageChannel: one(storageChannel, {
+        fields: [storagePolicy.storageChannelId],
+        references: [storageChannel.id],
+    }),
+    database: one(database, {
+        fields: [storagePolicy.databaseId],
+        references: [database.id],
+    }),
+}));
+
+export const storagePolicySchema = createSelectSchema(storagePolicy);
+export type StoragePolicy = z.infer<typeof storagePolicySchema>;
@@ -1,129 +1,51 @@
 "use server";
-import {eq} from "drizzle-orm";
-import {dispatchViaProvider} from "./providers";
-import type {EventPayload, DispatchResult, EventKind} from "./types";
-import * as drizzleDb from "@/db";
-import {db} from "@/db";
-import {notificationLog} from "@/db/schema/11_notification-log";
-import {NotificationChannel} from "@/db/schema/09_notification-channel";
-import {Json} from "drizzle-zod";
+
+import {eq} from 'drizzle-orm';
+import * as drizzleDb from '@/db';
+import {db} from '@/db';
+import type {StorageInput, StorageProviderKind, StorageResult,} from './types';
+import {dispatchViaProvider} from "@/features/storages/providers";

 export async function dispatchStorage(
-    payload: EventPayload,
+    input: StorageInput,
    policyId?: string,
    channelId?: string,
    organizationId?: string
-): Promise<DispatchResult> {
+): Promise<StorageResult> {
    try {
-        let channel: NotificationChannel | null = null;
+        if (!channelId) {

-        if (policyId) {
-            const policyDb = await db.query.alertPolicy.findFirst({
-                where: eq(drizzleDb.schemas.alertPolicy.id, policyId),
-                with: {
-                    notificationChannel: true
-                },
-            });

-            if (!policyDb || !policyDb.notificationChannel) {
-                return {
-                    success: false,
-                    channelId: "",
-                    provider: null,
-                    error: "Policy or associated channel not found",
-                };
-            }

-            if (!policyDb.enabled || !policyDb.notificationChannel.enabled) {
-                return {
-                    success: false,
-                    channelId: policyDb.notificationChannel.id,
-                    provider: policyDb.notificationChannel.provider as any,
-                    error: "Policy or channel is disabled",
-                };
-            }
-
-            channel = {
-                ...policyDb.notificationChannel,
-                config: policyDb.notificationChannel.config as Json,
-            };
-        }
-
-        if (channelId) {
-            const fetchedChannel = await db.query.notificationChannel.findFirst({
-                where: eq(drizzleDb.schemas.notificationChannel.id, channelId),
-            });
-
-            if (!fetchedChannel) {
-                return {
-                    success: false,
-                    channelId: channelId,
-                    provider: null,
-                    error: "Channel not found",
-                };
-            }
-
-            channel = {
-                ...fetchedChannel,
-                config: fetchedChannel.config as Json,
-            };
-        }
-
-        if (!channel) {
            return {
                success: false,
-                channelId: channelId || "",
                provider: null,
-                error: "No valid channel to dispatch notification",
+                error: 'No storage channel provided',
            };
        }

+        const channel = await db.query.storageChannel.findFirst({
+            where: eq(drizzleDb.schemas.storageChannel.id, channelId),
+        });

-        if (!channel.enabled) {
+        if (!channel || !channel.enabled) {
            return {
                success: false,
-                channelId: channelId || "",
-                provider: null,
-                error: "Channel not active",
+                provider: channel?.provider as StorageProviderKind,
+                error: 'Storage channel not found or disabled',
            };
        }

-        const result = await dispatchViaProvider(
-            channel.provider,
+        return await dispatchViaProvider(
+            channel.provider as StorageProviderKind,
            channel.config,
-            {...payload, timestamp: payload.timestamp || new Date()},
-            channel.id
+            input
        );
-
-        const [log] = await db
-            .insert(notificationLog)
-            .values({
-                channelId: channel.id,
-                policyId: policyId || null,
-                organizationId: organizationId || null,
-
-                provider: channel.provider,
-                providerName: channel.name,
-                event: payload.event as EventKind,
-
-                title: payload.title,
-                message: payload.message,
-                level: payload.level,
-                payload: payload.data || null,
-                success: result.success,
-                error: result.success ? null : result.error,
-                providerResponse: result.response || null,
-            })
-            .returning({id: notificationLog.id});
-
-        return {...result, channelId: channel.id};
-
    } catch (err: any) {
        return {
            success: false,
-            channelId: channelId || "",
            provider: null,
-            error: err?.message || "Unexpected error during dispatch",
+            error: err.message || 'Unexpected storage dispatch error',
        };
    }
 }
@@ -0,0 +1,10 @@
+import {DatabaseWith} from "@/db/schema/07_database";
+import {StorageChannel} from "@/db/schema/12_storage-channel";
+import {sendNotificationsBackupRestore} from "@/features/notifications/helpers";
+
+
+export async function storeFileBackup(database: DatabaseWith, file: Buffer)  {
+
+
+
+}
@@ -0,0 +1,54 @@
+import type {
+    StorageProviderKind,
+    StorageInput,
+    StorageResult,
+} from '../types';
+
+import {uploadLocal, getLocal, deleteLocal} from './local';
+
+type ProviderHandler = {
+    upload: (config: any, input: StorageInput & { action: 'upload' }) => Promise<StorageResult>;
+    get: (config: any, input: StorageInput & { action: 'get' }) => Promise<StorageResult>;
+    delete: (config: any, input: StorageInput & { action: 'delete' }) => Promise<StorageResult>;
+};
+
+const handlers: Record<StorageProviderKind, ProviderHandler> = {
+    local: {
+        upload: uploadLocal,
+        get: getLocal,
+        delete: deleteLocal,
+    },
+    // s3: {
+    //     upload: uploadS3,
+    //     get: getS3,
+    //     delete: deleteS3,
+    // },
+    // gcs: null as any,
+    // azure: null as any,
+};
+
+export async function dispatchViaProvider(
+    kind: StorageProviderKind,
+    config: any,
+    input: StorageInput
+): Promise<StorageResult> {
+    const provider = handlers[kind];
+
+    if (!provider) {
+        return {
+            success: false,
+            provider: kind,
+            error: `Unsupported storage provider: ${kind}`,
+        };
+    }
+
+    try {
+        return await provider[input.action](config, input as any);
+    } catch (err: any) {
+        return {
+            success: false,
+            provider: kind,
+            error: err.message || 'Storage provider error',
+        };
+    }
+}
@@ -0,0 +1,69 @@
+"use server"
+import {mkdir, writeFile, unlink, readFile} from 'fs/promises';
+import path from 'path';
+import {StorageDeleteInput, StorageGetInput, StorageResult, StorageUploadInput} from '../types';
+import fs from "node:fs";
+import {getServerUrl} from "@/utils/get-server-url";
+
+const BASE_DIR = "/private/uploads/files/";
+
+export async function uploadLocal(
+    config: { baseDir?: string },
+    input: { data: StorageUploadInput }
+): Promise<StorageResult> {
+    const base = config.baseDir || BASE_DIR;
+    const fullPath = path.join(process.cwd(), base, input.data.path);
+
+    await mkdir(fullPath, {recursive: true});
+    await writeFile(fullPath, input.data.file);
+
+    return {
+        success: true,
+        provider: 'local',
+        url: path.join(fullPath),
+    };
+}
+
+export async function getLocal(
+    config: { baseDir?: string },
+    input: { data: StorageGetInput }
+): Promise<StorageResult> {
+    const base = config.baseDir || BASE_DIR;
+    const filePath = path.join(base, input.data.path)
+    const fileName = path.basename(input.data.path);
+    const file = await readFile(filePath);
+
+
+    if (!fs.existsSync(filePath)) {
+        console.error("File not found at:", filePath);
+        return({
+            success: false,
+            provider: 'local',
+        });
+    }
+    const crypto = require("crypto");
+    const baseUrl = getServerUrl();
+
+    const expiresAt = Date.now() + 60 * 1000;
+    const token = crypto.createHash("sha256").update(`${fileName}${expiresAt}`).digest("hex");
+
+    return {
+        success: true,
+        provider: 'local',
+        file: file,
+        url: `${baseUrl}/api/files/${fileName}?token=${token}&expires=${expiresAt}`,
+    };
+}
+
+export async function deleteLocal(
+    config: { baseDir?: string },
+    input: { data: StorageDeleteInput }
+): Promise<StorageResult> {
+    const base = config.baseDir || BASE_DIR;
+    const fullPath = path.join(process.cwd(), base, input.data.path);
+    await unlink(fullPath);
+    return {
+        success: true,
+        provider: 'local',
+    };
+}
@@ -0,0 +1,39 @@
+export type StorageProviderKind =
+    | 'local'
+// | 's3'
+    ;
+
+export type StorageAction =
+    | 'upload'
+    | 'get'
+    | 'delete';
+
+export interface StorageUploadInput {
+    path: string;
+    file: Buffer | Uint8Array;
+    contentType?: string;
+}
+
+export interface StorageGetInput {
+    path: string;
+    signedUrl?: boolean;
+    expiresInSeconds?: number;
+}
+
+export interface StorageDeleteInput {
+    path: string;
+}
+
+export type StorageInput =
+    | { action: 'upload'; data: StorageUploadInput }
+    | { action: 'get'; data: StorageGetInput }
+    | { action: 'delete'; data: StorageDeleteInput };
+
+export interface StorageResult {
+    success: boolean;
+    provider: StorageProviderKind | null;
+    url?: string;
+    file?: Buffer;
+    error?: string;
+    response?: any;
+}
@@ -4,6 +4,9 @@ import {eq} from "drizzle-orm";
 import * as drizzleDb from "@/db";
 import {retentionJob} from "@/lib/tasks";
 import {generateRSAKeys} from "@/utils/rsa-keys";
+import {Provider} from "react";
+import type {ProviderKind} from "@/features/notifications/types";
+import {StorageProviderKind} from "@/features/storages/types";


 export async function init() {
@@ -47,6 +50,25 @@ async function createSettingsIfNotExist() {
        console.log("====Init Setting : Update ====");
        await db.update(drizzleDb.schemas.setting).set(configSettings).where(eq(drizzleDb.schemas.setting.name, "system"));
    }
+
+    const [existingLocalChannelStorage] = await db.select().from(drizzleDb.schemas.storageChannel).where(eq(drizzleDb.schemas.storageChannel.provider, "local")).limit(1);
+
+    const localChannelValues = {
+        provider: "local" as StorageProviderKind,
+        enabled: true,
+        name: "System",
+        config: {}
+    }
+
+    if (!existingLocalChannelStorage) {
+        console.log("====Local Storage : Create ====");
+        await db.insert(drizzleDb.schemas.storageChannel).values(localChannelValues);
+    } else {
+        console.log("====Local Storage : Update ====");
+        await db.update(drizzleDb.schemas.storageChannel).set(localChannelValues).where(eq(drizzleDb.schemas.storageChannel.provider, "local"));
+    }
+
+
 }

 async function createDefaultOrganization() {