External/portabase
1
0
Fork 0
mirror of https://github.com/Portabase/portabase.git synced 2026-07-14 11:16:13 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'feature' into feat/agent-update

Browse Source
This commit is contained in:
charlesgauthereau
2026-02-11 12:21:22 +01:00
parent d24e534828 d4cde2ba9c
commit d9badeacba
51 changed files with 3041 additions and 2808 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/docker.yml
+76 -51
View File
@@ -4,10 +4,6 @@ on:
workflow_call:
inputs:
image_name:
required: false
type: string
default: 'solucetechnologies/portabase'
image_name2:
required: false
type: string
default: 'portabase/portabase'
@@ -28,70 +24,99 @@ on:
required: true
DOCKER_PASSWORD:
required: true
DOCKER_USERNAME_2:
required: true
DOCKER_PASSWORD_2:
required: true
jobs:
build-and-push:
runs-on: ubuntu-latest
permissions:
packages: write
contents: read
build:
name: Build and push Docker images
runs-on: ${{ matrix.platform == 'linux/amd64' && 'ubuntu-latest' || matrix.platform == 'linux/arm64' && 'ubuntu-24.04-arm' }}
strategy:
fail-fast: false
matrix:
platform: [ linux/amd64, linux/arm64 ]
steps:
- name: Check out the repo
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Set tags
id: set-tags
run: |
REF_NAME=${GITHUB_REF#refs/tags/}
TAGS="${{ inputs.image_name }}:$REF_NAME"
TAGS2="${{ inputs.image_name2 }}:$REF_NAME"
if [[ "${{ inputs.add_latest }}" == "true" ]]; then
TAGS="$TAGS,${{ inputs.image_name }}:latest"
TAGS2="$TAGS2,${{ inputs.image_name2 }}:latest"
fi
echo "tags=$TAGS" >> $GITHUB_OUTPUT
echo "tags2=$TAGS2" >> $GITHUB_OUTPUT
- name: Set up QEMU (enables multi-arch emulation)
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Log in to Docker Hub account 1
- name: Login to Docker
uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Build and push to solucetechnologies/portabase
- name: Set image tag
id: set-tags
run: |
REF_NAME=${GITHUB_REF#refs/tags/}
if [ "${{ matrix.platform }}" = "linux/amd64" ]; then
IMAGE="${{ inputs.image_name }}:$REF_NAME-amd64"
else
IMAGE="${{ inputs.image_name }}:$REF_NAME-arm64"
fi
echo "image=$IMAGE" >> $GITHUB_OUTPUT
- name: Build and push image
uses: docker/build-push-action@v6
with:
context: .
file: ${{ inputs.dockerfile }}
platforms: linux/amd64,linux/arm64
platforms: ${{ matrix.platform }}
push: true
tags: ${{ steps.set-tags.outputs.tags }}
tags: ${{ steps.set-tags.outputs.image }}
target: ${{ inputs.target }}
- name: Log in to Docker Hub account 2
- name: Prepare artifact name
id: artifact
run: |
platform=${{ matrix.platform }}
echo "safe_platform=${platform//\//-}" >> $GITHUB_OUTPUT
echo "${{ steps.set-tags.outputs.image }}" > image.txt
- uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
with:
name: image-${{ steps.artifact.outputs.safe_platform }}
path: image.txt
if-no-files-found: warn
compression-level: 6
overwrite: false
include-hidden-files: false
create-manifest:
name: Create multi-arch Docker manifest (Portabase)
runs-on: ubuntu-latest
needs: build
steps:
- uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131
with:
name: image-linux-amd64
path: /tmp/digests/amd64
- uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131
with:
name: image-linux-arm64
path: /tmp/digests/arm64
- name: Login to Docker
uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a
with:
username: ${{ secrets.DOCKER_USERNAME_2 }}
password: ${{ secrets.DOCKER_PASSWORD_2 }}
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Build and push to portabase/portabase
uses: docker/build-push-action@v6
with:
context: .
file: ${{ inputs.dockerfile }}
platforms: linux/amd64,linux/arm64
push: true
tags: ${{ steps.set-tags.outputs.tags2 }}
target: ${{ inputs.target }}
- name: Create and push manifest list
working-directory: /tmp/digests
run: |
DOCKER_IMAGES="$(cat amd64/image.txt) $(cat arm64/image.txt)"
REF_NAME=${GITHUB_REF#refs/tags/}
MANIFEST_IMAGE="${{ inputs.image_name }}:$REF_NAME"
docker buildx imagetools create $DOCKER_IMAGES -t $MANIFEST_IMAGE
docker buildx imagetools inspect $MANIFEST_IMAGE
if [ "${{ inputs.add_latest }}" = "true" ]; then
docker buildx imagetools create $DOCKER_IMAGES -t ${{ inputs.image_name }}:latest
docker buildx imagetools inspect ${{ inputs.image_name }}:latest
fi
.github/workflows/release-candidate.yml
+2 -4
View File
@@ -15,10 +15,8 @@ jobs:
with:
add_latest: false
secrets:
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
DOCKER_USERNAME_2: ${{ secrets.DOCKER_USERNAME_2 }}
DOCKER_PASSWORD_2: ${{ secrets.DOCKER_PASSWORD_2 }}
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME_2 }}
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD_2 }}
github_release:
needs: docker_publish
.github/workflows/release.yml
+2 -4
View File
@@ -16,10 +16,8 @@ jobs:
with:
add_latest: true
secrets:
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
DOCKER_USERNAME_2: ${{ secrets.DOCKER_USERNAME_2 }}
DOCKER_PASSWORD_2: ${{ secrets.DOCKER_PASSWORD_2 }}
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME_2 }}
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD_2 }}
github_release:
needs: docker_publish
CITATION.cff
+2 -2
View File
@@ -26,5 +26,5 @@ keywords:
- web-ui
- agent
license: Apache-2.0
version: 1.2.4
date-released: "2026-01-29"
version: 1.2.5-rc.14
date-released: "2026-02-10"
app/api/agent/[agentId]/backup/helpers.ts
+46 -47
View File
@@ -1,53 +1,52 @@
import fs from "node:fs";
import forge from "node-forge";
import { NextResponse } from "next/server";
import { eq } from "drizzle-orm";
import { db } from "@/db";
import * as drizzleDb from "@/db";
export function withAgentCheck(handler: Function) {
return async (request: Request, context: { params: Promise<{ agentId: string }> }) => {
try {
const agentId = (await context.params).agentId;
const agent = await db.query.agent.findFirst({
where: eq(drizzleDb.schemas.agent.id, agentId),
});
if (!agent) {
return NextResponse.json(
{ error: "Agent not found" },
{ status: 404 }
);
}
return handler(request, { ...context, agent });
} catch (err) {
console.error("Error in agent middleware:", err);
return NextResponse.json(
{ error: "Internal server error" },
{ status: 500 }
);
}
};
}
export async function decryptedDump(file: File, aesKeyHex: string, ivHex: string, fileExtension: string): Promise<File> {
const privateKeyPem = fs.readFileSync("private/keys/server_private.pem", "utf8");
const privateKey = forge.pki.privateKeyFromPem(privateKeyPem);
// Decrypt AES key with RSA-OAEP
const encryptedAesKey = forge.util.hexToBytes(aesKeyHex);
const aesKey = privateKey.decrypt(encryptedAesKey, "RSA-OAEP", {
md: forge.md.sha256.create(),
mgf1: {md: forge.md.sha256.create()},
export async function getDatabaseOrThrow(generatedId: string) {
const database = await db.query.database.findFirst({
where: eq(drizzleDb.schemas.database.agentDatabaseId, generatedId),
with: {
project: true,
alertPolicies: true,
storagePolicies: true
}
});
// Read encrypted file content
const encryptedBuffer = Buffer.from(await file.arrayBuffer());
const iv = forge.util.hexToBytes(ivHex);
// AES decryption
const decipher = forge.cipher.createDecipher("AES-CBC", aesKey);
decipher.start({iv});
decipher.update(forge.util.createBuffer(encryptedBuffer.toString("binary")));
const success = decipher.finish();
if (!success) {
throw new Error("Decryption failed");
if (!database) {
throw NextResponse.json(
{ error: "Database associated with generatedId not found" },
{ status: 404 }
);
}
const decryptedBytes = decipher.output.getBytes();
const decryptedBuffer = Buffer.from(decryptedBytes, "binary");
// Return a File so you can use file.arrayBuffer() later
return new File(
[decryptedBuffer],
file.name.replace(/\.enc$/, fileExtension),
{type: "application/octet-stream"}
);
}
export function getFileExtension(dbType: string) {
switch (dbType) {
case "postgresql":
return ".dump";
case "mysql":
return ".sql";
default:
return ".dump";
}
}
return database;
}
app/api/agent/[agentId]/backup/route.ts
+112 -130
View File
@@ -1,89 +1,63 @@
import {NextResponse} from "next/server";
import {isUuidv4} from "@/utils/verify-uuid";
import {v4 as uuidv4} from "uuid";
import * as drizzleDb from "@/db";
import {db} from "@/db";
import {Backup} from "@/db/schema/07_database";
import {and, eq} from "drizzle-orm";
import * as drizzleDb from "@/db";
import {db as dbClient, db} from "@/db";
import {getDatabaseOrThrow, withAgentCheck} from "./helpers";
import {Backup} from "@/db/schema/07_database";
import {withUpdatedAt} from "@/db/utils";
import {decryptedDump, getFileExtension} from "./helpers";
import {eventEmitter} from "@/features/shared/event";
import {sendNotificationsBackupRestore} from "@/features/notifications/helpers";
import {storeBackupFiles} from "@/features/storages/helpers";
import {EventKind} from "@/features/notifications/types";
export async function POST(
request: Request,
{params}: { params: Promise<{ agentId: string }> }
) {
export type BodyPost = {
method: "manual" | "automatic"
generatedId: string
}
export type BodyPatch = {
backupId: string
status: "success" | "failed"
size: number
generatedId: string
}
export const POST = withAgentCheck(async (request: Request, {params, agent}: {
params: Promise<{ agentId: string }>,
agent: any
}) => {
try {
const contentType = request.headers.get("Content-Type");
if (!contentType || !contentType.includes("multipart/form-data")) {
return NextResponse.json(
{error: "Unsupported or missing Content-Type"},
{status: 400}
);
}
const agentId = (await params).agentId;
const formData = await request.formData();
const aesKeyHex = formData.get("aes_key") as string;
const ivHex = formData.get("iv") as string;
const generatedId = formData.get("generatedId") as string | null;
const method = formData.get("method") as string | null;
if (!generatedId || !isUuidv4(generatedId)) {
return NextResponse.json(
{error: "generatedId is not a valid UUID"},
{status: 400}
);
}
const agent = await db.query.agent.findFirst({
where: eq(drizzleDb.schemas.agent.id, agentId),
});
if (!agent) {
return NextResponse.json(
{error: "Agent not found"},
{status: 404}
);
}
const database = await db.query.database.findFirst({
where: eq(drizzleDb.schemas.database.agentDatabaseId, generatedId),
with: {
project: true,
alertPolicies: true,
storagePolicies: true
}
});
if (!database) {
return NextResponse.json(
{error: "Database associated with generatedId not found"},
{status: 404}
);
}
const body: BodyPost = await request.json();
const method = body.method
const database = await getDatabaseOrThrow(body.generatedId);
let backup: Backup | null | undefined = null;
if (method === "automatic") {
[backup] = await db
.insert(drizzleDb.schemas.backup)
.values({
status: 'ongoing',
databaseId: database.id,
})
.returning();
const ongoingBackup = await db.query.backup.findFirst({
where: and(
eq(drizzleDb.schemas.backup.status, 'ongoing'),
eq(drizzleDb.schemas.backup.databaseId, database.id),
),
});
if (!backup) {
if (!ongoingBackup) {
[backup] = await db
.insert(drizzleDb.schemas.backup)
.values({
status: 'ongoing',
databaseId: database.id,
})
.returning();
if (!backup) {
return NextResponse.json(
{error: "Unable to create an automatic backup"},
{status: 500}
);
}
} else {
return NextResponse.json(
{error: "Unable to create an automatic backup"},
{error: "A backup is already ongoing"},
{status: 500}
);
}
@@ -95,7 +69,6 @@ export async function POST(
),
});
if (!backup) {
return NextResponse.json(
{error: "Unable to find the corresponding backup"},
@@ -104,68 +77,77 @@ export async function POST(
}
}
const status = formData.get("status") as string | null;
if (status === "success") {
const file = formData.get("file") as File | null;
const extension = formData.get("extension") as string | null;
if (!aesKeyHex || !ivHex) {
return NextResponse.json({error: "Missing fields"}, {status: 400});
}
eventEmitter.emit('modification', {update: true});
if (!file) {
return NextResponse.json(
{error: "File is required for successful backup"},
{status: 400}
);
}
const fileSizeBytes = file.size;
// const fileExtension = '.' + (file.name.split('.').pop()?.toLowerCase() || '');
const fileExtension = extension ? extension : getFileExtension(database.dbms)
const decryptedFile = await decryptedDump(file, aesKeyHex, ivHex, fileExtension);
const uuid = uuidv4();
const fileName = `${uuid}${fileExtension}`;
const buffer = Buffer.from(await decryptedFile.arrayBuffer());
const storageResults = await storeBackupFiles(backup, database, buffer, fileName)
await sendNotificationsBackupRestore(database, "success_backup");
return NextResponse.json(
{
message: "Backup successfully uploaded",
},
{status: 200}
);
} else {
await db
.update(drizzleDb.schemas.backup)
.set(withUpdatedAt({status: 'failed'}))
.where(eq(drizzleDb.schemas.backup.id, backup.id));
await sendNotificationsBackupRestore(database, "error_backup");
return NextResponse.json(
{
message: "Backup successfully updated with status failed",
},
{status: 200}
);
}
return NextResponse.json(
{
message: "Init backup success",
backup: backup,
},
{status: 200}
);
} catch (error) {
console.error("Error in POST handler:", error);
console.error("Error in POST for INIT backup:", error);
return NextResponse.json(
{error: "Internal server error"},
{status: 500}
);
}
}
});
export const PATCH = withAgentCheck(async (request: Request, {params, agent}: {
params: Promise<{ agentId: string }>,
agent: any
}) => {
try {
const body: BodyPatch = await request.json();
const status = body.status
const backupId = body.backupId
const backupSize = body.size
const database = await getDatabaseOrThrow(body.generatedId);
const backup = await db.query.backup.findFirst({
where: eq(drizzleDb.schemas.backup.id, backupId),
});
if (!backup) {
return NextResponse.json(
{error: "No backup found"},
{status: 500}
);
}
const [backupUpdated] = await dbClient
.update(drizzleDb.schemas.backup)
.set(withUpdatedAt({
status: status,
fileSize: backupSize
}))
.where(eq(drizzleDb.schemas.backup.id, backup.id))
.returning();
eventEmitter.emit('modification', {update: true});
await sendNotificationsBackupRestore(database, `${status}_backup` as EventKind);
return NextResponse.json(
{
message: "Backup successfully updated",
backup: backupUpdated,
},
{status: 200}
);
} catch (error) {
console.error("Error in PATCH backup:", error);
return NextResponse.json(
{error: "Internal server error"},
{status: 500}
);
}
});
app/api/agent/[agentId]/backup/upload/init/route.ts
+76
View File
@@ -0,0 +1,76 @@
import {NextResponse} from "next/server";
import {and, eq} from "drizzle-orm";
import * as drizzleDb from "@/db";
import {db} from "@/db";
import {getDatabaseOrThrow, withAgentCheck} from "../../helpers";
import {isUuidv4} from "@/utils/verify-uuid";
import {eventEmitter} from "@/features/shared/event";
export type Body = {
generatedId: string
storageChannelId: string
backupId: string
}
export const POST = withAgentCheck(async (request: Request, {params, agent}: {
params: Promise<{ agentId: string }>,
agent: any
}) => {
try {
const body: Body = await request.json();
console.log("body", body);
const generatedId = body.generatedId;
const storageChannelId = body.storageChannelId;
const backupId = body.backupId;
if (!generatedId || !isUuidv4(generatedId)) {
return NextResponse.json(
{error: "generatedId is not a valid UUID"},
{status: 400}
);
}
const database = await getDatabaseOrThrow(generatedId);
const backup = await db.query.backup.findFirst({
where: and(
eq(drizzleDb.schemas.backup.id, backupId),
eq(drizzleDb.schemas.backup.databaseId, database.id),
),
});
if (!backup) {
return NextResponse.json(
{error: "Unable to find the corresponding backup"},
{status: 404}
);
}
const [backupStorage] = await db
.insert(drizzleDb.schemas.backupStorage)
.values({
backupId: backup.id,
storageChannelId: storageChannelId,
status: "pending",
})
.returning();
eventEmitter.emit('modification', {update: true});
return NextResponse.json(
{
message: "Backup storage successfully created",
backupStorage: backupStorage
},
{status: 200}
);
} catch (error) {
console.error("Error in POST for INIT backup:", error);
return NextResponse.json(
{error: "Internal server error"},
{status: 500}
);
}
});
app/api/agent/[agentId]/backup/upload/status/route.ts
+95
View File
@@ -0,0 +1,95 @@
import {NextResponse} from "next/server";
import {and, eq} from "drizzle-orm";
import * as drizzleDb from "@/db";
import {db as dbClient, db} from "@/db";
import {withUpdatedAt} from "@/db/utils";
import {getDatabaseOrThrow, withAgentCheck} from "../../helpers";
import {eventEmitter} from "@/features/shared/event";
export type Body = {
generatedId: string
status: "success" | "failed"
backupStorageId: string
path: string
size: number
backupId: string
}
export const PATCH = withAgentCheck(async (request: Request, {params, agent}: {
params: Promise<{ agentId: string }>,
agent: any
}) => {
try {
const body: Body = await request.json();
const generatedId = body.generatedId;
const status = body.status;
const filePath = body.path;
const fileSize = body.size;
const backupStorageId = body.backupStorageId;
const backupId = body.backupId;
console.log("body", body);
const database = await getDatabaseOrThrow(generatedId);
const backup = await db.query.backup.findFirst({
where: and(
eq(drizzleDb.schemas.backup.id, backupId),
eq(drizzleDb.schemas.backup.databaseId, database.id),
),
with: {
storages: true
}
});
if (!backup) {
return NextResponse.json(
{error: "Unable to find the corresponding backup"},
{status: 404}
);
}
const [backupStorage] = await dbClient
.update(drizzleDb.schemas.backupStorage)
.set(withUpdatedAt({
status: status,
path: filePath,
size: fileSize
}))
.where(eq(drizzleDb.schemas.backupStorage.id, backupStorageId))
.returning();
if (backup.storages.length > 0) {
const hasSuccessfulStorage = backup.storages.some(
(storage) => storage.status === "success"
);
if (hasSuccessfulStorage && backup.status !== "success") {
await db
.update(drizzleDb.schemas.backup)
.set({
status: "success",
fileSize: fileSize,
})
.where(eq(drizzleDb.schemas.backup.id, backup.id));
}
}
eventEmitter.emit('modification', {update: true});
return NextResponse.json({
message: "Backup status successfully updated",
backupStorage: backupStorage
},
{status: 200}
);
} catch (error) {
console.error("Error in POST for INIT backup:", error);
return NextResponse.json(
{error: "Internal server error"},
{status: 500}
);
}
});
app/api/agent/[agentId]/restore/route.ts
+1
View File
@@ -4,6 +4,7 @@ import * as drizzleDb from "@/db";
import {db} from "@/db";
import {and, eq} from "drizzle-orm";
import {sendNotificationsBackupRestore} from "@/features/notifications/helpers";
import {eventEmitter} from "@/features/shared/event";
export type BodyResultRestore = {
generatedId: string
app/api/agent/[agentId]/status/helpers.ts
+96 -52
View File
@@ -2,9 +2,9 @@ import {NextResponse} from "next/server";
import {Body} from "./route";
import {isUuidv4} from "@/utils/verify-uuid";
import {Agent} from "@/db/schema/08_agent";
import {Database} from "@/db/schema/07_database";
import {Database, DatabaseWith} from "@/db/schema/07_database";
import * as drizzleDb from "@/db";
import {db as dbClient} from "@/db";
import {db, db as dbClient} from "@/db";
import {and, eq, inArray} from "drizzle-orm";
import {dbmsEnumSchema, EDbmsSchema} from "@/db/schema/types";
import {withUpdatedAt} from "@/db/utils";
@@ -14,9 +14,11 @@ import {dispatchStorage} from "@/features/storages/dispatch";
export async function handleDatabases(body: Body, agent: Agent, lastContact: Date) {
const databasesResponse = [];
const formatDatabase = (database: Database, backupAction: boolean, restoreAction: boolean, UrlBackup: string) => ({
const formatDatabase = (database: DatabaseWith, backupAction: boolean, restoreAction: boolean, UrlBackup: string | null, storages: PingDatabaseStorageChannels[], urlMeta: string | null) => ({
generatedId: database.agentDatabaseId,
dbms: database.dbms,
storages: storages,
encrypt: false,
data: {
backup: {
action: backupAction,
@@ -25,6 +27,7 @@ export async function handleDatabases(body: Body, agent: Agent, lastContact: Dat
restore: {
action: restoreAction,
file: UrlBackup,
metaFile: urlMeta
},
},
});
@@ -32,12 +35,16 @@ export async function handleDatabases(body: Body, agent: Agent, lastContact: Dat
for (const db of body.databases) {
const existingDatabase = await dbClient.query.database.findFirst({
where: eq(drizzleDb.schemas.database.agentDatabaseId, db.generatedId)
where: eq(drizzleDb.schemas.database.agentDatabaseId, db.generatedId),
with: {
project: true
}
});
let backupAction: boolean = false
let restoreAction: boolean = false
let urlBackup: string = ""
let urlBackup: string | null = null;
let urlMeta: string | null = null
if (!existingDatabase) {
if (!isUuidv4(db.generatedId)) {
@@ -63,8 +70,11 @@ export async function handleDatabases(body: Body, agent: Agent, lastContact: Dat
})
.returning();
if (databaseCreated) {
databasesResponse.push(formatDatabase(databaseCreated, backupAction, restoreAction, urlBackup));
const storages = await getDatabaseStorageChannels(databaseCreated.id)
databasesResponse.push(formatDatabase(databaseCreated, backupAction, restoreAction, urlBackup, storages, null));
}
} else {
@@ -78,7 +88,6 @@ export async function handleDatabases(body: Body, agent: Agent, lastContact: Dat
.where(eq(drizzleDb.schemas.database.id, existingDatabase.id))
.returning();
const activeBackup = await dbClient.query.backup.findFirst({
where: and(
eq(drizzleDb.schemas.backup.databaseId, databaseUpdated.id),
@@ -86,7 +95,6 @@ export async function handleDatabases(body: Body, agent: Agent, lastContact: Dat
)
})
const restoration = await dbClient.query.restoration.findFirst({
where: and(eq(drizzleDb.schemas.restoration.databaseId, databaseUpdated.id), eq(drizzleDb.schemas.restoration.status, "waiting")),
with: {
@@ -94,7 +102,6 @@ export async function handleDatabases(body: Body, agent: Agent, lastContact: Dat
}
})
if (activeBackup && activeBackup.status == "waiting") {
backupAction = true
@@ -107,13 +114,11 @@ export async function handleDatabases(body: Body, agent: Agent, lastContact: Dat
if (restoration) {
restoreAction = true
if (!restoration.backupStorage || restoration.backupStorage.status != "success" || !restoration.backupStorage.path) {
restoreAction = false
continue;
}
const input: StorageInput = {
action: "get",
data: {
@@ -126,12 +131,26 @@ export async function handleDatabases(body: Body, agent: Agent, lastContact: Dat
}
};
const inputMeta: StorageInput = {
action: "get",
data: {
path: `${restoration.backupStorage.path}.meta`,
signedUrl: true,
},
metadata: {
storageId: restoration.backupStorage.storageChannelId,
fileKind: "backups"
}
};
try {
const result = await dispatchStorage(input, undefined, restoration.backupStorage.storageChannelId);
const resultMeta = await dispatchStorage(inputMeta, undefined, restoration.backupStorage.storageChannelId);
if (result.success) {
urlBackup = result.url ?? "";
urlBackup = result.url ?? null;
urlMeta = resultMeta.url ?? null
} else {
await dbClient
.update(drizzleDb.schemas.restoration)
@@ -151,53 +170,78 @@ export async function handleDatabases(body: Body, agent: Agent, lastContact: Dat
continue;
}
// const fileName = backupToRestore?.file
//
// let data: SafeActionResult<string, ZodString, readonly [], {
// _errors?: string[] | undefined;
// }, readonly [], ServerActionResult<string>, object> | undefined
//
// try {
//
// if (settings.storage == "local") {
// data = await getFileUrlPresignedLocal({fileName: fileName!})
// } else if (settings.storage == "s3") {
//
// data = await getFileUrlPreSignedS3Action(`backups/${backupToRestore?.database.project?.slug}/${fileName}`);
// }
//
// if (data?.data?.success) {
// urlBackup = data.data.value ?? "";
// } else {
// await dbClient
// .update(drizzleDb.schemas.restoration)
// .set({status: "failed"})
// .where(eq(drizzleDb.schemas.restoration.id, restoration.id));
//
// // @ts-ignore
// const errorMessage = data?.data?.actionError?.message || "Failed to get presigned URL";
// console.error("Restoration failed: ", errorMessage);
//
// continue;
// }
// } catch (err) {
// console.error("Restoration crashed unexpectedly:", err);
// await dbClient
// .update(drizzleDb.schemas.restoration)
// .set({status: "failed"})
// .where(eq(drizzleDb.schemas.restoration.id, restoration.id));
//
// continue;
// }
await dbClient
.update(drizzleDb.schemas.restoration)
.set({status: "ongoing"})
.where(eq(drizzleDb.schemas.restoration.id, restoration.id));
}
databasesResponse.push(formatDatabase(databaseUpdated, backupAction, restoreAction, urlBackup));
const storages = await getDatabaseStorageChannels(databaseUpdated.id)
databasesResponse.push(formatDatabase(databaseUpdated, backupAction, restoreAction, urlBackup, storages, urlMeta));
}
}
return databasesResponse;
}
type PingDatabaseStorageChannels = {
id: string;
config: any
provider: string
}
async function getDatabaseStorageChannels(databaseId: string): Promise<PingDatabaseStorageChannels[]> {
const database = await db.query.database.findFirst({
where: eq(drizzleDb.schemas.database.id, databaseId),
with: {
project: true,
retentionPolicy: true,
alertPolicies: true,
storagePolicies: true
}
});
if (!database) {
return []
}
const settings = await db.query.setting.findFirst({
where: eq(drizzleDb.schemas.setting.name, "system"),
with: {storageChannel: true},
});
const defaultStorageChannel: PingDatabaseStorageChannels[] = settings?.storageChannel
? [{
id: settings.storageChannel.id,
provider: settings.storageChannel.provider,
config: settings.storageChannel.config,
}]
: [];
const enabledDatabaseStorageChannels = await Promise.all(
(database.storagePolicies ?? [])
.filter(p => p.enabled)
.map(async policy => {
const storageChannel = await db.query.storageChannel.findFirst({
where: eq(drizzleDb.schemas.storageChannel.id, policy.storageChannelId),
});
if (!storageChannel) return null;
return {
id: storageChannel.id,
config: storageChannel.config,
provider: storageChannel.provider,
} as PingDatabaseStorageChannels;
})
);
const filteredChannels: PingDatabaseStorageChannels[] = enabledDatabaseStorageChannels.filter(
(c): c is PingDatabaseStorageChannels => c !== null
);
return filteredChannels.length > 0 ? filteredChannels : defaultStorageChannel;
}
app/api/agent/[agentId]/status/route.ts
+4
View File
@@ -6,6 +6,7 @@ import {EDbmsSchema} from "@/db/schema/types";
import {eq} from "drizzle-orm";
import {isUuidv4} from "@/utils/verify-uuid";
import {withUpdatedAt} from "@/db/utils";
import {eventEmitter} from "@/features/shared/event";
export type databaseAgent = {
name: string,
@@ -25,12 +26,14 @@ export async function POST(
) {
try {
const agentId = (await params).agentId
console.log(agentId)
const body: Body = await request.json();
const lastContact = new Date();
let message: string
if (!isUuidv4(agentId)) {
message = "agentId is not a valid uuid"
console.error(message)
return NextResponse.json(
{error: "agentId is not a valid uuid"},
{status: 500}
@@ -64,6 +67,7 @@ export async function POST(
databases: databasesResponse
}
return Response.json(response)
} catch (error) {
console.error('Error in POST handler:', error);
app/api/events/route.ts
+44
View File
@@ -0,0 +1,44 @@
import {auth} from "@/lib/auth/auth";
import {headers} from "next/headers";
import {NextResponse} from "next/server";
import {eventEmitter} from "@/features/shared/event";
export async function GET(request: Request) {
const session = await auth.api.getSession({
headers: await headers(),
});
if (!session) {
return NextResponse.json({error: "Unauthorized"}, {status: 403});
}
return new Response(
new ReadableStream({
start(controller) {
console.log('Stream started');
const handleModification = (data: any) => {
console.log('Modification event triggered:', data);
controller.enqueue(`event: modification\n`);
controller.enqueue(`data: ${JSON.stringify(data)}\n\n`);
};
eventEmitter.on('modification', handleModification);
request.signal.addEventListener('abort', () => {
console.log('Client disconnected');
controller.close();
eventEmitter.off('modification', handleModification);
});
},
}),
{
status: 200,
headers: {
'Content-Type': 'text/event-stream',
'Cache-Control': 'no-cache',
Connection: 'keep-alive',
},
}
);
}
app/api/files/backups/route.ts
+3 -4
View File
@@ -35,10 +35,9 @@ export async function GET(
const result = await dispatchStorage(input, undefined, storageId);
if (!result.success) {
return NextResponse.json({error: "Enable to get file from privided storage channel, an error occurred !"})
return NextResponse.json({error: "Enable to get file from provided storage channel, an error occurred !"})
}
const fileName = path.basename(pathFromUrl);
const crypto = require('crypto');
@@ -58,14 +57,14 @@ export async function GET(
);
}
if (!result.file || !Buffer.isBuffer(result.file)) {
if (!result.file || !(result.file instanceof Readable)) {
return NextResponse.json(
{error: "Invalid file payload"},
{status: 500}
);
}
const fileStream = Readable.from(result.file);
const fileStream = Readable.from(result.file as Readable);
const stream = new ReadableStream({
start(controller) {
app/api/files/images/[fileName]/route.ts
+2 -16
View File
@@ -26,20 +26,6 @@ export async function GET(
return NextResponse.json({error: "Missing storageId in search params"}, {status: 404})
}
// const settings = await db.query.setting.findFirst({
// where: eq(drizzleDb.schemas.setting.name, "system"),
// with: {
// storageChannel: true
// }
// });
//
// if (!settings || !settings.storageChannel) {
// return NextResponse.json({error: "Unable to get settings or no default storage channel"});
// }
const ext = fileName.split(".").pop()?.toLowerCase();
const contentType =
ext === "png"
@@ -69,7 +55,7 @@ export async function GET(
const result = await dispatchStorage(input, undefined, storageId);
if (!result.file || !Buffer.isBuffer(result.file)) {
if (!result.file || !(result.file instanceof Readable)) {
console.error(`An error occurred while getting file :`, result);
return NextResponse.json(
{error: "Invalid file payload"},
@@ -77,7 +63,7 @@ export async function GET(
);
}
const fileStream = Readable.from(result.file);
const fileStream = Readable.from(result.file as Readable);
const stream = new ReadableStream({
start(controller) {
app/api/tus/hooks/route.ts
+78
View File
@@ -0,0 +1,78 @@
import {NextResponse} from "next/server";
import fs from "fs";
import path from "path";
export async function POST(request: Request) {
try {
const body = await request.json();
const event = body.Event
const headers = event.HTTPRequest.Header
const uploadLength = headers["X-File-Size"]?.[0];
const uploadOffset = headers["Upload-Offset"]?.[0];
const status = headers["X-Status"]?.[0];
console.log(`Upload ID : ${event.Upload.ID} (${uploadOffset}/${uploadLength})`);
if (status === "success") {
if (
body.Type === "post-receive" &&
event.Upload.SizeIsDeferred === false &&
event.Upload.Offset === event.Upload.Size
) {
const id = event.Upload.ID;
const fileName = headers["X-File-Name"]?.[0];
const filePath = headers["X-File-Path"]?.[0];
if (!filePath) {
return NextResponse.json({error: "Missing X-File-Path"}, {status: 500});
}
const uploadDir = path.join(process.cwd(), "/private/uploads/");
const oldFilePath = path.join(uploadDir, "tmp", id);
const newFilePath = path.join(uploadDir, filePath);
fs.mkdirSync(path.dirname(newFilePath), {recursive: true});
let retries = 10;
while (!fs.existsSync(oldFilePath)) {
if (retries-- === 0) {
return NextResponse.json({error: `Upload file not found: ${oldFilePath}`}, {status: 500});
}
await new Promise(r => setTimeout(r, 200));
}
fs.renameSync(oldFilePath, newFilePath);
const infoFilePath = `${oldFilePath}.info`;
if (fs.existsSync(infoFilePath)) {
fs.unlinkSync(infoFilePath);
}
const metadataHeaderB64 = headers["Upload-Metadata"]?.[0];
if (metadataHeaderB64) {
const metadataHeader = Buffer.from(metadataHeaderB64, "base64").toString("utf-8");
if (metadataHeader) {
const tomlContent = metadataHeader
.split(",")
.map((pair) => {
const [key, value] = pair.split(" ");
const escapedValue = value.replace(/"/g, '\\"');
return `${key} = "${escapedValue}"`;
})
.join("\n");
const metaFilePath = `${newFilePath}.meta`;
fs.writeFileSync(metaFilePath, tomlContent, "utf-8");
}
}
}
}
return NextResponse.json({});
} catch (error) {
console.error("Hook error:", error);
return NextResponse.json({error: "Internal server error"}, {status: 500});
}
}
docker-compose.prod.yml
+5 -5
View File
@@ -3,11 +3,11 @@ name: portabase-prod
services:
app:
# build:
# context: .
# dockerfile: docker/dockerfile/Dockerfile
# target: prod
image: solucetechnologies/portabase:latest
build:
context: .
dockerfile: docker/dockerfile/Dockerfile
target: prod
image: portabase/portabase:1.2.5-rc.9
ports:
- '8887:80'
environment:
docker-compose.yml
+14
View File
@@ -17,5 +17,19 @@ services:
timeout: 5s
retries: 5
tusd:
image: tusproject/tusd:v2.8.0
ports:
- "1080:8080"
command: >
-upload-dir /data/uploads/tmp
-hooks-http http://localhost:8887/api/tus/hooks
-max-size 21474836480
-base-path /tus/files/
extra_hosts:
- "localhost:host-gateway"
volumes:
- ./private/uploads/tmp:/data/uploads/tmp
volumes:
postgres-data:
docker/dockerfile/Dockerfile
+56 -54
View File
@@ -1,53 +1,65 @@
FROM node:22-alpine AS base
#FROM node:22-bullseye AS base
FROM --platform=$BUILDPLATFORM node:22-bullseye AS base
RUN apt-get update && apt-get install -y \
curl \
ca-certificates \
bash \
tzdata \
libc6 \
build-essential \
g++ \
make \
autoconf \
automake \
libtool \
git \
nginx \
&& rm -rf /var/lib/apt/lists/*
FROM --platform=$BUILDPLATFORM golang:1.23-bookworm AS tusd-base
ARG TUSD_VERSION=2.8.0
RUN apt-get update && apt-get install -y --no-install-recommends git ca-certificates \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /build
RUN git clone https://github.com/tus/tusd.git . \
&& git checkout v${TUSD_VERSION}
RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} \
go build -ldflags="-s -w" -o /tusddist/tusd ./cmd/tusd
FROM scratch AS tusd-dist
COPY --from=tusd-base /tusddist/tusd /tusd
RUN apk add --update --no-cache \
libc6-compat \
openssl \
tzdata
FROM base AS build-env
RUN apk add --update --no-cache \
build-base \
g++ \
make \
libtool \
autoconf \
automake
RUN corepack enable && corepack prepare pnpm@latest --activate
FROM build-env AS deps
WORKDIR /app
COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./
RUN pnpm i --frozen-lockfile
FROM build-env AS dev
WORKDIR /app
COPY --from=deps /app/node_modules ./node_modules
COPY --from=tusd-base /usr/local/bin/tusd /usr/local/bin/tusd
WORKDIR /app
COPY . .
USER root
RUN chmod +x /app/docker/entrypoints/app-dev-entrypoint.sh
ENTRYPOINT ["sh","/app/docker/entrypoints/app-dev-entrypoint.sh"]
FROM build-env AS builder
WORKDIR /app
@@ -55,51 +67,41 @@ COPY --from=deps /app/node_modules ./node_modules
COPY . .
ENV NEXT_TELEMETRY_DISABLED=1
RUN pnpm run build
FROM base AS prod
WORKDIR /app
ENV NODE_ENV=production
ENV NEXT_TELEMETRY_DISABLED=1
ENV PORT=80
ENV HOSTNAME="0.0.0.0"
RUN addgroup --system --gid 1001 nodejs
RUN adduser --system --uid 1001 nextjs
COPY --from=builder /app/public ./public
RUN mkdir .next
RUN chown nextjs:nodejs .next
RUN mkdir -p /app/private/uploads
RUN chown -R nextjs:nodejs /app/private
RUN chown -R nextjs:nodejs /app/public
COPY --from=builder /app/next.config.ts ./
COPY --from=builder /app/portabase.config.ts ./
COPY --from=builder /app/drizzle.config.ts ./
COPY --from=builder --chown=nextjs:nodejs /app/.next/standalone ./
COPY --from=builder --chown=nextjs:nodejs /app/.next/static ./.next/static
COPY --chown=nextjs:nodejs src/db ./src/db
COPY --from=builder --chown=1001:1001 /app/.next/standalone ./
COPY --from=builder --chown=1001:1001 /app/.next/static ./.next/static
COPY --chown=1001:1001 src/db ./src/db
COPY --from=deps /app/node_modules ./node_modules
COPY --from=tusd-dist /tusd /usr/local/bin/tusd
RUN chmod +x /usr/local/bin/tusd
COPY docker/nginx/nginx.conf /etc/nginx/nginx.conf
RUN mkdir -p .next /app/private/uploads \
&& chown -R nextjs:nodejs .next /app/private /app/public
USER root
COPY ./docker/entrypoints/app-prod-entrypoint.sh /app/app-prod-entrypoint.sh
RUN chmod +x /app/app-prod-entrypoint.sh
EXPOSE 80
ENV PORT=80
ENV HOSTNAME="0.0.0.0"
USER nextjs
#USER nextjs
ENTRYPOINT ["sh","/app/app-prod-entrypoint.sh"]
docker/entrypoints/app-prod-entrypoint.sh
+8 -2
View File
@@ -7,8 +7,14 @@ else
echo "[WARN] No TZ provided, using default container timezone"
fi
mkdir -p /app/private/uploads/tmp
echo "▶ Starting tusd server..."
tusd --base-path /tus/files/ --upload-dir /app/private/uploads/tmp --hooks-http http://127.0.0.1:3000/api/tus/hooks --port 1080 --max-size 21474836480 &
node server.js
echo "▶ Starting Next.js server..."
PORT=3000 node server.js &
echo "▶ Starting nginx..."
exec nginx -g "daemon off;"
exec "$@"
docker/nginx/nginx.conf
+41
View File
@@ -0,0 +1,41 @@
events {}
http {
client_max_body_size 20G;
ignore_invalid_headers off;
server {
listen 80;
location /tus/ {
proxy_pass http://127.0.0.1:1080/tus/;
proxy_pass_request_headers on;
proxy_request_buffering off;
proxy_buffering off;
proxy_http_version 1.1;
proxy_set_header Host $http_host;
proxy_set_header X-Forwarded-Host $http_host;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
}
location / {
proxy_pass http://127.0.0.1:3000;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
proxy_set_header Host $http_host;
proxy_set_header X-Forwarded-Host $http_host;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
}
}
next.config.ts
+13 -1
View File
@@ -1,6 +1,8 @@
import type {NextConfig} from "next";
import {PORTABASE_DEFAULT_SETTINGS} from "./portabase.config";
const isDev = process.env.NODE_ENV === "development";
function buildCSPHeader(): string {
const {CSP} = PORTABASE_DEFAULT_SETTINGS.SECURITY;
@@ -46,8 +48,18 @@ const nextConfig: NextConfig = {
bodySizeLimit: "10gb",
},
proxyClientMaxBodySize: '10gb',
},
async rewrites() {
if (!isDev) return [];
return [
{
source: "/tus/:path*",
destination: "http://localhost:1080/tus/:path*",
},
];
},
async headers() {
return [
{
package.json
+7 -4
View File
@@ -1,6 +1,6 @@
{
"name": "portabase",
"version": "1.2.4",
"version": "1.2.5-rc.14",
"private": true,
"scripts": {
"dev": "next dev --turbopack -p 8887",
@@ -47,6 +47,7 @@
"@t3-oss/env-nextjs": "^0.13.4",
"@tanstack/react-query": "^5.76.1",
"@tanstack/react-table": "^8.21.3",
"@types/express": "^5.0.6",
"@types/nodemailer": "^6.4.17",
"@types/ws": "^8.18.1",
"@zenstackhq/runtime": "2.14.2",
@@ -62,12 +63,13 @@
"drizzle-orm": "^0.43.1",
"drizzle-zod": "^0.7.1",
"embla-carousel-react": "^8.6.0",
"express": "^5.2.1",
"googleapis": "^170.1.0",
"input-otp": "^1.4.2",
"lucide-react": "^0.553.0",
"minio": "^8.0.5",
"motion": "^12.23.24",
"next": "16.0.10",
"next": "16.1.5",
"next-safe-action": "^7.10.8",
"next-themes": "^0.4.6",
"node-cron": "^4.2.1",
@@ -111,8 +113,9 @@
"@types/react-dom": "^19.1.5",
"@zenstackhq/openapi": "^2.14.2",
"@zenstackhq/tanstack-query": "^2.14.2",
"baseline-browser-mapping": "^2.8.32",
"baseline-browser-mapping": "^2.9.19",
"drizzle-kit": "^0.31.1",
"esbuild": "^0.27.2",
"eslint": "^9.39.0",
"eslint-config-next": "^16.0.1",
"eslint-plugin-tailwindcss": "^3.18.0",
@@ -124,5 +127,5 @@
"typescript": "^5.8.3",
"zenstack": "2.14.2"
},
"packageManager": "pnpm@10.28.1+sha512.7d7dbbca9e99447b7c3bf7a73286afaaf6be99251eb9498baefa7d406892f67b879adb3a1d7e687fc4ccc1a388c7175fbaae567a26ab44d1067b54fcb0d6a316"
"packageManager": "pnpm@10.29.2+sha512.bef43fa759d91fd2da4b319a5a0d13ef7a45bb985a3d7342058470f9d2051a3ba8674e629672654686ef9443ad13a82da2beb9eeb3e0221c87b8154fff9d74b8"
}
pnpm-lock.yaml
Generated
+1677 -2239
View File
File diff suppressed because it is too large Load Diff
proxy.ts
+3 -1
View File
@@ -53,11 +53,13 @@ function checkRouteExists(pathname: string) {
const routePatterns = [
/^\/api\/agent\/[^/]+\/status\/?$/,
/^\/api\/agent\/[^/]+\/backup\/?$/,
/^\/api\/agent\/[^/]+\/backup\/upload\/init\/?$/,
/^\/api\/agent\/[^/]+\/backup\/upload\/status\/?$/,
/^\/api\/agent\/[^/]+\/restore\/?$/,
/^\/api\/files\/images\/[^/]+\/?$/,
/^\/api\/files\/backups\/?$/,
/^\/api\/tus\/hooks\/?$/,
/^\/api\/events\/?$/,
/^\/api\/init\/?$/,
/^\/api\/config\/?$/,
/^\/api\/google\/drive\/callback\/?$/,
];
server/api.ts
+6
View File
@@ -0,0 +1,6 @@
import express from "express";
import router from "../src/features/api/router";
export function mountApi(app: express.Express) {
app.use("/services/v1", router);
}
server/server.ts
+52
View File
@@ -0,0 +1,52 @@
import next from "next";
import express from "express";
import {mountApi} from "./api";
import http from "http";
const port = Number(process.env.PORT) || 8887;
const dev = process.env.NODE_ENV !== "production";
async function start() {
const nextApp = next({
dev,
hostname: "0.0.0.0",
turbopack: dev,
port
});
const handle = nextApp.getRequestHandler();
await nextApp.prepare();
const app = express();
app.use((req, res, next) => {
req.setTimeout(0);
res.setTimeout(0);
next();
});
mountApi(app);
app.use((req, res, next) => {
if (req.path.startsWith('/services/v1')) return next();
handle(req, res).catch((err) => {
console.error('Next.js App Router error:', err);
res.status(500).send('Internal Server Error');
});
});
const server = http.createServer(app);
server.setTimeout(0);
server.headersTimeout = 0;
server.requestTimeout = 0;
server.keepAliveTimeout = 0;
server.listen(port, "0.0.0.0", () => {
console.log(`NEXT APP → http://localhost:${port}`);
console.log(`API → http://localhost:${port}/services/v1`);
});
}
start();
src/components/wrappers/dashboard/admin/channels/channel/channel-form/providers/storages/forms/s3.form.tsx
+1 -1
View File
@@ -95,7 +95,7 @@ export const StorageS3Form = ({form}: StorageS3FormProps) => {
<FormField
control={form.control}
name="config.useSSL"
name="config.ssl"
render={({ field }) => (
<FormItem>
<FormLabel>Use SSL</FormLabel>
src/components/wrappers/dashboard/database/backup/actions/backup-actions-form.tsx
+1 -41
View File
@@ -220,18 +220,7 @@ export const BackupActionsForm = ({backup, action}: BackupActionsFormProps) => {
<div className="flex flex-row items-center gap-x-4 w-full">
{action === "delete" && (
// <ButtonWithLoading
// type="button"
// variant="destructive"
// onClick={() => mutationDeleteEntireBackup.mutateAsync()}
// isPending={mutationDeleteEntireBackup.isPending}
// disabled={mutationDeleteEntireBackup.isPending}
// >
// Delete entire backup
// </ButtonWithLoading>
<ButtonWithLoading
<ButtonWithLoading
type="button"
variant="destructive"
onClick={() => mutationDeleteEntireBackup.mutateAsync()}
@@ -240,35 +229,6 @@ export const BackupActionsForm = ({backup, action}: BackupActionsFormProps) => {
>
Delete entire backup
</ButtonWithLoading>
// <ButtonWithConfirm
// title={"Delete entire backup"}
// description={"Are you sure you want to delete this entire backup?"}
// button={{
// main: {
// type: "button",
// variant: "destructive",
// text: "Delete entire backup",
// },
// confirm: {
// className: "w-full",
// text: "Delete",
// icon: <Trash2/>,
// variant: "destructive",
// onClick: async () => {
// mutationDeleteEntireBackup.mutateAsync()
// },
// },
// cancel: {
// className: "w-full",
// text: "Cancel",
// icon: <Trash2/>,
// variant: "outline",
// },
// }}
// isPending={mutationDeleteEntireBackup.isPending}
// />
)}
{filteredBackupStorages.length > 0 && (
src/components/wrappers/dashboard/database/channels-policy/policy-form.tsx
+3 -4
View File
@@ -103,7 +103,6 @@ export const ChannelPoliciesForm = ({
return existing &&
(existing.eventKinds !== policy.eventKinds || existing.enabled !== policy.enabled);
});
const promises = kind === "notification"
? [
policiesToAdd.length > 0 ? await createAlertPoliciesAction({databaseId: database.id, alertPolicies: policiesToAdd}) : null,
@@ -128,9 +127,9 @@ export const ChannelPoliciesForm = ({
if (failedActions.length > 0) throw new Error(failedActions[0].data.actionError?.message || "One or more operations failed");
return {success: true};
},
onSuccess: () => {
toast.success("Policies saved successfully");
queryClient.invalidateQueries({queryKey: ["database-data", database.id]});
onSuccess: () => {
toast.success("Policies saved successfully");
queryClient.invalidateQueries({queryKey: ["database-data", database.id]});
},
onError: (error: any) => { toast.error(error.message || "Failed to save policies"); },
});
src/components/wrappers/dashboard/database/import/upload-backup.action.ts
+1 -3
View File
@@ -4,13 +4,11 @@ import {ServerActionResult} from "@/types/action-type";
import {db} from "@/db";
import * as drizzleDb from "@/db";
import {Backup} from "@/db/schema/07_database";
import {getFileExtension} from "../../../../../../app/api/agent/[agentId]/backup/helpers";
import {v4 as uuidv4} from "uuid";
import {eq} from "drizzle-orm";
import {uploadLocalPrivate, uploadS3Private} from "@/features/upload/private/upload.action";
import {z} from "zod";
import {env} from "@/env.mjs";
import {storeBackupFiles} from "@/features/storages/helpers";
import {getFileExtension} from "@/features/api/upload/helpers/file";
export const uploadBackupAction = userAction
src/db/schema/13_storage-policy.ts
+10 -2
View File
@@ -1,10 +1,13 @@
import {boolean, pgTable, uuid} from "drizzle-orm/pg-core";
import {timestamps} from "@/db/schema/00_common";
import {relations} from "drizzle-orm";
import {database} from "@/db/schema/07_database";
import {Backup, Database, database, Restoration, RetentionPolicy} from "@/db/schema/07_database";
import {createSelectSchema} from "drizzle-zod";
import {z} from "zod";
import {storageChannel} from "@/db/schema/12_storage-channel";
import {StorageChannel, storageChannel} from "@/db/schema/12_storage-channel";
import {Agent} from "@/db/schema/08_agent";
import {Project} from "@/db/schema/06_project";
import {AlertPolicy} from "@/db/schema/10_alert-policy";
export const storagePolicy = pgTable('storage_policy', {
id: uuid('id').defaultRandom().primaryKey(),
@@ -31,3 +34,8 @@ export const storagePolicyRelations = relations(storagePolicy, ({one}) => ({
export const storagePolicySchema = createSelectSchema(storagePolicy);
export type StoragePolicy = z.infer<typeof storagePolicySchema>;
export type StoragePolicyWith = StoragePolicy & {
storageChannel: StorageChannel;
};
src/features/agents/components/agent.dialog.tsx
+6 -1
View File
@@ -14,6 +14,7 @@ import {AgentType} from "@/features/agents/agents.schema";
import {GearIcon} from "@radix-ui/react-icons";
import {EmptyStatePlaceholder} from "@/components/wrappers/common/empty-state-placeholder";
import {useState} from "react";
import {useRouter} from "next/navigation";
type AgentDialogProps = {
agent?: AgentType & { id: string };
@@ -24,6 +25,7 @@ type AgentDialogProps = {
export const AgentDialog = ({agent, typeTrigger}: AgentDialogProps) => {
const [open, setOpen] = useState(false);
const isEdit = !!agent;
const router = useRouter();
const getTrigger = () => {
switch (typeTrigger) {
@@ -52,7 +54,10 @@ export const AgentDialog = ({agent, typeTrigger}: AgentDialogProps) => {
<DialogTitle>{isEdit ? `Edit ${agent.name}` : "Create new agent"}</DialogTitle>
</DialogHeader>
<AgentForm
onSuccess={() => setOpen(false)}
onSuccess={() => {
setOpen(false)
router.refresh()
}}
defaultValues={agent}
agentId={agent?.id}
/>
src/features/agents/components/agent.form.tsx
+1
View File
@@ -47,6 +47,7 @@ export const AgentForm = (props: agentFormProps) => {
});
const data = createAgent?.data?.data;
if (createAgent?.serverError || !data) {
toast.error(createAgent?.serverError);
return;
src/features/api/middleware.ts
+6
View File
@@ -0,0 +1,6 @@
import {Request, Response, NextFunction} from "express";
export function loggingMiddleware(req: Request, res: Response, next: NextFunction) {
console.log(`[API - SERVICES - V1] Received ${req.method} request : ${req.url} at ${new Date().toISOString()}`);
next();
}
src/features/api/router.ts
+10
View File
@@ -0,0 +1,10 @@
import express, { Router } from "express";
import uploadRouter from "./upload";
import { loggingMiddleware } from "@/features/api/middleware";
const router: Router = express.Router();
router.use(loggingMiddleware);
router.use("/upload", uploadRouter);
export default router;
src/features/api/upload/helpers/common.ts
+134
View File
@@ -0,0 +1,134 @@
import fs from "fs";
import {Backup, DatabaseWith} from "@/db/schema/07_database";
import {dispatchStorage} from "@/features/storages/dispatch";
import type {StorageInput, StorageResult} from "@/features/storages/types";
import {db} from "@/db";
import * as drizzleDb from "@/db";
import {eq} from "drizzle-orm";
import {withUpdatedAt} from "@/db/utils";
import {sendNotificationsBackupRestore} from "@/features/notifications/helpers";
import {eventEmitter} from "@/features/shared/event";
import forge from "node-forge";
import crypto from "node:crypto";
/**
* Save stream to a temporary file and upload to all storage providers in parallel.
*/
export default async function uploadTempFileToProviders(
backup: Backup,
database: DatabaseWith,
// inputStream: NodeJS.ReadableStream,
tmpPath: string,
fileName: string
): Promise<StorageResult[]> {
const stats = fs.statSync(tmpPath);
const fileSize = stats.size;
console.log(tmpPath);
const settings = await db.query.setting.findFirst({
where: eq(drizzleDb.schemas.setting.name, "system"),
with: {storageChannel: true},
});
const defaultPolicy = settings?.storageChannel
? [{
id: null,
storageChannelId: settings.storageChannel.id,
enabled: settings.storageChannel.enabled,
}]
: [];
const enabledPolicies = database.storagePolicies?.filter(p => p.enabled) ?? [];
const policies = enabledPolicies.length ? enabledPolicies : defaultPolicy;
if (!policies.length) {
await db.update(drizzleDb.schemas.backup)
.set(withUpdatedAt({status: "failed"}))
.where(eq(drizzleDb.schemas.backup.id, backup.id));
fs.existsSync(tmpPath) && fs.unlinkSync(tmpPath);
return [];
}
const storagePath = `backups/${database.project?.slug}/${fileName}`;
const results = await Promise.all(
policies.map(async (policy) => {
const fileStream = fs.createReadStream(tmpPath);
const [backupStorage] = await db.insert(drizzleDb.schemas.backupStorage)
.values({
backupId: backup.id,
storageChannelId: policy.storageChannelId,
status: "pending",
path: storagePath,
})
.returning();
const input: StorageInput = {
action: "upload",
data: {path: storagePath, file: fileStream, size: fileSize},
};
let result: StorageResult;
try {
result = policy.id
? await dispatchStorage(input, policy.id)
: await dispatchStorage(input, undefined, policy.storageChannelId);
} catch (err: any) {
console.error(err);
result = {success: false, provider: null, error: err.message};
}
await db.update(drizzleDb.schemas.backupStorage)
.set(withUpdatedAt({status: result.success ? "success" : "failed"}))
.where(eq(drizzleDb.schemas.backupStorage.id, backupStorage.id));
return result;
})
);
const backupStatus = results.some(r => r.success) ? "success" : "failed";
await db.update(drizzleDb.schemas.backup)
.set(withUpdatedAt({
fileSize: fileSize,
status: backupStatus
}))
.where(eq(drizzleDb.schemas.backup.id, backup.id));
fs.existsSync(tmpPath) && fs.unlinkSync(tmpPath);
console.log(results);
eventEmitter.emit('modification', {update: true});
if (!backupStatus) {
await sendNotificationsBackupRestore(database, "error_backup");
}
await sendNotificationsBackupRestore(database, "success_backup");
return results;
}
export function createDecryptionStream(
encryptedAesKeyHex: string,
ivHex: string
) {
const privateKeyPem = fs.readFileSync(
"private/keys/server_private.pem",
"utf8"
);
const privateKey = forge.pki.privateKeyFromPem(privateKeyPem);
const encryptedBytes = forge.util.hexToBytes(encryptedAesKeyHex);
const aesKeyBytes = privateKey.decrypt(encryptedBytes, "RSA-OAEP", {
md: forge.md.sha256.create(),
mgf1: {md: forge.md.sha256.create()},
});
const aesKey = Buffer.from(aesKeyBytes, "binary");
const iv = Buffer.from(ivHex, "hex");
return crypto.createDecipheriv("aes-256-cbc", aesKey, iv);
}
src/features/api/upload/helpers/file.ts
+27
View File
@@ -0,0 +1,27 @@
import fs from "fs";
import path from "path";
import {promisify} from "util";
import {pipeline} from "stream";
const pipelineAsync = promisify(pipeline);
const TMP_DIR = path.join(process.cwd(), "private/uploads/tmp");
fs.mkdirSync(TMP_DIR, { recursive: true });
export async function saveStreamToTempFile(stream: NodeJS.ReadableStream, fileName: string): Promise<string> {
const tmpPath = path.join(TMP_DIR, fileName);
const writeStream = fs.createWriteStream(tmpPath);
await pipelineAsync(stream, writeStream);
return tmpPath;
}
export function getFileExtension(dbType: string) {
switch (dbType) {
case "postgresql":
return ".dump";
case "mysql":
return ".sql";
default:
return ".dump";
}
}
src/features/api/upload/index.ts
+201
View File
@@ -0,0 +1,201 @@
import express, {Request, Response, Router} from "express";
import {v4 as uuidv4} from "uuid";
import {and, eq} from "drizzle-orm";
import * as drizzleDb from "@/db";
import {db} from "@/db";
import {isUuidv4} from "@/utils/verify-uuid";
import uploadTempFileToProviders, {createDecryptionStream} from "@/features/api/upload/helpers/common";
import {sendNotificationsBackupRestore} from "@/features/notifications/helpers";
import {Backup} from "@/db/schema/07_database";
import {withUpdatedAt} from "@/db/utils";
import {eventEmitter} from "@/features/shared/event";
import {getFileExtension, saveStreamToTempFile} from "@/features/api/upload/helpers/file";
const router: Router = express.Router();
// router.post("/:agentId", async (req: Request, res: Response) => {
// try {
// const agentId = req.params.agentId as string | undefined;
// const generatedId = req.headers["x-generated-id"] as string | undefined;
// const status = req.headers["x-status"] as string | undefined;
// const encryptedAesKeyHex = req.headers["x-aes-key"] as string | undefined;
// const ivHex = req.headers["x-iv"] as string | undefined;
// const method = (req.headers["x-method"] as string) ?? "manual";
// const extension = req.headers["x-extension"] as string | undefined;
//
//
// if (!generatedId || !encryptedAesKeyHex || !ivHex || !agentId || !status) {
// return res.status(400).json({error: "Missing required headers/params"});
// }
//
// if (!isUuidv4(generatedId)) {
// return res.status(400).json({error: "generatedId is not a valid UUID"});
// }
//
// const agent = await db.query.agent.findFirst({
// where: eq(drizzleDb.schemas.agent.id, agentId),
// });
//
// if (!agent) {
// return res.status(404).json({error: "Agent not found"});
// }
//
//
// const database = await db.query.database.findFirst({
// where: eq(drizzleDb.schemas.database.agentDatabaseId, generatedId),
// with: {
// project: true,
// storagePolicies: true,
// },
// });
//
// if (!database) {
// return res.status(404).json({error: "Database not found"});
// }
//
// let backup: Backup | null | undefined = null;
//
// if (method === "automatic") {
// [backup] = await db
// .insert(drizzleDb.schemas.backup)
// .values({
// status: "ongoing",
// databaseId: database.id,
// })
// .returning();
// } else {
// backup = await db.query.backup.findFirst({
// where: and(
// eq(drizzleDb.schemas.backup.databaseId, database.id),
// eq(drizzleDb.schemas.backup.status, "ongoing")
// ),
// });
// }
//
// if (!backup) {
// return res.status(404).json({error: "Backup not found"});
// }
//
//
// if (status === "success") {
//
// const decipher = createDecryptionStream(encryptedAesKeyHex, ivHex);
//
// const fileExt = extension || getFileExtension(database.dbms);
// const fileName = `${uuidv4()}${fileExt}`;
//
// const decryptedStream = req.pipe(decipher);
//
// const tmpPath = await saveStreamToTempFile(decryptedStream, fileName);
//
//
//
//
// if (!tmpPath) {
// return res.status(500).json({
// error: "Unable to save tmp backup file",
// });
// }
//
// uploadTempFileToProviders(backup, database, tmpPath, fileName);
//
// return res.json({success: true});
// } else {
// await db
// .update(drizzleDb.schemas.backup)
// .set(withUpdatedAt({status: 'failed'}))
// .where(eq(drizzleDb.schemas.backup.id, backup.id));
//
// eventEmitter.emit('modification', {update: true});
//
// await sendNotificationsBackupRestore(database, "error_backup");
//
// return res.status(200).json({
// error: "Backup successfully updated with status failed",
// });
// }
// } catch (err: any) {
// console.error("Upload error:", err);
// return res.status(500).json({
// error: "Upload failed",
// detail: err.message,
// });
// }
// });
router.post("/:agentId", async (req: Request, res: Response) => {
const agentId = req.params.agentId as string | undefined;
const generatedId = req.headers["x-generated-id"] as string | undefined;
const status = req.headers["x-status"] as string | undefined;
const encryptedAesKeyHex = req.headers["x-aes-key"] as string | undefined;
const ivHex = req.headers["x-iv"] as string | undefined;
const method = (req.headers["x-method"] as string) ?? "manual";
const extension = req.headers["x-extension"] as string | undefined;
if (!generatedId || !encryptedAesKeyHex || !ivHex || !agentId || !status) {
return res.status(400).json({error: "Missing required headers/params"});
}
if (!isUuidv4(generatedId)) {
return res.status(400).json({error: "generatedId is not a valid UUID"});
}
const agent = await db.query.agent.findFirst({ where: eq(drizzleDb.schemas.agent.id, agentId) });
if (!agent) return res.status(404).json({error: "Agent not found"});
const database = await db.query.database.findFirst({
where: eq(drizzleDb.schemas.database.agentDatabaseId, generatedId),
with: { project: true, storagePolicies: true },
});
if (!database) return res.status(404).json({error: "Database not found"});
let backup: Backup | null = null;
if (method === "automatic") {
[backup] = await db.insert(drizzleDb.schemas.backup).values({ status: "ongoing", databaseId: database.id }).returning();
} else {
backup = await db.query.backup.findFirst({
where: and(
eq(drizzleDb.schemas.backup.databaseId, database.id),
eq(drizzleDb.schemas.backup.status, "ongoing")
),
});
}
if (!backup) return res.status(404).json({error: "Backup not found"});
res.status(202).json({success: true, message: "Backup received, processing in background"});
(async () => {
try {
if (status === "success") {
const decipher = createDecryptionStream(encryptedAesKeyHex!, ivHex!);
const fileExt = extension || getFileExtension(database.dbms);
const fileName = `${uuidv4()}${fileExt}`;
const decryptedStream = req.pipe(decipher);
const tmpPath = await saveStreamToTempFile(decryptedStream, fileName);
if (!tmpPath) throw new Error("Unable to save tmp backup file");
await uploadTempFileToProviders(backup!, database, tmpPath, fileName);
} else {
await db.update(drizzleDb.schemas.backup)
.set(withUpdatedAt({status: 'failed'}))
.where(eq(drizzleDb.schemas.backup.id, backup!.id));
eventEmitter.emit('modification', {update: true});
await sendNotificationsBackupRestore(database, "error_backup");
}
} catch (err) {
console.error("Background backup processing failed:", err);
try {
await db.update(drizzleDb.schemas.backup)
.set(withUpdatedAt({status: 'failed'}))
.where(eq(drizzleDb.schemas.backup.id, backup!.id));
} catch (_) {
}
}
})();
});
export default router;
src/features/shared/event.ts
+3
View File
@@ -0,0 +1,3 @@
import { EventEmitter } from "events";
export const eventEmitter = new EventEmitter();
src/features/storages/helpers.ts
+1 -1
View File
@@ -11,7 +11,7 @@ import * as drizzleDb from "@/db";
import {withUpdatedAt} from "@/db/utils";
import {eq} from "drizzle-orm";
import {db} from "@/db";
import crypto, {createHash} from "crypto";
import {createHash} from "crypto";
import {getServerUrl} from "@/utils/get-server-url";
import path from "path";
src/features/storages/providers/google-drive/helpers.ts
-34
View File
@@ -1,36 +1,3 @@
// import {drive_v3, google} from "googleapis";
// import Drive = drive_v3.Drive;
// import {GoogleDriveConfig} from "@/features/storages/providers/google-drive/types";
//
//
// export async function getGoogleDriveClient(config: GoogleDriveConfig): Promise<Drive> {
// const auth = new google.auth.JWT({
// email: config.clientEmail,
// key: config.privateKey?.replace(/\\n/g, "\n"),
// scopes: ["https://www.googleapis.com/auth/drive"],
// });
//
// return google.drive({
// version: "v3",
// auth,
// });
// }
//
//
// export async function findFileByName(
// drive: any,
// name: string,
// folderId: string
// ): Promise<string | null> {
// const res = await drive.files.list({
// q: `name='${name}' and '${folderId}' in parents and trashed=false`,
// fields: "files(id)",
// pageSize: 1,
// });
//
// return res.data.files?.[0]?.id ?? null;
// }
import {drive_v3, google} from "googleapis";
import {GoogleDriveConfig} from "@/features/storages/providers/google-drive/types";
import Drive = drive_v3.Drive;
@@ -42,7 +9,6 @@ export async function getGoogleDriveClient(config: GoogleDriveConfig): Promise<D
const oauth2Client = new google.auth.OAuth2(
config.clientId,
config.clientSecret,
// config.redirectUri
baseUrl
);
src/features/storages/providers/google-drive/index.ts
+16 -5
View File
@@ -24,18 +24,27 @@ export async function uploadGoogleDrive(
? await ensureFolderPath(client, folderPath, config.folderId)
: config.folderId;
const existing = await findFileByName(client, fileName, folderId);
if (existing) return {success: false, provider: "google-drive", error: "File already exists"};
let fileStream: Readable;
const file = input.data.file;
if (Buffer.isBuffer(file) || file instanceof Uint8Array) {
fileStream = Readable.from(file);
} else if ((file as any).pipe) {
fileStream = file as Readable;
} else {
throw new Error("Unsupported file type for streaming upload");
}
await client.files.create({
requestBody: {name: fileName, parents: [folderId]},
media: {body: Readable.from(input.data.file as Buffer)},
media: {body: fileStream},
fields: "id",
supportsAllDrives: true,
});
if (input.data.url) {
const url = await generateFileUrl(input);
if (!url) {
@@ -70,9 +79,11 @@ export async function getGoogleDrive(
const res = await client.files.get(
{fileId, alt: "media", supportsAllDrives: true},
{responseType: "arraybuffer"}
{responseType: "stream"}
);
const stream = res.data as Readable;
if (input.data.signedUrl) {
const url = await generateFileUrl(input);
@@ -88,7 +99,7 @@ export async function getGoogleDrive(
return {
success: true,
provider: "google-drive",
file: Buffer.from(res.data as ArrayBuffer),
file: stream,
url: url,
};
}
@@ -96,7 +107,7 @@ export async function getGoogleDrive(
return {
success: true,
provider: "google-drive",
file: Buffer.from(res.data as ArrayBuffer),
file: stream,
};
}
src/features/storages/providers/google-drive/types.ts
-6
View File
@@ -1,9 +1,3 @@
// export type GoogleDriveConfig = {
// clientEmail: string;
// privateKey: string;
// folderId: string;
// };
export type GoogleDriveConfig = {
clientId: string;
src/features/storages/providers/index.ts
+1 -3
View File
@@ -1,7 +1,7 @@
import {
StorageProviderKind,
StorageInput,
StorageResult, StorageMetaData,
StorageResult,
} from '../types';
import {uploadLocal, getLocal, deleteLocal, pingLocal} from './local';
@@ -39,8 +39,6 @@ const handlers: Record<StorageProviderKind, ProviderHandler> = {
delete: deleteGoogleDrive,
ping: pingGoogleDrive,
}
// gcs: null as any,
// azure: null as any,
};
export async function dispatchViaProvider(
src/features/storages/providers/local.ts
+55 -41
View File
@@ -1,95 +1,109 @@
"use server"
import {mkdir, writeFile, unlink, readFile} from 'fs/promises';
import {mkdir, unlink} from 'fs/promises';
import path from 'path';
import {StorageDeleteInput, StorageGetInput, StorageMetaData, StorageResult, StorageUploadInput} from '../types';
import fs from "node:fs";
import {getServerUrl} from "@/utils/get-server-url";
import {generateFileUrl} from "@/features/storages/helpers";
import {Readable} from "node:stream";
const BASE_DIR = "/private/uploads/";
export async function uploadLocal(
config: { baseDir?: string },
input: { data: StorageUploadInput, metadata?: StorageMetaData }
input: { data: StorageUploadInput; metadata?: StorageMetaData }
): Promise<StorageResult> {
const base = config.baseDir || BASE_DIR;
const fullPath = path.join(process.cwd(), base, input.data.path);
const dir = path.dirname(fullPath);
await mkdir(dir, {recursive: true});
await writeFile(fullPath, input.data.file);
await mkdir(dir, { recursive: true });
if (input.data.url) {
const url = await generateFileUrl(input);
if (!url) {
return {
success: false,
provider: "local",
response: "Unable to get url file"
};
try {
const file = input.data.file;
if (Buffer.isBuffer(file)) {
await fs.promises.writeFile(fullPath, input.data.file);
} else if (file instanceof Readable) {
await new Promise<void>((resolve, reject) => {
const writable = fs.createWriteStream(fullPath);
file.pipe(writable);
writable.on("finish", resolve);
writable.on("error", reject);
});
} else {
return { success: false, provider: "local", error: "Unsupported file type. Must be Buffer or ReadableStream" };
}
return {
success: true,
provider: 'local',
url: url
};
if (input.data.url) {
const url = await generateFileUrl(input);
if (!url) {
return { success: false, provider: "local", response: "Unable to get URL" };
}
return { success: true, provider: "local", url };
}
return { success: true, provider: "local" };
} catch (err: any) {
try { await unlink(fullPath); } catch {}
return { success: false, provider: "local", error: err.message || "Upload failed" };
}
return {
success: true,
provider: 'local',
};
}
export async function getLocal(
config: { baseDir?: string },
input: { data: StorageGetInput, metadata: StorageMetaData }
input: { data: StorageGetInput; metadata: StorageMetaData }
): Promise<StorageResult> {
const base = config.baseDir || BASE_DIR;
const filePath = path.join(process.cwd(), base, input.data.path)
const fileName = path.basename(input.data.path);
const file = await readFile(filePath);
const filePath = path.join(process.cwd(), base, input.data.path);
if (!fs.existsSync(filePath)) {
console.error("File not found at:", filePath);
return ({
return {
success: false,
provider: 'local',
});
provider: "local",
error: "File not found",
};
}
let fileStream: fs.ReadStream | undefined;
try {
fileStream = fs.createReadStream(filePath);
} catch (err: any) {
console.error("Error creating read stream:", err);
return {
success: false,
provider: "local",
error: err.message,
};
}
if (input.data.signedUrl) {
const url = await generateFileUrl(input);
if (!url) {
return {
success: false,
provider: "local",
response: "Unable to get url file"
error: "Unable to generate signed URL",
};
}
return {
success: true,
provider: "local",
file: file,
url: url,
file: fileStream,
url,
};
}
return {
success: true,
provider: "local",
file: file,
file: fileStream,
};
}
export async function deleteLocal(
config: { baseDir?: string },
input: { data: StorageDeleteInput, metadata?: StorageMetaData }
src/features/storages/providers/s3.ts
+30 -42
View File
@@ -1,6 +1,6 @@
import * as Minio from "minio";
import {StorageDeleteInput, StorageGetInput, StorageMetaData, StorageResult, StorageUploadInput} from "../types";
import {generateFileUrl} from "@/features/storages/helpers";
import {Readable} from "node:stream";
type S3Config = {
endPointUrl: string;
@@ -9,7 +9,7 @@ type S3Config = {
secretKey: string;
bucketName: string;
port?: number;
useSSL?: boolean;
ssl?: boolean;
};
async function getS3Client(config: S3Config) {
@@ -19,7 +19,7 @@ async function getS3Client(config: S3Config) {
accessKey: config.accessKey,
secretKey: config.secretKey,
port: config.port ?? 443,
useSSL: config.useSSL ?? true,
useSSL: config.ssl ?? true,
});
}
@@ -40,68 +40,56 @@ export async function uploadS3(
await ensureBucket(config);
const key = `${BASE_DIR}${input.data.path}`;
const file = input.data.file;
let uploadStream: Readable;
if (Buffer.isBuffer(file) || file instanceof Uint8Array) {
uploadStream = Readable.from(file);
} else if ((file as any).pipe) {
uploadStream = file;
} else {
return {success: false, provider: "s3", error: "Unsupported file type for streaming upload"};
}
try {
await client.statObject(config.bucketName, key);
return {success: false, provider: "s3", error: "File already exists"};
} catch {
// continue if not found
const result = await client.putObject(config.bucketName, key, uploadStream, input.data.size);
} catch (err: any) {
return {success: false, provider: "s3", error: err.message};
}
await client.putObject(config.bucketName, key, input.data.file as Buffer);
if (input.data.url) {
const url = await generateFileUrl(input);
if (!url) {
return {
success: false,
provider: "s3",
response: "Unable to get url file"
};
}
return {
success: true,
provider: 's3',
url: url
};
}
return {
success: true,
provider: 's3',
};
return {success: true, provider: "s3"};
}
export async function getS3(config: S3Config, input: {
data: StorageGetInput,
metadata: StorageMetaData
}): Promise<StorageResult> {
const client = await getS3Client(config);
export async function getS3(
config: S3Config,
input: { data: StorageGetInput, metadata: StorageMetaData }
): Promise<StorageResult> {
const client = await getS3Client(config);
const key = `${BASE_DIR}${input.data.path}`;
try {
await client.statObject(config.bucketName, key);
} catch {
return {success: false, provider: "s3", error: "File not found"};
}
const presignedUrl = await client.presignedGetObject(config.bucketName, key, 60);
const fileStream = await client.getObject(config.bucketName, key);
const chunks: Buffer[] = [];
for await (const chunk of fileStream) chunks.push(chunk as Buffer);
const buffer = Buffer.concat(chunks);
let presignedUrl: string | undefined;
if (input.data.signedUrl) {
presignedUrl = await client.presignedGetObject(config.bucketName, key, input.data.expiresInSeconds ?? 60);
}
return {
success: true,
provider: "s3",
file: buffer,
file: fileStream as unknown as Buffer | Readable,
url: presignedUrl,
};
}
export async function deleteS3(config: S3Config, input: {
data: StorageDeleteInput,
metadata?: StorageMetaData
src/features/storages/types.ts
+6 -3
View File
@@ -1,3 +1,5 @@
import {Readable} from "node:stream";
export type StorageProviderKind =
| 'local'
| 's3'
@@ -20,9 +22,10 @@ export type StorageMetaData = {
export interface StorageUploadInput {
path: string;
file: Buffer | Uint8Array;
file: Readable | Buffer | Uint8Array;
url?: boolean;
contentType?: string;
size?: number;
}
export interface StorageGetInput {
@@ -37,7 +40,7 @@ export interface StorageDeleteInput {
export type StorageInput =
| { action: 'upload'; data: StorageUploadInput, metadata?: StorageMetaData }
| { action: 'get'; data: StorageGetInput, metadata: StorageMetaData}
| { action: 'get'; data: StorageGetInput, metadata: StorageMetaData }
| { action: 'delete'; data: StorageDeleteInput, metadata?: StorageMetaData }
| { action: 'ping'; };
@@ -45,7 +48,7 @@ export interface StorageResult {
success: boolean;
provider: StorageProviderKind | null;
url?: string;
file?: Buffer;
file?: Buffer | Readable;
error?: string;
response?: any;
}
src/lib/email/helpers.ts
+1
View File
@@ -1,4 +1,5 @@
import nodemailer from "nodemailer";
import {Server} from "./types"
export const createTransporter = (server: Server) => {
const portNumber = Number(server.port);
src/lib/email/index.ts
+1
View File
@@ -3,6 +3,7 @@ import {db} from "@/db";
import {eq} from "drizzle-orm";
import * as drizzleDb from "@/db";
import {createTransporter} from "@/lib/email/helpers";
import {Payload} from "@/lib/email/types";
export const sendEmail = async (data: Payload) => {
const settings = await db
src/lib/email/types.ts
+2 -2
View File
@@ -1,14 +1,14 @@
"use server";
type Payload = {
export type Payload = {
to: string;
from?: string;
subject: string;
html: any;
};
type Server = {
export type Server = {
host: string;
port: number;
user: string;
src/utils/s3-file-management.ts
+3 -3
View File
@@ -19,9 +19,9 @@ async function getS3Client() {
}
const baseConfig = {
endPoint: settings.s3EndPointUrl ?? "",
accessKey: settings.s3AccessKeyId ?? "",
secretKey: settings.s3SecretAccessKey ?? "",
endPoint: settings?.s3EndPointUrl ?? "",
accessKey: settings?.s3AccessKeyId ?? "",
secretKey: settings?.s3SecretAccessKey ?? "",
};
return new Minio.Client({