mirror of
https://github.com/Portabase/portabase.git
synced 2026-07-14 11:16:13 +02:00
fix: working on adding internal postgres database.
This commit is contained in:
@@ -1,6 +1,7 @@
|
|||||||
import {NextResponse} from "next/server";
|
import {NextResponse} from "next/server";
|
||||||
import fs from "fs";
|
import fs from "fs";
|
||||||
import path from "path";
|
import path from "path";
|
||||||
|
import {env} from "@/env.mjs";
|
||||||
|
|
||||||
|
|
||||||
export async function POST(request: Request) {
|
export async function POST(request: Request) {
|
||||||
@@ -29,7 +30,8 @@ export async function POST(request: Request) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
const uploadDir = path.join(process.cwd(), "/private/uploads/");
|
// const uploadDir = path.join(process.cwd(), "/private/uploads/");
|
||||||
|
const uploadDir = path.join(env.PRIVATE_PATH, "/uploads/");
|
||||||
|
|
||||||
const oldFilePath = path.join(uploadDir, "tmp", id);
|
const oldFilePath = path.join(uploadDir, "tmp", id);
|
||||||
const newFilePath = path.join(uploadDir, filePath);
|
const newFilePath = path.join(uploadDir, filePath);
|
||||||
|
|||||||
@@ -4,20 +4,21 @@ services:
|
|||||||
context: .
|
context: .
|
||||||
dockerfile: docker/dockerfile/Dockerfile
|
dockerfile: docker/dockerfile/Dockerfile
|
||||||
target: prod
|
target: prod
|
||||||
image: portabase/portabase:latest
|
# image: portabase/portabase:latest
|
||||||
ports:
|
ports:
|
||||||
- '8887:80'
|
- '8887:80'
|
||||||
environment:
|
environment:
|
||||||
TZ: "Europe/Paris"
|
TZ: "Europe/Paris"
|
||||||
env_file:
|
env_file:
|
||||||
- .env
|
- .env
|
||||||
|
volumes:
|
||||||
|
- portabase-data:/data
|
||||||
depends_on:
|
depends_on:
|
||||||
db:
|
db:
|
||||||
condition: service_healthy
|
condition: service_healthy
|
||||||
container_name: portabase-app-prod
|
container_name: portabase-app-prod
|
||||||
|
|
||||||
db:
|
db:
|
||||||
container_name: portabase-db-prod
|
|
||||||
image: postgres:16-alpine
|
image: postgres:16-alpine
|
||||||
ports:
|
ports:
|
||||||
- "5433:5432"
|
- "5433:5432"
|
||||||
@@ -34,4 +35,5 @@ services:
|
|||||||
retries: 5
|
retries: 5
|
||||||
|
|
||||||
volumes:
|
volumes:
|
||||||
|
portabase-data:
|
||||||
postgres-data:
|
postgres-data:
|
||||||
|
|||||||
@@ -18,6 +18,21 @@ RUN apt-get update && apt-get install -y \
|
|||||||
&& rm -rf /var/lib/apt/lists/*
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
|
||||||
|
RUN mkdir -p /etc/apt/keyrings \
|
||||||
|
&& curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc \
|
||||||
|
| gpg --dearmor -o /etc/apt/keyrings/postgresql.gpg \
|
||||||
|
&& echo "deb [signed-by=/etc/apt/keyrings/postgresql.gpg] http://apt.postgresql.org/pub/repos/apt bullseye-pgdg main" \
|
||||||
|
> /etc/apt/sources.list.d/pgdg.list
|
||||||
|
|
||||||
|
RUN apt-get update && apt-get install -y \
|
||||||
|
postgresql-18 \
|
||||||
|
postgresql-client-18 \
|
||||||
|
postgresql-contrib-18 \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
ENV PATH="/usr/lib/postgresql/18/bin:${PATH}"
|
||||||
|
|
||||||
|
|
||||||
FROM --platform=$BUILDPLATFORM golang:1.23-bookworm AS tusd-base
|
FROM --platform=$BUILDPLATFORM golang:1.23-bookworm AS tusd-base
|
||||||
|
|
||||||
ARG TUSD_VERSION=2.8.0
|
ARG TUSD_VERSION=2.8.0
|
||||||
@@ -76,6 +91,9 @@ ENV NODE_ENV=production
|
|||||||
ENV NEXT_TELEMETRY_DISABLED=1
|
ENV NEXT_TELEMETRY_DISABLED=1
|
||||||
ENV PORT=80
|
ENV PORT=80
|
||||||
ENV HOSTNAME="0.0.0.0"
|
ENV HOSTNAME="0.0.0.0"
|
||||||
|
ENV PGDATA=/data/postgres
|
||||||
|
ENV PRIVATE_PATH=/data/private
|
||||||
|
|
||||||
|
|
||||||
RUN addgroup --system --gid 1001 nodejs
|
RUN addgroup --system --gid 1001 nodejs
|
||||||
RUN adduser --system --uid 1001 nextjs
|
RUN adduser --system --uid 1001 nextjs
|
||||||
@@ -94,14 +112,16 @@ RUN chmod +x /usr/local/bin/tusd
|
|||||||
|
|
||||||
COPY docker/nginx/nginx.conf /etc/nginx/nginx.conf
|
COPY docker/nginx/nginx.conf /etc/nginx/nginx.conf
|
||||||
|
|
||||||
|
#RUN mkdir -p .next /app/private/uploads \
|
||||||
|
# && chown -R nextjs:nodejs .next /app/private /app/public
|
||||||
|
|
||||||
|
RUN mkdir -p .next /data/private/uploads \
|
||||||
|
&& chown -R nextjs:nodejs .next /data/private /app/public
|
||||||
|
|
||||||
RUN mkdir -p .next /app/private/uploads \
|
|
||||||
&& chown -R nextjs:nodejs .next /app/private /app/public
|
|
||||||
|
|
||||||
USER root
|
USER root
|
||||||
COPY ./docker/entrypoints/app-prod-entrypoint.sh /app/app-prod-entrypoint.sh
|
COPY ./docker/entrypoints/app-prod-entrypoint.sh /app/app-prod-entrypoint.sh
|
||||||
RUN chmod +x /app/app-prod-entrypoint.sh
|
RUN chmod +x /app/app-prod-entrypoint.sh
|
||||||
|
|
||||||
EXPOSE 80
|
EXPOSE 80
|
||||||
#USER nextjs
|
|
||||||
ENTRYPOINT ["sh","/app/app-prod-entrypoint.sh"]
|
ENTRYPOINT ["sh","/app/app-prod-entrypoint.sh"]
|
||||||
|
|||||||
@@ -7,9 +7,53 @@ else
|
|||||||
echo "[WARN] No TZ provided, using default container timezone"
|
echo "[WARN] No TZ provided, using default container timezone"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
mkdir -p /app/private/uploads/tmp
|
POSTGRES_BIN=$(ls -d /usr/lib/postgresql/*/bin | head -n 1)
|
||||||
|
|
||||||
|
if [ -z "$POSTGRES_BIN" ]; then
|
||||||
|
echo "PostgreSQL binaries not found"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
export PATH="$POSTGRES_BIN:$PATH"
|
||||||
|
|
||||||
|
if [ -z "$DATABASE_URL" ]; then
|
||||||
|
echo "[INFO] No DATABASE_URL provided, starting internal Postgres..."
|
||||||
|
|
||||||
|
mkdir -p "$PGDATA"
|
||||||
|
chown -R postgres:postgres "$PGDATA"
|
||||||
|
|
||||||
|
if [ ! -f "$PGDATA/PG_VERSION" ]; then
|
||||||
|
echo "[INFO] Initializing database cluster..."
|
||||||
|
su postgres -c "initdb -D '$PGDATA'"
|
||||||
|
fi
|
||||||
|
|
||||||
|
su postgres -c "pg_ctl -D '$PGDATA' -o \"-c listen_addresses='localhost'\" -w start"
|
||||||
|
|
||||||
|
until pg_isready -h 127.0.0.1 -p 5432; do
|
||||||
|
echo "Waiting for Postgres..."
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
|
||||||
|
DB_USER="${POSTGRES_USER:-portabase_user}"
|
||||||
|
DB_PASS="${POSTGRES_PASSWORD:-JaB6b1SUtIWYvt7srnOt}"
|
||||||
|
DB_NAME="${POSTGRES_DB:-portabase_db}"
|
||||||
|
|
||||||
|
USER_EXISTS=$(su postgres -c "psql -tAc \"SELECT 1 FROM pg_roles WHERE rolname='$DB_USER'\"")
|
||||||
|
if [ "$USER_EXISTS" != "1" ]; then
|
||||||
|
su postgres -c "psql -c \"CREATE USER $DB_USER WITH PASSWORD '$DB_PASS';\""
|
||||||
|
fi
|
||||||
|
|
||||||
|
DB_EXISTS=$(su - postgres -c "psql -tAc \"SELECT 1 FROM pg_database WHERE datname='$DB_NAME'\"")
|
||||||
|
if [ "$DB_EXISTS" != "1" ]; then
|
||||||
|
su postgres -c "psql -c \"CREATE DATABASE $DB_NAME OWNER $DB_USER;\""
|
||||||
|
fi
|
||||||
|
|
||||||
|
export DATABASE_URL="postgres://$DB_USER:$DB_PASS@127.0.0.1:5432/$DB_NAME"
|
||||||
|
fi
|
||||||
|
|
||||||
|
mkdir -p /data/private/uploads/tmp
|
||||||
echo "▶ Starting tusd server..."
|
echo "▶ Starting tusd server..."
|
||||||
tusd --base-path /tus/files/ --upload-dir /app/private/uploads/tmp --hooks-http http://127.0.0.1:3000/api/tus/hooks --port 1080 --max-size 21474836480 &
|
tusd --base-path /tus/files/ --upload-dir /data/private/uploads/tmp --hooks-http http://127.0.0.1:3000/api/tus/hooks --port 1080 --max-size 21474836480 &
|
||||||
|
|
||||||
echo "▶ Starting Next.js server..."
|
echo "▶ Starting Next.js server..."
|
||||||
PORT=3000 node server.js &
|
PORT=3000 node server.js &
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ import {v4 as uuidv4} from "uuid";
|
|||||||
import {eq} from "drizzle-orm";
|
import {eq} from "drizzle-orm";
|
||||||
import {z} from "zod";
|
import {z} from "zod";
|
||||||
import {storeBackupFiles} from "@/features/storages/helpers";
|
import {storeBackupFiles} from "@/features/storages/helpers";
|
||||||
import {getFileExtension} from "@/features/api/upload/helpers/file";
|
import {getFileExtension} from "@/utils/common";
|
||||||
|
|
||||||
|
|
||||||
export const uploadBackupAction = userAction
|
export const uploadBackupAction = userAction
|
||||||
@@ -46,18 +46,6 @@ export const uploadBackupAction = userAction
|
|||||||
const fileName = `${uuid}${fileExtension}`;
|
const fileName = `${uuid}${fileExtension}`;
|
||||||
const buffer = Buffer.from(arrayBuffer);
|
const buffer = Buffer.from(arrayBuffer);
|
||||||
|
|
||||||
// const [settings] = await db.select().from(drizzleDb.schemas.setting).where(eq(drizzleDb.schemas.setting.name, "system")).limit(1);
|
|
||||||
//
|
|
||||||
// if (!settings) {
|
|
||||||
// return {
|
|
||||||
// success: false,
|
|
||||||
// actionError: {
|
|
||||||
// message: "Settings not set",
|
|
||||||
// status: 500,
|
|
||||||
// cause: "Unknown error",
|
|
||||||
// },
|
|
||||||
// };
|
|
||||||
// }
|
|
||||||
|
|
||||||
const [backup] = await db
|
const [backup] = await db
|
||||||
.insert(drizzleDb.schemas.backup)
|
.insert(drizzleDb.schemas.backup)
|
||||||
@@ -68,30 +56,8 @@ export const uploadBackupAction = userAction
|
|||||||
})
|
})
|
||||||
.returning();
|
.returning();
|
||||||
|
|
||||||
|
|
||||||
await storeBackupFiles(backup, database, buffer, fileName)
|
await storeBackupFiles(backup, database, buffer, fileName)
|
||||||
|
|
||||||
|
|
||||||
// let success: boolean, message: string, filePath: string;
|
|
||||||
//
|
|
||||||
// const result =
|
|
||||||
// settings.storage === "local"
|
|
||||||
// ? await uploadLocalPrivate(fileName, buffer)
|
|
||||||
// : await uploadS3Private(`${database.project?.slug}/${fileName}`, buffer, env.S3_BUCKET_NAME!);
|
|
||||||
//
|
|
||||||
// ({success, message, filePath} = result);
|
|
||||||
//
|
|
||||||
// if (!success) {
|
|
||||||
// return {
|
|
||||||
// success: false,
|
|
||||||
// actionError: {
|
|
||||||
// message: "An error has occurred while uploading file",
|
|
||||||
// status: 500,
|
|
||||||
// cause: "Unknown error",
|
|
||||||
// },
|
|
||||||
// };
|
|
||||||
// }
|
|
||||||
|
|
||||||
return {
|
return {
|
||||||
success: true,
|
success: true,
|
||||||
value: backup,
|
value: backup,
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
import {createEnv} from "@t3-oss/env-nextjs";
|
import {createEnv} from "@t3-oss/env-nextjs";
|
||||||
import {z} from "zod";
|
import {z} from "zod";
|
||||||
import packageJson from "../package.json" with {type: "json"};
|
import packageJson from "../package.json" with {type: "json"};
|
||||||
|
import path from "path";
|
||||||
|
|
||||||
const {version} = packageJson;
|
const {version} = packageJson;
|
||||||
|
|
||||||
@@ -41,6 +42,8 @@ export const env = createEnv({
|
|||||||
.string()
|
.string()
|
||||||
.default(process.env.NODE_ENV === "production" ? "0 7 * * *" : "* * * * *"),
|
.default(process.env.NODE_ENV === "production" ? "0 7 * * *" : "* * * * *"),
|
||||||
|
|
||||||
|
PRIVATE_PATH: z.string(),
|
||||||
|
|
||||||
},
|
},
|
||||||
client: {
|
client: {
|
||||||
NEXT_PUBLIC_PROJECT_VERSION: z.string().optional(),
|
NEXT_PUBLIC_PROJECT_VERSION: z.string().optional(),
|
||||||
@@ -77,5 +80,6 @@ export const env = createEnv({
|
|||||||
|
|
||||||
RETENTION_CRON: process.env.RETENTION_CRON,
|
RETENTION_CRON: process.env.RETENTION_CRON,
|
||||||
|
|
||||||
|
PRIVATE_PATH: process.env.PRIVATE_PATH || path.join(process.cwd(), 'private')
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -1,6 +0,0 @@
|
|||||||
import {Request, Response, NextFunction} from "express";
|
|
||||||
|
|
||||||
export function loggingMiddleware(req: Request, res: Response, next: NextFunction) {
|
|
||||||
console.log(`[API - SERVICES - V1] Received ${req.method} request : ${req.url} at ${new Date().toISOString()}`);
|
|
||||||
next();
|
|
||||||
}
|
|
||||||
@@ -1,10 +0,0 @@
|
|||||||
import express, { Router } from "express";
|
|
||||||
import uploadRouter from "./upload";
|
|
||||||
import { loggingMiddleware } from "@/features/api/middleware";
|
|
||||||
|
|
||||||
const router: Router = express.Router();
|
|
||||||
|
|
||||||
router.use(loggingMiddleware);
|
|
||||||
router.use("/upload", uploadRouter);
|
|
||||||
|
|
||||||
export default router;
|
|
||||||
@@ -1,134 +0,0 @@
|
|||||||
import fs from "fs";
|
|
||||||
import {Backup, DatabaseWith} from "@/db/schema/07_database";
|
|
||||||
import {dispatchStorage} from "@/features/storages/dispatch";
|
|
||||||
import type {StorageInput, StorageResult} from "@/features/storages/types";
|
|
||||||
import {db} from "@/db";
|
|
||||||
import * as drizzleDb from "@/db";
|
|
||||||
import {eq} from "drizzle-orm";
|
|
||||||
import {withUpdatedAt} from "@/db/utils";
|
|
||||||
import {sendNotificationsBackupRestore} from "@/features/notifications/helpers";
|
|
||||||
import {eventEmitter} from "@/features/shared/event";
|
|
||||||
import forge from "node-forge";
|
|
||||||
import crypto from "node:crypto";
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Save stream to a temporary file and upload to all storage providers in parallel.
|
|
||||||
*/
|
|
||||||
export default async function uploadTempFileToProviders(
|
|
||||||
backup: Backup,
|
|
||||||
database: DatabaseWith,
|
|
||||||
// inputStream: NodeJS.ReadableStream,
|
|
||||||
tmpPath: string,
|
|
||||||
fileName: string
|
|
||||||
): Promise<StorageResult[]> {
|
|
||||||
|
|
||||||
const stats = fs.statSync(tmpPath);
|
|
||||||
const fileSize = stats.size;
|
|
||||||
console.log(tmpPath);
|
|
||||||
|
|
||||||
const settings = await db.query.setting.findFirst({
|
|
||||||
where: eq(drizzleDb.schemas.setting.name, "system"),
|
|
||||||
with: {storageChannel: true},
|
|
||||||
});
|
|
||||||
|
|
||||||
const defaultPolicy = settings?.storageChannel
|
|
||||||
? [{
|
|
||||||
id: null,
|
|
||||||
storageChannelId: settings.storageChannel.id,
|
|
||||||
enabled: settings.storageChannel.enabled,
|
|
||||||
}]
|
|
||||||
: [];
|
|
||||||
|
|
||||||
const enabledPolicies = database.storagePolicies?.filter(p => p.enabled) ?? [];
|
|
||||||
const policies = enabledPolicies.length ? enabledPolicies : defaultPolicy;
|
|
||||||
|
|
||||||
if (!policies.length) {
|
|
||||||
await db.update(drizzleDb.schemas.backup)
|
|
||||||
.set(withUpdatedAt({status: "failed"}))
|
|
||||||
.where(eq(drizzleDb.schemas.backup.id, backup.id));
|
|
||||||
fs.existsSync(tmpPath) && fs.unlinkSync(tmpPath);
|
|
||||||
return [];
|
|
||||||
}
|
|
||||||
|
|
||||||
const storagePath = `backups/${database.project?.slug}/${fileName}`;
|
|
||||||
|
|
||||||
const results = await Promise.all(
|
|
||||||
policies.map(async (policy) => {
|
|
||||||
const fileStream = fs.createReadStream(tmpPath);
|
|
||||||
|
|
||||||
const [backupStorage] = await db.insert(drizzleDb.schemas.backupStorage)
|
|
||||||
.values({
|
|
||||||
backupId: backup.id,
|
|
||||||
storageChannelId: policy.storageChannelId,
|
|
||||||
status: "pending",
|
|
||||||
path: storagePath,
|
|
||||||
})
|
|
||||||
.returning();
|
|
||||||
|
|
||||||
const input: StorageInput = {
|
|
||||||
action: "upload",
|
|
||||||
data: {path: storagePath, file: fileStream, size: fileSize},
|
|
||||||
};
|
|
||||||
|
|
||||||
let result: StorageResult;
|
|
||||||
|
|
||||||
try {
|
|
||||||
result = policy.id
|
|
||||||
? await dispatchStorage(input, policy.id)
|
|
||||||
: await dispatchStorage(input, undefined, policy.storageChannelId);
|
|
||||||
} catch (err: any) {
|
|
||||||
console.error(err);
|
|
||||||
result = {success: false, provider: null, error: err.message};
|
|
||||||
}
|
|
||||||
await db.update(drizzleDb.schemas.backupStorage)
|
|
||||||
.set(withUpdatedAt({status: result.success ? "success" : "failed"}))
|
|
||||||
.where(eq(drizzleDb.schemas.backupStorage.id, backupStorage.id));
|
|
||||||
|
|
||||||
return result;
|
|
||||||
})
|
|
||||||
);
|
|
||||||
|
|
||||||
const backupStatus = results.some(r => r.success) ? "success" : "failed";
|
|
||||||
await db.update(drizzleDb.schemas.backup)
|
|
||||||
.set(withUpdatedAt({
|
|
||||||
fileSize: fileSize,
|
|
||||||
status: backupStatus
|
|
||||||
}))
|
|
||||||
.where(eq(drizzleDb.schemas.backup.id, backup.id));
|
|
||||||
|
|
||||||
fs.existsSync(tmpPath) && fs.unlinkSync(tmpPath);
|
|
||||||
|
|
||||||
console.log(results);
|
|
||||||
eventEmitter.emit('modification', {update: true});
|
|
||||||
|
|
||||||
if (!backupStatus) {
|
|
||||||
await sendNotificationsBackupRestore(database, "error_backup");
|
|
||||||
}
|
|
||||||
await sendNotificationsBackupRestore(database, "success_backup");
|
|
||||||
|
|
||||||
return results;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
export function createDecryptionStream(
|
|
||||||
encryptedAesKeyHex: string,
|
|
||||||
ivHex: string
|
|
||||||
) {
|
|
||||||
const privateKeyPem = fs.readFileSync(
|
|
||||||
"private/keys/server_private.pem",
|
|
||||||
"utf8"
|
|
||||||
);
|
|
||||||
|
|
||||||
const privateKey = forge.pki.privateKeyFromPem(privateKeyPem);
|
|
||||||
|
|
||||||
const encryptedBytes = forge.util.hexToBytes(encryptedAesKeyHex);
|
|
||||||
const aesKeyBytes = privateKey.decrypt(encryptedBytes, "RSA-OAEP", {
|
|
||||||
md: forge.md.sha256.create(),
|
|
||||||
mgf1: {md: forge.md.sha256.create()},
|
|
||||||
});
|
|
||||||
|
|
||||||
const aesKey = Buffer.from(aesKeyBytes, "binary");
|
|
||||||
const iv = Buffer.from(ivHex, "hex");
|
|
||||||
|
|
||||||
return crypto.createDecipheriv("aes-256-cbc", aesKey, iv);
|
|
||||||
}
|
|
||||||
@@ -1,27 +0,0 @@
|
|||||||
import fs from "fs";
|
|
||||||
import path from "path";
|
|
||||||
import {promisify} from "util";
|
|
||||||
import {pipeline} from "stream";
|
|
||||||
|
|
||||||
const pipelineAsync = promisify(pipeline);
|
|
||||||
const TMP_DIR = path.join(process.cwd(), "private/uploads/tmp");
|
|
||||||
fs.mkdirSync(TMP_DIR, { recursive: true });
|
|
||||||
|
|
||||||
export async function saveStreamToTempFile(stream: NodeJS.ReadableStream, fileName: string): Promise<string> {
|
|
||||||
const tmpPath = path.join(TMP_DIR, fileName);
|
|
||||||
const writeStream = fs.createWriteStream(tmpPath);
|
|
||||||
await pipelineAsync(stream, writeStream);
|
|
||||||
return tmpPath;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
export function getFileExtension(dbType: string) {
|
|
||||||
switch (dbType) {
|
|
||||||
case "postgresql":
|
|
||||||
return ".dump";
|
|
||||||
case "mysql":
|
|
||||||
return ".sql";
|
|
||||||
default:
|
|
||||||
return ".dump";
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,201 +0,0 @@
|
|||||||
import express, {Request, Response, Router} from "express";
|
|
||||||
import {v4 as uuidv4} from "uuid";
|
|
||||||
import {and, eq} from "drizzle-orm";
|
|
||||||
import * as drizzleDb from "@/db";
|
|
||||||
import {db} from "@/db";
|
|
||||||
import {isUuidv4} from "@/utils/verify-uuid";
|
|
||||||
import uploadTempFileToProviders, {createDecryptionStream} from "@/features/api/upload/helpers/common";
|
|
||||||
import {sendNotificationsBackupRestore} from "@/features/notifications/helpers";
|
|
||||||
import {Backup} from "@/db/schema/07_database";
|
|
||||||
import {withUpdatedAt} from "@/db/utils";
|
|
||||||
import {eventEmitter} from "@/features/shared/event";
|
|
||||||
import {getFileExtension, saveStreamToTempFile} from "@/features/api/upload/helpers/file";
|
|
||||||
|
|
||||||
const router: Router = express.Router();
|
|
||||||
|
|
||||||
// router.post("/:agentId", async (req: Request, res: Response) => {
|
|
||||||
// try {
|
|
||||||
// const agentId = req.params.agentId as string | undefined;
|
|
||||||
// const generatedId = req.headers["x-generated-id"] as string | undefined;
|
|
||||||
// const status = req.headers["x-status"] as string | undefined;
|
|
||||||
// const encryptedAesKeyHex = req.headers["x-aes-key"] as string | undefined;
|
|
||||||
// const ivHex = req.headers["x-iv"] as string | undefined;
|
|
||||||
// const method = (req.headers["x-method"] as string) ?? "manual";
|
|
||||||
// const extension = req.headers["x-extension"] as string | undefined;
|
|
||||||
//
|
|
||||||
//
|
|
||||||
// if (!generatedId || !encryptedAesKeyHex || !ivHex || !agentId || !status) {
|
|
||||||
// return res.status(400).json({error: "Missing required headers/params"});
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// if (!isUuidv4(generatedId)) {
|
|
||||||
// return res.status(400).json({error: "generatedId is not a valid UUID"});
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// const agent = await db.query.agent.findFirst({
|
|
||||||
// where: eq(drizzleDb.schemas.agent.id, agentId),
|
|
||||||
// });
|
|
||||||
//
|
|
||||||
// if (!agent) {
|
|
||||||
// return res.status(404).json({error: "Agent not found"});
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
//
|
|
||||||
// const database = await db.query.database.findFirst({
|
|
||||||
// where: eq(drizzleDb.schemas.database.agentDatabaseId, generatedId),
|
|
||||||
// with: {
|
|
||||||
// project: true,
|
|
||||||
// storagePolicies: true,
|
|
||||||
// },
|
|
||||||
// });
|
|
||||||
//
|
|
||||||
// if (!database) {
|
|
||||||
// return res.status(404).json({error: "Database not found"});
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// let backup: Backup | null | undefined = null;
|
|
||||||
//
|
|
||||||
// if (method === "automatic") {
|
|
||||||
// [backup] = await db
|
|
||||||
// .insert(drizzleDb.schemas.backup)
|
|
||||||
// .values({
|
|
||||||
// status: "ongoing",
|
|
||||||
// databaseId: database.id,
|
|
||||||
// })
|
|
||||||
// .returning();
|
|
||||||
// } else {
|
|
||||||
// backup = await db.query.backup.findFirst({
|
|
||||||
// where: and(
|
|
||||||
// eq(drizzleDb.schemas.backup.databaseId, database.id),
|
|
||||||
// eq(drizzleDb.schemas.backup.status, "ongoing")
|
|
||||||
// ),
|
|
||||||
// });
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// if (!backup) {
|
|
||||||
// return res.status(404).json({error: "Backup not found"});
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
//
|
|
||||||
// if (status === "success") {
|
|
||||||
//
|
|
||||||
// const decipher = createDecryptionStream(encryptedAesKeyHex, ivHex);
|
|
||||||
//
|
|
||||||
// const fileExt = extension || getFileExtension(database.dbms);
|
|
||||||
// const fileName = `${uuidv4()}${fileExt}`;
|
|
||||||
//
|
|
||||||
// const decryptedStream = req.pipe(decipher);
|
|
||||||
//
|
|
||||||
// const tmpPath = await saveStreamToTempFile(decryptedStream, fileName);
|
|
||||||
//
|
|
||||||
//
|
|
||||||
//
|
|
||||||
//
|
|
||||||
// if (!tmpPath) {
|
|
||||||
// return res.status(500).json({
|
|
||||||
// error: "Unable to save tmp backup file",
|
|
||||||
// });
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// uploadTempFileToProviders(backup, database, tmpPath, fileName);
|
|
||||||
//
|
|
||||||
// return res.json({success: true});
|
|
||||||
// } else {
|
|
||||||
// await db
|
|
||||||
// .update(drizzleDb.schemas.backup)
|
|
||||||
// .set(withUpdatedAt({status: 'failed'}))
|
|
||||||
// .where(eq(drizzleDb.schemas.backup.id, backup.id));
|
|
||||||
//
|
|
||||||
// eventEmitter.emit('modification', {update: true});
|
|
||||||
//
|
|
||||||
// await sendNotificationsBackupRestore(database, "error_backup");
|
|
||||||
//
|
|
||||||
// return res.status(200).json({
|
|
||||||
// error: "Backup successfully updated with status failed",
|
|
||||||
// });
|
|
||||||
// }
|
|
||||||
// } catch (err: any) {
|
|
||||||
// console.error("Upload error:", err);
|
|
||||||
// return res.status(500).json({
|
|
||||||
// error: "Upload failed",
|
|
||||||
// detail: err.message,
|
|
||||||
// });
|
|
||||||
// }
|
|
||||||
// });
|
|
||||||
router.post("/:agentId", async (req: Request, res: Response) => {
|
|
||||||
const agentId = req.params.agentId as string | undefined;
|
|
||||||
const generatedId = req.headers["x-generated-id"] as string | undefined;
|
|
||||||
const status = req.headers["x-status"] as string | undefined;
|
|
||||||
const encryptedAesKeyHex = req.headers["x-aes-key"] as string | undefined;
|
|
||||||
const ivHex = req.headers["x-iv"] as string | undefined;
|
|
||||||
const method = (req.headers["x-method"] as string) ?? "manual";
|
|
||||||
const extension = req.headers["x-extension"] as string | undefined;
|
|
||||||
|
|
||||||
if (!generatedId || !encryptedAesKeyHex || !ivHex || !agentId || !status) {
|
|
||||||
return res.status(400).json({error: "Missing required headers/params"});
|
|
||||||
}
|
|
||||||
if (!isUuidv4(generatedId)) {
|
|
||||||
return res.status(400).json({error: "generatedId is not a valid UUID"});
|
|
||||||
}
|
|
||||||
|
|
||||||
const agent = await db.query.agent.findFirst({ where: eq(drizzleDb.schemas.agent.id, agentId) });
|
|
||||||
if (!agent) return res.status(404).json({error: "Agent not found"});
|
|
||||||
|
|
||||||
const database = await db.query.database.findFirst({
|
|
||||||
where: eq(drizzleDb.schemas.database.agentDatabaseId, generatedId),
|
|
||||||
with: { project: true, storagePolicies: true },
|
|
||||||
});
|
|
||||||
if (!database) return res.status(404).json({error: "Database not found"});
|
|
||||||
|
|
||||||
let backup: Backup | null = null;
|
|
||||||
if (method === "automatic") {
|
|
||||||
[backup] = await db.insert(drizzleDb.schemas.backup).values({ status: "ongoing", databaseId: database.id }).returning();
|
|
||||||
} else {
|
|
||||||
backup = await db.query.backup.findFirst({
|
|
||||||
where: and(
|
|
||||||
eq(drizzleDb.schemas.backup.databaseId, database.id),
|
|
||||||
eq(drizzleDb.schemas.backup.status, "ongoing")
|
|
||||||
),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
if (!backup) return res.status(404).json({error: "Backup not found"});
|
|
||||||
|
|
||||||
res.status(202).json({success: true, message: "Backup received, processing in background"});
|
|
||||||
|
|
||||||
(async () => {
|
|
||||||
try {
|
|
||||||
if (status === "success") {
|
|
||||||
const decipher = createDecryptionStream(encryptedAesKeyHex!, ivHex!);
|
|
||||||
const fileExt = extension || getFileExtension(database.dbms);
|
|
||||||
const fileName = `${uuidv4()}${fileExt}`;
|
|
||||||
const decryptedStream = req.pipe(decipher);
|
|
||||||
const tmpPath = await saveStreamToTempFile(decryptedStream, fileName);
|
|
||||||
|
|
||||||
if (!tmpPath) throw new Error("Unable to save tmp backup file");
|
|
||||||
|
|
||||||
await uploadTempFileToProviders(backup!, database, tmpPath, fileName);
|
|
||||||
} else {
|
|
||||||
await db.update(drizzleDb.schemas.backup)
|
|
||||||
.set(withUpdatedAt({status: 'failed'}))
|
|
||||||
.where(eq(drizzleDb.schemas.backup.id, backup!.id));
|
|
||||||
|
|
||||||
eventEmitter.emit('modification', {update: true});
|
|
||||||
await sendNotificationsBackupRestore(database, "error_backup");
|
|
||||||
}
|
|
||||||
} catch (err) {
|
|
||||||
console.error("Background backup processing failed:", err);
|
|
||||||
try {
|
|
||||||
await db.update(drizzleDb.schemas.backup)
|
|
||||||
.set(withUpdatedAt({status: 'failed'}))
|
|
||||||
.where(eq(drizzleDb.schemas.backup.id, backup!.id));
|
|
||||||
} catch (_) {
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})();
|
|
||||||
});
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
export default router;
|
|
||||||
|
|
||||||
|
|
||||||
@@ -1,4 +1,6 @@
|
|||||||
import fs from "node:fs";
|
import fs from "node:fs";
|
||||||
|
import {env} from "@/env.mjs";
|
||||||
|
import path from "path";
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -6,7 +8,8 @@ import fs from "node:fs";
|
|||||||
*/
|
*/
|
||||||
export function getPublicServerKeyContent() {
|
export function getPublicServerKeyContent() {
|
||||||
try {
|
try {
|
||||||
return fs.readFileSync("private/keys/server_public.pem", "utf8");
|
const keyPath = path.join(env.PRIVATE_PATH, '/keys/server_public.pem')
|
||||||
|
return fs.readFileSync(keyPath, "utf8");
|
||||||
} catch (error: any) {
|
} catch (error: any) {
|
||||||
console.error("Error :", error);
|
console.error("Error :", error);
|
||||||
return {
|
return {
|
||||||
@@ -22,7 +25,8 @@ export function getPublicServerKeyContent() {
|
|||||||
*/
|
*/
|
||||||
export function getMasterServerKeyContent() {
|
export function getMasterServerKeyContent() {
|
||||||
try {
|
try {
|
||||||
return fs.readFileSync("private/keys/master_key.bin");
|
const keyPath = path.join(env.PRIVATE_PATH, '/keys/master_key.bin')
|
||||||
|
return fs.readFileSync(keyPath);
|
||||||
} catch (error: any) {
|
} catch (error: any) {
|
||||||
console.error("Error :", error);
|
console.error("Error :", error);
|
||||||
return {
|
return {
|
||||||
|
|||||||
@@ -5,15 +5,17 @@ import {StorageDeleteInput, StorageGetInput, StorageMetaData, StorageResult, Sto
|
|||||||
import fs from "node:fs";
|
import fs from "node:fs";
|
||||||
import {generateFileUrl} from "@/features/storages/helpers";
|
import {generateFileUrl} from "@/features/storages/helpers";
|
||||||
import {Readable} from "node:stream";
|
import {Readable} from "node:stream";
|
||||||
|
import {env} from "@/env.mjs";
|
||||||
|
|
||||||
const BASE_DIR = "/private/uploads/";
|
const BASE_DIR = path.join(env.PRIVATE_PATH, '/uploads')
|
||||||
|
|
||||||
export async function uploadLocal(
|
export async function uploadLocal(
|
||||||
config: { baseDir?: string },
|
config: { baseDir?: string },
|
||||||
input: { data: StorageUploadInput; metadata?: StorageMetaData }
|
input: { data: StorageUploadInput; metadata?: StorageMetaData }
|
||||||
): Promise<StorageResult> {
|
): Promise<StorageResult> {
|
||||||
const base = config.baseDir || BASE_DIR;
|
const base = config.baseDir ? path.join(process.cwd(), config.baseDir ?? "") : BASE_DIR;
|
||||||
const fullPath = path.join(process.cwd(), base, input.data.path);
|
|
||||||
|
const fullPath = path.join(base, input.data.path);
|
||||||
const dir = path.dirname(fullPath);
|
const dir = path.dirname(fullPath);
|
||||||
|
|
||||||
await mkdir(dir, { recursive: true });
|
await mkdir(dir, { recursive: true });
|
||||||
@@ -52,8 +54,9 @@ export async function getLocal(
|
|||||||
config: { baseDir?: string },
|
config: { baseDir?: string },
|
||||||
input: { data: StorageGetInput; metadata: StorageMetaData }
|
input: { data: StorageGetInput; metadata: StorageMetaData }
|
||||||
): Promise<StorageResult> {
|
): Promise<StorageResult> {
|
||||||
const base = config.baseDir || BASE_DIR;
|
|
||||||
const filePath = path.join(process.cwd(), base, input.data.path);
|
const base = config.baseDir ? path.join(process.cwd(), config.baseDir ?? "") : BASE_DIR;
|
||||||
|
const filePath = path.join(base, input.data.path);
|
||||||
|
|
||||||
if (!fs.existsSync(filePath)) {
|
if (!fs.existsSync(filePath)) {
|
||||||
console.error("File not found at:", filePath);
|
console.error("File not found at:", filePath);
|
||||||
@@ -108,8 +111,10 @@ export async function deleteLocal(
|
|||||||
config: { baseDir?: string },
|
config: { baseDir?: string },
|
||||||
input: { data: StorageDeleteInput, metadata?: StorageMetaData }
|
input: { data: StorageDeleteInput, metadata?: StorageMetaData }
|
||||||
): Promise<StorageResult> {
|
): Promise<StorageResult> {
|
||||||
const base = config.baseDir || BASE_DIR;
|
|
||||||
const fullPath = path.join(process.cwd(), base, input.data.path);
|
const base = config.baseDir ? path.join(process.cwd(), config.baseDir ?? "") : BASE_DIR;
|
||||||
|
const fullPath = path.join(base, input.data.path);
|
||||||
|
|
||||||
await unlink(fullPath);
|
await unlink(fullPath);
|
||||||
return {
|
return {
|
||||||
success: true,
|
success: true,
|
||||||
@@ -121,8 +126,8 @@ export async function pingLocal(
|
|||||||
config: { baseDir?: string }
|
config: { baseDir?: string }
|
||||||
): Promise<StorageResult> {
|
): Promise<StorageResult> {
|
||||||
|
|
||||||
const base = config.baseDir || BASE_DIR;
|
const base = path.join(process.cwd(), config.baseDir ?? "") || BASE_DIR;
|
||||||
const fullPath = path.join(process.cwd(), base, "ping.txt");
|
const fullPath = path.join(base, "ping.txt");
|
||||||
|
|
||||||
await fs.promises.writeFile(fullPath, "ping");
|
await fs.promises.writeFile(fullPath, "ping");
|
||||||
await fs.promises.readFile(fullPath);
|
await fs.promises.readFile(fullPath);
|
||||||
|
|||||||
@@ -16,6 +16,9 @@ const privateS3Dir = "backups/";
|
|||||||
|
|
||||||
export async function uploadLocalPrivate(fileName: string, buffer: any) {
|
export async function uploadLocalPrivate(fileName: string, buffer: any) {
|
||||||
try {
|
try {
|
||||||
|
|
||||||
|
const privatePath = path.join(env.PRIVATE_PATH!, '/keys/master_key.bin')
|
||||||
|
|
||||||
await mkdir(path.join(process.cwd(), privateLocalDir), {recursive: true});
|
await mkdir(path.join(process.cwd(), privateLocalDir), {recursive: true});
|
||||||
await writeFile(path.join(process.cwd(), privateLocalDir, fileName), buffer);
|
await writeFile(path.join(process.cwd(), privateLocalDir, fileName), buffer);
|
||||||
|
|
||||||
@@ -96,26 +99,6 @@ export async function deleteLocalPrivate(fileName: string) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
// export async function getFileUrlPresignedLocal(fileName: string) {
|
|
||||||
// try {
|
|
||||||
// const filePath = path.join(privateLocalDir, fileName);
|
|
||||||
// await mkdir(path.join(process.cwd(), privateLocalDir), {recursive: true});
|
|
||||||
//
|
|
||||||
// if (!fs.existsSync(filePath)) {
|
|
||||||
// console.error("File not found at:", filePath);
|
|
||||||
// return `File not found at: ${filePath}`;
|
|
||||||
// }
|
|
||||||
// const crypto = require("crypto");
|
|
||||||
// const baseUrl = getServerUrl();
|
|
||||||
//
|
|
||||||
// const expiresAt = Date.now() + 60 * 1000; // expires in 1 minute
|
|
||||||
// const token = crypto.createHash("sha256").update(`${fileName}${expiresAt}`).digest("hex");
|
|
||||||
// return `${baseUrl}/api/files/${fileName}?token=${token}&expires=${expiresAt}`;
|
|
||||||
// } catch (error) {
|
|
||||||
// throw error;
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
|
|
||||||
export async function getFileUrlPresignedS3(fileName: string) {
|
export async function getFileUrlPresignedS3(fileName: string) {
|
||||||
try {
|
try {
|
||||||
return await createPresignedUrlToDownload({
|
return await createPresignedUrlToDownload({
|
||||||
|
|||||||
@@ -39,3 +39,13 @@ export function buildOrganizationWithMembers(
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
export function getFileExtension(dbType: string) {
|
||||||
|
switch (dbType) {
|
||||||
|
case "postgresql":
|
||||||
|
return ".dump";
|
||||||
|
case "mysql":
|
||||||
|
return ".sql";
|
||||||
|
default:
|
||||||
|
return ".dump";
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -3,6 +3,7 @@ import path from 'path';
|
|||||||
import {generateKeyPair} from 'crypto';
|
import {generateKeyPair} from 'crypto';
|
||||||
import {promisify} from 'util';
|
import {promisify} from 'util';
|
||||||
import {randomBytes} from 'crypto';
|
import {randomBytes} from 'crypto';
|
||||||
|
import {env} from "@/env.mjs";
|
||||||
|
|
||||||
const generateKeyPairAsync = promisify(generateKeyPair);
|
const generateKeyPairAsync = promisify(generateKeyPair);
|
||||||
|
|
||||||
@@ -13,9 +14,10 @@ const generateKeyPairAsync = promisify(generateKeyPair);
|
|||||||
* @param {string} [dir] path to directory
|
* @param {string} [dir] path to directory
|
||||||
* @returns {Promise<{privateKeyPath:string, publicKeyPath:string}>}
|
* @returns {Promise<{privateKeyPath:string, publicKeyPath:string}>}
|
||||||
*/
|
*/
|
||||||
export async function generateRSAKeys(dir = path.join(process.cwd(), 'private/keys')) {
|
export async function generateRSAKeys(dir = path.join(env.PRIVATE_PATH, '/keys')) {
|
||||||
await fs.mkdir(dir, {recursive: true});
|
await fs.mkdir(dir, {recursive: true});
|
||||||
|
|
||||||
|
|
||||||
const privateKeyPath = path.join(dir, 'server_private.pem');
|
const privateKeyPath = path.join(dir, 'server_private.pem');
|
||||||
const publicKeyPath = path.join(dir, 'server_public.pem');
|
const publicKeyPath = path.join(dir, 'server_public.pem');
|
||||||
|
|
||||||
@@ -47,7 +49,7 @@ export async function generateRSAKeys(dir = path.join(process.cwd(), 'private/ke
|
|||||||
* @param {string} [filePath] Path to store the key
|
* @param {string} [filePath] Path to store the key
|
||||||
* @returns {Promise<Buffer>} The master key
|
* @returns {Promise<Buffer>} The master key
|
||||||
*/
|
*/
|
||||||
export async function getOrCreateMasterKey(filePath = path.join(process.cwd(), 'private/keys', 'master_key.bin')) {
|
export async function getOrCreateMasterKey(filePath = path.join(env.PRIVATE_PATH, '/keys', 'master_key.bin')) {
|
||||||
|
|
||||||
await fs.mkdir(path.dirname(filePath), {recursive: true});
|
await fs.mkdir(path.dirname(filePath), {recursive: true});
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user