diff --git a/.gitleaks.toml b/.gitleaks.toml index 3b176363..cb21e78d 100644 --- a/.gitleaks.toml +++ b/.gitleaks.toml @@ -2,9 +2,7 @@ title = "Custom gitleaks config" [allowlist] # Global allowlist patterns (won’t be flagged) -regexes = [ - "s3SecretAccessKey: env.S3_SECRET_KEY ?? null", -] +regexes = [] paths = [ "src/utils/init.ts" ] diff --git a/CITATION.cff b/CITATION.cff index 940ae89e..ea9ad667 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -26,5 +26,5 @@ keywords: - web-ui - agent license: Apache-2.0 -version: 1.2.5 -date-released: "2026-02-13" +version: 1.2.8-rc.1 +date-released: "2026-02-17" diff --git a/README.md b/README.md index 35b9c790..75cf8dd3 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@

Portabase

- Portabase is a tool designed to simplify the backup and restoration of your database instances. It integrates seamlessly with Portabase agents for managing operations securely and efficiently. + Portabase is a tool designed to simplify the backup and restoration of your database instances. It integrates seamlessly with Portabase agents for managing operations securely and efficiently.

@@ -23,7 +23,6 @@ [![Self Hosted](https://img.shields.io/badge/self--hosted-yes-brightgreen)](https://github.com/Portabase/portabase) [![Open Source](https://img.shields.io/badge/open%20source-❤️-red)](https://github.com/Portabase/portabase) - [![NextJS][NextJS]][NextJS-url] [![BetterAuth][BetterAuth]][BetterAuth-url] [![Drizzle][Drizzle]][Drizzle-url] @@ -51,14 +50,13 @@ You have 4 ways to install Portabase: - Automated CLI (recommended) - [details](https://portabase.io/docs/dashboard/setup#cli) -- Docker Compose setup - [details](https://portabase.io/docs/dashboard/setup#docker) +- Docker Run - [details](https://portabase.io/docs/dashboard/setup#docker) +- Docker Compose setup - [details](https://portabase.io/docs/dashboard/setup#docker-compose) - Kubernetes with Helm (soon) - Development setup - [details](https://portabase.io/docs/dashboard/setup#development) **Ensure Docker is installed on your machine before getting started.** - - ## Contributors [![Contributors](https://contrib.rocks/image?repo=Portabase/portabase)](https://github.com/Portabase/portabase/graphs/contributors) diff --git a/app/(customer)/dashboard/(admin)/notifications/channels/page.tsx b/app/(customer)/dashboard/(admin)/notifications/channels/page.tsx index 28becfab..5b443bee 100644 --- a/app/(customer)/dashboard/(admin)/notifications/channels/page.tsx +++ b/app/(customer)/dashboard/(admin)/notifications/channels/page.tsx @@ -35,12 +35,10 @@ export default async function RoutePage(props: PageParams<{}>) { Notification channels - {/**/} - {/**/} diff --git a/app/api/tus/hooks/route.ts b/app/api/tus/hooks/route.ts index b69fa37d..c711270c 100644 --- a/app/api/tus/hooks/route.ts +++ b/app/api/tus/hooks/route.ts @@ -1,6 +1,7 @@ import {NextResponse} from "next/server"; import fs from "fs"; import path from "path"; +import {env} from "@/env.mjs"; export async function POST(request: Request) { @@ -29,7 +30,8 @@ export async function POST(request: Request) { } - const uploadDir = path.join(process.cwd(), "/private/uploads/"); + // const uploadDir = path.join(process.cwd(), "/private/uploads/"); + const uploadDir = path.join(env.PRIVATE_PATH, "/uploads/"); const oldFilePath = path.join(uploadDir, "tmp", id); const newFilePath = path.join(uploadDir, filePath); diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index aa46d0d1..302666c9 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -1,19 +1,18 @@ -name: portabase-prod - services: - app: - build: - context: . - dockerfile: docker/dockerfile/Dockerfile - target: prod - image: portabase/portabase:1.2.5-rc.9 +# build: +# context: . +# dockerfile: docker/dockerfile/Dockerfile +# target: prod + image: portabase/portabase:1.2.7-rc.1 ports: - '8887:80' environment: TZ: "Europe/Paris" env_file: - .env + volumes: + - portabase-data:/data depends_on: db: condition: service_healthy @@ -36,4 +35,5 @@ services: retries: 5 volumes: + portabase-data: postgres-data: diff --git a/docker/dockerfile/Dockerfile b/docker/dockerfile/Dockerfile index 4cb5c07b..0b15bdef 100644 --- a/docker/dockerfile/Dockerfile +++ b/docker/dockerfile/Dockerfile @@ -18,6 +18,21 @@ RUN apt-get update && apt-get install -y \ && rm -rf /var/lib/apt/lists/* +RUN mkdir -p /etc/apt/keyrings \ + && curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc \ + | gpg --dearmor -o /etc/apt/keyrings/postgresql.gpg \ + && echo "deb [signed-by=/etc/apt/keyrings/postgresql.gpg] http://apt.postgresql.org/pub/repos/apt bullseye-pgdg main" \ + > /etc/apt/sources.list.d/pgdg.list + +RUN apt-get update && apt-get install -y \ + postgresql-18 \ + postgresql-client-18 \ + postgresql-contrib-18 \ + && rm -rf /var/lib/apt/lists/* + +ENV PATH="/usr/lib/postgresql/18/bin:${PATH}" + + FROM --platform=$BUILDPLATFORM golang:1.23-bookworm AS tusd-base ARG TUSD_VERSION=2.8.0 @@ -76,6 +91,9 @@ ENV NODE_ENV=production ENV NEXT_TELEMETRY_DISABLED=1 ENV PORT=80 ENV HOSTNAME="0.0.0.0" +ENV PGDATA=/data/postgres +ENV PRIVATE_PATH=/data/private + RUN addgroup --system --gid 1001 nodejs RUN adduser --system --uid 1001 nextjs @@ -94,14 +112,16 @@ RUN chmod +x /usr/local/bin/tusd COPY docker/nginx/nginx.conf /etc/nginx/nginx.conf +#RUN mkdir -p .next /app/private/uploads \ +# && chown -R nextjs:nodejs .next /app/private /app/public + +RUN mkdir -p .next /data/private/uploads \ + && chown -R nextjs:nodejs .next /data/private /app/public -RUN mkdir -p .next /app/private/uploads \ - && chown -R nextjs:nodejs .next /app/private /app/public USER root COPY ./docker/entrypoints/app-prod-entrypoint.sh /app/app-prod-entrypoint.sh RUN chmod +x /app/app-prod-entrypoint.sh EXPOSE 80 -#USER nextjs ENTRYPOINT ["sh","/app/app-prod-entrypoint.sh"] diff --git a/docker/entrypoints/app-prod-entrypoint.sh b/docker/entrypoints/app-prod-entrypoint.sh index 5f69f7b8..2dd72312 100644 --- a/docker/entrypoints/app-prod-entrypoint.sh +++ b/docker/entrypoints/app-prod-entrypoint.sh @@ -7,9 +7,72 @@ else echo "[WARN] No TZ provided, using default container timezone" fi -mkdir -p /app/private/uploads/tmp +POSTGRES_BIN=$(ls -d /usr/lib/postgresql/*/bin | head -n 1) + +if [ -z "$POSTGRES_BIN" ]; then + echo "PostgreSQL binaries not found" + exit 1 +fi + +export PATH="$POSTGRES_BIN:$PATH" + +if [ -z "$DATABASE_URL" ]; then + echo "[INFO] No DATABASE_URL provided, starting internal Postgres..." + + mkdir -p "$PGDATA" + chown -R postgres:postgres "$PGDATA" + + if [ ! -f "$PGDATA/PG_VERSION" ]; then + echo "[INFO] Initializing database cluster..." + if ! su postgres -c "initdb -D '$PGDATA'" > /dev/null 2>&1; then + echo "[ERROR] initdb failed" + exit 1 + fi + fi + + if ! su postgres -c "pg_ctl -D '$PGDATA' \ + -o \"-c listen_addresses='localhost' -c logging_collector=on\" \ + -l $PGDATA/postgres.log -w start" > /dev/null 2>&1; then + echo "[ERROR] PostgreSQL failed to start" + exit 1 + fi + + until su postgres -c "pg_isready -h 127.0.0.1 -p 5432" > /dev/null 2>&1; do + sleep 1 + done + + echo "[INFO] PostgreSQL server is up and accepting connections" + + DB_USER="${POSTGRES_USER:-portabase_user}" + DB_PASS="${POSTGRES_PASSWORD:-JaB6b1SUtIWYvt7srnOt}" + DB_NAME="${POSTGRES_DB:-portabase_db}" + + USER_EXISTS=$(su postgres -c "psql -tAc \"SELECT 1 FROM pg_roles WHERE rolname='$DB_USER'\"" 2>/dev/null) + if [ "$USER_EXISTS" != "1" ]; then + if ! su postgres -c "psql -c \"CREATE USER $DB_USER WITH PASSWORD '$DB_PASS';\"" > /dev/null 2>&1; then + echo "[ERROR] Failed creating user" + exit 1 + fi + fi + + DB_EXISTS=$(su postgres -c "psql -tAc \"SELECT 1 FROM pg_database WHERE datname='$DB_NAME'\"" 2>/dev/null) + if [ "$DB_EXISTS" != "1" ]; then + if ! su postgres -c "psql -c \"CREATE DATABASE $DB_NAME OWNER $DB_USER;\"" > /dev/null 2>&1; then + echo "[ERROR] Failed creating database" + exit 1 + fi + fi + + export DATABASE_URL="postgres://$DB_USER:$DB_PASS@127.0.0.1:5432/$DB_NAME" + + echo "[SUCCESS] Internal PostgreSQL started successfully" + echo "[SUCCESS] Database: $DB_NAME | User: $DB_USER | Host: 127.0.0.1:5432" +fi + + +mkdir -p /data/private/uploads/tmp echo "▶ Starting tusd server..." -tusd --base-path /tus/files/ --upload-dir /app/private/uploads/tmp --hooks-http http://127.0.0.1:3000/api/tus/hooks --port 1080 --max-size 21474836480 & +tusd --base-path /tus/files/ --upload-dir /data/private/uploads/tmp --hooks-http http://127.0.0.1:3000/api/tus/hooks --port 1080 --max-size 21474836480 & echo "▶ Starting Next.js server..." PORT=3000 node server.js & diff --git a/package.json b/package.json index 743f6107..85d503f4 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "portabase", - "version": "1.2.5", + "version": "1.2.8-rc.1", "private": true, "scripts": { "dev": "next dev --turbopack -p 8887", diff --git a/portabase.config.ts b/portabase.config.ts index 2da80198..13584ac3 100644 --- a/portabase.config.ts +++ b/portabase.config.ts @@ -1,3 +1,15 @@ +export interface AuthProviderConfig { + id: "google" | "github" | "credential"; + isActive: boolean; + icon: string; + isManual?: boolean; + credentials?: { + clientId: string; + clientSecret: string; + }; +} + + export const PORTABASE_DEFAULT_SETTINGS = { SECURITY: { CSP: { @@ -50,7 +62,7 @@ export const PORTABASE_DEFAULT_SETTINGS = { FORM_ACTION: ["'self'"], FRAME_ANCESTORS: ["'none'"], BLOCK_ALL_MIXED_CONTENT: false, - UPGRADE_INSECURE_REQUESTS: true, + UPGRADE_INSECURE_REQUESTS: process.env.PROJECT_URL?.startsWith("https://") || false, }, PERMISSIONS_POLICY: { CAMERA: ["()"], @@ -61,3 +73,36 @@ export const PORTABASE_DEFAULT_SETTINGS = { }, }, }; + + +export const SUPPORTED_PROVIDERS: AuthProviderConfig[] = [ + { + id: "google", + icon: "hugeicons:chrome", + isActive: Boolean(process.env.AUTH_GOOGLE_METHOD), + // isManual: true, + credentials: { + clientId: process.env.AUTH_GOOGLE_ID || "", + clientSecret: process.env.AUTH_GOOGLE_SECRET || "", + }, + }, + { + id: "github", + icon: "iconoir:github", + isActive: Boolean(process.env.AUTH_GITHUB_METHOD), + credentials: { + clientId: process.env.AUTH_GITHUB_ID || "", + clientSecret: process.env.AUTH_GITHUB_SECRET || "", + }, + }, + { + id: "credential", + isActive: true, + icon: "proicons:key", + isManual: true, + credentials: { + clientId: "", + clientSecret: "", + }, + }, +]; diff --git a/release b/release index d4d58b2a..5e594fdb 100755 --- a/release +++ b/release @@ -4,7 +4,7 @@ set -e if [ -z "$1" ]; then echo "Usage: ./release " - echo "Example: ./release v1.0.0" + echo "Example: ./release 1.0.0" exit 1 fi diff --git a/src/components/wrappers/dashboard/admin/channels/channel/channel-form/channel-form.tsx b/src/components/wrappers/dashboard/admin/channels/channel/channel-form/channel-form.tsx index eb127cda..d6d78e73 100644 --- a/src/components/wrappers/dashboard/admin/channels/channel/channel-form/channel-form.tsx +++ b/src/components/wrappers/dashboard/admin/channels/channel/channel-form/channel-form.tsx @@ -224,7 +224,7 @@ export const ChannelForm = ({onSuccessAction, organization, defaultValues, kind}
- {kind == "storage" && ( + {(!isCreate || kind == "storage") && ( { mutationFn: async (backups: Backup[]) => { const results = await Promise.all( backups.map(async (backup) => { - if (backup.deletedAt == null) { + if (backup.deletedAt == null || backup.status == "ongoing") { const backupDeleted = await deleteBackupAction({ databaseId: backup.databaseId, backupId: backup.id, }) - // const backupDeleted = await deleteBackupAction({ - // backupId: backup.id, - // databaseId: backup.databaseId, - // status: backup.status, - // file: backup.file ?? "", - // projectSlug: props.database?.project?.slug! - // }); return { success: backupDeleted?.data?.success, message: backupDeleted?.data?.success @@ -136,15 +129,9 @@ export const DatabaseBackupList = (props: DatabaseBackupListProps) => { >Actions - { - const backupsToDelete = rows.map(row => row - ).filter(backup => backup.deletedAt == null) - if (backupsToDelete.length === 0) { - toast.error("No available backup selected for deletion."); - return; - } - mutationDeleteBackups.mutate(backupsToDelete); + mutationDeleteBackups.mutate(rows); setIsActionsOpen(false); }} onCancel={() => setIsActionsOpen(false)} diff --git a/src/env.mjs b/src/env.mjs index c18725ad..7cd1bef3 100644 --- a/src/env.mjs +++ b/src/env.mjs @@ -1,6 +1,7 @@ import { createEnv } from "@t3-oss/env-nextjs"; import { z } from "zod"; import packageJson from "../package.json" with { type: "json" }; +import path from "path"; const { version } = packageJson; @@ -14,8 +15,8 @@ export const env = createEnv({ PROJECT_NAME: z.string().optional(), PROJECT_DESCRIPTION: z.string().optional(), - PROJECT_URL: z.string().optional(), - PROJECT_SECRET: z.string().optional(), + PROJECT_URL: z.string().regex(/^https?:\/\//, "URL must start with http:// or https://"), + PROJECT_SECRET: z.string(), SMTP_PASSWORD: z.string().optional(), SMTP_FROM: z.string().optional(), @@ -31,15 +32,6 @@ export const env = createEnv({ AUTH_GITHUB_ID: z.string().optional(), AUTH_GITHUB_SECRET: z.string().optional(), - S3_ENDPOINT: z.string().optional(), - S3_ACCESS_KEY: z.string().optional(), - S3_SECRET_KEY: z.string().optional(), - S3_BUCKET_NAME: z.string().optional(), - S3_PORT: z.string().optional(), - S3_USE_SSL: z.string().optional(), - - STORAGE_TYPE: z.enum(["local", "s3"]).optional(), - RETENTION_CRON: z.string().default(process.env.NODE_ENV === "production" ? "0 7 * * *" : "* * * * *"), AUTH_OIDC_ID: z.string().optional().default("oidc"), @@ -59,6 +51,9 @@ export const env = createEnv({ AUTH_EMAIL_PASSWORD_ENABLED: z.string().optional().default("true"), AUTH_SIGNUP_ENABLED: z.string().optional().default("true"), AUTH_PASSKEY_ENABLED: z.string().optional().default("true"), + + PRIVATE_PATH: z.string().optional(), + }, client: { NEXT_PUBLIC_PROJECT_VERSION: z.string().optional(), @@ -87,15 +82,6 @@ export const env = createEnv({ AUTH_GITHUB_ID: process.env.AUTH_GITHUB_ID, AUTH_GITHUB_SECRET: process.env.AUTH_GITHUB_SECRET, - S3_ENDPOINT: process.env.S3_ENDPOINT, - S3_ACCESS_KEY: process.env.S3_ACCESS_KEY, - S3_SECRET_KEY: process.env.S3_SECRET_KEY, - S3_BUCKET_NAME: process.env.S3_BUCKET_NAME, - S3_PORT: process.env.S3_PORT, - S3_USE_SSL: process.env.S3_USE_SSL, - - STORAGE_TYPE: process.env.STORAGE_TYPE, - RETENTION_CRON: process.env.RETENTION_CRON, AUTH_OIDC_ID: process.env.AUTH_OIDC_ID, @@ -115,5 +101,7 @@ export const env = createEnv({ AUTH_EMAIL_PASSWORD_ENABLED: process.env.AUTH_EMAIL_PASSWORD_ENABLED, AUTH_SIGNUP_ENABLED: process.env.AUTH_SIGNUP_ENABLED, AUTH_PASSKEY_ENABLED: process.env.AUTH_PASSKEY_ENABLED, + + PRIVATE_PATH: process.env.PRIVATE_PATH || path.join(process.cwd(), 'private') }, }); diff --git a/src/features/api/middleware.ts b/src/features/api/middleware.ts deleted file mode 100644 index 06928e63..00000000 --- a/src/features/api/middleware.ts +++ /dev/null @@ -1,6 +0,0 @@ -import {Request, Response, NextFunction} from "express"; - -export function loggingMiddleware(req: Request, res: Response, next: NextFunction) { - console.log(`[API - SERVICES - V1] Received ${req.method} request : ${req.url} at ${new Date().toISOString()}`); - next(); -} \ No newline at end of file diff --git a/src/features/api/router.ts b/src/features/api/router.ts deleted file mode 100644 index d2e09797..00000000 --- a/src/features/api/router.ts +++ /dev/null @@ -1,10 +0,0 @@ -import express, { Router } from "express"; -import uploadRouter from "./upload"; -import { loggingMiddleware } from "@/features/api/middleware"; - -const router: Router = express.Router(); - -router.use(loggingMiddleware); -router.use("/upload", uploadRouter); - -export default router; \ No newline at end of file diff --git a/src/features/api/upload/helpers/common.ts b/src/features/api/upload/helpers/common.ts deleted file mode 100644 index cebbc76d..00000000 --- a/src/features/api/upload/helpers/common.ts +++ /dev/null @@ -1,134 +0,0 @@ -import fs from "fs"; -import {Backup, DatabaseWith} from "@/db/schema/07_database"; -import {dispatchStorage} from "@/features/storages/dispatch"; -import type {StorageInput, StorageResult} from "@/features/storages/types"; -import {db} from "@/db"; -import * as drizzleDb from "@/db"; -import {eq} from "drizzle-orm"; -import {withUpdatedAt} from "@/db/utils"; -import {sendNotificationsBackupRestore} from "@/features/notifications/helpers"; -import {eventEmitter} from "@/features/shared/event"; -import forge from "node-forge"; -import crypto from "node:crypto"; - -/** - * Save stream to a temporary file and upload to all storage providers in parallel. - */ -export default async function uploadTempFileToProviders( - backup: Backup, - database: DatabaseWith, - // inputStream: NodeJS.ReadableStream, - tmpPath: string, - fileName: string -): Promise { - - const stats = fs.statSync(tmpPath); - const fileSize = stats.size; - console.log(tmpPath); - - const settings = await db.query.setting.findFirst({ - where: eq(drizzleDb.schemas.setting.name, "system"), - with: {storageChannel: true}, - }); - - const defaultPolicy = settings?.storageChannel - ? [{ - id: null, - storageChannelId: settings.storageChannel.id, - enabled: settings.storageChannel.enabled, - }] - : []; - - const enabledPolicies = database.storagePolicies?.filter(p => p.enabled) ?? []; - const policies = enabledPolicies.length ? enabledPolicies : defaultPolicy; - - if (!policies.length) { - await db.update(drizzleDb.schemas.backup) - .set(withUpdatedAt({status: "failed"})) - .where(eq(drizzleDb.schemas.backup.id, backup.id)); - fs.existsSync(tmpPath) && fs.unlinkSync(tmpPath); - return []; - } - - const storagePath = `backups/${database.project?.slug}/${fileName}`; - - const results = await Promise.all( - policies.map(async (policy) => { - const fileStream = fs.createReadStream(tmpPath); - - const [backupStorage] = await db.insert(drizzleDb.schemas.backupStorage) - .values({ - backupId: backup.id, - storageChannelId: policy.storageChannelId, - status: "pending", - path: storagePath, - }) - .returning(); - - const input: StorageInput = { - action: "upload", - data: {path: storagePath, file: fileStream, size: fileSize}, - }; - - let result: StorageResult; - - try { - result = policy.id - ? await dispatchStorage(input, policy.id) - : await dispatchStorage(input, undefined, policy.storageChannelId); - } catch (err: any) { - console.error(err); - result = {success: false, provider: null, error: err.message}; - } - await db.update(drizzleDb.schemas.backupStorage) - .set(withUpdatedAt({status: result.success ? "success" : "failed"})) - .where(eq(drizzleDb.schemas.backupStorage.id, backupStorage.id)); - - return result; - }) - ); - - const backupStatus = results.some(r => r.success) ? "success" : "failed"; - await db.update(drizzleDb.schemas.backup) - .set(withUpdatedAt({ - fileSize: fileSize, - status: backupStatus - })) - .where(eq(drizzleDb.schemas.backup.id, backup.id)); - - fs.existsSync(tmpPath) && fs.unlinkSync(tmpPath); - - console.log(results); - eventEmitter.emit('modification', {update: true}); - - if (!backupStatus) { - await sendNotificationsBackupRestore(database, "error_backup"); - } - await sendNotificationsBackupRestore(database, "success_backup"); - - return results; -} - - -export function createDecryptionStream( - encryptedAesKeyHex: string, - ivHex: string -) { - const privateKeyPem = fs.readFileSync( - "private/keys/server_private.pem", - "utf8" - ); - - const privateKey = forge.pki.privateKeyFromPem(privateKeyPem); - - const encryptedBytes = forge.util.hexToBytes(encryptedAesKeyHex); - const aesKeyBytes = privateKey.decrypt(encryptedBytes, "RSA-OAEP", { - md: forge.md.sha256.create(), - mgf1: {md: forge.md.sha256.create()}, - }); - - const aesKey = Buffer.from(aesKeyBytes, "binary"); - const iv = Buffer.from(ivHex, "hex"); - - return crypto.createDecipheriv("aes-256-cbc", aesKey, iv); -} \ No newline at end of file diff --git a/src/features/api/upload/helpers/file.ts b/src/features/api/upload/helpers/file.ts deleted file mode 100644 index 8f6dbc05..00000000 --- a/src/features/api/upload/helpers/file.ts +++ /dev/null @@ -1,27 +0,0 @@ -import fs from "fs"; -import path from "path"; -import {promisify} from "util"; -import {pipeline} from "stream"; - -const pipelineAsync = promisify(pipeline); -const TMP_DIR = path.join(process.cwd(), "private/uploads/tmp"); -fs.mkdirSync(TMP_DIR, { recursive: true }); - -export async function saveStreamToTempFile(stream: NodeJS.ReadableStream, fileName: string): Promise { - const tmpPath = path.join(TMP_DIR, fileName); - const writeStream = fs.createWriteStream(tmpPath); - await pipelineAsync(stream, writeStream); - return tmpPath; -} - - -export function getFileExtension(dbType: string) { - switch (dbType) { - case "postgresql": - return ".dump"; - case "mysql": - return ".sql"; - default: - return ".dump"; - } -} \ No newline at end of file diff --git a/src/features/api/upload/index.ts b/src/features/api/upload/index.ts deleted file mode 100644 index a0a3ae08..00000000 --- a/src/features/api/upload/index.ts +++ /dev/null @@ -1,201 +0,0 @@ -import express, {Request, Response, Router} from "express"; -import {v4 as uuidv4} from "uuid"; -import {and, eq} from "drizzle-orm"; -import * as drizzleDb from "@/db"; -import {db} from "@/db"; -import {isUuidv4} from "@/utils/verify-uuid"; -import uploadTempFileToProviders, {createDecryptionStream} from "@/features/api/upload/helpers/common"; -import {sendNotificationsBackupRestore} from "@/features/notifications/helpers"; -import {Backup} from "@/db/schema/07_database"; -import {withUpdatedAt} from "@/db/utils"; -import {eventEmitter} from "@/features/shared/event"; -import {getFileExtension, saveStreamToTempFile} from "@/features/api/upload/helpers/file"; - -const router: Router = express.Router(); - -// router.post("/:agentId", async (req: Request, res: Response) => { -// try { -// const agentId = req.params.agentId as string | undefined; -// const generatedId = req.headers["x-generated-id"] as string | undefined; -// const status = req.headers["x-status"] as string | undefined; -// const encryptedAesKeyHex = req.headers["x-aes-key"] as string | undefined; -// const ivHex = req.headers["x-iv"] as string | undefined; -// const method = (req.headers["x-method"] as string) ?? "manual"; -// const extension = req.headers["x-extension"] as string | undefined; -// -// -// if (!generatedId || !encryptedAesKeyHex || !ivHex || !agentId || !status) { -// return res.status(400).json({error: "Missing required headers/params"}); -// } -// -// if (!isUuidv4(generatedId)) { -// return res.status(400).json({error: "generatedId is not a valid UUID"}); -// } -// -// const agent = await db.query.agent.findFirst({ -// where: eq(drizzleDb.schemas.agent.id, agentId), -// }); -// -// if (!agent) { -// return res.status(404).json({error: "Agent not found"}); -// } -// -// -// const database = await db.query.database.findFirst({ -// where: eq(drizzleDb.schemas.database.agentDatabaseId, generatedId), -// with: { -// project: true, -// storagePolicies: true, -// }, -// }); -// -// if (!database) { -// return res.status(404).json({error: "Database not found"}); -// } -// -// let backup: Backup | null | undefined = null; -// -// if (method === "automatic") { -// [backup] = await db -// .insert(drizzleDb.schemas.backup) -// .values({ -// status: "ongoing", -// databaseId: database.id, -// }) -// .returning(); -// } else { -// backup = await db.query.backup.findFirst({ -// where: and( -// eq(drizzleDb.schemas.backup.databaseId, database.id), -// eq(drizzleDb.schemas.backup.status, "ongoing") -// ), -// }); -// } -// -// if (!backup) { -// return res.status(404).json({error: "Backup not found"}); -// } -// -// -// if (status === "success") { -// -// const decipher = createDecryptionStream(encryptedAesKeyHex, ivHex); -// -// const fileExt = extension || getFileExtension(database.dbms); -// const fileName = `${uuidv4()}${fileExt}`; -// -// const decryptedStream = req.pipe(decipher); -// -// const tmpPath = await saveStreamToTempFile(decryptedStream, fileName); -// -// -// -// -// if (!tmpPath) { -// return res.status(500).json({ -// error: "Unable to save tmp backup file", -// }); -// } -// -// uploadTempFileToProviders(backup, database, tmpPath, fileName); -// -// return res.json({success: true}); -// } else { -// await db -// .update(drizzleDb.schemas.backup) -// .set(withUpdatedAt({status: 'failed'})) -// .where(eq(drizzleDb.schemas.backup.id, backup.id)); -// -// eventEmitter.emit('modification', {update: true}); -// -// await sendNotificationsBackupRestore(database, "error_backup"); -// -// return res.status(200).json({ -// error: "Backup successfully updated with status failed", -// }); -// } -// } catch (err: any) { -// console.error("Upload error:", err); -// return res.status(500).json({ -// error: "Upload failed", -// detail: err.message, -// }); -// } -// }); -router.post("/:agentId", async (req: Request, res: Response) => { - const agentId = req.params.agentId as string | undefined; - const generatedId = req.headers["x-generated-id"] as string | undefined; - const status = req.headers["x-status"] as string | undefined; - const encryptedAesKeyHex = req.headers["x-aes-key"] as string | undefined; - const ivHex = req.headers["x-iv"] as string | undefined; - const method = (req.headers["x-method"] as string) ?? "manual"; - const extension = req.headers["x-extension"] as string | undefined; - - if (!generatedId || !encryptedAesKeyHex || !ivHex || !agentId || !status) { - return res.status(400).json({error: "Missing required headers/params"}); - } - if (!isUuidv4(generatedId)) { - return res.status(400).json({error: "generatedId is not a valid UUID"}); - } - - const agent = await db.query.agent.findFirst({ where: eq(drizzleDb.schemas.agent.id, agentId) }); - if (!agent) return res.status(404).json({error: "Agent not found"}); - - const database = await db.query.database.findFirst({ - where: eq(drizzleDb.schemas.database.agentDatabaseId, generatedId), - with: { project: true, storagePolicies: true }, - }); - if (!database) return res.status(404).json({error: "Database not found"}); - - let backup: Backup | null = null; - if (method === "automatic") { - [backup] = await db.insert(drizzleDb.schemas.backup).values({ status: "ongoing", databaseId: database.id }).returning(); - } else { - backup = await db.query.backup.findFirst({ - where: and( - eq(drizzleDb.schemas.backup.databaseId, database.id), - eq(drizzleDb.schemas.backup.status, "ongoing") - ), - }); - } - if (!backup) return res.status(404).json({error: "Backup not found"}); - - res.status(202).json({success: true, message: "Backup received, processing in background"}); - - (async () => { - try { - if (status === "success") { - const decipher = createDecryptionStream(encryptedAesKeyHex!, ivHex!); - const fileExt = extension || getFileExtension(database.dbms); - const fileName = `${uuidv4()}${fileExt}`; - const decryptedStream = req.pipe(decipher); - const tmpPath = await saveStreamToTempFile(decryptedStream, fileName); - - if (!tmpPath) throw new Error("Unable to save tmp backup file"); - - await uploadTempFileToProviders(backup!, database, tmpPath, fileName); - } else { - await db.update(drizzleDb.schemas.backup) - .set(withUpdatedAt({status: 'failed'})) - .where(eq(drizzleDb.schemas.backup.id, backup!.id)); - - eventEmitter.emit('modification', {update: true}); - await sendNotificationsBackupRestore(database, "error_backup"); - } - } catch (err) { - console.error("Background backup processing failed:", err); - try { - await db.update(drizzleDb.schemas.backup) - .set(withUpdatedAt({status: 'failed'})) - .where(eq(drizzleDb.schemas.backup.id, backup!.id)); - } catch (_) { - } - } - })(); -}); - - - -export default router; - - diff --git a/src/features/dashboard/backup/columns.tsx b/src/features/dashboard/backup/columns.tsx index ba8aaa9e..0995e5cb 100644 --- a/src/features/dashboard/backup/columns.tsx +++ b/src/features/dashboard/backup/columns.tsx @@ -8,7 +8,7 @@ import {cn} from "@/lib/utils"; import {Tooltip, TooltipContent, TooltipProvider, TooltipTrigger} from "@/components/ui/tooltip"; import {MemberWithUser} from "@/db/schema/03_organization"; import {formatLocalizedDate} from "@/utils/date-formatting"; -import {formatBytes, isImportedFilename} from "@/utils/text"; +import {formatBytes} from "@/utils/text"; import {DatabaseActionsCell} from "@/components/wrappers/dashboard/database/backup/actions/backup-actions-cell"; import { Badge as BadgeC } from "@/components/ui/badge"; @@ -22,7 +22,17 @@ export function backupColumns( { id: "availability", cell: ({row}) => { - const colorStatus = row.original.deletedAt != null ? "bg-red-400 border-red-600" : "bg-green-400 border-green-600"; + const statusColors: Record = { + waiting: "bg-gray-400 border-gray-600", + ongoing: "bg-orange-400 border-orange-600", + success: "bg-green-400 border-green-600", + }; + + const colorStatus = + row.original.deletedAt != null + ? "bg-red-400 border-red-600" + : statusColors[row.original.status] ?? "bg-gray-400 border-gray-600"; + return ( diff --git a/src/features/dashboard/restore/restore.action.ts b/src/features/dashboard/restore/restore.action.ts index 6ae72a74..794b8e58 100644 --- a/src/features/dashboard/restore/restore.action.ts +++ b/src/features/dashboard/restore/restore.action.ts @@ -7,12 +7,6 @@ import * as drizzleDb from "@/db"; import {db} from "@/db"; import {and, eq} from "drizzle-orm"; import {Backup, Restoration} from "@/db/schema/07_database"; -import { - deleteFileS3Private, - deleteLocalPrivate, -} from "@/features/upload/private/upload.action"; -import {env} from "@/env.mjs"; -import {withUpdatedAt} from "@/db/utils"; export const deleteRestoreAction = userAction .schema( @@ -48,84 +42,6 @@ export const deleteRestoreAction = userAction } }); - -export const deleteBackupAction = userAction - .schema( - z.object({ - backupId: z.string(), - databaseId: z.string(), - projectSlug: z.string(), - status: z.enum(["ongoing", "failed", "success", "waiting"]), - file: z.string(), - }) - ) - .action(async ({parsedInput}): Promise> => { - try { - - - const [settings] = await db.select().from(drizzleDb.schemas.setting).where(eq(drizzleDb.schemas.setting.name, "system")).limit(1); - if (!settings) { - return { - success: false, - actionError: { - message: "No settings found.", - status: 404, - cause: "No settings found.", - messageParams: {message: "Error deleting the backup"}, - }, - }; - } - - await db - .update(drizzleDb.schemas.backup) - .set(withUpdatedAt({ - deletedAt: new Date(), - status: parsedInput.status == "ongoing" ? "failed" : parsedInput.status - })) - .where(and(eq(drizzleDb.schemas.backup.id, parsedInput.backupId), eq(drizzleDb.schemas.backup.databaseId, parsedInput.databaseId))) - - let success: boolean, message: string; - - if (parsedInput.file) { - const result = - settings.storage === "local" - ? await deleteLocalPrivate(parsedInput.file) - : await deleteFileS3Private(`${parsedInput.projectSlug}/${parsedInput.file}`, env.S3_BUCKET_NAME!); - - ({success, message} = result); - - if (!success) { - return { - success: false, - actionError: { - message: message, - status: 404, - cause: "Unable to delete backup from storage", - messageParams: {message: "Error deleting the backup"}, - }, - }; - } - } - return { - success: true, - actionSuccess: { - message: `Backup deleted successfully (ref: ${parsedInput.backupId}).`, - }, - }; - } catch (error) { - return { - success: false, - actionError: { - message: "Failed to delete backup.", - status: 500, - cause: error instanceof Error ? error.message : "Unknown error", - messageParams: {message: "Error deleting the backup"}, - }, - }; - } - }); - - export const rerunRestorationAction = userAction .schema( z.object({ @@ -175,46 +91,3 @@ export const rerunRestorationAction = userAction }; } }); - - -export const createRestorationAction = userAction - .schema( - z.object({ - backupId: z.string(), - databaseId: z.string(), - }) - ) - .action(async ({parsedInput}): Promise> => { - try { - const restorationData = await db - .insert(drizzleDb.schemas.restoration) - .values({ - databaseId: parsedInput.databaseId, - backupId: parsedInput.backupId, - status: "waiting", - }) - .returning() - .execute(); - - const createdRestoration = restorationData[0]; - - return { - success: true, - value: createdRestoration, - actionSuccess: { - message: "Restoration has been successfully created.", - messageParams: {restorationId: createdRestoration.id}, - }, - }; - } catch (error) { - return { - success: false, - actionError: { - message: "Failed to create restoration.", - status: 500, - cause: error instanceof Error ? error.message : "Unknown error", - messageParams: {message: "Error creating the restoration"}, - }, - }; - } - }); diff --git a/src/features/keys/keys.action.ts b/src/features/keys/keys.action.ts index 894595fe..5f935554 100644 --- a/src/features/keys/keys.action.ts +++ b/src/features/keys/keys.action.ts @@ -1,4 +1,6 @@ import fs from "node:fs"; +import {env} from "@/env.mjs"; +import path from "path"; /** @@ -6,7 +8,8 @@ import fs from "node:fs"; */ export function getPublicServerKeyContent() { try { - return fs.readFileSync("private/keys/server_public.pem", "utf8"); + const keyPath = path.join(env.PRIVATE_PATH, '/keys/server_public.pem') + return fs.readFileSync(keyPath, "utf8"); } catch (error: any) { console.error("Error :", error); return { @@ -22,7 +25,8 @@ export function getPublicServerKeyContent() { */ export function getMasterServerKeyContent() { try { - return fs.readFileSync("private/keys/master_key.bin"); + const keyPath = path.join(env.PRIVATE_PATH, '/keys/master_key.bin') + return fs.readFileSync(keyPath); } catch (error: any) { console.error("Error :", error); return { diff --git a/src/features/projects/components/project.dialog.tsx b/src/features/projects/components/project.dialog.tsx index 5076e96c..9d8ee6d1 100644 --- a/src/features/projects/components/project.dialog.tsx +++ b/src/features/projects/components/project.dialog.tsx @@ -16,6 +16,7 @@ import {Organization} from "@/db/schema/03_organization"; import {ProjectWith} from "@/db/schema/06_project"; import {GearIcon} from "@radix-ui/react-icons"; import {EmptyStatePlaceholder} from "@/components/wrappers/common/empty-state-placeholder"; +import {useRouter} from "next/navigation"; type ProjectDialogProps = { databases: DatabaseWith[]; @@ -33,6 +34,7 @@ export const ProjectDialog = ({ isEmpty = false }: ProjectDialogProps) => { const [open, setOpen] = useState(false); + const router = useRouter(); return ( @@ -60,7 +62,10 @@ export const ProjectDialog = ({ {isEdit ? `Edit ${project?.name}` : "Create new project"} setOpen(false)} + onSuccess={() => { + setOpen(false) + router.refresh() + }} databases={databases} organization={organization} defaultValues={project ? { diff --git a/src/features/projects/components/project.form.tsx b/src/features/projects/components/project.form.tsx index 67b8726a..e1cad22b 100644 --- a/src/features/projects/components/project.form.tsx +++ b/src/features/projects/components/project.form.tsx @@ -22,11 +22,6 @@ export type projectFormProps = { }; - - - - - export const ProjectForm = (props: projectFormProps) => { const router = useRouter(); const queryClient = useQueryClient(); diff --git a/src/features/storages/dispatch.ts b/src/features/storages/dispatch.ts index 5dba383f..32b87fc8 100644 --- a/src/features/storages/dispatch.ts +++ b/src/features/storages/dispatch.ts @@ -74,7 +74,10 @@ export async function dispatchStorage( }; } - if (channelData) channel = {...channelData, config: channelData.config as Json}; + if (channelData) { + // @ts-ignore + channel = {...channelData, config: channelData.config as Json}; + } if (!channel) { return { diff --git a/src/features/storages/providers/local.ts b/src/features/storages/providers/local.ts index aba89bd2..e9d803e9 100644 --- a/src/features/storages/providers/local.ts +++ b/src/features/storages/providers/local.ts @@ -5,15 +5,17 @@ import {StorageDeleteInput, StorageGetInput, StorageMetaData, StorageResult, Sto import fs from "node:fs"; import {generateFileUrl} from "@/features/storages/helpers"; import {Readable} from "node:stream"; +import {env} from "@/env.mjs"; -const BASE_DIR = "/private/uploads/"; +const BASE_DIR = path.join(env.PRIVATE_PATH, '/uploads') export async function uploadLocal( config: { baseDir?: string }, input: { data: StorageUploadInput; metadata?: StorageMetaData } ): Promise { - const base = config.baseDir || BASE_DIR; - const fullPath = path.join(process.cwd(), base, input.data.path); + const base = config.baseDir ? path.join(process.cwd(), config.baseDir ?? "") : BASE_DIR; + + const fullPath = path.join(base, input.data.path); const dir = path.dirname(fullPath); await mkdir(dir, { recursive: true }); @@ -52,8 +54,9 @@ export async function getLocal( config: { baseDir?: string }, input: { data: StorageGetInput; metadata: StorageMetaData } ): Promise { - const base = config.baseDir || BASE_DIR; - const filePath = path.join(process.cwd(), base, input.data.path); + + const base = config.baseDir ? path.join(process.cwd(), config.baseDir ?? "") : BASE_DIR; + const filePath = path.join(base, input.data.path); if (!fs.existsSync(filePath)) { console.error("File not found at:", filePath); @@ -108,8 +111,10 @@ export async function deleteLocal( config: { baseDir?: string }, input: { data: StorageDeleteInput, metadata?: StorageMetaData } ): Promise { - const base = config.baseDir || BASE_DIR; - const fullPath = path.join(process.cwd(), base, input.data.path); + + const base = config.baseDir ? path.join(process.cwd(), config.baseDir ?? "") : BASE_DIR; + const fullPath = path.join(base, input.data.path); + await unlink(fullPath); return { success: true, @@ -121,8 +126,8 @@ export async function pingLocal( config: { baseDir?: string } ): Promise { - const base = config.baseDir || BASE_DIR; - const fullPath = path.join(process.cwd(), base, "ping.txt"); + const base = path.join(process.cwd(), config.baseDir ?? "") || BASE_DIR; + const fullPath = path.join(base, "ping.txt"); await fs.promises.writeFile(fullPath, "ping"); await fs.promises.readFile(fullPath); diff --git a/src/features/updates/hooks/use-update-check.ts b/src/features/updates/hooks/use-update-check.ts index 43adf34f..43fa2ad4 100644 --- a/src/features/updates/hooks/use-update-check.ts +++ b/src/features/updates/hooks/use-update-check.ts @@ -19,8 +19,6 @@ export const useUpdateCheck = () => { staleTime: 1000 * 60 * 60, }); - console.log("newRelease", newRelease); - useEffect(() => { if (!newRelease) return; diff --git a/src/features/upload/private/upload.action.ts b/src/features/upload/private/upload.action.ts deleted file mode 100644 index 03642236..00000000 --- a/src/features/upload/private/upload.action.ts +++ /dev/null @@ -1,215 +0,0 @@ -"use server"; - -import {mkdir, writeFile} from "fs/promises"; -import path from "path"; -import * as fs from "node:fs"; -import {getServerUrl} from "@/utils/get-server-url"; -import {createPresignedUrlToDownload, deleteFileFromBucket, saveFileInBucket} from "@/utils/s3-file-management"; -import {env} from "@/env.mjs"; -import {z} from "zod"; -import {ServerActionResult} from "@/types/action-type"; -import {unlink} from "fs/promises"; -import {action} from "@/lib/safe-actions/actions"; - -const privateLocalDir = "private/uploads/files/"; -const privateS3Dir = "backups/"; - -export async function uploadLocalPrivate(fileName: string, buffer: any) { - try { - await mkdir(path.join(process.cwd(), privateLocalDir), {recursive: true}); - await writeFile(path.join(process.cwd(), privateLocalDir, fileName), buffer); - - return { - success: true, - message: "File uploaded successfully", - filePath: path.join(privateLocalDir, fileName), - }; - } catch (error) { - console.error("Error occurred:", error); - throw new Error("An error occurred while importing the private file"); - } -} - -export async function uploadS3Private(fileName: string, buffer: any, bucketName: string) { - try { - - await saveFileInBucket({ - bucketName, - fileName: `${privateS3Dir}${fileName}`, - file: buffer, - }); - - return { - success: true, - filePath: `${privateS3Dir}${fileName}`, - message: "File uploaded successfully", - }; - } catch (error) { - console.error("Error occurred:", error); - throw new Error("An error occurred while importing the private file"); - } -} - - -export async function deleteFileS3Private(fileName: string, bucketName: string) { - try { - - await deleteFileFromBucket({ - bucketName, - fileName: `${privateS3Dir}${fileName}`, - }); - - return { - success: true, - message: "File deleted successfully", - }; - } catch (error) { - console.error("Error occurred:", error); - throw new Error("An error occurred while deleting the private file"); - } -} - - -/** - * Delete a file from local private storage - */ -export async function deleteLocalPrivate(fileName: string) { - try { - const filePath = path.join(process.cwd(), privateLocalDir, fileName); - await unlink(filePath); - - return { - success: true, - message: `File '${fileName}' deleted successfully`, - }; - } catch (error: any) { - if (error.code === "ENOENT") { - return { - success: false, - message: `File '${fileName}' not found`, - }; - } - - console.error("Error occurred while deleting file:", error); - throw new Error("An error occurred while deleting the file"); - } -} - - -// export async function getFileUrlPresignedLocal(fileName: string) { -// try { -// const filePath = path.join(privateLocalDir, fileName); -// await mkdir(path.join(process.cwd(), privateLocalDir), {recursive: true}); -// -// if (!fs.existsSync(filePath)) { -// console.error("File not found at:", filePath); -// return `File not found at: ${filePath}`; -// } -// const crypto = require("crypto"); -// const baseUrl = getServerUrl(); -// -// const expiresAt = Date.now() + 60 * 1000; // expires in 1 minute -// const token = crypto.createHash("sha256").update(`${fileName}${expiresAt}`).digest("hex"); -// return `${baseUrl}/api/files/${fileName}?token=${token}&expires=${expiresAt}`; -// } catch (error) { -// throw error; -// } -// } - -export async function getFileUrlPresignedS3(fileName: string) { - try { - return await createPresignedUrlToDownload({ - bucketName: env.S3_BUCKET_NAME!, - fileName: fileName, - }); - } catch (error) { - throw error; - } -} - - -export const getFileUrlPresignedLocal = action - .schema(z.object({ - dir: z.string().optional(), - fileName: z.string() - })) - .action(async ({parsedInput}): Promise> => { - try { - const filePath = path.join(parsedInput.dir ? parsedInput.dir : privateLocalDir, parsedInput.fileName); - await mkdir(path.join(process.cwd(), privateLocalDir), {recursive: true}); - - if (!fs.existsSync(filePath)) { - console.error("File not found at:", filePath); - throw new Error(`File not found at: ${filePath}`); - } - const crypto = require("crypto"); - const baseUrl = getServerUrl(); - - const expiresAt = Date.now() + 60 * 1000; // expires in 1 minute - const token = crypto.createHash("sha256").update(`${parsedInput.fileName}${expiresAt}`).digest("hex"); - - return { - success: true, - value: `${baseUrl}/api/files/${parsedInput.fileName}?token=${token}&expires=${expiresAt}`, - actionSuccess: { - message: "Successfully retrieved presigned URL Local", - messageParams: {fileName: parsedInput.fileName}, - }, - }; - } catch (error) { - - return { - success: false, - actionError: { - message: "Failed to generate presigned URL", - status: 500, - cause: error instanceof Error ? error.message : "Unknown error", - messageParams: {fileName: parsedInput.fileName}, - }, - }; - } - }); - - -export const getFileUrlPreSignedS3Action = action - .schema(z.string()) - .action(async ({parsedInput}): Promise> => { - try { - const data = await createPresignedUrlToDownload({ - bucketName: env.S3_BUCKET_NAME!, - fileName: parsedInput, - }); - - return { - success: true, - value: data.url, - actionSuccess: { - message: "Successfully retrieved presigned URL", - messageParams: {fileName: parsedInput}, - }, - }; - } catch (error) { - const isNotFound = error instanceof Error && error.message.includes("File does not exist"); - - const logContext = { - file: parsedInput, - reason: error instanceof Error ? error.message : "Unknown", - }; - - console.error("Presigned URL generation failed:", logContext); - - return { - success: false, - actionError: { - message: isNotFound - ? "File not found in S3 bucket" - : "Failed to generate presigned URL", - status: isNotFound ? 404 : 500, - cause: error instanceof Error ? error.message : "Unknown error", - messageParams: {fileName: parsedInput}, - }, - }; - } - }); - - diff --git a/src/lib/auth/auth.ts b/src/lib/auth/auth.ts index 9f982f1d..21a9ecf1 100644 --- a/src/lib/auth/auth.ts +++ b/src/lib/auth/auth.ts @@ -269,6 +269,8 @@ export const auth = betterAuth({ session: { create: { before: async (session, context) => { + + const userId = session.userId; let memberships = await db.query.member.findMany({ @@ -308,6 +310,9 @@ export const auth = betterAuth({ // } // }, after: async (session) => { + + console.log("session", session); + const user = await db.query.user.findFirst({ where: eq(drizzleDb.schemas.user.id, session.userId), }); @@ -321,6 +326,12 @@ export const auth = betterAuth({ return; } + const lastDiff = user.lastConnectedAt + ? new Date(session.createdAt).getTime() - new Date(user.lastConnectedAt).getTime() + : Infinity; + + if (lastDiff < 30000) return; + if (user.role === "pending") return; const deviceInfo = getDeviceDetails(session.userAgent); @@ -605,7 +616,8 @@ export const checkSlugOrganization = async (slug: string) => { }); return status; - } catch {} + } catch { + } }; export const getActiveMember = async () => { @@ -628,5 +640,6 @@ export const setActiveOrganization = async (slug: string) => { organizationSlug: slug, }, }); - } catch {} + } catch { + } }; diff --git a/src/lib/tasks/cleaning/index.ts b/src/lib/tasks/cleaning/index.ts new file mode 100644 index 00000000..192f6d4a --- /dev/null +++ b/src/lib/tasks/cleaning/index.ts @@ -0,0 +1,28 @@ +import {db} from "@/db"; +import {and, eq, isNotNull, isNull} from "drizzle-orm"; +import * as drizzleDb from "@/db"; +import {withUpdatedAt} from "@/db/utils"; + +export const backupCleanTask = async () => { + try { + const backups = await db.query.backup.findMany({ + where: and( + isNotNull(drizzleDb.schemas.backup.deletedAt), + eq(drizzleDb.schemas.backup.status, "ongoing") + ) + }); + + console.log(`Backups to clean: ${backups.length}`); + + for (const backup of backups) { + await db.update(drizzleDb.schemas.backup).set(withUpdatedAt({ + status: "failed", + })) + .where(eq(drizzleDb.schemas.backup.id, backup.id)); + } + + } catch (e: any) { + console.error("Backup cleanup failed:", e); + throw e; + } +}; \ No newline at end of file diff --git a/src/lib/tasks/database/utils/delete.ts b/src/lib/tasks/database/utils/delete.ts index 5bb1654e..09c2992d 100644 --- a/src/lib/tasks/database/utils/delete.ts +++ b/src/lib/tasks/database/utils/delete.ts @@ -70,6 +70,8 @@ export const deleteBackupCronAction = action .update(drizzleDb.schemas.backup) .set(withUpdatedAt({ deletedAt: new Date(), + status: backup.status == "ongoing" ? "failed" : backup.status + })) .where(and(eq(drizzleDb.schemas.backup.id, parsedInput.backupId), eq(drizzleDb.schemas.backup.databaseId, parsedInput.databaseId))) diff --git a/src/lib/tasks/index.ts b/src/lib/tasks/index.ts index a4d2566c..430bbaae 100644 --- a/src/lib/tasks/index.ts +++ b/src/lib/tasks/index.ts @@ -1,6 +1,7 @@ import cron from "node-cron"; import {retentionCleanTask} from "@/lib/tasks/database"; import {env} from "@/env.mjs"; +import {backupCleanTask} from "@/lib/tasks/cleaning"; export const retentionJob = cron.schedule(env.RETENTION_CRON, async () => { @@ -10,4 +11,13 @@ export const retentionJob = cron.schedule(env.RETENTION_CRON, async () => { } catch (err) { console.error(`[CRON] Error:`, err); } +}); + +export const cleaningJob = cron.schedule("* * * * *", async () => { + try { + console.log("Cleaning Job : Starting task"); + await backupCleanTask(); + } catch (err) { + console.error(`[CRON] Error:`, err); + } }); \ No newline at end of file diff --git a/src/utils/common.ts b/src/utils/common.ts index c0dad09f..5b2a50db 100644 --- a/src/utils/common.ts +++ b/src/utils/common.ts @@ -39,3 +39,13 @@ export function buildOrganizationWithMembers( +export function getFileExtension(dbType: string) { + switch (dbType) { + case "postgresql": + return ".dump"; + case "mysql": + return ".sql"; + default: + return ".dump"; + } +} \ No newline at end of file diff --git a/src/utils/edge_key.ts b/src/utils/edge_key.ts index 7be159c6..fee5ef21 100644 --- a/src/utils/edge_key.ts +++ b/src/utils/edge_key.ts @@ -3,7 +3,6 @@ import {getMasterServerKeyContent} from "@/features/keys/keys.action"; export async function generateEdgeKey(serverUrl: string, agentId: string): Promise { const masterKey = getMasterServerKeyContent() - console.log("Master server key: ", masterKey) const edgeKeyData = { serverUrl, agentId, diff --git a/src/utils/init.ts b/src/utils/init.ts index 9344b763..75dbaf58 100644 --- a/src/utils/init.ts +++ b/src/utils/init.ts @@ -2,7 +2,7 @@ import {env} from "@/env.mjs"; import {db, makeMigration} from "@/db"; import {eq} from "drizzle-orm"; import * as drizzleDb from "@/db"; -import {retentionJob} from "@/lib/tasks"; +import {cleaningJob, retentionJob} from "@/lib/tasks"; import {generateRSAKeys, getOrCreateMasterKey} from "@/utils/rsa-keys"; import {StorageProviderKind} from "@/features/storages/types"; @@ -17,6 +17,7 @@ export async function init() { await createSettingsIfNotExist() console.log("====Initialization completed===="); await setupCronJobs() + await setupCleaningJobs() } async function setupCronJobs() { @@ -25,6 +26,12 @@ async function setupCronJobs() { console.log("==== Cron job started ===="); } +async function setupCleaningJobs() { + console.log("==== Setting up Cleaning Jobs ===="); + cleaningJob.start(); + console.log("==== Cleaning job started ===="); +} + async function createSettingsIfNotExist() { await db.transaction(async (tx) => { const systemSettingsValues = { diff --git a/src/utils/rsa-keys.ts b/src/utils/rsa-keys.ts index 4f51effc..f3e5cdd3 100644 --- a/src/utils/rsa-keys.ts +++ b/src/utils/rsa-keys.ts @@ -3,6 +3,7 @@ import path from 'path'; import {generateKeyPair} from 'crypto'; import {promisify} from 'util'; import {randomBytes} from 'crypto'; +import {env} from "@/env.mjs"; const generateKeyPairAsync = promisify(generateKeyPair); @@ -13,9 +14,10 @@ const generateKeyPairAsync = promisify(generateKeyPair); * @param {string} [dir] path to directory * @returns {Promise<{privateKeyPath:string, publicKeyPath:string}>} */ -export async function generateRSAKeys(dir = path.join(process.cwd(), 'private/keys')) { +export async function generateRSAKeys(dir = path.join(env.PRIVATE_PATH, '/keys')) { await fs.mkdir(dir, {recursive: true}); + const privateKeyPath = path.join(dir, 'server_private.pem'); const publicKeyPath = path.join(dir, 'server_public.pem'); @@ -47,7 +49,7 @@ export async function generateRSAKeys(dir = path.join(process.cwd(), 'private/ke * @param {string} [filePath] Path to store the key * @returns {Promise} The master key */ -export async function getOrCreateMasterKey(filePath = path.join(process.cwd(), 'private/keys', 'master_key.bin')) { +export async function getOrCreateMasterKey(filePath = path.join(env.PRIVATE_PATH, '/keys', 'master_key.bin')) { await fs.mkdir(path.dirname(filePath), {recursive: true}); @@ -61,8 +63,6 @@ export async function getOrCreateMasterKey(filePath = path.join(process.cwd(), ' const key = randomBytes(32); // 256-bit key - console.log(key) - await fs.writeFile(filePath, key, {mode: 0o600}); console.log(`Master key generated at ${filePath}`); diff --git a/src/utils/s3-file-management.ts b/src/utils/s3-file-management.ts deleted file mode 100644 index 1a07aeba..00000000 --- a/src/utils/s3-file-management.ts +++ /dev/null @@ -1,243 +0,0 @@ -import * as Minio from "minio"; -import {env} from "@/env.mjs"; -import internal from "node:stream"; -import stream from "node:stream"; -import * as drizzleDb from "@/db"; -import {db} from "@/db"; -import {eq} from "drizzle-orm"; - - -async function getS3Client() { - const settings = await db - .select() - .from(drizzleDb.schemas.setting) - .where(eq(drizzleDb.schemas.setting.name, "system")) - .then((res) => res[0]); - - if (!settings) { - throw new Error("S3 settings not found in database."); - } - - const baseConfig = { - endPoint: settings?.s3EndPointUrl ?? "", - accessKey: settings?.s3AccessKeyId ?? "", - secretKey: settings?.s3SecretAccessKey ?? "", - }; - - return new Minio.Client({ - ...baseConfig, - port: Number(env.S3_PORT ?? 0), - useSSL: env.S3_USE_SSL === "true", - }); -} - -export async function checkMinioAlive() { - try { - console.log("Check MinioAlive"); - const s3Client = await getS3Client(); - const buckets = await s3Client.listBuckets(); - console.log("MinIO is up and running. Buckets:", buckets); - return {message: true}; - } catch (error) { - console.error("Error connecting to MinIO:", error); - return {error: error}; - } -} - -export async function createBucketIfNotExists(bucketName: string) { - const s3Client = await getS3Client(); - - const bucketExists = await s3Client.bucketExists(bucketName); - if (!bucketExists) { - console.log(`Creating bucket ${bucketName}`); - await s3Client.makeBucket(bucketName); - } -} - -/** - * Delete a file from a bucket - * @param bucketName name of the bucket - * @param fileName name of the file - * @returns true if deleted, false if not - */ -export async function deleteFileFromBucket({ - bucketName, - fileName, - }: { - bucketName: string; - fileName: string; -}): Promise { - const s3Client = await getS3Client(); - - try { - const fileExists = await checkFileExistsInBucket({ bucketName, fileName }); - - if (!fileExists) { - console.warn(`File not found: ${bucketName}/${fileName}`); - return false; - } - - await s3Client.removeObject(bucketName, fileName); - console.log(`Deleted file: ${bucketName}/${fileName}`); - return true; - } catch (error: any) { - console.error("Error deleting file from bucket:", { - bucketName, - fileName, - error: error.message, - }); - return false; - } -} - - -/** - * Save file in S3 bucket - * @param bucketName name of the bucket - * @param fileName name of the file - * @param file file to save - */ -export async function saveFileInBucket({bucketName, fileName, file}: { - bucketName: string; - fileName: string; - file: Buffer | internal.Readable -}) { - await checkMinioAlive(); - await createBucketIfNotExists(bucketName); - const fileExists = await checkFileExistsInBucket({ - bucketName, - fileName, - }); - console.log("File exists:", fileExists); - if (fileExists) { - throw new Error("File already exists"); - } - const s3Client = await getS3Client(); - - return await s3Client.putObject(bucketName, fileName, file); -} - - -export async function getObjectFromClient({ - bucketName, - fileName, - }: { - bucketName: string; - fileName: string; -}): Promise { - const s3 = await getS3Client(); - return await s3.getObject(bucketName, fileName); -} - - - -export async function checkFileExistsInBucket({ - bucketName, - fileName, - }: { - bucketName: string; - fileName: string; -}): Promise { - const s3 = await getS3Client(); - try { - const stat = await s3.statObject(bucketName, fileName); - return !!stat; - } catch (error: any) { - if (error.code === 'NoSuchKey' || error.message?.includes('not found')) { - return false; - } - return false; - } -} - -/** - * Generate presigned urls for uploading files to S3 - * @param files files to upload - * @returns promise with array of presigned urls - */ -export async function createPresignedUrlToUpload({ - bucketName, - fileName, - expiry = 60 * 60, // 1 hour - }: { - bucketName: string; - fileName: string; - expiry?: number; -}) { - // Create bucket if it doesn't exist - await createBucketIfNotExists(bucketName); - const s3Client = await getS3Client(); - - return await s3Client.presignedPutObject(bucketName, fileName, expiry); -} - -export async function createPublicBucket({bucketName}: { bucketName: string }) { - const s3Client = await getS3Client(); - - try { - const exists = await s3Client.bucketExists(bucketName); - if (!exists) { - await s3Client.makeBucket(bucketName); - console.log(`Bucket ${bucketName} created successfully.`); - } else { - console.log(`Bucket ${bucketName} already exists.`); - } - - const policy = { - Version: "2012-10-17", - Statement: [ - { - Effect: "Allow", - Principal: "*", - Action: "s3:GetObject", - Resource: `arn:aws:s3:::${bucketName}/*`, - }, - ], - }; - - await s3Client.setBucketPolicy(bucketName, JSON.stringify(policy)); - console.log(`Bucket ${bucketName} is now public.`); - } catch (error) { - console.error("Error creating bucket:", error); - } -} - -/** - * Generate a presigned URL for downloading a file from a private S3 bucket - * @param bucketName name of the bucket - * @param fileName name of the file - * @param expiry expiry time in seconds (default 1 hour) - * @returns presigned download URL - */ -export async function createPresignedUrlToDownload({ - bucketName, - fileName, - expiry = 60 * 60, - }: { - bucketName: string; - fileName: string; - expiry?: number; -}) { - try { - const s3Client = await getS3Client(); - - console.debug("Checking if file exists in bucket:", {bucketName, fileName}); - - const fileExists = await checkFileExistsInBucket({bucketName, fileName}); - - if (!fileExists) { - console.warn("File does not exist:", {bucketName, fileName}); - throw new Error("File does not exist in the bucket."); - } - const presignedUrl = await s3Client.presignedGetObject(bucketName, fileName, expiry); - console.debug("Generated pre signed URL:", presignedUrl); - return {url: presignedUrl}; - } catch (err: any) { - console.error("Error in createPreSignedUrlToDownload:", { - bucketName, - fileName, - errorMessage: err?.message, - }); - throw {error: err.message ?? "Unknown error"}; - } -}