Merge pull request #369 from Portabase/fix/upload-tar-gz

fix: upload-tar-gz
This commit is contained in:
Charles GTE
2026-07-11 11:35:54 +02:00
committed by GitHub
7 changed files with 375 additions and 31 deletions
+2
View File
@@ -83,6 +83,7 @@
"sharp": "^0.34.5",
"sonner": "^2.0.7",
"tailwind-merge": "^3.6.0",
"tar-stream": "^3.2.0",
"uuid": "^11.1.1",
"zod": "4.3.6"
},
@@ -98,6 +99,7 @@
"@types/pg": "^8.20.0",
"@types/react": "^19.2.15",
"@types/react-dom": "^19.2.3",
"@types/tar-stream": "^3.1.4",
"drizzle-kit": "^0.31.10",
"eslint": "^9.39.4",
"eslint-config-next": "^16.2.6",
+148
View File
@@ -212,6 +212,9 @@ importers:
tailwind-merge:
specifier: ^3.6.0
version: 3.6.0
tar-stream:
specifier: ^3.2.0
version: 3.2.0
uuid:
specifier: ^11.1.1
version: 11.1.1
@@ -252,6 +255,9 @@ importers:
'@types/react-dom':
specifier: ^19.2.3
version: 19.2.3(@types/react@19.2.17)
'@types/tar-stream':
specifier: ^3.1.4
version: 3.1.4
drizzle-kit:
specifier: ^0.31.10
version: 0.31.10
@@ -3404,6 +3410,9 @@ packages:
'@types/request@2.48.13':
resolution: {integrity: sha512-FGJ6udDNUCjd19pp0Q3iTiDkwhYup7J8hpMW9c4k53NrccQFFWKRho6hvtPPEhnXWKvukfwAlB6DbDz4yhH5Gg==}
'@types/tar-stream@3.1.4':
resolution: {integrity: sha512-921gW0+g29mCJX0fRvqeHzBlE/XclDaAG0Ousy1LCghsOhvaKacDeRGEVzQP9IPfKn8Vysy7FEXAIxycpc/CMg==}
'@types/tough-cookie@4.0.5':
resolution: {integrity: sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==}
@@ -3851,6 +3860,14 @@ packages:
resolution: {integrity: sha512-qIj0G9wZbMGNLjLmg1PT6v2mE9AH2zlnADJD/2tC6E00hgmhUOfEB6greHPAfLRSufHqROIUTkw6E+M3lH0PTQ==}
engines: {node: '>= 0.4'}
b4a@1.8.1:
resolution: {integrity: sha512-aiqre1Nr0B/6DgE2N5vwTc+2/oQZ4Wh1t4NznYY4E00y8LCt6NqdRv81so00oo27D8MVKTpUa/MwUUtBLXCoDw==}
peerDependencies:
react-native-b4a: '*'
peerDependenciesMeta:
react-native-b4a:
optional: true
balanced-match@1.0.2:
resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==}
@@ -3858,6 +3875,43 @@ packages:
resolution: {integrity: sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==}
engines: {node: 18 || 20 || >=22}
bare-events@2.9.1:
resolution: {integrity: sha512-Z0oHEHAFDZkffN8Qc39zNZjQlMDkPJRyyyZieU1VH7u8c5S+qHZ2S8ixdKIAxEjfHO7FJxXmJWgteOghVanIsg==}
peerDependencies:
bare-abort-controller: '*'
peerDependenciesMeta:
bare-abort-controller:
optional: true
bare-fs@4.7.4:
resolution: {integrity: sha512-y1kC+ffIx/tPLdTE693uNjHfzTfr+ravR5tvWlMXe25nELbkqV400S71qHDwbkAQ1FVEZobB1NFRzFbCCcyBCQ==}
engines: {bare: '>=1.16.0'}
peerDependencies:
bare-buffer: '*'
peerDependenciesMeta:
bare-buffer:
optional: true
bare-path@3.1.1:
resolution: {integrity: sha512-JprUlveX3QjApC1cTpsUOiscADftCGVWkzitbHsRqv84hzYwYHw2mbluddsq5TvI8mH/8Ov1f4BiMAdcB0oYnQ==}
bare-stream@2.13.3:
resolution: {integrity: sha512-Kc+brLqvEqGkjyfiwJmImAOqLZL7OsoLKuavx+hJjgVV3nLTOjloJyPMFxjUPerGGHrNH0fLU06jjykMLWrERQ==}
peerDependencies:
bare-abort-controller: '*'
bare-buffer: '*'
bare-events: '*'
peerDependenciesMeta:
bare-abort-controller:
optional: true
bare-buffer:
optional: true
bare-events:
optional: true
bare-url@2.4.5:
resolution: {integrity: sha512-K+y9xF1tN+CdPu4qWwr0QiK1Al07eFPGYK5M2pDXcmHdMdgC/tT/bpmMe1hrmRHaidKLkXrC+cRNYf3XVDUhSQ==}
base64-js@1.5.1:
resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==}
@@ -4865,6 +4919,9 @@ packages:
eventemitter3@5.0.4:
resolution: {integrity: sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw==}
events-universal@1.0.1:
resolution: {integrity: sha512-LUd5euvbMLpwOF8m6ivPCbhQeSiYVNb8Vs0fQ8QjXo0JTkEHpz8pxdQf0gStltaPpw0Cca8b39KxvK9cfKRiAw==}
events@3.3.0:
resolution: {integrity: sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==}
engines: {node: '>=0.8.x'}
@@ -4910,6 +4967,9 @@ packages:
resolution: {integrity: sha512-jt2DW/aNFNwke7AUd+Z+e6pz39KO5rzdbbFCg2sGafS4mk13MI7Z8O5z9cADNn5lhGODIgLwug6TZO2ctf7kcw==}
engines: {node: '>=6.0.0'}
fast-fifo@1.3.2:
resolution: {integrity: sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==}
fast-glob@3.3.1:
resolution: {integrity: sha512-kNFPyjhh5cKjrUltxs+wFx+ZkbRaxxmZ+X0ZU31SOsxCEtP9VPgtq2teZw1DebupL5GmDaNQ6yKMMVcM41iqDg==}
engines: {node: '>=8.6.0'}
@@ -7062,6 +7122,9 @@ packages:
stream-shift@1.0.3:
resolution: {integrity: sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ==}
streamx@2.28.0:
resolution: {integrity: sha512-1Yowhzjf0ivGMrTIkY9hav5TxobO9qIVqUE41fiCGMGgc3CLlf4MY+9AHmZqBWgDTue0fY9zWjYFVyf6Diuobw==}
strict-uri-encode@2.0.0:
resolution: {integrity: sha512-QwiXZgpRcKkhTj2Scnn++4PKtWsH0kpzZ62L2R6c/LUVYv7hVnZqcg2+sMuT6R7Jusu1vviK/MFsu6kNJfWlEQ==}
engines: {node: '>=4'}
@@ -7186,15 +7249,24 @@ packages:
resolution: {integrity: sha512-uxc/zpqFg6x7C8vOE7lh6Lbda8eEL9zmVm/PLeTPBRhh1xCgdWaQ+J1CUieGpIfm2HdtsUpRv+HshiasBMcc6A==}
engines: {node: '>=6'}
tar-stream@3.2.0:
resolution: {integrity: sha512-ojzvCvVaNp6aOTFmG7jaRD0meowIAuPc3cMMhSgKiVWws1GyHbGd/xvnyuRKcKlMpt3qvxx6r0hreCNITP9hIg==}
teeny-request@9.0.0:
resolution: {integrity: sha512-resvxdc6Mgb7YEThw6G6bExlXKkv6+YbuzGg9xuXxSgxJF7Ozs+o8Y9+2R3sArdWdW8nOokoQb1yrpFB0pQK2g==}
engines: {node: '>=14'}
teex@1.0.1:
resolution: {integrity: sha512-eYE6iEI62Ni1H8oIa7KlDU6uQBtqr4Eajni3wX7rpfXD8ysFx8z0+dri+KWEPWpBsxXfxu58x/0jvTVT1ekOSg==}
terser@5.48.0:
resolution: {integrity: sha512-J/9An6vs9Us6wKRriSFXBWdRZapREHqFzdNUKk0pmu804EMR6dr6winwo7e5JDxN4xahxQsuysyYFwlwj4XN/Q==}
engines: {node: '>=10'}
hasBin: true
text-decoder@1.2.7:
resolution: {integrity: sha512-vlLytXkeP4xvEq2otHeJfSQIRyWxo/oZGEbXrtEEF9Hnmrdly59sUbzZ/QgyWuLYHctCHxFF4tRQZNQ9k60ExQ==}
thenify-all@1.6.0:
resolution: {integrity: sha512-RNxQH/qI8/t3thXJDwcstUO4zeqo64+Uy/+sNVRBx4Xn2OX+OZ9oP+iJnNFqplFra2ZUVeKCSa2oVWi3T4uVmA==}
engines: {node: '>=0.8'}
@@ -10480,6 +10552,10 @@ snapshots:
'@types/tough-cookie': 4.0.5
form-data: 2.5.6
'@types/tar-stream@3.1.4':
dependencies:
'@types/node': 22.20.0
'@types/tough-cookie@4.0.5': {}
'@types/webpack@5.28.5(esbuild@0.25.10)(postcss@8.5.10)':
@@ -10966,10 +11042,41 @@ snapshots:
axobject-query@4.1.0: {}
b4a@1.8.1: {}
balanced-match@1.0.2: {}
balanced-match@4.0.4: {}
bare-events@2.9.1: {}
bare-fs@4.7.4:
dependencies:
bare-events: 2.9.1
bare-path: 3.1.1
bare-stream: 2.13.3(bare-events@2.9.1)
bare-url: 2.4.5
fast-fifo: 1.3.2
transitivePeerDependencies:
- bare-abort-controller
- react-native-b4a
bare-path@3.1.1: {}
bare-stream@2.13.3(bare-events@2.9.1):
dependencies:
b4a: 1.8.1
streamx: 2.28.0
teex: 1.0.1
optionalDependencies:
bare-events: 2.9.1
transitivePeerDependencies:
- react-native-b4a
bare-url@2.4.5:
dependencies:
bare-path: 3.1.1
base64-js@1.5.1: {}
base64id@2.0.0: {}
@@ -12105,6 +12212,12 @@ snapshots:
eventemitter3@5.0.4: {}
events-universal@1.0.1:
dependencies:
bare-events: 2.9.1
transitivePeerDependencies:
- bare-abort-controller
events@3.3.0: {}
eventsource-parser@3.1.0: {}
@@ -12175,6 +12288,8 @@ snapshots:
fast-equals@5.4.0: {}
fast-fifo@1.3.2: {}
fast-glob@3.3.1:
dependencies:
'@nodelib/fs.stat': 2.0.5
@@ -14522,6 +14637,15 @@ snapshots:
stream-shift@1.0.3: {}
streamx@2.28.0:
dependencies:
events-universal: 1.0.1
fast-fifo: 1.3.2
text-decoder: 1.2.7
transitivePeerDependencies:
- bare-abort-controller
- react-native-b4a
strict-uri-encode@2.0.0: {}
string-width@4.2.3:
@@ -14693,6 +14817,17 @@ snapshots:
tapable@2.3.3: {}
tar-stream@3.2.0:
dependencies:
b4a: 1.8.1
bare-fs: 4.7.4
fast-fifo: 1.3.2
streamx: 2.28.0
transitivePeerDependencies:
- bare-abort-controller
- bare-buffer
- react-native-b4a
teeny-request@9.0.0:
dependencies:
http-proxy-agent: 5.0.0
@@ -14704,6 +14839,13 @@ snapshots:
- encoding
- supports-color
teex@1.0.1:
dependencies:
streamx: 2.28.0
transitivePeerDependencies:
- bare-abort-controller
- react-native-b4a
terser@5.48.0:
dependencies:
'@jridgewell/source-map': 0.3.11
@@ -14711,6 +14853,12 @@ snapshots:
commander: 2.20.3
source-map-support: 0.5.21
text-decoder@1.2.7:
dependencies:
b4a: 1.8.1
transitivePeerDependencies:
- react-native-b4a
thenify-all@1.6.0:
dependencies:
thenify: 3.3.1
+3 -3
View File
@@ -134,9 +134,9 @@ const getRootError = (
const errors = errorCodes.map((error) => {
switch (error) {
case "file-invalid-type":
const acceptedTypes = Object.values(limits.accept ?? {})
.flat()
.join(", ");
const acceptedTypes = Array.from(
new Set(Object.values(limits.accept ?? {}).flat()),
).join(", ");
return `only ${acceptedTypes} are allowed`;
case "file-too-large":
const maxMb = limits.maxSize
+37 -5
View File
@@ -8,7 +8,7 @@ import {v4 as uuidv4} from "uuid";
import {and, eq, isNull} from "drizzle-orm";
import {z} from "zod";
import {storeBackupFiles} from "@/features/storages/utils/storages.helpers";
import {getFileExtension} from "@/utils/common";
import {inspectUpload, InvalidUploadError, packToTarGz} from "@/features/database/utils/archive-inspect";
export const uploadBackupAction = userAction
@@ -41,13 +41,45 @@ export const uploadBackupAction = userAction
};
}
const fileExtension = getFileExtension(database.dbms)
const MAX_UPLOAD_BYTES = 2 * 1024 * 1024 * 1024; // 2 GB
if (file.size > MAX_UPLOAD_BYTES) {
return {
success: false,
actionError: {
message: "File exceeds the 2 GB upload limit.",
status: 413,
cause: "File too large",
},
};
}
const arrayBuffer = await file.arrayBuffer();
const buffer = Buffer.from(arrayBuffer);
let inspection;
try {
inspection = await inspectUpload(buffer, database.dbms);
} catch (error) {
if (error instanceof InvalidUploadError) {
return {
success: false,
actionError: {
message: error.message,
status: 400,
cause: "Invalid backup archive",
},
};
}
throw error;
}
let fileBuffer: Buffer = buffer;
if (inspection.kind === "wrap" && inspection.innerName) {
fileBuffer = await packToTarGz(buffer, inspection.innerName);
}
const uuid = uuidv4();
const fileName = `${uuid}${fileExtension}`;
const buffer = Buffer.from(arrayBuffer);
const fileName = `${uuid}${inspection.storeExtension}`;
const [backup] = await db
@@ -59,7 +91,7 @@ export const uploadBackupAction = userAction
})
.returning();
await storeBackupFiles(backup, database, buffer, fileName)
await storeBackupFiles(backup, database, fileBuffer, fileName)
return {
success: true,
@@ -59,9 +59,9 @@ export const UploadBackupZone = ({onSuccessAction, database}: UploadRetentionZon
const acceptDbImportFiles = getFileHeadersBasedOnDbms(database.dbms)
const fileKindDescription = Object.values(acceptDbImportFiles)
.flat()
.join(", ");
const fileKindDescription = Array.from(
new Set(Object.values(acceptDbImportFiles).flat())
).join(", ");
return (
@@ -75,7 +75,7 @@ export const UploadBackupZone = ({onSuccessAction, database}: UploadRetentionZon
maxFiles={1}
description="Import database backup"
fileKind={`Database file (${fileKindDescription})`}
dragMessage="Click or drag a database dump here"
dragMessage="Click or drag a database dump or .tar.gz here"
onFileDropAction={(file: File) => setFile(file)}
/>
)}
@@ -0,0 +1,126 @@
import { Readable } from "node:stream";
import { createGunzip, createGzip } from "node:zlib";
import * as tar from "tar-stream";
import { getArchiveEntryMatcher, getFileExtension } from "@/utils/common";
import { logger } from "@/lib/logger";
const log = logger.child({ module: "features/database/archive-inspect" });
export type UploadKind = "raw" | "targz" | "wrap";
export type InspectResult = {
kind: UploadKind;
storeExtension: string;
innerName?: string;
};
export class InvalidUploadError extends Error {
constructor(message: string) {
super(message);
this.name = "InvalidUploadError";
}
}
const GZIP_MAGIC_0 = 0x1f;
const GZIP_MAGIC_1 = 0x8b;
function isGzip(buffer: Buffer): boolean {
return (
buffer.length >= 2 &&
buffer[0] === GZIP_MAGIC_0 &&
buffer[1] === GZIP_MAGIC_1
);
}
function listTarEntries(buffer: Buffer): Promise<string[]> {
return new Promise<string[]>((resolve, reject) => {
const names: string[] = [];
const extract = tar.extract();
extract.on("entry", (header, stream, next) => {
names.push(header.name);
stream.on("end", next);
stream.on("error", reject);
stream.resume();
});
extract.on("finish", () => resolve(names));
extract.on("error", reject);
Readable.from(buffer)
.pipe(createGunzip())
.on("error", reject)
.pipe(extract);
});
}
export async function inspectUpload(
buffer: Buffer,
dbms: string,
): Promise<InspectResult> {
log.info({ dbms, size: buffer.length }, "Inspecting uploaded backup");
if (!isGzip(buffer)) {
const innerName = `backup${getFileExtension(dbms)}`;
log.info(
{ dbms, innerName },
"Native (non-gzip) upload, will wrap into tar.gz before storing",
);
return { kind: "wrap", storeExtension: ".tar.gz", innerName };
}
let entries: string[];
try {
entries = await listTarEntries(buffer);
} catch (error) {
log.info(
{ dbms, error: error instanceof Error ? error.message : "unknown" },
"Gzip is not a tar archive, treating as raw dump",
);
return { kind: "raw", storeExtension: getFileExtension(dbms) };
}
if (entries.length === 0) {
log.info({ dbms }, "Tar archive has no entries, treating as raw dump");
return { kind: "raw", storeExtension: getFileExtension(dbms) };
}
const matcher = getArchiveEntryMatcher(dbms);
const hasValidEntry = entries.some((name) => matcher.test(name));
if (!hasValidEntry) {
log.warn(
{ dbms, entries },
"Tar archive does not contain a valid backup for dbms",
);
throw new InvalidUploadError(
`Archive does not contain a valid ${dbms} backup.`,
);
}
log.info(
{ dbms, entryCount: entries.length },
"Valid tar.gz backup detected",
);
return { kind: "targz", storeExtension: ".tar.gz" };
}
export function packToTarGz(buffer: Buffer, innerName: string): Promise<Buffer> {
return new Promise<Buffer>((resolve, reject) => {
const pack = tar.pack();
const gzip = createGzip();
const chunks: Buffer[] = [];
gzip.on("data", (chunk: Buffer) => chunks.push(chunk));
gzip.on("end", () => resolve(Buffer.concat(chunks)));
gzip.on("error", reject);
pack.on("error", reject);
pack.pipe(gzip);
pack.entry({ name: innerName }, buffer, (err) => {
if (err) {
reject(err);
return;
}
pack.finalize();
});
});
}
+55 -19
View File
@@ -50,41 +50,77 @@ export function getFileExtension(dbType: string) {
}
}
const GZIP_ACCEPT: Record<string, string[]> = {
"application/gzip": [".tar.gz", ".tgz"],
"application/x-gzip": [".tar.gz", ".tgz"],
};
function mergeAccept(
base: Record<string, string[]>,
extra: Record<string, string[]>,
): Record<string, string[]> {
const out: Record<string, string[]> = { ...base };
for (const [mime, exts] of Object.entries(extra)) {
out[mime] = Array.from(new Set([...(out[mime] ?? []), ...exts]));
}
return out;
}
export function getFileHeadersBasedOnDbms(
dbType: string,
): Record<string, string[]> {
let base: Record<string, string[]>;
switch (dbType) {
case "postgresql":
return {
"application/octet-stream": [".dump"],
};
base = { "application/octet-stream": [".dump"] };
break;
case "mysql":
case "mariadb":
return {
base = {
"application/sql": [".sql"],
"application/x-sql": [".sql"],
};
break;
case "mongodb":
return {
"application/gzip": [".archive.gz"],
};
base = { "application/gzip": [".archive.gz"] };
break;
case "firebird":
return {
"application/octet-stream": [".fbk"],
};
base = { "application/octet-stream": [".fbk"] };
break;
case "valkey":
case "redis":
return {
"application/octet-stream": [".rdb"],
};
base = { "application/octet-stream": [".rdb"] };
break;
case "sqlite":
return {
"application/octet-stream": [".backup"],
};
base = { "application/octet-stream": [".backup"] };
break;
case "mssql":
return {
"application/octet-stream": [".bacpac"],
};
base = { "application/octet-stream": [".bacpac"] };
break;
default:
throw new Error(`Unsupported database type: ${dbType}`);
}
return mergeAccept(base, GZIP_ACCEPT);
}
export function getArchiveEntryMatcher(dbType: string): RegExp {
switch (dbType) {
case "postgresql":
return /(\.dump$|\.dmp$|(^|\/)toc\.dat$)/i;
case "mysql":
case "mariadb":
return /\.sql$/i;
case "mongodb":
return /(\.archive(\.gz)?$|\.bson(\.gz)?$)/i;
case "valkey":
case "redis":
return /\.rdb$/i;
case "sqlite":
return /(\.backup$|\.sqlite$|\.db$)/i;
case "firebird":
return /\.fbk$/i;
case "mssql":
return /\.bacpac$/i;
default:
throw new Error(`Unsupported database type: ${dbType}`);
}