mirror of
https://github.com/Portabase/portabase.git
synced 2026-07-14 11:16:13 +02:00
Feature/google cloud storage (#316)
* environment setup and reproduce * rmv my dashboard * Stop tracking my-dashboard * gitignore: ignore all .env files * working UI, not yet tested * Update credential validation Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> * tested with fake-gcs-server * fix: migration * fix: migration, init for gcs and refactoring * fix: get object gcs --------- Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> Co-authored-by: Charles GTE <charlesgte31@gmail.com>
This commit is contained in:
Hayzie Chu
co-authored by
coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>
coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>
Charles GTE
coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>
coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>
Charles GTE
parent
79832f0d0b
commit
8f44071069
@@ -44,6 +44,10 @@ import {
|
||||
import {
|
||||
StorageBlobForm
|
||||
} from "@/features/channel/components/storages/az-blob.form";
|
||||
import {
|
||||
StorageGoogleCloudStorageForm
|
||||
} from "@/features/channel/components/storages/google-cloud-storage/google-cloud-storage.form";
|
||||
|
||||
|
||||
export type ChannelKind = "notification" | "storage";
|
||||
|
||||
@@ -109,6 +113,8 @@ export const renderChannelForm = (provider: string | undefined, form: UseFormRet
|
||||
return <StorageS3Form form={form}/>
|
||||
case "google-drive":
|
||||
return <StorageGoogleDriveForm form={form}/>
|
||||
case "google-cloud-storage":
|
||||
return <StorageGoogleCloudStorageForm form={form}/>
|
||||
case "blob":
|
||||
return <StorageBlobForm form={form}/>
|
||||
case "local":
|
||||
|
||||
@@ -7,7 +7,7 @@ export const storageProviders: ProviderIconTypes[] = [
|
||||
{value: "s3", label: "S3", icon: S3Icon},
|
||||
{value: "google-drive", label: "Google Drive", icon: GoogleDriveIcon},
|
||||
{value: "blob", label: "Azure Blob Storage", icon: BlobIcon},
|
||||
{value: "gcs", label: "Google Cloud Storage", icon: GCSIcon, preview: true},
|
||||
{value: "google-cloud-storage", label: "Google Cloud Storage", icon: GCSIcon},
|
||||
]
|
||||
|
||||
export function S3Icon(props: SVGProps<SVGSVGElement>) {
|
||||
|
||||
+103
@@ -0,0 +1,103 @@
|
||||
import { UseFormReturn } from "react-hook-form";
|
||||
import {
|
||||
FormControl,
|
||||
FormField,
|
||||
FormItem,
|
||||
FormLabel,
|
||||
FormMessage,
|
||||
} from "@/components/ui/form";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { Separator } from "@/components/ui/separator";
|
||||
import { Textarea } from "@/components/ui/textarea";
|
||||
|
||||
type StorageGoogleCloudStorageFormProps = {
|
||||
form: UseFormReturn<any, any, any>;
|
||||
};
|
||||
|
||||
export const StorageGoogleCloudStorageForm = ({
|
||||
form,
|
||||
}: StorageGoogleCloudStorageFormProps) => {
|
||||
return (
|
||||
<>
|
||||
<Separator className="my-1" />
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="config.projectId"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>Project ID *</FormLabel>
|
||||
<FormControl>
|
||||
<Input {...field} value={field.value ?? ""} placeholder="e.g. my-gcp-project" />
|
||||
</FormControl>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="config.bucketName"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>Bucket name *</FormLabel>
|
||||
<FormControl>
|
||||
<Input {...field} value={field.value ?? ""} placeholder="e.g. backups-prod" />
|
||||
</FormControl>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="config.clientEmail"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>Client email *</FormLabel>
|
||||
<FormControl>
|
||||
<Input
|
||||
{...field}
|
||||
value={field.value ?? ""}
|
||||
placeholder="e.g. service-account@my-gcp-project.iam.gserviceaccount.com"
|
||||
/>
|
||||
</FormControl>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="config.apiEndpoint"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>Endpoint URL</FormLabel>
|
||||
<FormControl>
|
||||
<Input
|
||||
{...field}
|
||||
value={field.value ?? ""}
|
||||
placeholder="e.g. http://localhost:4443"
|
||||
/>
|
||||
</FormControl>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="config.privateKey"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>Private key *</FormLabel>
|
||||
<FormControl>
|
||||
<Textarea
|
||||
{...field}
|
||||
value={field.value ?? ""}
|
||||
rows={5}
|
||||
placeholder="-----BEGIN PRIVATE KEY-----\n...\n-----END PRIVATE KEY-----\n"
|
||||
/>
|
||||
</FormControl>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
</>
|
||||
);
|
||||
};
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
import {z} from "zod";
|
||||
|
||||
export const GoogleCloudStorageChannelConfigSchema = z.object({
|
||||
projectId: z.string().trim().min(1, "Project ID is required"),
|
||||
bucketName: z.string().trim().min(1, "Bucket name is required"),
|
||||
clientEmail: z.email("Client email must be a valid email").trim(),
|
||||
privateKey: z.string().trim().min(1, "Private key is required"),
|
||||
apiEndpoint: z.preprocess(
|
||||
(v) => (v === "" ? undefined : v),
|
||||
z.string().url("Endpoint URL must be a valid URL").optional(),
|
||||
),
|
||||
});
|
||||
@@ -0,0 +1,179 @@
|
||||
import {Storage} from "@google-cloud/storage";
|
||||
|
||||
import {Readable} from "node:stream";
|
||||
import {GoogleCloudStorageConfig} from "@/features/channel/components/storages/google-cloud-storage/types";
|
||||
import {
|
||||
StorageCopyInput,
|
||||
StorageDeleteInput,
|
||||
StorageGetInput,
|
||||
StorageMetaData,
|
||||
StorageResult,
|
||||
StorageUploadInput
|
||||
} from "@/features/storages/types";
|
||||
|
||||
async function getGoogleCloudStorageClient(config: GoogleCloudStorageConfig) {
|
||||
return new Storage({
|
||||
projectId: config.projectId,
|
||||
...(config.apiEndpoint ? {apiEndpoint: config.apiEndpoint} : {}),
|
||||
credentials: {
|
||||
client_email: config.clientEmail,
|
||||
private_key: config.privateKey.replace(/\\n/g, "\n"),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
const BASE_DIR = "";
|
||||
|
||||
async function ensureBucket(config: GoogleCloudStorageConfig) {
|
||||
const client = await getGoogleCloudStorageClient(config);
|
||||
const [exists] = await client.bucket(config.bucketName).exists();
|
||||
if (!exists) throw new Error(`Bucket "${config.bucketName}" does not exist`);
|
||||
}
|
||||
|
||||
export async function uploadGoogleCloudStorage(
|
||||
config: GoogleCloudStorageConfig,
|
||||
input: { data: StorageUploadInput, metadata?: StorageMetaData }
|
||||
): Promise<StorageResult> {
|
||||
const client = await getGoogleCloudStorageClient(config);
|
||||
await ensureBucket(config);
|
||||
|
||||
const key = `${BASE_DIR}${input.data.path}`;
|
||||
const file = input.data.file;
|
||||
|
||||
let uploadStream: Readable;
|
||||
if (Buffer.isBuffer(file) || file instanceof Uint8Array) {
|
||||
uploadStream = Readable.from(file);
|
||||
} else if ((file as any).pipe) {
|
||||
uploadStream = file as Readable;
|
||||
} else {
|
||||
return {success: false, provider: "google-cloud-storage", error: "Unsupported file type for streaming upload"};
|
||||
}
|
||||
|
||||
try {
|
||||
const writeStream = client
|
||||
.bucket(config.bucketName)
|
||||
.file(key)
|
||||
.createWriteStream({contentType: input.data.contentType});
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
uploadStream
|
||||
.pipe(writeStream)
|
||||
.on("finish", resolve)
|
||||
.on("error", reject);
|
||||
});
|
||||
} catch (err: any) {
|
||||
return {success: false, provider: "google-cloud-storage", error: err.message};
|
||||
}
|
||||
|
||||
return {success: true, provider: "google-cloud-storage"};
|
||||
}
|
||||
|
||||
export async function getGoogleCloudStorage(
|
||||
config: GoogleCloudStorageConfig,
|
||||
input: { data: StorageGetInput, metadata: StorageMetaData }
|
||||
): Promise<StorageResult> {
|
||||
const client = await getGoogleCloudStorageClient(config);
|
||||
const key = `${BASE_DIR}${input.data.path}`;
|
||||
const file = client.bucket(config.bucketName).file(key);
|
||||
|
||||
const [exists] = await file.exists();
|
||||
if (!exists) return {success: false, provider: "google-cloud-storage", error: "File not found"};
|
||||
|
||||
const fileStream = file.createReadStream();
|
||||
|
||||
let signedUrl: string | undefined;
|
||||
if (input.data.signedUrl) {
|
||||
if (config.apiEndpoint) {
|
||||
signedUrl = `${config.apiEndpoint.replace(/\/$/, "")}/${config.bucketName}/${encodeURI(key)}`;
|
||||
} else {
|
||||
const [url] = await file.getSignedUrl({
|
||||
action: "read",
|
||||
expires: Date.now() + (input.data.expiresInSeconds ?? 60) * 1000,
|
||||
});
|
||||
signedUrl = url;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
provider: "google-cloud-storage",
|
||||
file: fileStream as unknown as Buffer | Readable,
|
||||
url: signedUrl,
|
||||
};
|
||||
}
|
||||
|
||||
export async function deleteGoogleCloudStorage(config: GoogleCloudStorageConfig, input: {
|
||||
data: StorageDeleteInput,
|
||||
metadata?: StorageMetaData
|
||||
}): Promise<StorageResult> {
|
||||
const client = await getGoogleCloudStorageClient(config);
|
||||
const key = `${BASE_DIR}${input.data.path}`;
|
||||
|
||||
try {
|
||||
await client.bucket(config.bucketName).file(key).delete();
|
||||
return {success: true, provider: "google-cloud-storage"};
|
||||
} catch (err: any) {
|
||||
return {success: false, provider: "google-cloud-storage", error: err.message};
|
||||
}
|
||||
}
|
||||
|
||||
export async function pingGoogleCloudStorage(config: GoogleCloudStorageConfig): Promise<StorageResult> {
|
||||
try {
|
||||
const client = await getGoogleCloudStorageClient(config);
|
||||
const bucket = client.bucket(config.bucketName);
|
||||
|
||||
const [exists] = await bucket.exists();
|
||||
if (!exists) return {
|
||||
success: false,
|
||||
provider: "google-cloud-storage",
|
||||
response: "Bucket does not exist"
|
||||
};
|
||||
|
||||
const key = `${BASE_DIR}ping.txt`;
|
||||
const file = bucket.file(key);
|
||||
await file.save(Buffer.from("ping"));
|
||||
await file.download();
|
||||
await file.delete();
|
||||
|
||||
return {
|
||||
success: true,
|
||||
provider: "google-cloud-storage",
|
||||
response: "Google Cloud Storage OK"
|
||||
};
|
||||
} catch (err: any) {
|
||||
return {
|
||||
success: false,
|
||||
provider: "google-cloud-storage",
|
||||
response: err.message
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export async function copyGoogleCloudStorage(
|
||||
config: GoogleCloudStorageConfig,
|
||||
input: {
|
||||
data: StorageCopyInput,
|
||||
},
|
||||
): Promise<StorageResult> {
|
||||
const client = await getGoogleCloudStorageClient(config);
|
||||
await ensureBucket(config);
|
||||
|
||||
const sourceKey = `${BASE_DIR}${input.data.from}`;
|
||||
const destinationKey = `${BASE_DIR}${input.data.to}`;
|
||||
|
||||
try {
|
||||
const bucket = client.bucket(config.bucketName);
|
||||
await bucket.file(sourceKey).copy(bucket.file(destinationKey));
|
||||
|
||||
return {
|
||||
success: true,
|
||||
provider: "google-cloud-storage",
|
||||
};
|
||||
} catch (err: any) {
|
||||
return {
|
||||
success: false,
|
||||
provider: "google-cloud-storage",
|
||||
error: err.message,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
export type GoogleCloudStorageConfig = {
|
||||
projectId: string;
|
||||
bucketName: string;
|
||||
clientEmail: string;
|
||||
privateKey: string;
|
||||
apiEndpoint: string;
|
||||
};
|
||||
@@ -14,6 +14,13 @@ import {
|
||||
pingGoogleDrive,
|
||||
uploadGoogleDrive
|
||||
} from "@/features/channel/components/storages/google-drive";
|
||||
import {
|
||||
copyGoogleCloudStorage,
|
||||
deleteGoogleCloudStorage,
|
||||
getGoogleCloudStorage,
|
||||
pingGoogleCloudStorage,
|
||||
uploadGoogleCloudStorage
|
||||
} from "@/features/channel/components/storages/google-cloud-storage";
|
||||
|
||||
type ProviderHandler = {
|
||||
upload: (config: any, input: StorageInput & { action: 'upload' }) => Promise<StorageResult>;
|
||||
@@ -51,6 +58,13 @@ const handlers: Record<StorageProviderKind, ProviderHandler> = {
|
||||
delete: deleteBlob,
|
||||
ping: pingBlob,
|
||||
copy: copyBlob,
|
||||
},
|
||||
"google-cloud-storage": {
|
||||
upload: uploadGoogleCloudStorage,
|
||||
get: getGoogleCloudStorage,
|
||||
delete: deleteGoogleCloudStorage,
|
||||
ping: pingGoogleCloudStorage,
|
||||
copy: copyGoogleCloudStorage,
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
Reference in New Issue
Block a user