Merge pull request #328 from jjingi/feature/Integrate-Azure-Blob

Feature/integrate azure blob
2026-07-14 11:16:13 +02:00 · 2026-06-21 15:49:52 +02:00
parent 402d3f3d9d afd9a8eff2
commit bf0ed69b60
18 changed files with 4633 additions and 1042 deletions
@@ -45,4 +45,12 @@ export-keycloak:
 	@docker cp kc-exporter:/tmp/kc-export/. ./seeds/keycloak/
 	@docker rm -f kc-exporter >/dev/null 2>&1
 	@docker compose -f docker-compose.func.yml start keycloak >/dev/null 2>&1
-	@echo "Keycloak configuration and users exported to seeds/keycloak/*.json"
+	@echo "Keycloak configuration and users exported to seeds/keycloak/*.json"
+
+seed-blob:
+	@chmod +x seeds/azurite/azurite-seed.sh
+	@bash seeds/azurite/azurite-seed.sh
+
+list-blob:
+	@chmod +x seeds/azurite/azurite-list.sh
+	@bash seeds/azurite/azurite-list.sh
@@ -41,5 +41,25 @@ services:
      - "8025:8025"
      - "1025:1025"

+  azurite:
+    image: mcr.microsoft.com/azure-storage/azurite
+    command: azurite --blobHost 0.0.0.0 --skipApiVersionCheck
+    ports:
+      - "10000:10000"
+    volumes:
+      - azurite-data:/data
+    extra_hosts:
+      - "localhost:host-gateway"
+    networks:
+      - portabase
+
+    #  DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;
+
 volumes:
  postgres-data:
+  azurite-data:
+
+networks:
+  portabase:
+    name: portabase_network
+    external: true
@@ -16,6 +16,7 @@
  },
  "dependencies": {
    "@asteasolutions/zod-to-openapi": "^8.5.0",
+    "@azure/storage-blob": "^12.32.0",
    "@better-auth/api-key": "1.6.11",
    "@better-auth/core": "1.6.11",
    "@better-auth/passkey": "1.6.11",
@@ -138,5 +139,5 @@
    "typescript": "^5.9.3",
    "zenstack": "2.14.2"
  },
-  "packageManager": "pnpm@11.5.2+sha512.71c631e382066efc25625d5cf029075de07b61b37f6e27350fbd84b1bda5864c8c1967adc280776b45c30a715c0359a3be08fef42d5bb09e2b99029979692916"
+  "packageManager": "pnpm@11.8.0+sha512.c1f5e7c4cb241c8f174b743851d82f42b802324afc8b0f116b96adb15aa06664948dde36960a3ba1079ba5b4b29dd0140135b94b5b5f5263592249d68e555f26"
 }
@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+#
+# azurite-list.sh — enumerate every container and blob in a local Azurite emulator.
+#
+# Usage:
+#   ./azurite-list.sh            # list containers + blobs
+#   ./azurite-list.sh -v         # also print blob size, last-modified, content-type
+#
+
+set -euo pipefail
+
+export AZURE_STORAGE_CONNECTION_STRING="DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;"
+
+VERBOSE=0
+[[ "${1:-}" == "-v" || "${1:-}" == "--verbose" ]] && VERBOSE=1
+
+command -v az >/dev/null 2>&1 || { echo "ERROR: 'az' CLI not found in PATH." >&2; exit 1; }
+command -v jq >/dev/null 2>&1 || { echo "ERROR: 'jq' not found in PATH." >&2; exit 1; }
+
+if ! az storage container list --num-results 1 >/dev/null 2>&1; then
+  echo "ERROR: Cannot reach Azurite at 127.0.0.1:10000. Is the container running?" >&2
+  echo "       Start it with: docker start azurite" >&2
+  exit 1
+fi
+
+containers=$(az storage container list --query "[].name" -o tsv)
+
+if [[ -z "$containers" ]]; then
+  echo "(no containers found in this emulator)"
+  exit 0
+fi
+
+while IFS= read -r container; do
+  echo "📦 container: $container"
+
+  if [[ "$VERBOSE" -eq 1 ]]; then
+    az storage blob list --container-name "$container" \
+      --query "[].{name:name, bytes:properties.contentLength, modified:properties.lastModified, type:properties.contentSettings.contentType}" \
+      -o json \
+    | jq -r '.[] | "   • \(.name)  [\(.bytes) bytes]  \(.type // "?")  \(.modified)"'
+  else
+    az storage blob list --container-name "$container" \
+      --query "[].{name:name, bytes:properties.contentLength}" \
+      -o json \
+    | jq -r '.[] | "   • \(.name)  [\(.bytes) bytes]"'
+  fi
+
+  count=$(az storage blob list --container-name "$container" --query "length(@)" -o tsv)
+  [[ "$count" -eq 0 ]] && echo "   (empty)"
+  echo
+done <<< "$containers"
@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+#
+# azurite-seed.sh — seed a local Azurite emulator with a container + sample blob.
+#
+# Usage:
+#   ./azurite-seed.sh                 # create container "portabase" + sample blob
+#   ./azurite-seed.sh my-container    # override container name
+#
+
+set -euo pipefail
+
+export AZURE_STORAGE_CONNECTION_STRING="DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;"
+
+CONTAINER="${1:-portabase}"
+BLOB_NAME="hello.txt"
+
+command -v az >/dev/null 2>&1 || { echo "ERROR: 'az' CLI not found in PATH." >&2; exit 1; }
+
+# Reachability check — surface the real az error instead of guessing "container down".
+if ! err=$(az storage container list --num-results 1 2>&1 >/dev/null); then
+  echo "ERROR: Azurite query failed:" >&2
+  printf '%s\n' "$err" | sed 's/^/       /' >&2
+  echo "       Is Azurite up? Start it with: docker compose up -d azurite" >&2
+  exit 1
+fi
+
+# Create container (idempotent — az returns created:false if it already exists).
+az storage container create --name "$CONTAINER" -o none
+echo "📦 container ready: $CONTAINER"
+
+# Upload a sample blob from a temp file.
+tmp=$(mktemp)
+trap 'rm -f "$tmp"' EXIT
+printf 'hello from azurite seed — %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" > "$tmp"
+
+az storage blob upload \
+  --container-name "$CONTAINER" \
+  --name "$BLOB_NAME" \
+  --file "$tmp" \
+  --overwrite true \
+  --no-progress \
+  -o none
+echo "   • uploaded blob: $BLOB_NAME"
+
+echo
+echo "Seed complete. Verify with: make list-blob"
@@ -0,0 +1 @@
+ALTER TYPE "public"."provider_storage_kind" ADD VALUE 'blob';
@@ -449,6 +449,13 @@
      "when": 1781854323074,
      "tag": "0063_open_proudstar",
      "breakpoints": true
+    },
+    {
+      "idx": 64,
+      "version": "7",
+      "when": 1781960184847,
+      "tag": "0064_orange_richard_fisk",
+      "breakpoints": true
    }
  ]
 }
@@ -5,7 +5,7 @@ import {relations} from "drizzle-orm";
 import {createSelectSchema} from "drizzle-zod";
 import {z} from "zod";

-export const providerStorageKindEnum = pgEnum('provider_storage_kind', ['local', 's3', 'google-drive']);
+export const providerStorageKindEnum = pgEnum('provider_storage_kind', ['local', 's3', 'google-drive', 'blob']);

 export const storageChannel = pgTable('storage_channel', {
    id: uuid("id").defaultRandom().primaryKey(),
@@ -12,6 +12,7 @@ import {S3ChannelConfigSchema} from "@/features/channel/storages/s3.schema";
 import {GoogleDriveChannelConfigSchema} from "@/features/channel/storages/google-drive/google-drive.schema";
 import {LocalChannelConfigSchema} from "@/features/channel/storages/local.schema";
 import {TeamsChannelConfigSchema} from "@/features/channel/notifications/teams.schema";
+import {BlobChannelConfigSchema} from "@/features/channel/storages/az-blob.schema";


 const BaseChannelFormSchema = z.object({
@@ -74,6 +75,10 @@ export const StorageChannelFormSchema = z.discriminatedUnion("provider", [
        provider: z.literal("google-drive"),
        config: GoogleDriveChannelConfigSchema,
    }),
+    BaseChannelFormSchema.extend({
+        provider: z.literal("blob"),
+        config: BlobChannelConfigSchema,
+    }),
    BaseChannelFormSchema.extend({
        provider: z.literal("local"),
        config: LocalChannelConfigSchema
@@ -41,6 +41,9 @@ import {
 import {
    StorageGoogleDriveForm
 } from "@/features/channel/storages/google-drive/google-drive.form";
+import {
+    StorageBlobForm
+} from "@/features/channel/storages/az-blob.form";

 export type ChannelKind = "notification" | "storage";

@@ -106,6 +109,8 @@ export const renderChannelForm = (provider: string | undefined, form: UseFormRet
            return <StorageS3Form form={form}/>
        case "google-drive":
            return <StorageGoogleDriveForm form={form}/>
+        case "blob":
+            return <StorageBlobForm form={form}/>
        case "local":
            return <></>
        default:
@@ -6,7 +6,7 @@ export const storageProviders: ProviderIconTypes[] = [
    {value: "local", label: "Local", icon: Server},
    {value: "s3", label: "S3", icon: S3Icon},
    {value: "google-drive", label: "Google Drive", icon: GoogleDriveIcon},
-    {value: "blob", label: "Azure Blob Storage", icon: BlobIcon, preview: true},
+    {value: "blob", label: "Azure Blob Storage", icon: BlobIcon},
    {value: "gcs", label: "Google Cloud Storage", icon: GCSIcon, preview: true},
 ]

@@ -0,0 +1,87 @@
+import { UseFormReturn } from "react-hook-form";
+import {
+  FormControl,
+  FormField,
+  FormItem,
+  FormLabel,
+  FormMessage,
+} from "@/components/ui/form";
+import { Input } from "@/components/ui/input";
+import { Separator } from "@/components/ui/separator";
+import { PasswordInput } from "@/components/ui/password-input";
+type StorageBlobFormProps = {
+  form: UseFormReturn<any, any, any>;
+};
+
+export const StorageBlobForm = ({ form }: StorageBlobFormProps) => {
+  return (
+    <>
+      <Separator className="my-1" />
+      <FormField
+        control={form.control}
+        name="config.accountName"
+        render={({ field }) => (
+          <FormItem>
+            <FormLabel>Account Name *</FormLabel>
+            <FormControl>
+              <Input {...field} placeholder="e.g. mystorageaccount" />
+            </FormControl>
+            <FormMessage />
+          </FormItem>
+        )}
+      />
+      <FormField
+        control={form.control}
+        name="config.accountKey"
+        render={({ field }) => (
+          <FormItem>
+            <FormLabel>Account Key</FormLabel>
+            <FormControl>
+              <PasswordInput {...field} placeholder="e.g. base64-encoded-key" />
+            </FormControl>
+            <FormMessage />
+          </FormItem>
+        )}
+      />
+      <FormField
+        control={form.control}
+        name="config.connectionString"
+        render={({ field }) => (
+          <FormItem>
+            <FormLabel>Connection String</FormLabel>
+            <FormControl>
+              <PasswordInput {...field} placeholder="e.g. DefaultEndpointsProtocol=https;..." />
+            </FormControl>
+            <FormMessage />
+          </FormItem>
+        )}
+      />
+      <FormField
+        control={form.control}
+        name="config.containerName"
+        render={({ field }) => (
+          <FormItem>
+            <FormLabel>Container Name *</FormLabel>
+            <FormControl>
+              <Input {...field} placeholder="e.g. backups-prod" />
+            </FormControl>
+            <FormMessage />
+          </FormItem>
+        )}
+      />
+      <FormField
+        control={form.control}
+        name="config.endpointUrl"
+        render={({ field }) => (
+          <FormItem>
+            <FormLabel>Endpoint URL</FormLabel>
+            <FormControl>
+              <Input {...field} placeholder="e.g. https://myaccount.blob.core.windows.net" />
+            </FormControl>
+            <FormMessage />
+          </FormItem>
+        )}
+      />
+    </>
+  );
+};
@@ -0,0 +1,15 @@
+import {z} from "zod";
+
+export const BlobChannelConfigSchema = z.object({
+    accountName: z.string().min(1, "Account name is required"),
+    accountKey: z.string().optional(),
+    connectionString: z.string().optional(),
+    containerName: z.string().min(1, "Container name is required"),
+    endpointUrl: z.preprocess(
+        (v) => (v === "" ? undefined : v),
+        z.string().url("Endpoint URL must be a valid URL").optional(),
+    ),
+}).refine(
+    (data) => data.accountKey || data.connectionString,
+    {message: "Either account key or connection string is required"}
+);
@@ -0,0 +1,185 @@
+import {BlobSASPermissions, BlobServiceClient, StorageSharedKeyCredential} from "@azure/storage-blob";
+import {
+    StorageCopyInput,
+    StorageDeleteInput,
+    StorageGetInput,
+    StorageMetaData,
+    StorageResult,
+    StorageUploadInput
+} from '@/features/storages/storages.types';
+import {Readable} from "node:stream";
+
+type BlobConfig = {
+    accountName: string;
+    accountKey?: string;
+    connectionString?: string;
+    containerName: string;
+    endpointUrl?: string;
+};
+
+async function getBlobClient(config: BlobConfig) {
+    if (config.connectionString) {
+        return BlobServiceClient.fromConnectionString(config.connectionString);
+    }
+
+    const url = config.endpointUrl ?? `https://${config.accountName}.blob.core.windows.net`;
+    const credential = new StorageSharedKeyCredential(config.accountName, config.accountKey!);
+    return new BlobServiceClient(url, credential);
+}
+
+const BASE_DIR = "";
+
+
+async function ensureContainer(config: BlobConfig) {
+    const client = await getBlobClient(config);
+    const containerClient = client.getContainerClient(config.containerName);
+    await containerClient.createIfNotExists();
+}
+
+export async function uploadBlob(
+    config: BlobConfig,
+    input: { data: StorageUploadInput, metadata?: StorageMetaData }
+): Promise<StorageResult> {
+    const client = await getBlobClient(config);
+    await ensureContainer(config);
+
+    const key = `${BASE_DIR}${input.data.path}`;
+    const file = input.data.file;
+
+    let uploadStream: Readable;
+    if (Buffer.isBuffer(file) || file instanceof Uint8Array) {
+        uploadStream = Readable.from(file);
+    } else if ((file as any).pipe) {
+        uploadStream = file;
+    } else {
+        return {success: false, provider: "blob", error: "Unsupported file type for streaming upload"};
+    }
+
+    try {
+        const containerClient = client.getContainerClient(config.containerName);
+        const blockBlobClient = containerClient.getBlockBlobClient(key);
+        await blockBlobClient.uploadStream(uploadStream);
+    } catch (err: any) {
+        return {success: false, provider: "blob", error: err.message};
+    }
+
+    return {success: true, provider: "blob"};
+}
+
+
+export async function getBlob(
+    config: BlobConfig,
+    input: { data: StorageGetInput, metadata: StorageMetaData }
+): Promise<StorageResult> {
+    const client = await getBlobClient(config);
+    const key = `${BASE_DIR}${input.data.path}`;
+
+    const containerClient = client.getContainerClient(config.containerName);
+    const blockBlobClient = containerClient.getBlockBlobClient(key);
+
+    if (!(await blockBlobClient.exists())) {
+        return {success: false, provider: "blob", error: "File not found"};
+    }
+
+    const downloadResponse = await blockBlobClient.download();
+    const fileStream = downloadResponse.readableStreamBody as unknown as Readable;
+
+    let presignedUrl: string | undefined;
+    if (input.data.signedUrl) {
+        presignedUrl = await blockBlobClient.generateSasUrl({
+            expiresOn: new Date(Date.now() + (input.data.expiresInSeconds ?? 60) * 1000),
+            permissions: BlobSASPermissions.parse("r"),
+        });
+    }
+
+    return {
+        success: true,
+        provider: "blob",
+        file: fileStream,
+        url: presignedUrl,
+    };
+}
+
+
+export async function deleteBlob(config: BlobConfig, input: {
+    data: StorageDeleteInput,
+    metadata?: StorageMetaData
+}): Promise<StorageResult> {
+    const client = await getBlobClient(config);
+    const key = `${BASE_DIR}${input.data.path}`;
+
+    try {
+        const containerClient = client.getContainerClient(config.containerName);
+        const blockBlobClient = containerClient.getBlockBlobClient(key);
+        await blockBlobClient.delete();
+        return {success: true, provider: "blob"};
+    } catch (err: any) {
+        return {success: false, provider: "blob", error: err.message};
+    }
+}
+
+export async function pingBlob(config: BlobConfig): Promise<StorageResult> {
+    try {
+        const client = await getBlobClient(config);
+        const containerClient = client.getContainerClient(config.containerName);
+        const exists = await containerClient.exists();
+        if (!exists) return {
+            success: false,
+            provider: "blob",
+            response: "Container does not exist"
+        };
+        const key = `${BASE_DIR}ping-${Date.now()}-${Math.random().toString(36).slice(2)}.txt`;
+        const blockBlobClient = containerClient.getBlockBlobClient(key);
+        try {
+            await blockBlobClient.upload(Buffer.from("ping"), 4);
+            await blockBlobClient.download();
+        } finally {
+            await blockBlobClient.delete().catch(() => undefined);
+        }
+
+        return {
+            success: true,
+            provider: "blob",
+            response: "Blob storage OK"
+        };
+    } catch (err: any) {
+        return {
+            success: false,
+            provider: "blob",
+            response: err.message
+        };
+    }
+}
+
+
+export async function copyBlob(
+    config: BlobConfig,
+    input: {
+        data: StorageCopyInput,
+    },
+): Promise<StorageResult> {
+    const client = await getBlobClient(config);
+    await ensureContainer(config);
+
+    const sourceKey = `${BASE_DIR}${input.data.from}`;
+    const destinationKey = `${BASE_DIR}${input.data.to}`;
+
+    try {
+        const containerClient = client.getContainerClient(config.containerName);
+        const sourceBlob = containerClient.getBlockBlobClient(sourceKey);
+        const destBlob = containerClient.getBlockBlobClient(destinationKey);
+        const poller = await destBlob.beginCopyFromURL(sourceBlob.url);
+        await poller.pollUntilDone();
+
+        return {
+            success: true,
+            provider: "blob",
+        };
+    } catch (err: any) {
+        return {
+            success: false,
+            provider: "blob",
+            error: err.message,
+        };
+    }
+}
@@ -6,6 +6,7 @@ import {

 import {uploadLocal, getLocal, deleteLocal, pingLocal, copyLocal} from './local';
 import {copyS3, deleteS3, getS3, pingS3, uploadS3} from "@/features/channel/storages/s3";
+import {copyBlob, deleteBlob, getBlob, pingBlob, uploadBlob} from "@/features/channel/storages/az-blob";
 import {
    copyGoogleDrive,
    deleteGoogleDrive,
@@ -43,6 +44,13 @@ const handlers: Record<StorageProviderKind, ProviderHandler> = {
        delete: deleteGoogleDrive,
        ping: pingGoogleDrive,
        copy: copyGoogleDrive,
+    },
+    blob: {
+        upload: uploadBlob,
+        get: getBlob,
+        delete: deleteBlob,
+        ping: pingBlob,
+        copy: copyBlob,
    }
 };

@@ -4,6 +4,7 @@ export type StorageProviderKind =
    | 'local'
    | 's3'
    | 'google-drive'
+    | 'blob'
    ;

 export type StorageAction =
				`@@ -0,0 +1 @@`
				`ALTER TYPE "public"."provider_storage_kind" ADD VALUE 'blob';`