feat: add POST /api/v1/fetch-urls endpoint for server-side URL import

Accepts { urls: string[] } (1-50), fetches each URL with SSRF protection via safeFetch, validates as image, saves to workspace, generates WebP preview for non-browser formats, and returns results with download URLs. Uses p-queue with concurrency 4 to parallelize fetches.
2026-08-03 07:46:42 +02:00 · 2026-05-11 21:24:56 +08:00
parent a2be47bd68
commit d38101b8ea
4 changed files with 572 additions and 0 deletions
@@ -23,6 +23,7 @@ import { auditLogRoutes } from "./routes/audit-log.js";
 import { registerBatchRoutes } from "./routes/batch.js";
 import { docsRoutes } from "./routes/docs.js";
 import { registerFeatureRoutes } from "./routes/features.js";
+import { registerFetchUrlsRoute } from "./routes/fetch-urls.js";
 import { fileRoutes } from "./routes/files.js";
 import { registerMemeTemplates } from "./routes/meme-templates.js";
 import { registerPipelineRoutes } from "./routes/pipeline.js";
@@ -165,6 +166,9 @@ await registerToolRoutes(app);
 // Batch processing routes (must be after tool routes so the registry is populated)
 await registerBatchRoutes(app);

+// URL fetch routes (server-side image fetching with SSRF protection)
+await registerFetchUrlsRoute(app);
+
 // Pipeline routes (must be after tool routes so the registry is populated)
 await registerPipelineRoutes(app);

@@ -0,0 +1,246 @@
+/**
+ * Fetch URLs route.
+ *
+ * POST /api/v1/fetch-urls
+ *
+ * Accepts a JSON body with { urls: string[] } (1-50 URLs).
+ * Fetches each URL server-side with SSRF protection, validates as an image,
+ * saves to a workspace, generates a preview for non-browser formats, and
+ * returns results with download URLs.
+ */
+import { randomUUID } from "node:crypto";
+import { writeFile } from "node:fs/promises";
+import { basename, join } from "node:path";
+import type { FastifyInstance } from "fastify";
+import PQueue from "p-queue";
+import sharp from "sharp";
+import { z } from "zod";
+import { validateImageBuffer } from "../lib/file-validation.js";
+import { sanitizeFilename } from "../lib/filename.js";
+import {
+  FETCH_TIMEOUT_MS,
+  MAX_URL_FETCH_SIZE,
+  MAX_URLS_PER_REQUEST,
+  safeFetch,
+  URL_FETCH_CONCURRENCY,
+} from "../lib/ssrf.js";
+import { createWorkspace } from "../lib/workspace.js";
+
+/** Formats browsers can display natively (no preview needed). */
+const BROWSER_PREVIEWABLE = new Set([
+  "image/jpeg",
+  "image/png",
+  "image/gif",
+  "image/webp",
+  "image/svg+xml",
+  "image/bmp",
+  "image/avif",
+]);
+
+/** Map detected format string to MIME type. */
+const FORMAT_TO_MIME: Record<string, string> = {
+  jpeg: "image/jpeg",
+  png: "image/png",
+  gif: "image/gif",
+  webp: "image/webp",
+  svg: "image/svg+xml",
+  bmp: "image/bmp",
+  avif: "image/avif",
+  tiff: "image/tiff",
+  heif: "image/heic",
+  jxl: "image/jxl",
+  ico: "image/x-icon",
+  psd: "image/vnd.adobe.photoshop",
+  raw: "image/x-dcraw",
+  tga: "image/x-tga",
+  exr: "image/x-exr",
+  hdr: "image/vnd.radiance",
+  jp2: "image/jp2",
+  qoi: "image/x-qoi",
+  eps: "application/postscript",
+  dds: "image/x-dds",
+  cur: "image/x-icon",
+  dpx: "image/x-dpx",
+  fits: "image/fits",
+  ppm: "image/x-portable-pixmap",
+  pgm: "image/x-portable-graymap",
+  pbm: "image/x-portable-bitmap",
+  pfm: "image/x-portable-floatmap",
+};
+
+const fetchUrlsSchema = z.object({
+  urls: z
+    .array(z.string().url("Each entry must be a valid URL"))
+    .min(1, "At least one URL is required")
+    .max(MAX_URLS_PER_REQUEST, `Maximum ${MAX_URLS_PER_REQUEST} URLs per request`),
+});
+
+interface SuccessResult {
+  success: true;
+  url: string;
+  filename: string;
+  contentType: string;
+  size: number;
+  width: number;
+  height: number;
+  downloadUrl: string;
+  previewUrl: string | null;
+}
+
+interface FailureResult {
+  success: false;
+  url: string;
+  error: string;
+}
+
+type FetchResult = SuccessResult | FailureResult;
+
+/**
+ * Extract a usable filename from a URL path, falling back to a UUID-based name.
+ */
+function filenameFromUrl(url: string): string {
+  try {
+    const pathname = new URL(url).pathname;
+    const base = basename(pathname);
+    // Decode percent-encoded characters
+    const decoded = decodeURIComponent(base);
+    // Only use it if it looks like a file with an extension
+    if (decoded?.includes(".") && decoded.length <= 255) {
+      return decoded;
+    }
+  } catch {
+    // ignore parse errors
+  }
+  return `image-${randomUUID().slice(0, 8)}`;
+}
+
+export async function registerFetchUrlsRoute(app: FastifyInstance): Promise<void> {
+  app.post("/api/v1/fetch-urls", async (request, reply) => {
+    // Validate body
+    const parsed = fetchUrlsSchema.safeParse(request.body);
+    if (!parsed.success) {
+      const messages = parsed.error.issues.map((i) => i.message).join("; ");
+      return reply.status(400).send({ error: messages });
+    }
+
+    const { urls } = parsed.data;
+    const jobId = randomUUID();
+    const workspace = await createWorkspace(jobId);
+    const outputDir = join(workspace, "output");
+
+    const queue = new PQueue({ concurrency: URL_FETCH_CONCURRENCY });
+
+    // Pre-allocate result slots to preserve order
+    const resultSlots: FetchResult[] = new Array(urls.length);
+
+    await Promise.all(
+      urls.map((url, index) =>
+        queue.add(async () => {
+          resultSlots[index] = await fetchSingleUrl(url, jobId, outputDir);
+        }),
+      ),
+    );
+
+    return reply.send({ results: resultSlots });
+  });
+}
+
+async function fetchSingleUrl(url: string, jobId: string, outputDir: string): Promise<FetchResult> {
+  try {
+    // Fetch with SSRF protection and timeout
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS);
+
+    let response: Response;
+    try {
+      response = await safeFetch(url, controller.signal);
+    } finally {
+      clearTimeout(timeout);
+    }
+
+    if (!response.ok) {
+      return {
+        success: false,
+        url,
+        error: `HTTP ${response.status} ${response.statusText}`,
+      };
+    }
+
+    // Read body with size limit
+    const chunks: Uint8Array[] = [];
+    let totalSize = 0;
+
+    if (!response.body) {
+      return { success: false, url, error: "Empty response body" };
+    }
+
+    const reader = response.body.getReader();
+    try {
+      while (true) {
+        const { done, value } = await reader.read();
+        if (done) break;
+        totalSize += value.byteLength;
+        if (totalSize > MAX_URL_FETCH_SIZE) {
+          reader.cancel();
+          return {
+            success: false,
+            url,
+            error: `File exceeds maximum size of ${MAX_URL_FETCH_SIZE / (1024 * 1024)}MB`,
+          };
+        }
+        chunks.push(value);
+      }
+    } finally {
+      reader.releaseLock();
+    }
+
+    const buffer = Buffer.concat(chunks);
+    if (buffer.length === 0) {
+      return { success: false, url, error: "Empty response body" };
+    }
+
+    // Derive filename from URL
+    const rawFilename = filenameFromUrl(url);
+    const filename = sanitizeFilename(rawFilename);
+
+    // Validate as an image
+    const validation = await validateImageBuffer(buffer, filename);
+    if (!validation.valid) {
+      return { success: false, url, error: validation.reason };
+    }
+
+    // Save to output directory
+    await writeFile(join(outputDir, filename), buffer);
+
+    const contentType = FORMAT_TO_MIME[validation.format] ?? "application/octet-stream";
+    const downloadUrl = `/api/v1/download/${jobId}/${encodeURIComponent(filename)}`;
+
+    // Generate preview for non-browser formats
+    let previewUrl: string | null = null;
+    if (!BROWSER_PREVIEWABLE.has(contentType)) {
+      try {
+        const previewBuffer = await sharp(buffer).webp({ quality: 80 }).toBuffer();
+        const previewFilename = `preview-${filename.replace(/\.[^.]+$/, "")}.webp`;
+        await writeFile(join(outputDir, previewFilename), previewBuffer);
+        previewUrl = `/api/v1/download/${jobId}/${encodeURIComponent(previewFilename)}`;
+      } catch {
+        // Preview generation failed -- non-fatal, skip preview
+      }
+    }
+
+    return {
+      success: true,
+      url,
+      filename,
+      contentType,
+      size: buffer.length,
+      width: validation.width,
+      height: validation.height,
+      downloadUrl,
+      previewUrl,
+    };
+  } catch (err) {
+    const message = err instanceof Error ? err.message : "Unknown error";
+    return { success: false, url, error: message };
+  }
+}