feat: add URL-based image import (single + bulk)

Add a fourth image ingestion path: importing images by URL. Backend: - POST /api/v1/fetch-urls endpoint with SSRF protection, image validation, preview generation, and p-queue concurrency - SSRF utility blocking private IPs, validating redirect hops, with comprehensive IPv4/IPv6 range coverage Frontend: - Always-visible URL input in the dropzone for quick single-image import - Bulk URL import modal with smart URL parsing (lists, markdown, HTML), per-URL progress tracking, retry on failure, and batch add - useUrlImport hook managing the full fetch lifecycle Tests: 48 new tests (23 SSRF unit, 10 URL parser unit, 15 integration)
2026-08-03 07:46:42 +02:00 · 2026-05-11 22:41:47 +08:00
parent b06906025c 04e7b21f31
commit 095e3d9488
13 changed files with 1788 additions and 2 deletions
@@ -23,6 +23,7 @@ import { auditLogRoutes } from "./routes/audit-log.js";
 import { registerBatchRoutes } from "./routes/batch.js";
 import { docsRoutes } from "./routes/docs.js";
 import { registerFeatureRoutes } from "./routes/features.js";
+import { registerFetchUrlsRoute } from "./routes/fetch-urls.js";
 import { fileRoutes } from "./routes/files.js";
 import { registerMemeTemplates } from "./routes/meme-templates.js";
 import { registerPipelineRoutes } from "./routes/pipeline.js";
@@ -165,6 +166,9 @@ await registerToolRoutes(app);
 // Batch processing routes (must be after tool routes so the registry is populated)
 await registerBatchRoutes(app);

+// URL fetch routes (server-side image fetching with SSRF protection)
+await registerFetchUrlsRoute(app);
+
 // Pipeline routes (must be after tool routes so the registry is populated)
 await registerPipelineRoutes(app);

@@ -0,0 +1,96 @@
+import { lookup } from "node:dns/promises";
+import { isIP } from "node:net";
+
+function isPrivateIPv4(ip: string): boolean {
+  const parts = ip.split(".").map(Number);
+  if (parts.length !== 4 || parts.some((p) => Number.isNaN(p))) return false;
+  const [a, b] = parts;
+  if (a === 10) return true;
+  if (a === 172 && b >= 16 && b <= 31) return true;
+  if (a === 192 && b === 168) return true;
+  if (a === 127) return true;
+  if (a === 169 && b === 254) return true;
+  if (a === 0) return true;
+  if (a === 100 && b >= 64 && b <= 127) return true;
+  if (a === 192 && b === 0 && parts[2] === 0) return true;
+  if (a === 198 && (b === 18 || b === 19)) return true;
+  if (a >= 240) return true;
+  return false;
+}
+
+function isPrivateIPv6(ip: string): boolean {
+  const normalized = ip.replace(/^\[|]$/g, "").toLowerCase();
+  if (normalized === "::1") return true;
+  if (normalized === "::") return true;
+  if (normalized.startsWith("fe80:")) return true;
+  if (normalized.startsWith("fc") || normalized.startsWith("fd")) return true;
+  if (normalized.startsWith("2001:db8:")) return true;
+  if (normalized.includes("::ffff:")) {
+    const v4 = normalized.split("::ffff:")[1];
+    if (v4 && isPrivateIPv4(v4)) return true;
+  }
+  return false;
+}
+
+async function resolveAndCheck(hostname: string): Promise<void> {
+  const bare = hostname.replace(/^\[|]$/g, "");
+  if (isIP(bare)) {
+    if (isPrivateIPv4(bare) || isPrivateIPv6(bare)) {
+      throw new Error("URL resolves to a private or reserved IP address");
+    }
+    return;
+  }
+
+  const result = await lookup(hostname, { all: true });
+  const addresses = Array.isArray(result) ? result : [result];
+  for (const entry of addresses) {
+    const addr = entry.address;
+    if (isPrivateIPv4(addr) || isPrivateIPv6(addr)) {
+      throw new Error("URL resolves to a private or reserved IP address");
+    }
+  }
+}
+
+export async function validateFetchUrl(url: string): Promise<void> {
+  let parsed: URL;
+  try {
+    parsed = new URL(url);
+  } catch {
+    throw new Error("Invalid URL");
+  }
+
+  if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
+    throw new Error("Only HTTP and HTTPS URLs are supported");
+  }
+
+  await resolveAndCheck(parsed.hostname);
+}
+
+export const MAX_REDIRECTS = 5;
+export const FETCH_TIMEOUT_MS = 30_000;
+export const MAX_URL_FETCH_SIZE = 50 * 1024 * 1024;
+export const MAX_URLS_PER_REQUEST = 50;
+export const URL_FETCH_CONCURRENCY = 4;
+
+export async function safeFetch(url: string, signal?: AbortSignal): Promise<Response> {
+  let currentUrl = url;
+  for (let i = 0; i <= MAX_REDIRECTS; i++) {
+    await validateFetchUrl(currentUrl);
+    const res = await fetch(currentUrl, {
+      signal,
+      redirect: "manual",
+      headers: { "User-Agent": "SnapOtter/1.0 (image-fetch)" },
+    });
+
+    if (res.status >= 300 && res.status < 400) {
+      const location = res.headers.get("location");
+      if (!location) throw new Error("Redirect without Location header");
+      await res.body?.cancel();
+      currentUrl = new URL(location, currentUrl).href;
+      continue;
+    }
+
+    return res;
+  }
+  throw new Error("Too many redirects");
+}
@@ -0,0 +1,279 @@
+/**
+ * Fetch URLs route.
+ *
+ * POST /api/v1/fetch-urls
+ *
+ * Accepts a JSON body with { urls: string[] } (1-50 URLs).
+ * Fetches each URL server-side with SSRF protection, validates as an image,
+ * saves to a workspace, generates a preview for non-browser formats, and
+ * returns results with download URLs.
+ */
+import { randomUUID } from "node:crypto";
+import { writeFile } from "node:fs/promises";
+import { basename, join } from "node:path";
+import type { FastifyInstance } from "fastify";
+import PQueue from "p-queue";
+import sharp from "sharp";
+import { z } from "zod";
+import { validateImageBuffer } from "../lib/file-validation.js";
+import { sanitizeFilename } from "../lib/filename.js";
+import {
+  FETCH_TIMEOUT_MS,
+  MAX_URL_FETCH_SIZE,
+  MAX_URLS_PER_REQUEST,
+  safeFetch,
+  URL_FETCH_CONCURRENCY,
+} from "../lib/ssrf.js";
+import { createWorkspace } from "../lib/workspace.js";
+
+/** Formats browsers can display natively (no preview needed). */
+const BROWSER_PREVIEWABLE = new Set([
+  "image/jpeg",
+  "image/png",
+  "image/gif",
+  "image/webp",
+  "image/svg+xml",
+  "image/bmp",
+  "image/avif",
+]);
+
+/** Map detected format string to MIME type. */
+const FORMAT_TO_MIME: Record<string, string> = {
+  jpeg: "image/jpeg",
+  png: "image/png",
+  gif: "image/gif",
+  webp: "image/webp",
+  svg: "image/svg+xml",
+  bmp: "image/bmp",
+  avif: "image/avif",
+  tiff: "image/tiff",
+  heif: "image/heic",
+  jxl: "image/jxl",
+  ico: "image/x-icon",
+  psd: "image/vnd.adobe.photoshop",
+  raw: "image/x-dcraw",
+  tga: "image/x-tga",
+  exr: "image/x-exr",
+  hdr: "image/vnd.radiance",
+  jp2: "image/jp2",
+  qoi: "image/x-qoi",
+  eps: "application/postscript",
+  dds: "image/x-dds",
+  cur: "image/x-icon",
+  dpx: "image/x-dpx",
+  fits: "image/fits",
+  ppm: "image/x-portable-pixmap",
+  pgm: "image/x-portable-graymap",
+  pbm: "image/x-portable-bitmap",
+  pfm: "image/x-portable-floatmap",
+};
+
+const fetchUrlsSchema = z.object({
+  urls: z
+    .array(z.string().url("Each entry must be a valid URL"))
+    .min(1, "At least one URL is required")
+    .max(MAX_URLS_PER_REQUEST, `Maximum ${MAX_URLS_PER_REQUEST} URLs per request`),
+});
+
+interface SuccessResult {
+  success: true;
+  url: string;
+  filename: string;
+  contentType: string;
+  size: number;
+  width: number;
+  height: number;
+  downloadUrl: string;
+  previewUrl: string | null;
+}
+
+interface FailureResult {
+  success: false;
+  url: string;
+  error: string;
+}
+
+type FetchResult = SuccessResult | FailureResult;
+
+/**
+ * Extract a usable filename from a URL path, falling back to a UUID-based name.
+ */
+function filenameFromUrl(url: string): string {
+  try {
+    const pathname = new URL(url).pathname;
+    const base = basename(pathname);
+    // Decode percent-encoded characters
+    const decoded = decodeURIComponent(base);
+    // Only use it if it looks like a file with an extension
+    if (decoded?.includes(".") && decoded.length <= 255) {
+      return decoded;
+    }
+  } catch {
+    // ignore parse errors
+  }
+  return `image-${randomUUID().slice(0, 8)}`;
+}
+
+/**
+ * Return a filename that does not collide with any name already in `used`.
+ * Appends `_1`, `_2`, etc. before the extension when a collision is found.
+ * Mirrors the deduplication logic in batch.ts.
+ */
+function getUniqueName(name: string, used: Set<string>): string {
+  if (!used.has(name)) {
+    used.add(name);
+    return name;
+  }
+  const dotIdx = name.lastIndexOf(".");
+  const base = dotIdx > 0 ? name.slice(0, dotIdx) : name;
+  const ext = dotIdx > 0 ? name.slice(dotIdx) : "";
+  let counter = 1;
+  let candidate = `${base}_${counter}${ext}`;
+  while (used.has(candidate)) {
+    counter++;
+    candidate = `${base}_${counter}${ext}`;
+  }
+  used.add(candidate);
+  return candidate;
+}
+
+export async function registerFetchUrlsRoute(app: FastifyInstance): Promise<void> {
+  app.post("/api/v1/fetch-urls", async (request, reply) => {
+    // Validate body
+    const parsed = fetchUrlsSchema.safeParse(request.body);
+    if (!parsed.success) {
+      const messages = parsed.error.issues.map((i) => i.message).join("; ");
+      return reply.status(400).send({ error: messages });
+    }
+
+    const { urls } = parsed.data;
+    const jobId = randomUUID();
+    const workspace = await createWorkspace(jobId);
+    const outputDir = join(workspace, "output");
+
+    const queue = new PQueue({ concurrency: URL_FETCH_CONCURRENCY });
+
+    // Track filenames to prevent collisions when multiple URLs resolve to the
+    // same name (e.g. https://a.com/photo.jpg and https://b.com/photo.jpg).
+    const usedFilenames = new Set<string>();
+
+    // Pre-allocate result slots to preserve order
+    const resultSlots: FetchResult[] = new Array(urls.length);
+
+    await Promise.all(
+      urls.map((url, index) =>
+        queue.add(async () => {
+          resultSlots[index] = await fetchSingleUrl(url, jobId, outputDir, usedFilenames);
+        }),
+      ),
+    );
+
+    return reply.send({ results: resultSlots });
+  });
+}
+
+async function fetchSingleUrl(
+  url: string,
+  jobId: string,
+  outputDir: string,
+  usedFilenames: Set<string>,
+): Promise<FetchResult> {
+  try {
+    // Fetch with SSRF protection and timeout
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS);
+
+    let response: Response;
+    try {
+      response = await safeFetch(url, controller.signal);
+    } finally {
+      clearTimeout(timeout);
+    }
+
+    if (!response.ok) {
+      return {
+        success: false,
+        url,
+        error: `HTTP ${response.status} ${response.statusText}`,
+      };
+    }
+
+    // Read body with size limit
+    const chunks: Uint8Array[] = [];
+    let totalSize = 0;
+
+    if (!response.body) {
+      return { success: false, url, error: "Empty response body" };
+    }
+
+    const reader = response.body.getReader();
+    try {
+      while (true) {
+        const { done, value } = await reader.read();
+        if (done) break;
+        totalSize += value.byteLength;
+        if (totalSize > MAX_URL_FETCH_SIZE) {
+          reader.cancel();
+          return {
+            success: false,
+            url,
+            error: `File exceeds maximum size of ${MAX_URL_FETCH_SIZE / (1024 * 1024)}MB`,
+          };
+        }
+        chunks.push(value);
+      }
+    } finally {
+      reader.releaseLock();
+    }
+
+    const buffer = Buffer.concat(chunks);
+    if (buffer.length === 0) {
+      return { success: false, url, error: "Empty response body" };
+    }
+
+    // Derive filename from URL, deduplicating to prevent overwrites when
+    // multiple URLs resolve to the same name (all URLs share one workspace).
+    const rawFilename = filenameFromUrl(url);
+    const filename = getUniqueName(sanitizeFilename(rawFilename), usedFilenames);
+
+    // Validate as an image
+    const validation = await validateImageBuffer(buffer, filename);
+    if (!validation.valid) {
+      return { success: false, url, error: validation.reason };
+    }
+
+    // Save to output directory
+    await writeFile(join(outputDir, filename), buffer);
+
+    const contentType = FORMAT_TO_MIME[validation.format] ?? "application/octet-stream";
+    const downloadUrl = `/api/v1/download/${jobId}/${encodeURIComponent(filename)}`;
+
+    // Generate preview for non-browser formats
+    let previewUrl: string | null = null;
+    if (!BROWSER_PREVIEWABLE.has(contentType)) {
+      try {
+        const previewBuffer = await sharp(buffer).webp({ quality: 80 }).toBuffer();
+        const previewFilename = `preview-${filename.replace(/\.[^.]+$/, "")}.webp`;
+        await writeFile(join(outputDir, previewFilename), previewBuffer);
+        previewUrl = `/api/v1/download/${jobId}/${encodeURIComponent(previewFilename)}`;
+      } catch {
+        // Preview generation failed -- non-fatal, skip preview
+      }
+    }
+
+    return {
+      success: true,
+      url,
+      filename,
+      contentType,
+      size: buffer.length,
+      width: validation.width,
+      height: validation.height,
+      downloadUrl,
+      previewUrl,
+    };
+  } catch (err) {
+    const message = err instanceof Error ? err.message : "Unknown error";
+    return { success: false, url, error: message };
+  }
+}