feat: add POST /api/v1/fetch-urls endpoint for server-side URL import

Accepts { urls: string[] } (1-50), fetches each URL with SSRF protection via safeFetch, validates as image, saves to workspace, generates WebP preview for non-browser formats, and returns results with download URLs. Uses p-queue with concurrency 4 to parallelize fetches.
2026-08-03 07:46:42 +02:00 · 2026-05-11 21:24:56 +08:00
parent a2be47bd68
commit d38101b8ea
4 changed files with 572 additions and 0 deletions
@@ -23,6 +23,7 @@ import { auditLogRoutes } from "./routes/audit-log.js";
 import { registerBatchRoutes } from "./routes/batch.js";
 import { docsRoutes } from "./routes/docs.js";
 import { registerFeatureRoutes } from "./routes/features.js";
+import { registerFetchUrlsRoute } from "./routes/fetch-urls.js";
 import { fileRoutes } from "./routes/files.js";
 import { registerMemeTemplates } from "./routes/meme-templates.js";
 import { registerPipelineRoutes } from "./routes/pipeline.js";
@@ -165,6 +166,9 @@ await registerToolRoutes(app);
 // Batch processing routes (must be after tool routes so the registry is populated)
 await registerBatchRoutes(app);

+// URL fetch routes (server-side image fetching with SSRF protection)
+await registerFetchUrlsRoute(app);
+
 // Pipeline routes (must be after tool routes so the registry is populated)
 await registerPipelineRoutes(app);

@@ -0,0 +1,246 @@
+/**
+ * Fetch URLs route.
+ *
+ * POST /api/v1/fetch-urls
+ *
+ * Accepts a JSON body with { urls: string[] } (1-50 URLs).
+ * Fetches each URL server-side with SSRF protection, validates as an image,
+ * saves to a workspace, generates a preview for non-browser formats, and
+ * returns results with download URLs.
+ */
+import { randomUUID } from "node:crypto";
+import { writeFile } from "node:fs/promises";
+import { basename, join } from "node:path";
+import type { FastifyInstance } from "fastify";
+import PQueue from "p-queue";
+import sharp from "sharp";
+import { z } from "zod";
+import { validateImageBuffer } from "../lib/file-validation.js";
+import { sanitizeFilename } from "../lib/filename.js";
+import {
+  FETCH_TIMEOUT_MS,
+  MAX_URL_FETCH_SIZE,
+  MAX_URLS_PER_REQUEST,
+  safeFetch,
+  URL_FETCH_CONCURRENCY,
+} from "../lib/ssrf.js";
+import { createWorkspace } from "../lib/workspace.js";
+
+/** Formats browsers can display natively (no preview needed). */
+const BROWSER_PREVIEWABLE = new Set([
+  "image/jpeg",
+  "image/png",
+  "image/gif",
+  "image/webp",
+  "image/svg+xml",
+  "image/bmp",
+  "image/avif",
+]);
+
+/** Map detected format string to MIME type. */
+const FORMAT_TO_MIME: Record<string, string> = {
+  jpeg: "image/jpeg",
+  png: "image/png",
+  gif: "image/gif",
+  webp: "image/webp",
+  svg: "image/svg+xml",
+  bmp: "image/bmp",
+  avif: "image/avif",
+  tiff: "image/tiff",
+  heif: "image/heic",
+  jxl: "image/jxl",
+  ico: "image/x-icon",
+  psd: "image/vnd.adobe.photoshop",
+  raw: "image/x-dcraw",
+  tga: "image/x-tga",
+  exr: "image/x-exr",
+  hdr: "image/vnd.radiance",
+  jp2: "image/jp2",
+  qoi: "image/x-qoi",
+  eps: "application/postscript",
+  dds: "image/x-dds",
+  cur: "image/x-icon",
+  dpx: "image/x-dpx",
+  fits: "image/fits",
+  ppm: "image/x-portable-pixmap",
+  pgm: "image/x-portable-graymap",
+  pbm: "image/x-portable-bitmap",
+  pfm: "image/x-portable-floatmap",
+};
+
+const fetchUrlsSchema = z.object({
+  urls: z
+    .array(z.string().url("Each entry must be a valid URL"))
+    .min(1, "At least one URL is required")
+    .max(MAX_URLS_PER_REQUEST, `Maximum ${MAX_URLS_PER_REQUEST} URLs per request`),
+});
+
+interface SuccessResult {
+  success: true;
+  url: string;
+  filename: string;
+  contentType: string;
+  size: number;
+  width: number;
+  height: number;
+  downloadUrl: string;
+  previewUrl: string | null;
+}
+
+interface FailureResult {
+  success: false;
+  url: string;
+  error: string;
+}
+
+type FetchResult = SuccessResult | FailureResult;
+
+/**
+ * Extract a usable filename from a URL path, falling back to a UUID-based name.
+ */
+function filenameFromUrl(url: string): string {
+  try {
+    const pathname = new URL(url).pathname;
+    const base = basename(pathname);
+    // Decode percent-encoded characters
+    const decoded = decodeURIComponent(base);
+    // Only use it if it looks like a file with an extension
+    if (decoded?.includes(".") && decoded.length <= 255) {
+      return decoded;
+    }
+  } catch {
+    // ignore parse errors
+  }
+  return `image-${randomUUID().slice(0, 8)}`;
+}
+
+export async function registerFetchUrlsRoute(app: FastifyInstance): Promise<void> {
+  app.post("/api/v1/fetch-urls", async (request, reply) => {
+    // Validate body
+    const parsed = fetchUrlsSchema.safeParse(request.body);
+    if (!parsed.success) {
+      const messages = parsed.error.issues.map((i) => i.message).join("; ");
+      return reply.status(400).send({ error: messages });
+    }
+
+    const { urls } = parsed.data;
+    const jobId = randomUUID();
+    const workspace = await createWorkspace(jobId);
+    const outputDir = join(workspace, "output");
+
+    const queue = new PQueue({ concurrency: URL_FETCH_CONCURRENCY });
+
+    // Pre-allocate result slots to preserve order
+    const resultSlots: FetchResult[] = new Array(urls.length);
+
+    await Promise.all(
+      urls.map((url, index) =>
+        queue.add(async () => {
+          resultSlots[index] = await fetchSingleUrl(url, jobId, outputDir);
+        }),
+      ),
+    );
+
+    return reply.send({ results: resultSlots });
+  });
+}
+
+async function fetchSingleUrl(url: string, jobId: string, outputDir: string): Promise<FetchResult> {
+  try {
+    // Fetch with SSRF protection and timeout
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS);
+
+    let response: Response;
+    try {
+      response = await safeFetch(url, controller.signal);
+    } finally {
+      clearTimeout(timeout);
+    }
+
+    if (!response.ok) {
+      return {
+        success: false,
+        url,
+        error: `HTTP ${response.status} ${response.statusText}`,
+      };
+    }
+
+    // Read body with size limit
+    const chunks: Uint8Array[] = [];
+    let totalSize = 0;
+
+    if (!response.body) {
+      return { success: false, url, error: "Empty response body" };
+    }
+
+    const reader = response.body.getReader();
+    try {
+      while (true) {
+        const { done, value } = await reader.read();
+        if (done) break;
+        totalSize += value.byteLength;
+        if (totalSize > MAX_URL_FETCH_SIZE) {
+          reader.cancel();
+          return {
+            success: false,
+            url,
+            error: `File exceeds maximum size of ${MAX_URL_FETCH_SIZE / (1024 * 1024)}MB`,
+          };
+        }
+        chunks.push(value);
+      }
+    } finally {
+      reader.releaseLock();
+    }
+
+    const buffer = Buffer.concat(chunks);
+    if (buffer.length === 0) {
+      return { success: false, url, error: "Empty response body" };
+    }
+
+    // Derive filename from URL
+    const rawFilename = filenameFromUrl(url);
+    const filename = sanitizeFilename(rawFilename);
+
+    // Validate as an image
+    const validation = await validateImageBuffer(buffer, filename);
+    if (!validation.valid) {
+      return { success: false, url, error: validation.reason };
+    }
+
+    // Save to output directory
+    await writeFile(join(outputDir, filename), buffer);
+
+    const contentType = FORMAT_TO_MIME[validation.format] ?? "application/octet-stream";
+    const downloadUrl = `/api/v1/download/${jobId}/${encodeURIComponent(filename)}`;
+
+    // Generate preview for non-browser formats
+    let previewUrl: string | null = null;
+    if (!BROWSER_PREVIEWABLE.has(contentType)) {
+      try {
+        const previewBuffer = await sharp(buffer).webp({ quality: 80 }).toBuffer();
+        const previewFilename = `preview-${filename.replace(/\.[^.]+$/, "")}.webp`;
+        await writeFile(join(outputDir, previewFilename), previewBuffer);
+        previewUrl = `/api/v1/download/${jobId}/${encodeURIComponent(previewFilename)}`;
+      } catch {
+        // Preview generation failed -- non-fatal, skip preview
+      }
+    }
+
+    return {
+      success: true,
+      url,
+      filename,
+      contentType,
+      size: buffer.length,
+      width: validation.width,
+      height: validation.height,
+      downloadUrl,
+      previewUrl,
+    };
+  } catch (err) {
+    const message = err instanceof Error ? err.message : "Unknown error";
+    return { success: false, url, error: message };
+  }
+}
@@ -0,0 +1,318 @@
+/**
+ * Integration tests for the fetch-urls route.
+ *
+ * Spins up a local HTTP server to serve test fixtures, and mocks the SSRF
+ * validation to allow localhost connections during tests.
+ */
+
+import { readFileSync } from "node:fs";
+import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http";
+import { join } from "node:path";
+import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
+
+// Mock the SSRF validation to allow localhost in tests.
+// We keep the real safeFetch logic but skip the private-IP DNS check.
+vi.mock("../../apps/api/src/lib/ssrf.js", async (importOriginal) => {
+  const original = (await importOriginal()) as Record<string, unknown>;
+  return {
+    ...original,
+    // validateFetchUrl that allows localhost for tests
+    validateFetchUrl: async (_url: string) => {
+      // No-op: allow all URLs in tests (including localhost)
+    },
+    // safeFetch that skips SSRF validation but still does the real fetch
+    safeFetch: async (url: string, signal?: AbortSignal) => {
+      const MAX_REDIRECTS = 5;
+      let currentUrl = url;
+      for (let i = 0; i <= MAX_REDIRECTS; i++) {
+        const res = await fetch(currentUrl, {
+          signal,
+          redirect: "manual",
+          headers: { "User-Agent": "SnapOtter/1.0 (image-fetch)" },
+        });
+        if (res.status >= 300 && res.status < 400) {
+          const location = res.headers.get("location");
+          if (!location) throw new Error("Redirect without Location header");
+          currentUrl = new URL(location, currentUrl).href;
+          if (i === MAX_REDIRECTS) throw new Error("Too many redirects");
+          continue;
+        }
+        return res;
+      }
+      throw new Error("Too many redirects");
+    },
+  };
+});
+
+import { buildTestApp, loginAsAdmin, type TestApp } from "./test-server.js";
+
+const FIXTURES = join(__dirname, "..", "fixtures");
+const JPG = readFileSync(join(FIXTURES, "test-100x100.jpg"));
+
+let testApp: TestApp;
+let app: TestApp["app"];
+let adminToken: string;
+let mockServer: Server;
+let mockPort: number;
+
+function startMockServer(): Promise<{ server: Server; port: number }> {
+  return new Promise((resolve) => {
+    const server = createServer((req: IncomingMessage, res: ServerResponse) => {
+      const url = req.url ?? "";
+
+      if (url === "/photo.jpg") {
+        res.writeHead(200, { "Content-Type": "image/jpeg" });
+        res.end(JPG);
+        return;
+      }
+
+      if (url === "/not-image.txt") {
+        res.writeHead(200, { "Content-Type": "text/plain" });
+        res.end("This is not an image");
+        return;
+      }
+
+      if (url === "/redirect") {
+        res.writeHead(302, { Location: "/photo.jpg" });
+        res.end();
+        return;
+      }
+
+      if (url === "/missing.jpg") {
+        res.writeHead(404);
+        res.end("Not Found");
+        return;
+      }
+
+      res.writeHead(404);
+      res.end("Not Found");
+    });
+
+    server.listen(0, "127.0.0.1", () => {
+      const addr = server.address();
+      const port = typeof addr === "object" && addr ? addr.port : 0;
+      resolve({ server, port });
+    });
+  });
+}
+
+beforeAll(async () => {
+  const mock = await startMockServer();
+  mockServer = mock.server;
+  mockPort = mock.port;
+
+  testApp = await buildTestApp();
+  app = testApp.app;
+  adminToken = await loginAsAdmin(app);
+}, 30_000);
+
+afterAll(async () => {
+  await testApp.cleanup();
+  await new Promise<void>((resolve) => mockServer.close(() => resolve()));
+}, 10_000);
+
+describe("POST /api/v1/fetch-urls", () => {
+  it("fetches a valid image URL and returns metadata + download URL", async () => {
+    const res = await app.inject({
+      method: "POST",
+      url: "/api/v1/fetch-urls",
+      headers: {
+        authorization: `Bearer ${adminToken}`,
+      },
+      payload: {
+        urls: [`http://127.0.0.1:${mockPort}/photo.jpg`],
+      },
+    });
+
+    expect(res.statusCode).toBe(200);
+    const body = JSON.parse(res.body);
+    expect(body.results).toHaveLength(1);
+
+    const result = body.results[0];
+    expect(result.success).toBe(true);
+    expect(result.url).toBe(`http://127.0.0.1:${mockPort}/photo.jpg`);
+    expect(result.filename).toBe("photo.jpg");
+    expect(result.contentType).toBe("image/jpeg");
+    expect(result.size).toBeGreaterThan(0);
+    expect(result.width).toBe(100);
+    expect(result.height).toBe(100);
+    expect(result.downloadUrl).toMatch(/^\/api\/v1\/download\/.+\/photo\.jpg$/);
+    expect(result.previewUrl).toBeNull(); // JPEG is browser-previewable
+  });
+
+  it("returns failure for a 404 URL", async () => {
+    const res = await app.inject({
+      method: "POST",
+      url: "/api/v1/fetch-urls",
+      headers: {
+        authorization: `Bearer ${adminToken}`,
+      },
+      payload: {
+        urls: [`http://127.0.0.1:${mockPort}/missing.jpg`],
+      },
+    });
+
+    expect(res.statusCode).toBe(200);
+    const body = JSON.parse(res.body);
+    expect(body.results).toHaveLength(1);
+
+    const result = body.results[0];
+    expect(result.success).toBe(false);
+    expect(result.error).toContain("404");
+  });
+
+  it("returns failure for non-image content", async () => {
+    const res = await app.inject({
+      method: "POST",
+      url: "/api/v1/fetch-urls",
+      headers: {
+        authorization: `Bearer ${adminToken}`,
+      },
+      payload: {
+        urls: [`http://127.0.0.1:${mockPort}/not-image.txt`],
+      },
+    });
+
+    expect(res.statusCode).toBe(200);
+    const body = JSON.parse(res.body);
+    expect(body.results).toHaveLength(1);
+
+    const result = body.results[0];
+    expect(result.success).toBe(false);
+    expect(result.error).toBeTruthy();
+  });
+
+  it("handles mixed batch with successes and failures", async () => {
+    const res = await app.inject({
+      method: "POST",
+      url: "/api/v1/fetch-urls",
+      headers: {
+        authorization: `Bearer ${adminToken}`,
+      },
+      payload: {
+        urls: [
+          `http://127.0.0.1:${mockPort}/photo.jpg`,
+          `http://127.0.0.1:${mockPort}/missing.jpg`,
+          `http://127.0.0.1:${mockPort}/not-image.txt`,
+        ],
+      },
+    });
+
+    expect(res.statusCode).toBe(200);
+    const body = JSON.parse(res.body);
+    expect(body.results).toHaveLength(3);
+
+    // Results preserve order
+    expect(body.results[0].success).toBe(true);
+    expect(body.results[0].filename).toBe("photo.jpg");
+
+    expect(body.results[1].success).toBe(false);
+    expect(body.results[1].error).toContain("404");
+
+    expect(body.results[2].success).toBe(false);
+  });
+
+  it("returns 400 for an empty URL array", async () => {
+    const res = await app.inject({
+      method: "POST",
+      url: "/api/v1/fetch-urls",
+      headers: {
+        authorization: `Bearer ${adminToken}`,
+      },
+      payload: {
+        urls: [],
+      },
+    });
+
+    expect(res.statusCode).toBe(400);
+    const body = JSON.parse(res.body);
+    expect(body.error).toBeTruthy();
+  });
+
+  it("returns 400 for more than 50 URLs", async () => {
+    const urls = Array.from({ length: 51 }, (_, i) => `http://example.com/img${i}.jpg`);
+    const res = await app.inject({
+      method: "POST",
+      url: "/api/v1/fetch-urls",
+      headers: {
+        authorization: `Bearer ${adminToken}`,
+      },
+      payload: { urls },
+    });
+
+    expect(res.statusCode).toBe(400);
+    const body = JSON.parse(res.body);
+    expect(body.error).toBeTruthy();
+  });
+
+  it("follows redirects to fetch the final image", async () => {
+    const res = await app.inject({
+      method: "POST",
+      url: "/api/v1/fetch-urls",
+      headers: {
+        authorization: `Bearer ${adminToken}`,
+      },
+      payload: {
+        urls: [`http://127.0.0.1:${mockPort}/redirect`],
+      },
+    });
+
+    expect(res.statusCode).toBe(200);
+    const body = JSON.parse(res.body);
+    expect(body.results).toHaveLength(1);
+
+    const result = body.results[0];
+    expect(result.success).toBe(true);
+    expect(result.contentType).toBe("image/jpeg");
+    expect(result.size).toBe(JPG.length);
+  });
+
+  it("download URL serves the actual image", async () => {
+    // First, fetch the URL to get a downloadUrl
+    const fetchRes = await app.inject({
+      method: "POST",
+      url: "/api/v1/fetch-urls",
+      headers: {
+        authorization: `Bearer ${adminToken}`,
+      },
+      payload: {
+        urls: [`http://127.0.0.1:${mockPort}/photo.jpg`],
+      },
+    });
+
+    const body = JSON.parse(fetchRes.body);
+    const downloadUrl = body.results[0].downloadUrl;
+    expect(downloadUrl).toBeTruthy();
+
+    // Now download the file
+    const downloadRes = await app.inject({
+      method: "GET",
+      url: downloadUrl,
+      headers: {
+        authorization: `Bearer ${adminToken}`,
+      },
+    });
+
+    expect(downloadRes.statusCode).toBe(200);
+    expect(downloadRes.headers["content-type"]).toBe("image/jpeg");
+    // The downloaded buffer should match the original fixture
+    expect(downloadRes.rawPayload.length).toBe(JPG.length);
+  });
+
+  it("returns 400 for invalid URL format", async () => {
+    const res = await app.inject({
+      method: "POST",
+      url: "/api/v1/fetch-urls",
+      headers: {
+        authorization: `Bearer ${adminToken}`,
+      },
+      payload: {
+        urls: ["not-a-valid-url"],
+      },
+    });
+
+    expect(res.statusCode).toBe(400);
+    const body = JSON.parse(res.body);
+    expect(body.error).toBeTruthy();
+  });
+});
@@ -37,6 +37,7 @@ import { apiKeyRoutes } from "../../apps/api/src/routes/api-keys.js";
 import { auditLogRoutes } from "../../apps/api/src/routes/audit-log.js";
 import { registerBatchRoutes } from "../../apps/api/src/routes/batch.js";
 import { docsRoutes } from "../../apps/api/src/routes/docs.js";
+import { registerFetchUrlsRoute } from "../../apps/api/src/routes/fetch-urls.js";
 import { fileRoutes } from "../../apps/api/src/routes/files.js";
 import { registerMemeTemplates } from "../../apps/api/src/routes/meme-templates.js";
 import { registerPipelineRoutes } from "../../apps/api/src/routes/pipeline.js";
@@ -100,6 +101,9 @@ export async function buildTestApp(): Promise<TestApp> {
  // Batch processing routes
  await registerBatchRoutes(app);

+  // URL fetch routes
+  await registerFetchUrlsRoute(app);
+
  // Pipeline routes
  await registerPipelineRoutes(app);