mirror of
https://github.com/snapotter-hq/SnapOtter.git
synced 2026-08-03 07:46:42 +02:00
feat: add URL-based image import (single + bulk)
Add a fourth image ingestion path: importing images by URL. Backend: - POST /api/v1/fetch-urls endpoint with SSRF protection, image validation, preview generation, and p-queue concurrency - SSRF utility blocking private IPs, validating redirect hops, with comprehensive IPv4/IPv6 range coverage Frontend: - Always-visible URL input in the dropzone for quick single-image import - Bulk URL import modal with smart URL parsing (lists, markdown, HTML), per-URL progress tracking, retry on failure, and batch add - useUrlImport hook managing the full fetch lifecycle Tests: 48 new tests (23 SSRF unit, 10 URL parser unit, 15 integration)
This commit is contained in:
SnapOtter
@@ -23,6 +23,7 @@ import { auditLogRoutes } from "./routes/audit-log.js";
|
||||
import { registerBatchRoutes } from "./routes/batch.js";
|
||||
import { docsRoutes } from "./routes/docs.js";
|
||||
import { registerFeatureRoutes } from "./routes/features.js";
|
||||
import { registerFetchUrlsRoute } from "./routes/fetch-urls.js";
|
||||
import { fileRoutes } from "./routes/files.js";
|
||||
import { registerMemeTemplates } from "./routes/meme-templates.js";
|
||||
import { registerPipelineRoutes } from "./routes/pipeline.js";
|
||||
@@ -165,6 +166,9 @@ await registerToolRoutes(app);
|
||||
// Batch processing routes (must be after tool routes so the registry is populated)
|
||||
await registerBatchRoutes(app);
|
||||
|
||||
// URL fetch routes (server-side image fetching with SSRF protection)
|
||||
await registerFetchUrlsRoute(app);
|
||||
|
||||
// Pipeline routes (must be after tool routes so the registry is populated)
|
||||
await registerPipelineRoutes(app);
|
||||
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
import { lookup } from "node:dns/promises";
|
||||
import { isIP } from "node:net";
|
||||
|
||||
function isPrivateIPv4(ip: string): boolean {
|
||||
const parts = ip.split(".").map(Number);
|
||||
if (parts.length !== 4 || parts.some((p) => Number.isNaN(p))) return false;
|
||||
const [a, b] = parts;
|
||||
if (a === 10) return true;
|
||||
if (a === 172 && b >= 16 && b <= 31) return true;
|
||||
if (a === 192 && b === 168) return true;
|
||||
if (a === 127) return true;
|
||||
if (a === 169 && b === 254) return true;
|
||||
if (a === 0) return true;
|
||||
if (a === 100 && b >= 64 && b <= 127) return true;
|
||||
if (a === 192 && b === 0 && parts[2] === 0) return true;
|
||||
if (a === 198 && (b === 18 || b === 19)) return true;
|
||||
if (a >= 240) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
function isPrivateIPv6(ip: string): boolean {
|
||||
const normalized = ip.replace(/^\[|]$/g, "").toLowerCase();
|
||||
if (normalized === "::1") return true;
|
||||
if (normalized === "::") return true;
|
||||
if (normalized.startsWith("fe80:")) return true;
|
||||
if (normalized.startsWith("fc") || normalized.startsWith("fd")) return true;
|
||||
if (normalized.startsWith("2001:db8:")) return true;
|
||||
if (normalized.includes("::ffff:")) {
|
||||
const v4 = normalized.split("::ffff:")[1];
|
||||
if (v4 && isPrivateIPv4(v4)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
async function resolveAndCheck(hostname: string): Promise<void> {
|
||||
const bare = hostname.replace(/^\[|]$/g, "");
|
||||
if (isIP(bare)) {
|
||||
if (isPrivateIPv4(bare) || isPrivateIPv6(bare)) {
|
||||
throw new Error("URL resolves to a private or reserved IP address");
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const result = await lookup(hostname, { all: true });
|
||||
const addresses = Array.isArray(result) ? result : [result];
|
||||
for (const entry of addresses) {
|
||||
const addr = entry.address;
|
||||
if (isPrivateIPv4(addr) || isPrivateIPv6(addr)) {
|
||||
throw new Error("URL resolves to a private or reserved IP address");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function validateFetchUrl(url: string): Promise<void> {
|
||||
let parsed: URL;
|
||||
try {
|
||||
parsed = new URL(url);
|
||||
} catch {
|
||||
throw new Error("Invalid URL");
|
||||
}
|
||||
|
||||
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
|
||||
throw new Error("Only HTTP and HTTPS URLs are supported");
|
||||
}
|
||||
|
||||
await resolveAndCheck(parsed.hostname);
|
||||
}
|
||||
|
||||
export const MAX_REDIRECTS = 5;
|
||||
export const FETCH_TIMEOUT_MS = 30_000;
|
||||
export const MAX_URL_FETCH_SIZE = 50 * 1024 * 1024;
|
||||
export const MAX_URLS_PER_REQUEST = 50;
|
||||
export const URL_FETCH_CONCURRENCY = 4;
|
||||
|
||||
export async function safeFetch(url: string, signal?: AbortSignal): Promise<Response> {
|
||||
let currentUrl = url;
|
||||
for (let i = 0; i <= MAX_REDIRECTS; i++) {
|
||||
await validateFetchUrl(currentUrl);
|
||||
const res = await fetch(currentUrl, {
|
||||
signal,
|
||||
redirect: "manual",
|
||||
headers: { "User-Agent": "SnapOtter/1.0 (image-fetch)" },
|
||||
});
|
||||
|
||||
if (res.status >= 300 && res.status < 400) {
|
||||
const location = res.headers.get("location");
|
||||
if (!location) throw new Error("Redirect without Location header");
|
||||
await res.body?.cancel();
|
||||
currentUrl = new URL(location, currentUrl).href;
|
||||
continue;
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
throw new Error("Too many redirects");
|
||||
}
|
||||
@@ -0,0 +1,279 @@
|
||||
/**
|
||||
* Fetch URLs route.
|
||||
*
|
||||
* POST /api/v1/fetch-urls
|
||||
*
|
||||
* Accepts a JSON body with { urls: string[] } (1-50 URLs).
|
||||
* Fetches each URL server-side with SSRF protection, validates as an image,
|
||||
* saves to a workspace, generates a preview for non-browser formats, and
|
||||
* returns results with download URLs.
|
||||
*/
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import { basename, join } from "node:path";
|
||||
import type { FastifyInstance } from "fastify";
|
||||
import PQueue from "p-queue";
|
||||
import sharp from "sharp";
|
||||
import { z } from "zod";
|
||||
import { validateImageBuffer } from "../lib/file-validation.js";
|
||||
import { sanitizeFilename } from "../lib/filename.js";
|
||||
import {
|
||||
FETCH_TIMEOUT_MS,
|
||||
MAX_URL_FETCH_SIZE,
|
||||
MAX_URLS_PER_REQUEST,
|
||||
safeFetch,
|
||||
URL_FETCH_CONCURRENCY,
|
||||
} from "../lib/ssrf.js";
|
||||
import { createWorkspace } from "../lib/workspace.js";
|
||||
|
||||
/** Formats browsers can display natively (no preview needed). */
|
||||
const BROWSER_PREVIEWABLE = new Set([
|
||||
"image/jpeg",
|
||||
"image/png",
|
||||
"image/gif",
|
||||
"image/webp",
|
||||
"image/svg+xml",
|
||||
"image/bmp",
|
||||
"image/avif",
|
||||
]);
|
||||
|
||||
/** Map detected format string to MIME type. */
|
||||
const FORMAT_TO_MIME: Record<string, string> = {
|
||||
jpeg: "image/jpeg",
|
||||
png: "image/png",
|
||||
gif: "image/gif",
|
||||
webp: "image/webp",
|
||||
svg: "image/svg+xml",
|
||||
bmp: "image/bmp",
|
||||
avif: "image/avif",
|
||||
tiff: "image/tiff",
|
||||
heif: "image/heic",
|
||||
jxl: "image/jxl",
|
||||
ico: "image/x-icon",
|
||||
psd: "image/vnd.adobe.photoshop",
|
||||
raw: "image/x-dcraw",
|
||||
tga: "image/x-tga",
|
||||
exr: "image/x-exr",
|
||||
hdr: "image/vnd.radiance",
|
||||
jp2: "image/jp2",
|
||||
qoi: "image/x-qoi",
|
||||
eps: "application/postscript",
|
||||
dds: "image/x-dds",
|
||||
cur: "image/x-icon",
|
||||
dpx: "image/x-dpx",
|
||||
fits: "image/fits",
|
||||
ppm: "image/x-portable-pixmap",
|
||||
pgm: "image/x-portable-graymap",
|
||||
pbm: "image/x-portable-bitmap",
|
||||
pfm: "image/x-portable-floatmap",
|
||||
};
|
||||
|
||||
const fetchUrlsSchema = z.object({
|
||||
urls: z
|
||||
.array(z.string().url("Each entry must be a valid URL"))
|
||||
.min(1, "At least one URL is required")
|
||||
.max(MAX_URLS_PER_REQUEST, `Maximum ${MAX_URLS_PER_REQUEST} URLs per request`),
|
||||
});
|
||||
|
||||
interface SuccessResult {
|
||||
success: true;
|
||||
url: string;
|
||||
filename: string;
|
||||
contentType: string;
|
||||
size: number;
|
||||
width: number;
|
||||
height: number;
|
||||
downloadUrl: string;
|
||||
previewUrl: string | null;
|
||||
}
|
||||
|
||||
interface FailureResult {
|
||||
success: false;
|
||||
url: string;
|
||||
error: string;
|
||||
}
|
||||
|
||||
type FetchResult = SuccessResult | FailureResult;
|
||||
|
||||
/**
|
||||
* Extract a usable filename from a URL path, falling back to a UUID-based name.
|
||||
*/
|
||||
function filenameFromUrl(url: string): string {
|
||||
try {
|
||||
const pathname = new URL(url).pathname;
|
||||
const base = basename(pathname);
|
||||
// Decode percent-encoded characters
|
||||
const decoded = decodeURIComponent(base);
|
||||
// Only use it if it looks like a file with an extension
|
||||
if (decoded?.includes(".") && decoded.length <= 255) {
|
||||
return decoded;
|
||||
}
|
||||
} catch {
|
||||
// ignore parse errors
|
||||
}
|
||||
return `image-${randomUUID().slice(0, 8)}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return a filename that does not collide with any name already in `used`.
|
||||
* Appends `_1`, `_2`, etc. before the extension when a collision is found.
|
||||
* Mirrors the deduplication logic in batch.ts.
|
||||
*/
|
||||
function getUniqueName(name: string, used: Set<string>): string {
|
||||
if (!used.has(name)) {
|
||||
used.add(name);
|
||||
return name;
|
||||
}
|
||||
const dotIdx = name.lastIndexOf(".");
|
||||
const base = dotIdx > 0 ? name.slice(0, dotIdx) : name;
|
||||
const ext = dotIdx > 0 ? name.slice(dotIdx) : "";
|
||||
let counter = 1;
|
||||
let candidate = `${base}_${counter}${ext}`;
|
||||
while (used.has(candidate)) {
|
||||
counter++;
|
||||
candidate = `${base}_${counter}${ext}`;
|
||||
}
|
||||
used.add(candidate);
|
||||
return candidate;
|
||||
}
|
||||
|
||||
export async function registerFetchUrlsRoute(app: FastifyInstance): Promise<void> {
|
||||
app.post("/api/v1/fetch-urls", async (request, reply) => {
|
||||
// Validate body
|
||||
const parsed = fetchUrlsSchema.safeParse(request.body);
|
||||
if (!parsed.success) {
|
||||
const messages = parsed.error.issues.map((i) => i.message).join("; ");
|
||||
return reply.status(400).send({ error: messages });
|
||||
}
|
||||
|
||||
const { urls } = parsed.data;
|
||||
const jobId = randomUUID();
|
||||
const workspace = await createWorkspace(jobId);
|
||||
const outputDir = join(workspace, "output");
|
||||
|
||||
const queue = new PQueue({ concurrency: URL_FETCH_CONCURRENCY });
|
||||
|
||||
// Track filenames to prevent collisions when multiple URLs resolve to the
|
||||
// same name (e.g. https://a.com/photo.jpg and https://b.com/photo.jpg).
|
||||
const usedFilenames = new Set<string>();
|
||||
|
||||
// Pre-allocate result slots to preserve order
|
||||
const resultSlots: FetchResult[] = new Array(urls.length);
|
||||
|
||||
await Promise.all(
|
||||
urls.map((url, index) =>
|
||||
queue.add(async () => {
|
||||
resultSlots[index] = await fetchSingleUrl(url, jobId, outputDir, usedFilenames);
|
||||
}),
|
||||
),
|
||||
);
|
||||
|
||||
return reply.send({ results: resultSlots });
|
||||
});
|
||||
}
|
||||
|
||||
async function fetchSingleUrl(
|
||||
url: string,
|
||||
jobId: string,
|
||||
outputDir: string,
|
||||
usedFilenames: Set<string>,
|
||||
): Promise<FetchResult> {
|
||||
try {
|
||||
// Fetch with SSRF protection and timeout
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS);
|
||||
|
||||
let response: Response;
|
||||
try {
|
||||
response = await safeFetch(url, controller.signal);
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
return {
|
||||
success: false,
|
||||
url,
|
||||
error: `HTTP ${response.status} ${response.statusText}`,
|
||||
};
|
||||
}
|
||||
|
||||
// Read body with size limit
|
||||
const chunks: Uint8Array[] = [];
|
||||
let totalSize = 0;
|
||||
|
||||
if (!response.body) {
|
||||
return { success: false, url, error: "Empty response body" };
|
||||
}
|
||||
|
||||
const reader = response.body.getReader();
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
totalSize += value.byteLength;
|
||||
if (totalSize > MAX_URL_FETCH_SIZE) {
|
||||
reader.cancel();
|
||||
return {
|
||||
success: false,
|
||||
url,
|
||||
error: `File exceeds maximum size of ${MAX_URL_FETCH_SIZE / (1024 * 1024)}MB`,
|
||||
};
|
||||
}
|
||||
chunks.push(value);
|
||||
}
|
||||
} finally {
|
||||
reader.releaseLock();
|
||||
}
|
||||
|
||||
const buffer = Buffer.concat(chunks);
|
||||
if (buffer.length === 0) {
|
||||
return { success: false, url, error: "Empty response body" };
|
||||
}
|
||||
|
||||
// Derive filename from URL, deduplicating to prevent overwrites when
|
||||
// multiple URLs resolve to the same name (all URLs share one workspace).
|
||||
const rawFilename = filenameFromUrl(url);
|
||||
const filename = getUniqueName(sanitizeFilename(rawFilename), usedFilenames);
|
||||
|
||||
// Validate as an image
|
||||
const validation = await validateImageBuffer(buffer, filename);
|
||||
if (!validation.valid) {
|
||||
return { success: false, url, error: validation.reason };
|
||||
}
|
||||
|
||||
// Save to output directory
|
||||
await writeFile(join(outputDir, filename), buffer);
|
||||
|
||||
const contentType = FORMAT_TO_MIME[validation.format] ?? "application/octet-stream";
|
||||
const downloadUrl = `/api/v1/download/${jobId}/${encodeURIComponent(filename)}`;
|
||||
|
||||
// Generate preview for non-browser formats
|
||||
let previewUrl: string | null = null;
|
||||
if (!BROWSER_PREVIEWABLE.has(contentType)) {
|
||||
try {
|
||||
const previewBuffer = await sharp(buffer).webp({ quality: 80 }).toBuffer();
|
||||
const previewFilename = `preview-${filename.replace(/\.[^.]+$/, "")}.webp`;
|
||||
await writeFile(join(outputDir, previewFilename), previewBuffer);
|
||||
previewUrl = `/api/v1/download/${jobId}/${encodeURIComponent(previewFilename)}`;
|
||||
} catch {
|
||||
// Preview generation failed -- non-fatal, skip preview
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
url,
|
||||
filename,
|
||||
contentType,
|
||||
size: buffer.length,
|
||||
width: validation.width,
|
||||
height: validation.height,
|
||||
downloadUrl,
|
||||
previewUrl,
|
||||
};
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : "Unknown error";
|
||||
return { success: false, url, error: message };
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,8 @@
|
||||
import { FileImage, ImageUp, Upload } from "lucide-react";
|
||||
import { type DragEvent, useCallback, useEffect, useState } from "react";
|
||||
import { useUrlImport } from "@/hooks/use-url-import";
|
||||
import { cn } from "@/lib/utils";
|
||||
import { UrlImportModal } from "./url-import-modal";
|
||||
|
||||
const IMAGE_EXTENSIONS = new Set([
|
||||
"jpg",
|
||||
@@ -79,6 +81,7 @@ export function isImageFile(file: File): boolean {
|
||||
|
||||
interface DropzoneProps {
|
||||
onFiles?: (files: File[]) => void;
|
||||
onUrlImport?: (file: File) => void;
|
||||
accept?: string;
|
||||
multiple?: boolean;
|
||||
/** Files that have already been dropped (for showing count & list). */
|
||||
@@ -96,6 +99,7 @@ function expandAccept(accept?: string): string | undefined {
|
||||
|
||||
export function Dropzone({
|
||||
onFiles,
|
||||
onUrlImport,
|
||||
accept,
|
||||
multiple = true,
|
||||
currentFiles = [],
|
||||
@@ -103,6 +107,27 @@ export function Dropzone({
|
||||
}: DropzoneProps) {
|
||||
const resolvedAccept = expandAccept(accept);
|
||||
const [isDragging, setIsDragging] = useState(false);
|
||||
const [urlInput, setUrlInput] = useState("");
|
||||
const [urlLoading, setUrlLoading] = useState(false);
|
||||
const [urlError, setUrlError] = useState<string | null>(null);
|
||||
const [showBulkModal, setShowBulkModal] = useState(false);
|
||||
|
||||
const { importSingleUrl } = useUrlImport();
|
||||
|
||||
const handleUrlSubmit = useCallback(async () => {
|
||||
const url = urlInput.trim();
|
||||
if (!url) return;
|
||||
setUrlLoading(true);
|
||||
setUrlError(null);
|
||||
const file = await importSingleUrl(url);
|
||||
if (file) {
|
||||
setUrlInput("");
|
||||
onUrlImport?.(file);
|
||||
} else {
|
||||
setUrlError("Could not fetch image from URL");
|
||||
}
|
||||
setUrlLoading(false);
|
||||
}, [urlInput, importSingleUrl, onUrlImport]);
|
||||
|
||||
const handleDrag = useCallback((e: DragEvent) => {
|
||||
e.preventDefault();
|
||||
@@ -224,6 +249,58 @@ export function Dropzone({
|
||||
PNG, JPG, WebP, HEIC, RAW, PSD, and 65+ formats
|
||||
</p>
|
||||
|
||||
{!compact && onUrlImport && (
|
||||
<>
|
||||
<div className="flex items-center gap-2 w-full max-w-xs">
|
||||
<div className="h-px flex-1 bg-border" />
|
||||
<span className="text-xs text-muted-foreground">or</span>
|
||||
<div className="h-px flex-1 bg-border" />
|
||||
</div>
|
||||
<div className="flex gap-2 w-full max-w-sm">
|
||||
<input
|
||||
type="url"
|
||||
value={urlInput}
|
||||
onChange={(e) => {
|
||||
setUrlInput(e.target.value);
|
||||
setUrlError(null);
|
||||
}}
|
||||
onKeyDown={(e) => {
|
||||
if (e.key === "Enter") {
|
||||
e.preventDefault();
|
||||
handleUrlSubmit();
|
||||
}
|
||||
}}
|
||||
onClick={(e) => e.stopPropagation()}
|
||||
placeholder="Paste image URL..."
|
||||
className="flex-1 rounded-md border border-border bg-background px-3 py-1.5 text-sm text-foreground placeholder:text-muted-foreground focus:border-primary focus:outline-none"
|
||||
disabled={urlLoading}
|
||||
/>
|
||||
<button
|
||||
type="button"
|
||||
onClick={(e) => {
|
||||
e.stopPropagation();
|
||||
handleUrlSubmit();
|
||||
}}
|
||||
disabled={urlLoading || !urlInput.trim()}
|
||||
className="rounded-md bg-primary px-3 py-1.5 text-sm font-medium text-primary-foreground hover:bg-primary/90 disabled:opacity-50"
|
||||
>
|
||||
{urlLoading ? "..." : "Add"}
|
||||
</button>
|
||||
</div>
|
||||
{urlError && <p className="text-xs text-destructive">{urlError}</p>}
|
||||
<button
|
||||
type="button"
|
||||
onClick={(e) => {
|
||||
e.stopPropagation();
|
||||
setShowBulkModal(true);
|
||||
}}
|
||||
className="text-xs text-primary hover:text-primary/80"
|
||||
>
|
||||
Import multiple URLs...
|
||||
</button>
|
||||
</>
|
||||
)}
|
||||
|
||||
{hasMultipleFiles && (
|
||||
<div className="flex flex-col items-center gap-2 mt-1">
|
||||
<span className="inline-flex items-center gap-1.5 px-3 py-1 rounded-full bg-primary/10 text-primary text-xs font-medium">
|
||||
@@ -244,6 +321,16 @@ export function Dropzone({
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{showBulkModal && (
|
||||
<UrlImportModal
|
||||
onClose={() => setShowBulkModal(false)}
|
||||
onImport={(files) => {
|
||||
for (const file of files) onUrlImport?.(file);
|
||||
setShowBulkModal(false);
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
</section>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,228 @@
|
||||
import { AlertCircle, Check, Clock, Link, Loader2, RotateCw, X } from "lucide-react";
|
||||
import { useCallback, useEffect, useState } from "react";
|
||||
import { type UrlImportEntry, useUrlImport } from "@/hooks/use-url-import";
|
||||
import { extractUrls } from "@/lib/url-parser";
|
||||
|
||||
// ── Types ──────────────────────────────────────────────────────
|
||||
|
||||
interface UrlImportModalProps {
|
||||
onClose: () => void;
|
||||
onImport: (files: File[]) => void;
|
||||
}
|
||||
|
||||
// ── Helpers ────────────────────────────────────────────────────
|
||||
|
||||
function StatusIcon({ status }: { status: UrlImportEntry["status"] }) {
|
||||
switch (status) {
|
||||
case "pending":
|
||||
return <Clock className="h-4 w-4 text-muted-foreground" />;
|
||||
case "fetching":
|
||||
return <Loader2 className="h-4 w-4 text-primary animate-spin" />;
|
||||
case "ready":
|
||||
return <Check className="h-4 w-4 text-emerald-500" />;
|
||||
case "failed":
|
||||
return <AlertCircle className="h-4 w-4 text-destructive" />;
|
||||
}
|
||||
}
|
||||
|
||||
function formatSize(bytes: number): string {
|
||||
if (bytes < 1024) return `${bytes} B`;
|
||||
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
|
||||
return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
|
||||
}
|
||||
|
||||
function filenameFromUrl(url: string): string {
|
||||
try {
|
||||
return new URL(url).pathname.split("/").pop() || url;
|
||||
} catch {
|
||||
return url;
|
||||
}
|
||||
}
|
||||
|
||||
// ── Component ──────────────────────────────────────────────────
|
||||
|
||||
export function UrlImportModal({ onClose, onImport }: UrlImportModalProps) {
|
||||
const [text, setText] = useState("");
|
||||
const [adding, setAdding] = useState(false);
|
||||
|
||||
const { entries, importing, importUrls, addReadyFiles, retryUrl, cancel, reset, readyCount } =
|
||||
useUrlImport();
|
||||
|
||||
const hasResults = entries.length > 0;
|
||||
|
||||
const handleImport = useCallback(() => {
|
||||
const urls = extractUrls(text);
|
||||
if (urls.length === 0) return;
|
||||
importUrls(urls);
|
||||
}, [text, importUrls]);
|
||||
|
||||
const handleAdd = useCallback(async () => {
|
||||
if (readyCount === 0) return;
|
||||
setAdding(true);
|
||||
try {
|
||||
const files = await addReadyFiles();
|
||||
if (files.length > 0) {
|
||||
onImport(files);
|
||||
onClose();
|
||||
}
|
||||
} finally {
|
||||
setAdding(false);
|
||||
}
|
||||
}, [readyCount, addReadyFiles, onImport, onClose]);
|
||||
|
||||
const handleBack = useCallback(() => {
|
||||
reset();
|
||||
}, [reset]);
|
||||
|
||||
const handleClose = useCallback(() => {
|
||||
cancel();
|
||||
onClose();
|
||||
}, [cancel, onClose]);
|
||||
|
||||
useEffect(() => {
|
||||
const handler = (e: KeyboardEvent) => {
|
||||
if (e.key === "Escape") handleClose();
|
||||
};
|
||||
window.addEventListener("keydown", handler);
|
||||
return () => window.removeEventListener("keydown", handler);
|
||||
}, [handleClose]);
|
||||
|
||||
return (
|
||||
<div className="fixed inset-0 z-50 flex items-center justify-center">
|
||||
{/* Overlay */}
|
||||
<div
|
||||
aria-hidden="true"
|
||||
className="absolute inset-0 bg-black/60 cursor-default"
|
||||
onClick={handleClose}
|
||||
/>
|
||||
|
||||
{/* Modal card */}
|
||||
<div
|
||||
role="dialog"
|
||||
aria-modal="true"
|
||||
className="relative z-10 w-full max-w-lg bg-background border border-border rounded-xl shadow-xl flex flex-col mx-4"
|
||||
>
|
||||
{/* Header */}
|
||||
<div className="flex items-center gap-3 px-4 py-3 border-b border-border shrink-0">
|
||||
<Link className="h-5 w-5 text-primary" />
|
||||
<h2 className="text-sm font-semibold text-foreground flex-1">Import from URLs</h2>
|
||||
<button
|
||||
type="button"
|
||||
onClick={handleClose}
|
||||
aria-label="Close"
|
||||
className="p-1.5 rounded-lg hover:bg-muted text-muted-foreground"
|
||||
>
|
||||
<X className="h-4 w-4" />
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{/* Body */}
|
||||
<div className="px-4 py-3 flex flex-col gap-3">
|
||||
{/* Textarea */}
|
||||
<textarea
|
||||
value={text}
|
||||
onChange={(e) => setText(e.target.value)}
|
||||
placeholder={
|
||||
"https://example.com/photo1.jpg\nhttps://example.com/photo2.png\n- https://example.com/photo3.webp\n[My image](https://example.com/photo4.jpg)"
|
||||
}
|
||||
className="w-full min-h-[120px] max-h-[240px] resize-y rounded-lg border border-border bg-muted px-3 py-2 text-sm font-mono text-foreground placeholder:text-muted-foreground focus:outline-none focus:ring-2 focus:ring-primary/50"
|
||||
/>
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Supports plain URLs, bulleted lists, numbered lists, and markdown links
|
||||
</p>
|
||||
|
||||
{/* Progress list */}
|
||||
{hasResults && (
|
||||
<div className="max-h-[200px] overflow-y-auto rounded-lg border border-border divide-y divide-border">
|
||||
{entries.map((entry, i) => (
|
||||
<div key={entry.url} className="flex items-center gap-2.5 px-3 py-2 text-sm">
|
||||
<StatusIcon status={entry.status} />
|
||||
<span className="flex-1 truncate text-foreground">
|
||||
{entry.filename || filenameFromUrl(entry.url)}
|
||||
</span>
|
||||
{entry.status === "failed" && (
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => retryUrl(i)}
|
||||
className="p-1 rounded hover:bg-muted text-muted-foreground"
|
||||
title="Retry"
|
||||
>
|
||||
<RotateCw className="h-3.5 w-3.5" />
|
||||
</button>
|
||||
)}
|
||||
{entry.status === "ready" && entry.size != null && (
|
||||
<span className="text-xs text-muted-foreground whitespace-nowrap">
|
||||
{formatSize(entry.size)}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Footer */}
|
||||
<div className="flex items-center justify-between px-4 py-3 border-t border-border shrink-0">
|
||||
<span className="text-xs text-muted-foreground">
|
||||
{hasResults && !importing ? `${readyCount} of ${entries.length} ready` : ""}
|
||||
</span>
|
||||
<div className="flex items-center gap-2">
|
||||
{hasResults && !importing ? (
|
||||
<>
|
||||
<button
|
||||
type="button"
|
||||
onClick={handleBack}
|
||||
className="px-4 py-2 text-sm rounded-lg border border-border text-foreground hover:bg-muted"
|
||||
>
|
||||
Back
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={handleAdd}
|
||||
disabled={readyCount === 0 || adding}
|
||||
className="px-4 py-2 text-sm rounded-lg bg-primary text-primary-foreground font-medium hover:bg-primary/90 disabled:opacity-50 disabled:cursor-not-allowed flex items-center gap-2"
|
||||
>
|
||||
{adding ? (
|
||||
<>
|
||||
<Loader2 className="h-4 w-4 animate-spin" />
|
||||
Adding...
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
Add {readyCount} Image{readyCount !== 1 ? "s" : ""}
|
||||
</>
|
||||
)}
|
||||
</button>
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<button
|
||||
type="button"
|
||||
onClick={handleClose}
|
||||
className="px-4 py-2 text-sm rounded-lg border border-border text-foreground hover:bg-muted"
|
||||
>
|
||||
Cancel
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={handleImport}
|
||||
disabled={text.trim().length === 0 || importing}
|
||||
className="px-4 py-2 text-sm rounded-lg bg-primary text-primary-foreground font-medium hover:bg-primary/90 disabled:opacity-50 disabled:cursor-not-allowed flex items-center gap-2"
|
||||
>
|
||||
{importing ? (
|
||||
<>
|
||||
<Loader2 className="h-4 w-4 animate-spin" />
|
||||
Importing...
|
||||
</>
|
||||
) : (
|
||||
"Import"
|
||||
)}
|
||||
</button>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,220 @@
|
||||
import { useCallback, useRef, useState } from "react";
|
||||
import { formatHeaders } from "@/lib/api";
|
||||
|
||||
// ── Types ──────────────────────────────────────────────────────
|
||||
|
||||
export interface UrlImportEntry {
|
||||
url: string;
|
||||
status: "pending" | "fetching" | "ready" | "failed";
|
||||
filename?: string;
|
||||
size?: number;
|
||||
width?: number;
|
||||
height?: number;
|
||||
downloadUrl?: string;
|
||||
previewUrl?: string | null;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
interface FetchUrlResult {
|
||||
success: boolean;
|
||||
url: string;
|
||||
filename?: string;
|
||||
contentType?: string;
|
||||
size?: number;
|
||||
width?: number;
|
||||
height?: number;
|
||||
downloadUrl?: string;
|
||||
previewUrl?: string | null;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
interface FetchUrlsResponse {
|
||||
results: FetchUrlResult[];
|
||||
}
|
||||
|
||||
// ── Hook ───────────────────────────────────────────────────────
|
||||
|
||||
export function useUrlImport() {
|
||||
const [entries, setEntries] = useState<UrlImportEntry[]>([]);
|
||||
const [importing, setImporting] = useState(false);
|
||||
const abortRef = useRef<AbortController | null>(null);
|
||||
|
||||
// -- helpers --
|
||||
|
||||
const fetchUrls = useCallback(
|
||||
async (urls: string[], signal?: AbortSignal): Promise<FetchUrlsResponse> => {
|
||||
const headers = formatHeaders();
|
||||
headers.set("Content-Type", "application/json");
|
||||
const res = await fetch("/api/v1/fetch-urls", {
|
||||
method: "POST",
|
||||
headers,
|
||||
body: JSON.stringify({ urls }),
|
||||
signal,
|
||||
});
|
||||
if (!res.ok) {
|
||||
const body = await res.json().catch(() => ({}));
|
||||
throw new Error((body as Record<string, string>).error || `Fetch failed: ${res.status}`);
|
||||
}
|
||||
return res.json();
|
||||
},
|
||||
[],
|
||||
);
|
||||
|
||||
const resultToEntry = useCallback((result: FetchUrlResult): UrlImportEntry => {
|
||||
if (result.success) {
|
||||
return {
|
||||
url: result.url,
|
||||
status: "ready",
|
||||
filename: result.filename,
|
||||
size: result.size,
|
||||
width: result.width,
|
||||
height: result.height,
|
||||
downloadUrl: result.downloadUrl,
|
||||
previewUrl: result.previewUrl,
|
||||
};
|
||||
}
|
||||
return {
|
||||
url: result.url,
|
||||
status: "failed",
|
||||
error: result.error,
|
||||
};
|
||||
}, []);
|
||||
|
||||
const downloadAsFile = useCallback(
|
||||
async (downloadUrl: string, filename: string, signal?: AbortSignal): Promise<File> => {
|
||||
const res = await fetch(downloadUrl, { headers: formatHeaders(), signal });
|
||||
if (!res.ok) throw new Error(`Download failed: ${res.status}`);
|
||||
const blob = await res.blob();
|
||||
return new File([blob], filename, { type: blob.type });
|
||||
},
|
||||
[],
|
||||
);
|
||||
|
||||
// -- public API --
|
||||
|
||||
const importUrls = useCallback(
|
||||
async (urls: string[]) => {
|
||||
if (urls.length === 0) return;
|
||||
|
||||
abortRef.current?.abort();
|
||||
const controller = new AbortController();
|
||||
abortRef.current = controller;
|
||||
|
||||
setEntries(urls.map((url) => ({ url, status: "fetching" })));
|
||||
setImporting(true);
|
||||
|
||||
try {
|
||||
const { results } = await fetchUrls(urls, controller.signal);
|
||||
|
||||
if (controller.signal.aborted) return;
|
||||
|
||||
setEntries(results.map(resultToEntry));
|
||||
} catch (err) {
|
||||
if ((err as Error).name === "AbortError") return;
|
||||
|
||||
setEntries(
|
||||
urls.map((url) => ({
|
||||
url,
|
||||
status: "failed" as const,
|
||||
error: (err as Error).message,
|
||||
})),
|
||||
);
|
||||
} finally {
|
||||
if (!controller.signal.aborted) {
|
||||
setImporting(false);
|
||||
}
|
||||
}
|
||||
},
|
||||
[fetchUrls, resultToEntry],
|
||||
);
|
||||
|
||||
const importSingleUrl = useCallback(
|
||||
async (url: string): Promise<File | null> => {
|
||||
const controller = new AbortController();
|
||||
abortRef.current = controller;
|
||||
try {
|
||||
const { results } = await fetchUrls([url], controller.signal);
|
||||
const result = results[0];
|
||||
if (!result?.success || !result.downloadUrl || !result.filename) return null;
|
||||
return await downloadAsFile(result.downloadUrl, result.filename, controller.signal);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
},
|
||||
[fetchUrls, downloadAsFile],
|
||||
);
|
||||
|
||||
const addReadyFiles = useCallback(async (): Promise<File[]> => {
|
||||
const ready = entries.filter(
|
||||
(e): e is UrlImportEntry & { downloadUrl: string; filename: string } =>
|
||||
e.status === "ready" && !!e.downloadUrl && !!e.filename,
|
||||
);
|
||||
|
||||
const settled = await Promise.allSettled(
|
||||
ready.map((e) => downloadAsFile(e.downloadUrl, e.filename)),
|
||||
);
|
||||
|
||||
return settled
|
||||
.filter((r): r is PromiseFulfilledResult<File> => r.status === "fulfilled")
|
||||
.map((r) => r.value);
|
||||
}, [entries, downloadAsFile]);
|
||||
|
||||
const retryUrl = useCallback(
|
||||
async (index: number) => {
|
||||
let url: string | undefined;
|
||||
setEntries((prev) => {
|
||||
url = prev[index]?.url;
|
||||
if (!url) return prev;
|
||||
return prev.map((e, i) =>
|
||||
i === index ? { ...e, status: "fetching" as const, error: undefined } : e,
|
||||
);
|
||||
});
|
||||
if (!url) return;
|
||||
|
||||
try {
|
||||
const { results } = await fetchUrls([url]);
|
||||
const result = results[0];
|
||||
if (!result) return;
|
||||
|
||||
setEntries((prev) => prev.map((e, i) => (i === index ? resultToEntry(result) : e)));
|
||||
} catch (err) {
|
||||
setEntries((prev) =>
|
||||
prev.map((e, i) =>
|
||||
i === index ? { ...e, status: "failed" as const, error: (err as Error).message } : e,
|
||||
),
|
||||
);
|
||||
}
|
||||
},
|
||||
[fetchUrls, resultToEntry],
|
||||
);
|
||||
|
||||
const cancel = useCallback(() => {
|
||||
abortRef.current?.abort();
|
||||
abortRef.current = null;
|
||||
setEntries([]);
|
||||
setImporting(false);
|
||||
}, []);
|
||||
|
||||
const reset = useCallback(() => {
|
||||
setEntries([]);
|
||||
setImporting(false);
|
||||
}, []);
|
||||
|
||||
// -- derived counts --
|
||||
|
||||
const readyCount = entries.filter((e) => e.status === "ready").length;
|
||||
const failedCount = entries.filter((e) => e.status === "failed").length;
|
||||
|
||||
return {
|
||||
entries,
|
||||
importing,
|
||||
importUrls,
|
||||
importSingleUrl,
|
||||
addReadyFiles,
|
||||
retryUrl,
|
||||
cancel,
|
||||
reset,
|
||||
readyCount,
|
||||
failedCount,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
function isValidHttpUrl(str: string): boolean {
|
||||
try {
|
||||
const url = new URL(str);
|
||||
return url.protocol === "http:" || url.protocol === "https:";
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export function extractUrls(input: string): string[] {
|
||||
const urls: string[] = [];
|
||||
|
||||
for (const rawLine of input.split("\n")) {
|
||||
let line = rawLine.trim();
|
||||
if (!line) continue;
|
||||
|
||||
// Strip numbered list prefixes: "1. ", "2) ", "3 "
|
||||
line = line.replace(/^\d+[.)]?\s+/, "");
|
||||
// Strip bullet prefixes: "- ", "* ", "+ "
|
||||
line = line.replace(/^[-*+]\s+/, "");
|
||||
|
||||
// Extract from markdown links: [text](url)
|
||||
const mdMatch = line.match(/\[.*?]\((https?:\/\/[^)]+)\)/);
|
||||
if (mdMatch) {
|
||||
urls.push(mdMatch[1]);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Extract from HTML img tags: <img src="url">
|
||||
const imgMatch = line.match(/<img[^>]+src=["'](https?:\/\/[^"']+)["']/i);
|
||||
if (imgMatch) {
|
||||
urls.push(imgMatch[1]);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (isValidHttpUrl(line)) {
|
||||
urls.push(line);
|
||||
}
|
||||
}
|
||||
|
||||
return [...new Set(urls)];
|
||||
}
|
||||
@@ -283,6 +283,13 @@ export function ToolPage() {
|
||||
[setFiles, reset],
|
||||
);
|
||||
|
||||
const handleUrlImport = useCallback(
|
||||
(file: File) => {
|
||||
addFiles([file]);
|
||||
},
|
||||
[addFiles],
|
||||
);
|
||||
|
||||
const handleUndo = useCallback(() => {
|
||||
undoProcessing();
|
||||
setEraserSliderInitPos(null);
|
||||
@@ -434,7 +441,15 @@ export function ToolPage() {
|
||||
// Custom results panel (find-duplicates, etc.)
|
||||
if (displayMode === "custom-results" && registryEntry?.ResultsPanel) {
|
||||
if (!hasFile)
|
||||
return <Dropzone onFiles={handleFiles} accept="image/*" multiple currentFiles={files} />;
|
||||
return (
|
||||
<Dropzone
|
||||
onFiles={handleFiles}
|
||||
onUrlImport={handleUrlImport}
|
||||
accept="image/*"
|
||||
multiple
|
||||
currentFiles={files}
|
||||
/>
|
||||
);
|
||||
const ResultsPanel = registryEntry.ResultsPanel;
|
||||
return (
|
||||
<Suspense fallback={<div className="text-sm text-muted-foreground">Loading...</div>}>
|
||||
@@ -637,7 +652,15 @@ export function ToolPage() {
|
||||
);
|
||||
}
|
||||
|
||||
return <Dropzone onFiles={handleFiles} accept="image/*" multiple currentFiles={files} />;
|
||||
return (
|
||||
<Dropzone
|
||||
onFiles={handleFiles}
|
||||
onUrlImport={handleUrlImport}
|
||||
accept="image/*"
|
||||
multiple
|
||||
currentFiles={files}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
// Navigation arrows (shared between mobile/desktop)
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
import { expect, test } from "./helpers";
|
||||
|
||||
test.describe("URL Image Import", () => {
|
||||
test("inline URL input is visible on tool page", async ({ loggedInPage: page }) => {
|
||||
await page.goto("/resize");
|
||||
|
||||
await expect(page.getByPlaceholder("Paste image URL...")).toBeVisible();
|
||||
});
|
||||
|
||||
test("bulk import modal opens and closes", async ({ loggedInPage: page }) => {
|
||||
await page.goto("/resize");
|
||||
|
||||
// Open the bulk import modal
|
||||
await page.getByText("Import multiple URLs...").click();
|
||||
|
||||
// Assert the modal title is visible
|
||||
await expect(page.getByText("Import from URLs")).toBeVisible();
|
||||
|
||||
// Close the modal via Cancel
|
||||
await page.getByRole("button", { name: "Cancel" }).click();
|
||||
|
||||
// Assert the modal title is no longer visible
|
||||
await expect(page.getByText("Import from URLs")).not.toBeVisible();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,496 @@
|
||||
/**
|
||||
* Integration tests for the fetch-urls route.
|
||||
*
|
||||
* Spins up a local HTTP server to serve test fixtures, and mocks the SSRF
|
||||
* validation to allow localhost connections during tests.
|
||||
*/
|
||||
|
||||
import { readFileSync } from "node:fs";
|
||||
import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http";
|
||||
import { join } from "node:path";
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
|
||||
// Mock the SSRF validation to allow localhost in tests.
|
||||
// We keep the real safeFetch logic but skip the private-IP DNS check.
|
||||
vi.mock("../../apps/api/src/lib/ssrf.js", async (importOriginal) => {
|
||||
const original = (await importOriginal()) as Record<string, unknown>;
|
||||
return {
|
||||
...original,
|
||||
// validateFetchUrl that allows localhost for tests
|
||||
validateFetchUrl: async (_url: string) => {
|
||||
// No-op: allow all URLs in tests (including localhost)
|
||||
},
|
||||
// safeFetch that skips SSRF validation but still does the real fetch
|
||||
safeFetch: async (url: string, signal?: AbortSignal) => {
|
||||
const MAX_REDIRECTS = 5;
|
||||
let currentUrl = url;
|
||||
for (let i = 0; i <= MAX_REDIRECTS; i++) {
|
||||
const res = await fetch(currentUrl, {
|
||||
signal,
|
||||
redirect: "manual",
|
||||
headers: { "User-Agent": "SnapOtter/1.0 (image-fetch)" },
|
||||
});
|
||||
if (res.status >= 300 && res.status < 400) {
|
||||
const location = res.headers.get("location");
|
||||
if (!location) throw new Error("Redirect without Location header");
|
||||
currentUrl = new URL(location, currentUrl).href;
|
||||
if (i === MAX_REDIRECTS) throw new Error("Too many redirects");
|
||||
continue;
|
||||
}
|
||||
return res;
|
||||
}
|
||||
throw new Error("Too many redirects");
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
import { buildTestApp, loginAsAdmin, type TestApp } from "./test-server.js";
|
||||
|
||||
const FIXTURES = join(__dirname, "..", "fixtures");
|
||||
const JPG = readFileSync(join(FIXTURES, "test-100x100.jpg"));
|
||||
const TIFF = readFileSync(join(FIXTURES, "formats", "sample.tiff"));
|
||||
|
||||
let testApp: TestApp;
|
||||
let app: TestApp["app"];
|
||||
let adminToken: string;
|
||||
let mockServer: Server;
|
||||
let mockPort: number;
|
||||
|
||||
function startMockServer(): Promise<{ server: Server; port: number }> {
|
||||
return new Promise((resolve) => {
|
||||
const server = createServer((req: IncomingMessage, res: ServerResponse) => {
|
||||
const url = req.url ?? "";
|
||||
|
||||
if (url === "/photo.jpg") {
|
||||
res.writeHead(200, { "Content-Type": "image/jpeg" });
|
||||
res.end(JPG);
|
||||
return;
|
||||
}
|
||||
|
||||
if (url === "/not-image.txt") {
|
||||
res.writeHead(200, { "Content-Type": "text/plain" });
|
||||
res.end("This is not an image");
|
||||
return;
|
||||
}
|
||||
|
||||
if (url === "/redirect") {
|
||||
res.writeHead(302, { Location: "/photo.jpg" });
|
||||
res.end();
|
||||
return;
|
||||
}
|
||||
|
||||
if (url === "/missing.jpg") {
|
||||
res.writeHead(404);
|
||||
res.end("Not Found");
|
||||
return;
|
||||
}
|
||||
|
||||
if (url === "/photo.tiff") {
|
||||
res.writeHead(200, { "Content-Type": "image/tiff" });
|
||||
res.end(TIFF);
|
||||
return;
|
||||
}
|
||||
|
||||
if (url === "/empty") {
|
||||
res.writeHead(200, { "Content-Type": "image/jpeg" });
|
||||
res.end();
|
||||
return;
|
||||
}
|
||||
|
||||
if (url === "/server-error") {
|
||||
res.writeHead(500, { "Content-Type": "text/plain" });
|
||||
res.end("Internal Server Error");
|
||||
return;
|
||||
}
|
||||
|
||||
if (url === "/slow-close") {
|
||||
// Return a valid response with no body stream at all
|
||||
res.writeHead(200, { "Content-Type": "image/jpeg", "Content-Length": "0" });
|
||||
res.end();
|
||||
return;
|
||||
}
|
||||
|
||||
res.writeHead(404);
|
||||
res.end("Not Found");
|
||||
});
|
||||
|
||||
server.listen(0, "127.0.0.1", () => {
|
||||
const addr = server.address();
|
||||
const port = typeof addr === "object" && addr ? addr.port : 0;
|
||||
resolve({ server, port });
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
const mock = await startMockServer();
|
||||
mockServer = mock.server;
|
||||
mockPort = mock.port;
|
||||
|
||||
testApp = await buildTestApp();
|
||||
app = testApp.app;
|
||||
adminToken = await loginAsAdmin(app);
|
||||
}, 30_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await testApp.cleanup();
|
||||
await new Promise<void>((resolve) => mockServer.close(() => resolve()));
|
||||
}, 10_000);
|
||||
|
||||
describe("POST /api/v1/fetch-urls", () => {
|
||||
it("fetches a valid image URL and returns metadata + download URL", async () => {
|
||||
const res = await app.inject({
|
||||
method: "POST",
|
||||
url: "/api/v1/fetch-urls",
|
||||
headers: {
|
||||
authorization: `Bearer ${adminToken}`,
|
||||
},
|
||||
payload: {
|
||||
urls: [`http://127.0.0.1:${mockPort}/photo.jpg`],
|
||||
},
|
||||
});
|
||||
|
||||
expect(res.statusCode).toBe(200);
|
||||
const body = JSON.parse(res.body);
|
||||
expect(body.results).toHaveLength(1);
|
||||
|
||||
const result = body.results[0];
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.url).toBe(`http://127.0.0.1:${mockPort}/photo.jpg`);
|
||||
expect(result.filename).toBe("photo.jpg");
|
||||
expect(result.contentType).toBe("image/jpeg");
|
||||
expect(result.size).toBeGreaterThan(0);
|
||||
expect(result.width).toBe(100);
|
||||
expect(result.height).toBe(100);
|
||||
expect(result.downloadUrl).toMatch(/^\/api\/v1\/download\/.+\/photo\.jpg$/);
|
||||
expect(result.previewUrl).toBeNull(); // JPEG is browser-previewable
|
||||
});
|
||||
|
||||
it("returns failure for a 404 URL", async () => {
|
||||
const res = await app.inject({
|
||||
method: "POST",
|
||||
url: "/api/v1/fetch-urls",
|
||||
headers: {
|
||||
authorization: `Bearer ${adminToken}`,
|
||||
},
|
||||
payload: {
|
||||
urls: [`http://127.0.0.1:${mockPort}/missing.jpg`],
|
||||
},
|
||||
});
|
||||
|
||||
expect(res.statusCode).toBe(200);
|
||||
const body = JSON.parse(res.body);
|
||||
expect(body.results).toHaveLength(1);
|
||||
|
||||
const result = body.results[0];
|
||||
expect(result.success).toBe(false);
|
||||
expect(result.error).toContain("404");
|
||||
});
|
||||
|
||||
it("returns failure for non-image content", async () => {
|
||||
const res = await app.inject({
|
||||
method: "POST",
|
||||
url: "/api/v1/fetch-urls",
|
||||
headers: {
|
||||
authorization: `Bearer ${adminToken}`,
|
||||
},
|
||||
payload: {
|
||||
urls: [`http://127.0.0.1:${mockPort}/not-image.txt`],
|
||||
},
|
||||
});
|
||||
|
||||
expect(res.statusCode).toBe(200);
|
||||
const body = JSON.parse(res.body);
|
||||
expect(body.results).toHaveLength(1);
|
||||
|
||||
const result = body.results[0];
|
||||
expect(result.success).toBe(false);
|
||||
expect(result.error).toBeTruthy();
|
||||
});
|
||||
|
||||
it("handles mixed batch with successes and failures", async () => {
|
||||
const res = await app.inject({
|
||||
method: "POST",
|
||||
url: "/api/v1/fetch-urls",
|
||||
headers: {
|
||||
authorization: `Bearer ${adminToken}`,
|
||||
},
|
||||
payload: {
|
||||
urls: [
|
||||
`http://127.0.0.1:${mockPort}/photo.jpg`,
|
||||
`http://127.0.0.1:${mockPort}/missing.jpg`,
|
||||
`http://127.0.0.1:${mockPort}/not-image.txt`,
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
expect(res.statusCode).toBe(200);
|
||||
const body = JSON.parse(res.body);
|
||||
expect(body.results).toHaveLength(3);
|
||||
|
||||
// Results preserve order
|
||||
expect(body.results[0].success).toBe(true);
|
||||
expect(body.results[0].filename).toBe("photo.jpg");
|
||||
|
||||
expect(body.results[1].success).toBe(false);
|
||||
expect(body.results[1].error).toContain("404");
|
||||
|
||||
expect(body.results[2].success).toBe(false);
|
||||
});
|
||||
|
||||
it("returns 400 for an empty URL array", async () => {
|
||||
const res = await app.inject({
|
||||
method: "POST",
|
||||
url: "/api/v1/fetch-urls",
|
||||
headers: {
|
||||
authorization: `Bearer ${adminToken}`,
|
||||
},
|
||||
payload: {
|
||||
urls: [],
|
||||
},
|
||||
});
|
||||
|
||||
expect(res.statusCode).toBe(400);
|
||||
const body = JSON.parse(res.body);
|
||||
expect(body.error).toBeTruthy();
|
||||
});
|
||||
|
||||
it("returns 400 for more than 50 URLs", async () => {
|
||||
const urls = Array.from({ length: 51 }, (_, i) => `http://example.com/img${i}.jpg`);
|
||||
const res = await app.inject({
|
||||
method: "POST",
|
||||
url: "/api/v1/fetch-urls",
|
||||
headers: {
|
||||
authorization: `Bearer ${adminToken}`,
|
||||
},
|
||||
payload: { urls },
|
||||
});
|
||||
|
||||
expect(res.statusCode).toBe(400);
|
||||
const body = JSON.parse(res.body);
|
||||
expect(body.error).toBeTruthy();
|
||||
});
|
||||
|
||||
it("follows redirects to fetch the final image", async () => {
|
||||
const res = await app.inject({
|
||||
method: "POST",
|
||||
url: "/api/v1/fetch-urls",
|
||||
headers: {
|
||||
authorization: `Bearer ${adminToken}`,
|
||||
},
|
||||
payload: {
|
||||
urls: [`http://127.0.0.1:${mockPort}/redirect`],
|
||||
},
|
||||
});
|
||||
|
||||
expect(res.statusCode).toBe(200);
|
||||
const body = JSON.parse(res.body);
|
||||
expect(body.results).toHaveLength(1);
|
||||
|
||||
const result = body.results[0];
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.contentType).toBe("image/jpeg");
|
||||
expect(result.size).toBe(JPG.length);
|
||||
});
|
||||
|
||||
it("download URL serves the actual image", async () => {
|
||||
// First, fetch the URL to get a downloadUrl
|
||||
const fetchRes = await app.inject({
|
||||
method: "POST",
|
||||
url: "/api/v1/fetch-urls",
|
||||
headers: {
|
||||
authorization: `Bearer ${adminToken}`,
|
||||
},
|
||||
payload: {
|
||||
urls: [`http://127.0.0.1:${mockPort}/photo.jpg`],
|
||||
},
|
||||
});
|
||||
|
||||
const body = JSON.parse(fetchRes.body);
|
||||
const downloadUrl = body.results[0].downloadUrl;
|
||||
expect(downloadUrl).toBeTruthy();
|
||||
|
||||
// Now download the file
|
||||
const downloadRes = await app.inject({
|
||||
method: "GET",
|
||||
url: downloadUrl,
|
||||
headers: {
|
||||
authorization: `Bearer ${adminToken}`,
|
||||
},
|
||||
});
|
||||
|
||||
expect(downloadRes.statusCode).toBe(200);
|
||||
expect(downloadRes.headers["content-type"]).toBe("image/jpeg");
|
||||
// The downloaded buffer should match the original fixture
|
||||
expect(downloadRes.rawPayload.length).toBe(JPG.length);
|
||||
});
|
||||
|
||||
it("deduplicates filenames when multiple URLs resolve to the same name", async () => {
|
||||
const res = await app.inject({
|
||||
method: "POST",
|
||||
url: "/api/v1/fetch-urls",
|
||||
headers: {
|
||||
authorization: `Bearer ${adminToken}`,
|
||||
},
|
||||
payload: {
|
||||
urls: [`http://127.0.0.1:${mockPort}/photo.jpg`, `http://127.0.0.1:${mockPort}/photo.jpg`],
|
||||
},
|
||||
});
|
||||
|
||||
expect(res.statusCode).toBe(200);
|
||||
const body = JSON.parse(res.body);
|
||||
expect(body.results).toHaveLength(2);
|
||||
|
||||
expect(body.results[0].success).toBe(true);
|
||||
expect(body.results[1].success).toBe(true);
|
||||
|
||||
// Filenames must differ so one does not overwrite the other
|
||||
const names = [body.results[0].filename, body.results[1].filename];
|
||||
expect(new Set(names).size).toBe(2);
|
||||
expect(names).toContain("photo.jpg");
|
||||
expect(names).toContain("photo_1.jpg");
|
||||
|
||||
// Download URLs must also differ
|
||||
expect(body.results[0].downloadUrl).not.toBe(body.results[1].downloadUrl);
|
||||
|
||||
// Both download URLs should serve valid content
|
||||
for (const result of body.results) {
|
||||
const dl = await app.inject({
|
||||
method: "GET",
|
||||
url: result.downloadUrl,
|
||||
headers: { authorization: `Bearer ${adminToken}` },
|
||||
});
|
||||
expect(dl.statusCode).toBe(200);
|
||||
expect(dl.rawPayload.length).toBe(JPG.length);
|
||||
}
|
||||
});
|
||||
|
||||
it("returns 400 for invalid URL format", async () => {
|
||||
const res = await app.inject({
|
||||
method: "POST",
|
||||
url: "/api/v1/fetch-urls",
|
||||
headers: {
|
||||
authorization: `Bearer ${adminToken}`,
|
||||
},
|
||||
payload: {
|
||||
urls: ["not-a-valid-url"],
|
||||
},
|
||||
});
|
||||
|
||||
expect(res.statusCode).toBe(400);
|
||||
const body = JSON.parse(res.body);
|
||||
expect(body.error).toBeTruthy();
|
||||
});
|
||||
|
||||
it("generates a preview for non-browser-previewable formats", async () => {
|
||||
const res = await app.inject({
|
||||
method: "POST",
|
||||
url: "/api/v1/fetch-urls",
|
||||
headers: {
|
||||
authorization: `Bearer ${adminToken}`,
|
||||
},
|
||||
payload: {
|
||||
urls: [`http://127.0.0.1:${mockPort}/photo.tiff`],
|
||||
},
|
||||
});
|
||||
|
||||
expect(res.statusCode).toBe(200);
|
||||
const body = JSON.parse(res.body);
|
||||
expect(body.results).toHaveLength(1);
|
||||
|
||||
const result = body.results[0];
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.contentType).toBe("image/tiff");
|
||||
expect(result.previewUrl).toBeTruthy();
|
||||
expect(result.previewUrl).toContain("preview-");
|
||||
expect(result.previewUrl).toContain(".webp");
|
||||
|
||||
// Preview URL should serve a valid webp image
|
||||
const previewRes = await app.inject({
|
||||
method: "GET",
|
||||
url: result.previewUrl,
|
||||
headers: {
|
||||
authorization: `Bearer ${adminToken}`,
|
||||
},
|
||||
});
|
||||
expect(previewRes.statusCode).toBe(200);
|
||||
});
|
||||
|
||||
it("returns failure for empty response body", async () => {
|
||||
const res = await app.inject({
|
||||
method: "POST",
|
||||
url: "/api/v1/fetch-urls",
|
||||
headers: {
|
||||
authorization: `Bearer ${adminToken}`,
|
||||
},
|
||||
payload: {
|
||||
urls: [`http://127.0.0.1:${mockPort}/empty`],
|
||||
},
|
||||
});
|
||||
|
||||
expect(res.statusCode).toBe(200);
|
||||
const body = JSON.parse(res.body);
|
||||
expect(body.results).toHaveLength(1);
|
||||
expect(body.results[0].success).toBe(false);
|
||||
expect(body.results[0].error).toContain("Empty");
|
||||
});
|
||||
|
||||
it("returns failure for 500 server error", async () => {
|
||||
const res = await app.inject({
|
||||
method: "POST",
|
||||
url: "/api/v1/fetch-urls",
|
||||
headers: {
|
||||
authorization: `Bearer ${adminToken}`,
|
||||
},
|
||||
payload: {
|
||||
urls: [`http://127.0.0.1:${mockPort}/server-error`],
|
||||
},
|
||||
});
|
||||
|
||||
expect(res.statusCode).toBe(200);
|
||||
const body = JSON.parse(res.body);
|
||||
expect(body.results).toHaveLength(1);
|
||||
expect(body.results[0].success).toBe(false);
|
||||
expect(body.results[0].error).toContain("500");
|
||||
});
|
||||
|
||||
it("returns failure when fetch throws a network error", async () => {
|
||||
// Port 1 is almost guaranteed to refuse connections, triggering the outer
|
||||
// catch block (lines 275-278 in fetch-urls.ts).
|
||||
const res = await app.inject({
|
||||
method: "POST",
|
||||
url: "/api/v1/fetch-urls",
|
||||
headers: {
|
||||
authorization: `Bearer ${adminToken}`,
|
||||
},
|
||||
payload: {
|
||||
urls: ["http://127.0.0.1:1/unreachable.jpg"],
|
||||
},
|
||||
});
|
||||
|
||||
expect(res.statusCode).toBe(200);
|
||||
const body = JSON.parse(res.body);
|
||||
expect(body.results).toHaveLength(1);
|
||||
expect(body.results[0].success).toBe(false);
|
||||
expect(body.results[0].error).toBeTruthy();
|
||||
});
|
||||
|
||||
it("returns failure for zero-length content", async () => {
|
||||
const res = await app.inject({
|
||||
method: "POST",
|
||||
url: "/api/v1/fetch-urls",
|
||||
headers: {
|
||||
authorization: `Bearer ${adminToken}`,
|
||||
},
|
||||
payload: {
|
||||
urls: [`http://127.0.0.1:${mockPort}/slow-close`],
|
||||
},
|
||||
});
|
||||
|
||||
expect(res.statusCode).toBe(200);
|
||||
const body = JSON.parse(res.body);
|
||||
expect(body.results).toHaveLength(1);
|
||||
expect(body.results[0].success).toBe(false);
|
||||
expect(body.results[0].error).toContain("Empty");
|
||||
});
|
||||
});
|
||||
@@ -37,6 +37,7 @@ import { apiKeyRoutes } from "../../apps/api/src/routes/api-keys.js";
|
||||
import { auditLogRoutes } from "../../apps/api/src/routes/audit-log.js";
|
||||
import { registerBatchRoutes } from "../../apps/api/src/routes/batch.js";
|
||||
import { docsRoutes } from "../../apps/api/src/routes/docs.js";
|
||||
import { registerFetchUrlsRoute } from "../../apps/api/src/routes/fetch-urls.js";
|
||||
import { fileRoutes } from "../../apps/api/src/routes/files.js";
|
||||
import { registerMemeTemplates } from "../../apps/api/src/routes/meme-templates.js";
|
||||
import { registerPipelineRoutes } from "../../apps/api/src/routes/pipeline.js";
|
||||
@@ -100,6 +101,9 @@ export async function buildTestApp(): Promise<TestApp> {
|
||||
// Batch processing routes
|
||||
await registerBatchRoutes(app);
|
||||
|
||||
// URL fetch routes
|
||||
await registerFetchUrlsRoute(app);
|
||||
|
||||
// Pipeline routes
|
||||
await registerPipelineRoutes(app);
|
||||
|
||||
|
||||
@@ -0,0 +1,210 @@
|
||||
import { beforeEach, describe, expect, it, type Mock, vi } from "vitest";
|
||||
import { MAX_REDIRECTS, safeFetch, validateFetchUrl } from "../../../apps/api/src/lib/ssrf.js";
|
||||
|
||||
describe("validateFetchUrl", () => {
|
||||
it("allows valid public HTTP URL", async () => {
|
||||
await expect(
|
||||
validateFetchUrl("https://images.unsplash.com/photo.jpg"),
|
||||
).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it("allows valid public HTTP URL without TLS", async () => {
|
||||
await expect(validateFetchUrl("http://example.com/image.png")).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it("rejects non-HTTP schemes", async () => {
|
||||
await expect(validateFetchUrl("ftp://example.com/image.jpg")).rejects.toThrow(
|
||||
"Only HTTP and HTTPS",
|
||||
);
|
||||
await expect(validateFetchUrl("file:///etc/passwd")).rejects.toThrow("Only HTTP and HTTPS");
|
||||
await expect(validateFetchUrl("data:image/png;base64,abc")).rejects.toThrow(
|
||||
"Only HTTP and HTTPS",
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects localhost and loopback", async () => {
|
||||
await expect(validateFetchUrl("http://127.0.0.1/image.jpg")).rejects.toThrow("private");
|
||||
await expect(validateFetchUrl("http://localhost/image.jpg")).rejects.toThrow("private");
|
||||
await expect(validateFetchUrl("http://[::1]/image.jpg")).rejects.toThrow("private");
|
||||
});
|
||||
|
||||
it("rejects private network ranges", async () => {
|
||||
await expect(validateFetchUrl("http://10.0.0.1/image.jpg")).rejects.toThrow("private");
|
||||
await expect(validateFetchUrl("http://172.16.0.1/image.jpg")).rejects.toThrow("private");
|
||||
await expect(validateFetchUrl("http://192.168.1.1/image.jpg")).rejects.toThrow("private");
|
||||
});
|
||||
|
||||
it("rejects link-local addresses", async () => {
|
||||
await expect(validateFetchUrl("http://169.254.169.254/latest/meta-data/")).rejects.toThrow(
|
||||
"private",
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects CG-NAT range (100.64.0.0/10)", async () => {
|
||||
await expect(validateFetchUrl("http://100.64.0.1/image.jpg")).rejects.toThrow("private");
|
||||
await expect(validateFetchUrl("http://100.127.255.255/image.jpg")).rejects.toThrow("private");
|
||||
});
|
||||
|
||||
it("rejects IETF protocol assignments (192.0.0.0/24)", async () => {
|
||||
await expect(validateFetchUrl("http://192.0.0.1/image.jpg")).rejects.toThrow("private");
|
||||
});
|
||||
|
||||
it("rejects benchmarking range (198.18.0.0/15)", async () => {
|
||||
await expect(validateFetchUrl("http://198.18.0.1/image.jpg")).rejects.toThrow("private");
|
||||
await expect(validateFetchUrl("http://198.19.255.255/image.jpg")).rejects.toThrow("private");
|
||||
});
|
||||
|
||||
it("rejects reserved/class E range (240.0.0.0/4)", async () => {
|
||||
await expect(validateFetchUrl("http://240.0.0.1/image.jpg")).rejects.toThrow("private");
|
||||
await expect(validateFetchUrl("http://255.255.255.255/image.jpg")).rejects.toThrow("private");
|
||||
});
|
||||
|
||||
it("rejects IPv6 unspecified address", async () => {
|
||||
await expect(validateFetchUrl("http://[::]/image.jpg")).rejects.toThrow("private");
|
||||
});
|
||||
|
||||
it("rejects IPv6 documentation range (2001:db8::/32)", async () => {
|
||||
await expect(validateFetchUrl("http://[2001:db8::1]/image.jpg")).rejects.toThrow("private");
|
||||
await expect(validateFetchUrl("http://[2001:DB8::1]/image.jpg")).rejects.toThrow("private");
|
||||
});
|
||||
|
||||
it("allows a public IP address directly in URL", async () => {
|
||||
// Exercises the early-return path in resolveAndCheck when hostname is a
|
||||
// non-private IP literal (covers the `return` after the isIP check).
|
||||
await expect(validateFetchUrl("http://8.8.8.8/image.jpg")).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it("rejects invalid URLs", async () => {
|
||||
await expect(validateFetchUrl("not-a-url")).rejects.toThrow();
|
||||
await expect(validateFetchUrl("")).rejects.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* Tests that require DNS mocking to exercise resolveAndCheck paths that only
|
||||
* trigger when the hostname is a non-IP string and lookup returns results.
|
||||
*/
|
||||
describe("validateFetchUrl with DNS mocking", () => {
|
||||
const originalLookup = vi.hoisted(() => {
|
||||
return { fn: null as null | ((...args: unknown[]) => unknown) };
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
vi.mock("node:dns/promises", async (importOriginal) => {
|
||||
const orig = (await importOriginal()) as Record<string, unknown>;
|
||||
originalLookup.fn = orig.lookup as (...args: unknown[]) => unknown;
|
||||
return {
|
||||
...orig,
|
||||
lookup: vi.fn((...args: unknown[]) => originalLookup.fn?.(...args)),
|
||||
};
|
||||
});
|
||||
|
||||
it("rejects hostname that resolves to IPv4-mapped IPv6 with private IPv4", async () => {
|
||||
// Covers isPrivateIPv6 lines 28-31 (::ffff: mapped address path)
|
||||
const dns = await import("node:dns/promises");
|
||||
vi.mocked(dns.lookup).mockResolvedValueOnce([
|
||||
{ address: "::ffff:127.0.0.1", family: 6 },
|
||||
] as never);
|
||||
await expect(validateFetchUrl("http://mapped-v6.example.com/image.jpg")).rejects.toThrow(
|
||||
"private",
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects hostname resolving to IPv4-mapped IPv6 with 10.x private", async () => {
|
||||
const dns = await import("node:dns/promises");
|
||||
vi.mocked(dns.lookup).mockResolvedValueOnce([
|
||||
{ address: "::ffff:10.0.0.1", family: 6 },
|
||||
] as never);
|
||||
await expect(validateFetchUrl("http://mapped-ten.example.com/image.jpg")).rejects.toThrow(
|
||||
"private",
|
||||
);
|
||||
});
|
||||
|
||||
it("handles DNS lookup returning a single result object", async () => {
|
||||
// Covers the Array.isArray fallback branch (line 45: wrapping non-array in [])
|
||||
const dns = await import("node:dns/promises");
|
||||
vi.mocked(dns.lookup).mockResolvedValueOnce({
|
||||
address: "203.0.113.1",
|
||||
family: 4,
|
||||
} as never);
|
||||
await expect(
|
||||
validateFetchUrl("http://single-result.example.com/image.jpg"),
|
||||
).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it("rejects when DNS returns multiple addresses with one private", async () => {
|
||||
const dns = await import("node:dns/promises");
|
||||
vi.mocked(dns.lookup).mockResolvedValueOnce([
|
||||
{ address: "203.0.113.1", family: 4 },
|
||||
{ address: "10.0.0.1", family: 4 },
|
||||
] as never);
|
||||
await expect(validateFetchUrl("http://dual-addr.example.com/image.jpg")).rejects.toThrow(
|
||||
"private",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("safeFetch", () => {
|
||||
let mockFetch: Mock;
|
||||
|
||||
beforeEach(() => {
|
||||
mockFetch = vi.fn();
|
||||
vi.stubGlobal("fetch", mockFetch);
|
||||
});
|
||||
|
||||
function mockResponse(status: number, headers?: Record<string, string>): Response {
|
||||
return {
|
||||
status,
|
||||
headers: new Headers(headers),
|
||||
body: { cancel: vi.fn() },
|
||||
} as unknown as Response;
|
||||
}
|
||||
|
||||
it("returns response for a direct (non-redirect) fetch", async () => {
|
||||
mockFetch.mockResolvedValueOnce(mockResponse(200));
|
||||
const res = await safeFetch("https://example.com/image.jpg");
|
||||
expect(res.status).toBe(200);
|
||||
expect(mockFetch).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("follows a redirect chain within MAX_REDIRECTS", async () => {
|
||||
// 3 redirects then a 200
|
||||
mockFetch
|
||||
.mockResolvedValueOnce(mockResponse(302, { location: "https://example.com/hop1" }))
|
||||
.mockResolvedValueOnce(mockResponse(301, { location: "https://example.com/hop2" }))
|
||||
.mockResolvedValueOnce(mockResponse(307, { location: "https://example.com/final" }))
|
||||
.mockResolvedValueOnce(mockResponse(200));
|
||||
|
||||
const res = await safeFetch("https://example.com/start");
|
||||
expect(res.status).toBe(200);
|
||||
expect(mockFetch).toHaveBeenCalledTimes(4);
|
||||
});
|
||||
|
||||
it("throws when redirect chain exceeds MAX_REDIRECTS", async () => {
|
||||
// Return redirects for every call (MAX_REDIRECTS + 1 iterations, all redirects)
|
||||
for (let i = 0; i <= MAX_REDIRECTS; i++) {
|
||||
mockFetch.mockResolvedValueOnce(
|
||||
mockResponse(302, { location: `https://example.com/hop${i + 1}` }),
|
||||
);
|
||||
}
|
||||
|
||||
await expect(safeFetch("https://example.com/start")).rejects.toThrow("Too many redirects");
|
||||
});
|
||||
|
||||
it("rejects a redirect to a private IP", async () => {
|
||||
mockFetch.mockResolvedValueOnce(mockResponse(302, { location: "http://127.0.0.1/evil" }));
|
||||
|
||||
await expect(safeFetch("https://example.com/image.jpg")).rejects.toThrow("private");
|
||||
});
|
||||
|
||||
it("throws when redirect has no Location header", async () => {
|
||||
mockFetch.mockResolvedValueOnce(mockResponse(302));
|
||||
|
||||
await expect(safeFetch("https://example.com/image.jpg")).rejects.toThrow(
|
||||
"Redirect without Location header",
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,72 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { extractUrls } from "../../../apps/web/src/lib/url-parser.js";
|
||||
|
||||
describe("extractUrls", () => {
|
||||
it("extracts plain URLs one per line", () => {
|
||||
const input = "https://example.com/a.jpg\nhttps://example.com/b.png";
|
||||
expect(extractUrls(input)).toEqual(["https://example.com/a.jpg", "https://example.com/b.png"]);
|
||||
});
|
||||
|
||||
it("strips numbered list prefixes", () => {
|
||||
const input =
|
||||
"1. https://example.com/a.jpg\n2) https://example.com/b.png\n3 https://example.com/c.webp";
|
||||
expect(extractUrls(input)).toEqual([
|
||||
"https://example.com/a.jpg",
|
||||
"https://example.com/b.png",
|
||||
"https://example.com/c.webp",
|
||||
]);
|
||||
});
|
||||
|
||||
it("strips bullet prefixes", () => {
|
||||
const input =
|
||||
"- https://example.com/a.jpg\n* https://example.com/b.png\n+ https://example.com/c.webp";
|
||||
expect(extractUrls(input)).toEqual([
|
||||
"https://example.com/a.jpg",
|
||||
"https://example.com/b.png",
|
||||
"https://example.com/c.webp",
|
||||
]);
|
||||
});
|
||||
|
||||
it("extracts URLs from markdown links", () => {
|
||||
const input = "[Photo 1](https://example.com/a.jpg)\n[Photo 2](https://example.com/b.png)";
|
||||
expect(extractUrls(input)).toEqual(["https://example.com/a.jpg", "https://example.com/b.png"]);
|
||||
});
|
||||
|
||||
it("extracts URLs from HTML img tags", () => {
|
||||
const input = '<img src="https://example.com/a.jpg">\n<img src="https://example.com/b.png" />';
|
||||
expect(extractUrls(input)).toEqual(["https://example.com/a.jpg", "https://example.com/b.png"]);
|
||||
});
|
||||
|
||||
it("handles mixed formats", () => {
|
||||
const input = `1. https://example.com/a.jpg
|
||||
- [Photo](https://example.com/b.png)
|
||||
<img src="https://example.com/c.webp">
|
||||
https://example.com/d.avif`;
|
||||
expect(extractUrls(input)).toEqual([
|
||||
"https://example.com/a.jpg",
|
||||
"https://example.com/b.png",
|
||||
"https://example.com/c.webp",
|
||||
"https://example.com/d.avif",
|
||||
]);
|
||||
});
|
||||
|
||||
it("deduplicates URLs", () => {
|
||||
const input = "https://example.com/a.jpg\nhttps://example.com/a.jpg";
|
||||
expect(extractUrls(input)).toEqual(["https://example.com/a.jpg"]);
|
||||
});
|
||||
|
||||
it("filters out non-HTTP URLs", () => {
|
||||
const input = "ftp://example.com/a.jpg\nhttps://example.com/b.png\nnot-a-url";
|
||||
expect(extractUrls(input)).toEqual(["https://example.com/b.png"]);
|
||||
});
|
||||
|
||||
it("returns empty array for empty input", () => {
|
||||
expect(extractUrls("")).toEqual([]);
|
||||
expect(extractUrls(" \n \n ")).toEqual([]);
|
||||
});
|
||||
|
||||
it("preserves URLs with query parameters", () => {
|
||||
const input = "https://example.com/photo?id=123&size=large";
|
||||
expect(extractUrls(input)).toEqual(["https://example.com/photo?id=123&size=large"]);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user