diff --git a/apps/api/src/routes/tools/content-aware-resize.ts b/apps/api/src/routes/tools/content-aware-resize.ts index 22c8fe70..4fd240ad 100644 --- a/apps/api/src/routes/tools/content-aware-resize.ts +++ b/apps/api/src/routes/tools/content-aware-resize.ts @@ -7,10 +7,20 @@ import { z } from "zod"; import { autoOrient } from "../../lib/auto-orient.js"; import { validateImageBuffer } from "../../lib/file-validation.js"; import { createWorkspace } from "../../lib/workspace.js"; -import { updateSingleFileProgress } from "../progress.js"; import { registerToolProcessFn } from "../tool-factory.js"; -/** Content-aware resize (seam carving) route. */ +const settingsSchema = z.object({ + width: z.number().positive().optional(), + height: z.number().positive().optional(), + protectFaces: z.boolean().default(false), + blurRadius: z.number().min(0).max(20).default(4), + sobelThreshold: z.number().min(1).max(20).default(2), + square: z.boolean().default(false), +}); + +type Settings = z.infer; + +/** Content-aware resize (seam carving via caire) route. */ export function registerContentAwareResize(app: FastifyInstance) { app.post( "/api/v1/tools/content-aware-resize", @@ -18,7 +28,6 @@ export function registerContentAwareResize(app: FastifyInstance) { let fileBuffer: Buffer | null = null; let filename = "image"; let settingsRaw: string | null = null; - let clientJobId: string | null = null; try { const parts = request.parts(); @@ -32,8 +41,6 @@ export function registerContentAwareResize(app: FastifyInstance) { filename = basename(part.filename ?? "image"); } else if (part.fieldname === "settings") { settingsRaw = part.value as string; - } else if (part.fieldname === "clientJobId") { - clientJobId = part.value as string; } } } catch (err) { @@ -52,15 +59,37 @@ export function registerContentAwareResize(app: FastifyInstance) { return reply.status(400).send({ error: `Invalid image: ${validation.reason}` }); } + // Validate settings + let settings: Settings; + try { + const parsed = settingsRaw ? JSON.parse(settingsRaw) : {}; + const result = settingsSchema.safeParse(parsed); + if (!result.success) { + return reply.status(400).send({ + error: "Invalid settings", + details: result.error.issues.map((i) => ({ + path: i.path.join("."), + message: i.message, + })), + }); + } + settings = result.data; + } catch { + return reply.status(400).send({ error: "Settings must be valid JSON" }); + } + + if (!settings.square && !settings.width && !settings.height) { + return reply.status(400).send({ + error: "Either width, height, or square mode must be specified", + }); + } + try { - const settings = settingsRaw ? JSON.parse(settingsRaw) : {}; request.log.info( { toolId: "content-aware-resize", imageSize: fileBuffer.length, - width: settings.width, - height: settings.height, - protectFaces: settings.protectFaces, + ...settings, }, "Starting content-aware resize", ); @@ -75,43 +104,21 @@ export function registerContentAwareResize(app: FastifyInstance) { const inputPath = join(workspacePath, "input", filename); await writeFile(inputPath, fileBuffer); - // Process - const jobIdForProgress = clientJobId; - const onProgress = jobIdForProgress - ? (percent: number, stage: string) => { - updateSingleFileProgress({ - jobId: jobIdForProgress, - phase: "processing", - stage, - percent, - }); - } - : undefined; - - const result = await seamCarve( - fileBuffer, - join(workspacePath, "output"), - { - width: settings.width, - height: settings.height, - protectFaces: settings.protectFaces ?? true, - }, - onProgress, - ); + // Process with caire + const result = await seamCarve(fileBuffer, join(workspacePath, "output"), { + width: settings.width, + height: settings.height, + protectFaces: settings.protectFaces, + blurRadius: settings.blurRadius, + sobelThreshold: settings.sobelThreshold, + square: settings.square, + }); // Save output const outputFilename = `${filename.replace(/\.[^.]+$/, "")}_seam.png`; const outputPath = join(workspacePath, "output", outputFilename); await writeFile(outputPath, result.buffer); - if (clientJobId) { - updateSingleFileProgress({ - jobId: clientJobId, - phase: "complete", - percent: 100, - }); - } - return reply.send({ jobId, downloadUrl: `/api/v1/download/${jobId}/${encodeURIComponent(outputFilename)}`, @@ -130,24 +137,22 @@ export function registerContentAwareResize(app: FastifyInstance) { }, ); - // Register in the pipeline/batch registry so this tool can be used - // as a step in automation pipelines (without progress callbacks). + // Register in the pipeline/batch registry registerToolProcessFn({ toolId: "content-aware-resize", - settingsSchema: z.object({ - width: z.number().positive().optional(), - height: z.number().positive().optional(), - protectFaces: z.boolean().default(true), - }), + settingsSchema, process: async (inputBuffer, settings, filename) => { - const s = settings as { width?: number; height?: number; protectFaces?: boolean }; + const s = settings as Settings; const orientedBuffer = await autoOrient(inputBuffer); const jobId = randomUUID(); const workspacePath = await createWorkspace(jobId); const result = await seamCarve(orientedBuffer, join(workspacePath, "output"), { width: s.width, height: s.height, - protectFaces: s.protectFaces ?? true, + protectFaces: s.protectFaces, + blurRadius: s.blurRadius, + sobelThreshold: s.sobelThreshold, + square: s.square, }); const outputFilename = `${filename.replace(/\.[^.]+$/, "")}_seam.png`; return { buffer: result.buffer, filename: outputFilename, contentType: "image/png" }; diff --git a/apps/web/src/components/tools/resize-settings.tsx b/apps/web/src/components/tools/resize-settings.tsx index 0bc3ac9e..ae105591 100644 --- a/apps/web/src/components/tools/resize-settings.tsx +++ b/apps/web/src/components/tools/resize-settings.tsx @@ -1,5 +1,5 @@ import { SOCIAL_MEDIA_PRESETS } from "@stirling-image/shared"; -import { Download, Link, Unlink } from "lucide-react"; +import { Download, Info, Link, Unlink } from "lucide-react"; import { useCallback, useEffect, useRef, useState } from "react"; import { ProgressCard } from "@/components/common/progress-card"; import { useToolProcessor } from "@/hooks/use-tool-processor"; @@ -17,6 +17,17 @@ const FIT_LABELS: Record = { // Group presets by platform const platforms = [...new Set(SOCIAL_MEDIA_PRESETS.map((p) => p.platform))]; +function HintIcon({ text }: { text: string }) { + return ( + + + + {text} + + + ); +} + export interface ResizeControlsProps { onChange?: (settings: Record) => void; } @@ -31,7 +42,10 @@ export function ResizeControls({ onChange }: ResizeControlsProps) { const [lockAspect, setLockAspect] = useState(true); const [withoutEnlargement, setWithoutEnlargement] = useState(false); const [contentAware, setContentAware] = useState(false); - const [protectFaces, setProtectFaces] = useState(true); + const [protectFaces, setProtectFaces] = useState(false); + const [blurRadius, setBlurRadius] = useState(4); + const [sobelThreshold, setSobelThreshold] = useState(2); + const [squareMode, setSquareMode] = useState(false); const onChangeRef = useRef(onChange); useEffect(() => { @@ -42,9 +56,14 @@ export function ResizeControls({ onChange }: ResizeControlsProps) { const settings: Record = {}; if (contentAware) { settings.contentAware = true; - if (width) settings.width = Number(width); - if (height) settings.height = Number(height); + if (!squareMode) { + if (width) settings.width = Number(width); + if (height) settings.height = Number(height); + } settings.protectFaces = protectFaces; + settings.blurRadius = blurRadius; + settings.sobelThreshold = sobelThreshold; + settings.square = squareMode; } else if (tab === "scale") { settings.percentage = Number(percentage); } else { @@ -54,7 +73,19 @@ export function ResizeControls({ onChange }: ResizeControlsProps) { settings.withoutEnlargement = withoutEnlargement; } onChangeRef.current?.(settings); - }, [tab, width, height, percentage, fit, withoutEnlargement, contentAware, protectFaces]); + }, [ + tab, + width, + height, + percentage, + fit, + withoutEnlargement, + contentAware, + protectFaces, + blurRadius, + sobelThreshold, + squareMode, + ]); const handlePreset = (preset: (typeof SOCIAL_MEDIA_PRESETS)[number]) => { const key = `${preset.platform}-${preset.name}`; @@ -84,7 +115,8 @@ export function ResizeControls({ onChange }: ResizeControlsProps) { value={width} onChange={(e) => setWidth(e.target.value)} placeholder="Auto" - className="w-full mt-0.5 px-2 py-1.5 rounded border border-border bg-background text-sm text-foreground" + disabled={squareMode && contentAware} + className="w-full mt-0.5 px-2 py-1.5 rounded border border-border bg-background text-sm text-foreground disabled:opacity-50" /> - - - {/* Content-aware inputs */} - {contentAware && ( -
- {dimensionInputs} - - {/* Protect faces */} - -
- )} - {/* Standard resize tabs */} {!contentAware && ( <> @@ -196,7 +204,7 @@ export function ResizeControls({ onChange }: ResizeControlsProps) { > {preset.name} - {preset.width} × {preset.height} + {preset.width} x {preset.height} ); @@ -205,16 +213,7 @@ export function ResizeControls({ onChange }: ResizeControlsProps) { ))} - {/* Don't enlarge */} - + {enlargementCheckbox} )} @@ -240,16 +239,7 @@ export function ResizeControls({ onChange }: ResizeControlsProps) { - {/* Don't enlarge */} - + {enlargementCheckbox} )} @@ -289,6 +279,98 @@ export function ResizeControls({ onChange }: ResizeControlsProps) { )} )} + + {/* Content-aware section - positioned below standard resize */} +
+
+
+ Content-aware +
+ +
+ + {/* Content-aware options (expanded when toggled) */} + {contentAware && ( +
+ {/* Dimensions */} + {dimensionInputs} + + {/* Square mode */} + + + {/* Face protection */} + + + {/* Blur radius */} +
+
+ + {blurRadius} +
+ setBlurRadius(Number(e.target.value))} + className="w-full mt-1 h-1.5 rounded-full appearance-none bg-muted accent-primary" + /> +
+ + {/* Sobel threshold */} +
+
+ + {sobelThreshold} +
+ setSobelThreshold(Number(e.target.value))} + className="w-full mt-1 h-1.5 rounded-full appearance-none bg-muted accent-primary" + /> +
+
+ )} +
); } @@ -323,7 +405,7 @@ export function ResizeSettings() { hasFile && !processing && (isContentAware - ? Boolean(settings.width) || Boolean(settings.height) + ? Boolean(settings.width) || Boolean(settings.height) || Boolean(settings.square) : tab === "scale" ? Number(settings.percentage) > 0 : Boolean(settings.width) || Boolean(settings.height)); diff --git a/apps/web/src/hooks/use-tool-processor.ts b/apps/web/src/hooks/use-tool-processor.ts index 11c29a7d..ebf1535d 100644 --- a/apps/web/src/hooks/use-tool-processor.ts +++ b/apps/web/src/hooks/use-tool-processor.ts @@ -29,6 +29,10 @@ const IDLE_PROGRESS: ToolProgress = { // smart-crop is category "ai" but uses Sharp (no Python), so it's excluded. const AI_PYTHON_TOOLS = new Set(PYTHON_SIDECAR_TOOLS); +// Tools that take a few seconds (not instant like Sharp, not minutes like AI). +// Uses a smoother progress: upload 0-40%, then a gradual fill during processing. +const MEDIUM_TOOLS = new Set(["content-aware-resize"]); + export function useToolProcessor(toolId: string) { const { processing, @@ -49,11 +53,14 @@ export function useToolProcessor(toolId: string) { const eventSourceRef = useRef(null); const isAiTool = AI_PYTHON_TOOLS.has(toolId); + const isMediumTool = MEDIUM_TOOLS.has(toolId); + const processingTimerRef = useRef | null>(null); // Clean up on unmount useEffect(() => { return () => { if (elapsedRef.current) clearInterval(elapsedRef.current); + if (processingTimerRef.current) clearInterval(processingTimerRef.current); if (eventSourceRef.current) eventSourceRef.current.close(); if (xhrRef.current) xhrRef.current.abort(); }; @@ -134,12 +141,13 @@ export function useToolProcessor(toolId: string) { const xhr = new XMLHttpRequest(); xhrRef.current = xhr; - // Timeout: 60s for fast tools, 5 min for AI tools + // Timeout: 60s for fast/medium tools, 5 min for AI tools xhr.timeout = isAiTool ? 300_000 : 60_000; - // For AI tools: upload = 0-15%, processing = 15-100% (continuous, no reset) + // For AI tools: upload = 0-15%, processing = 15-100% (SSE-driven) + // For medium tools: upload = 0-40%, processing = 40-95% (gradual fill) // For fast tools: upload = 0-100%, processing = brief 100% hold - const UPLOAD_WEIGHT = isAiTool ? 15 : 100; + const UPLOAD_WEIGHT = isAiTool ? 15 : isMediumTool ? 40 : 100; xhr.upload.onprogress = (event) => { if (event.lengthComputable) { @@ -158,10 +166,25 @@ export function useToolProcessor(toolId: string) { percent: UPLOAD_WEIGHT, stage: isAiTool ? "Starting..." : "Processing...", })); + + // Medium tools: gradually fill from upload weight to 95% over ~15s + if (isMediumTool) { + const start = UPLOAD_WEIGHT; + const target = 95; + const step = (target - start) / 30; // 30 ticks over ~15s + processingTimerRef.current = setInterval(() => { + setProgress((prev) => { + if (prev.phase !== "processing") return prev; + const next = Math.min(target, prev.percent + step); + return { ...prev, percent: next }; + }); + }, 500); + } }; xhr.onload = () => { if (elapsedRef.current) clearInterval(elapsedRef.current); + if (processingTimerRef.current) clearInterval(processingTimerRef.current); if (eventSourceRef.current) { eventSourceRef.current.close(); eventSourceRef.current = null; @@ -201,6 +224,7 @@ export function useToolProcessor(toolId: string) { xhr.onerror = () => { if (elapsedRef.current) clearInterval(elapsedRef.current); + if (processingTimerRef.current) clearInterval(processingTimerRef.current); if (eventSourceRef.current) { eventSourceRef.current.close(); eventSourceRef.current = null; @@ -212,6 +236,7 @@ export function useToolProcessor(toolId: string) { xhr.ontimeout = () => { if (elapsedRef.current) clearInterval(elapsedRef.current); + if (processingTimerRef.current) clearInterval(processingTimerRef.current); if (eventSourceRef.current) { eventSourceRef.current.close(); eventSourceRef.current = null; @@ -227,7 +252,7 @@ export function useToolProcessor(toolId: string) { }); xhr.send(formData); }, - [toolId, isAiTool, setProcessing, setError, setProcessedUrl, setSizes, setJobId], + [toolId, isAiTool, isMediumTool, setProcessing, setError, setProcessedUrl, setSizes, setJobId], ); const processAllFiles = useCallback( diff --git a/docker/Dockerfile b/docker/Dockerfile index be594c25..f857a2a8 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -36,13 +36,27 @@ RUN --mount=type=cache,id=turbo-cache,target=/app/.turbo \ pnpm --filter @stirling-image/web build # ============================================ -# Stage 2: Platform-specific base images +# Stage 2: Build caire (content-aware resize) +# ============================================ +FROM golang:1.23-bookworm AS caire-builder + +RUN apt-get update && apt-get install -y --no-install-recommends \ + libwayland-dev libx11-dev libx11-xcb-dev libxkbcommon-x11-dev \ + libgles2-mesa-dev libegl1-mesa-dev libffi-dev libxcursor-dev \ + libxrandr-dev libxinerama-dev libxi-dev libxxf86vm-dev \ + libvulkan-dev libxfixes-dev pkg-config \ + && rm -rf /var/lib/apt/lists/* + +RUN go install github.com/esimov/caire/cmd/caire@v1.5.0 + +# ============================================ +# Stage 3: Platform-specific base images # ============================================ FROM node:22-bookworm AS base-linux-arm64 FROM nvidia/cuda:12.6.3-runtime-ubuntu24.04 AS base-linux-amd64 # ============================================ -# Stage 3: Production runtime +# Stage 4: Production runtime # ============================================ ARG TARGETOS ARG TARGETARCH @@ -78,8 +92,13 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ tesseract-ocr tesseract-ocr-eng tesseract-ocr-deu tesseract-ocr-fra tesseract-ocr-spa \ build-essential \ libgl1 libglib2.0-0 \ + libegl1 libwayland-egl1 libwayland-client0 libwayland-cursor0 \ + libxkbcommon-x11-0 libxkbcommon0 libxcursor1 \ && rm -rf /var/lib/apt/lists/* +# Caire binary (content-aware seam carving) +COPY --from=caire-builder /go/bin/caire /usr/local/bin/caire + # Python venv - Layer 1: Base packages (rarely change, ~3 GB) RUN python3 -m venv /opt/venv && \ /opt/venv/bin/pip install --upgrade pip && \ @@ -116,8 +135,6 @@ RUN if [ "$TARGETARCH" = "amd64" ]; then \ /opt/venv/bin/pip install mediapipe==0.10.18 \ ; fi -RUN /opt/venv/bin/pip install seam-carving==1.1.0 - # Pre-download and verify all ML models # Note: on amd64, paddlepaddle-gpu can't import without the CUDA driver (only # available at runtime). The download script gracefully skips PaddleOCR model diff --git a/docker/download_models.py b/docker/download_models.py index d635e7f9..19324d88 100644 --- a/docker/download_models.py +++ b/docker/download_models.py @@ -110,9 +110,8 @@ def smoke_test(): from PIL import Image import cv2 import numpy - import seam_carving from rembg import new_session - print(" CPU imports OK (Pillow, cv2, numpy, seam_carving, rembg)") + print(" CPU imports OK (Pillow, cv2, numpy, rembg)") # MediaPipe is CPU-only, should always import import mediapipe as mp diff --git a/packages/ai/python/requirements-gpu.txt b/packages/ai/python/requirements-gpu.txt index 3a9d773e..95fb1199 100644 --- a/packages/ai/python/requirements-gpu.txt +++ b/packages/ai/python/requirements-gpu.txt @@ -7,4 +7,3 @@ onnxruntime-gpu==1.20.1 numpy==1.26.4 Pillow==11.1.0 opencv-python-headless==4.10.0.84 -seam-carving==1.1.0 diff --git a/packages/ai/python/requirements.txt b/packages/ai/python/requirements.txt index 88395384..1487732f 100644 --- a/packages/ai/python/requirements.txt +++ b/packages/ai/python/requirements.txt @@ -7,4 +7,3 @@ onnxruntime==1.20.1 numpy==1.26.4 Pillow==11.1.0 opencv-python-headless==4.10.0.84 -seam-carving==1.1.0 diff --git a/packages/ai/python/seam_carve.py b/packages/ai/python/seam_carve.py deleted file mode 100644 index 51a8dc2a..00000000 --- a/packages/ai/python/seam_carve.py +++ /dev/null @@ -1,161 +0,0 @@ -""" -Content-aware image resize using seam carving. -Uses the seam-carving library (li-plus) with optional face protection via MediaPipe. - -Args: - sys.argv[1]: input image path - sys.argv[2]: output image path - sys.argv[3]: JSON settings string with keys: - - width (int, optional): target width - - height (int, optional): target height - - protectFaces (bool, optional): enable face detection for protection mask -""" - -import json -import sys - - -def emit_progress(percent, stage): - """Emit structured progress to stderr for bridge.ts to capture.""" - print(json.dumps({"progress": percent, "stage": stage}), file=sys.stderr, flush=True) - - -def build_face_mask(img_array): - """Detect faces with MediaPipe and return a boolean keep_mask.""" - import numpy as np - - try: - import mediapipe as mp - except ImportError: - emit_progress(20, "MediaPipe not available, skipping face protection") - return None - - h, w = img_array.shape[:2] - mask = np.zeros((h, w), dtype=bool) - - face_detection = mp.solutions.face_detection - detector = face_detection.FaceDetection(model_selection=1, min_detection_confidence=0.5) - - try: - results = detector.process(img_array) - if not results.detections: - emit_progress(20, "No faces detected") - return None - - for detection in results.detections: - bbox = detection.location_data.relative_bounding_box - x = int(bbox.xmin * w) - y = int(bbox.ymin * h) - bw = int(bbox.width * w) - bh = int(bbox.height * h) - - # Add 20% padding around face - pad_x = int(bw * 0.2) - pad_y = int(bh * 0.2) - x1 = max(0, x - pad_x) - y1 = max(0, y - pad_y) - x2 = min(w, x + bw + pad_x) - y2 = min(h, y + bh + pad_y) - - mask[y1:y2, x1:x2] = True - - emit_progress(20, f"Detected {len(results.detections)} face(s)") - return mask - finally: - detector.close() - - -def main(): - if len(sys.argv) < 4: - print(json.dumps({"success": False, "error": "Usage: seam_carve.py "})) - sys.exit(1) - - input_path = sys.argv[1] - output_path = sys.argv[2] - - try: - settings = json.loads(sys.argv[3]) - except (json.JSONDecodeError, ValueError): - print(json.dumps({"success": False, "error": "Invalid settings JSON"})) - sys.exit(1) - - target_width = settings.get("width") - target_height = settings.get("height") - protect_faces = settings.get("protectFaces", False) - - try: - import numpy as np - from PIL import Image - except ImportError: - print(json.dumps({"success": False, "error": "Pillow/numpy not installed"})) - sys.exit(1) - - try: - import seam_carving - except ImportError: - print(json.dumps({"success": False, "error": "seam-carving package not installed"})) - sys.exit(1) - - try: - emit_progress(0, "Loading image") - img = Image.open(input_path).convert("RGB") - img_array = np.array(img) - src_h, src_w = img_array.shape[:2] - - # Default to source dimensions if not specified - if target_width is None: - target_width = src_w - if target_height is None: - target_height = src_h - - # Validate: shrink only - if target_width > src_w or target_height > src_h: - print(json.dumps({ - "success": False, - "error": f"Content-aware resize only supports shrinking. Source is {src_w}x{src_h}, target is {target_width}x{target_height}." - })) - sys.exit(1) - - # Nothing to do - if target_width == src_w and target_height == src_h: - img.save(output_path) - print(json.dumps({"success": True, "width": src_w, "height": src_h})) - return - - # Warn about large images - if src_w > 3000 or src_h > 3000: - emit_progress(5, "Large image detected, this may take a while") - - # Face protection mask - keep_mask = None - if protect_faces: - emit_progress(10, "Detecting faces") - keep_mask = build_face_mask(img_array) - - emit_progress(25, "Starting seam carving") - - # seam_carving.resize takes size as (width, height) - result = seam_carving.resize( - img_array, - (target_width, target_height), - energy_mode="backward", - order="width-first", - keep_mask=keep_mask, - ) - - emit_progress(90, "Saving result") - Image.fromarray(result).save(output_path) - - print(json.dumps({ - "success": True, - "width": result.shape[1], - "height": result.shape[0], - })) - - except Exception as e: - print(json.dumps({"success": False, "error": str(e)})) - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/packages/ai/src/seam-carving.ts b/packages/ai/src/seam-carving.ts index 5dd72e9a..69ad6dd8 100644 --- a/packages/ai/src/seam-carving.ts +++ b/packages/ai/src/seam-carving.ts @@ -1,11 +1,19 @@ -import { readFile, writeFile } from "node:fs/promises"; +import { execFile } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { readFile, rm, writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { type ProgressCallback, runPythonWithProgress } from "./bridge.js"; +import { promisify } from "node:util"; +import sharp from "sharp"; + +const execFileAsync = promisify(execFile); export interface SeamCarveOptions { width?: number; height?: number; protectFaces?: boolean; + blurRadius?: number; + sobelThreshold?: number; + square?: boolean; } export interface SeamCarveResult { @@ -14,31 +22,77 @@ export interface SeamCarveResult { height: number; } +/** + * Discover the caire binary. Checks PATH (Docker installs to /usr/local/bin) + * and the CAIRE_PATH env var for local development. + */ +let cachedCairePath: string | null = null; + +async function findCaire(): Promise { + if (cachedCairePath) return cachedCairePath; + + const candidates = process.env.CAIRE_PATH ? [process.env.CAIRE_PATH, "caire"] : ["caire"]; + + for (const cmd of candidates) { + try { + await execFileAsync(cmd, ["-help"], { timeout: 5_000 }); + cachedCairePath = cmd; + return cmd; + } catch { + // try next + } + } + throw new Error( + "caire binary not found. Install via: go install github.com/esimov/caire/cmd/caire@v1.5.0", + ); +} + +/** + * Content-aware resize using caire (Go seam carving engine). + * Supports both shrinking and enlarging via seam removal/insertion. + */ export async function seamCarve( inputBuffer: Buffer, outputDir: string, options: SeamCarveOptions = {}, - onProgress?: ProgressCallback, ): Promise { - const inputPath = join(outputDir, "input_seam_carve.png"); - const outputPath = join(outputDir, "output_seam_carve.png"); + const cairePath = await findCaire(); + const id = randomUUID(); + const inputPath = join(outputDir, `caire-in-${id}.png`); + const outputPath = join(outputDir, `caire-out-${id}.png`); - await writeFile(inputPath, inputBuffer); - const { stdout } = await runPythonWithProgress( - "seam_carve.py", - [inputPath, outputPath, JSON.stringify(options)], - { onProgress }, - ); + try { + await writeFile(inputPath, inputBuffer); - const result = JSON.parse(stdout); - if (!result.success) { - throw new Error(result.error || "Content-aware resize failed"); + // Build caire arguments + const args = ["-in", inputPath, "-out", outputPath, "-preview=false"]; + + if (options.square) { + // Caire -square requires -width and -height set to the shortest edge + const meta = await sharp(inputBuffer).metadata(); + const shortest = Math.min(meta.width ?? 0, meta.height ?? 0); + args.push("-square", "-width", String(shortest), "-height", String(shortest)); + } else { + if (options.width) args.push("-width", String(options.width)); + if (options.height) args.push("-height", String(options.height)); + } + + if (options.protectFaces) args.push("-face"); + if (options.blurRadius !== undefined) args.push("-blur", String(options.blurRadius)); + if (options.sobelThreshold !== undefined) args.push("-sobel", String(options.sobelThreshold)); + + await execFileAsync(cairePath, args, { timeout: 60_000 }); + + const buffer = await readFile(outputPath); + const meta = await sharp(buffer).metadata(); + + return { + buffer, + width: meta.width ?? 0, + height: meta.height ?? 0, + }; + } finally { + await rm(inputPath, { force: true }).catch(() => {}); + await rm(outputPath, { force: true }).catch(() => {}); } - - const buffer = await readFile(outputPath); - return { - buffer, - width: result.width, - height: result.height, - }; } diff --git a/packages/shared/src/constants.ts b/packages/shared/src/constants.ts index ccf77f82..d4dcb211 100644 --- a/packages/shared/src/constants.ts +++ b/packages/shared/src/constants.ts @@ -390,5 +390,4 @@ export const PYTHON_SIDECAR_TOOLS = [ "blur-faces", "erase-object", "ocr", - "content-aware-resize", ] as const; diff --git a/tests/e2e/__screenshots__/visual-regression.spec.ts/resize-empty-desktop.png b/tests/e2e/__screenshots__/visual-regression.spec.ts/resize-empty-desktop.png index ef9a249d..4fe49458 100644 Binary files a/tests/e2e/__screenshots__/visual-regression.spec.ts/resize-empty-desktop.png and b/tests/e2e/__screenshots__/visual-regression.spec.ts/resize-empty-desktop.png differ diff --git a/tests/e2e/__screenshots__/visual-regression.spec.ts/resize-empty-mobile.png b/tests/e2e/__screenshots__/visual-regression.spec.ts/resize-empty-mobile.png index 9ac45694..25a718fa 100644 Binary files a/tests/e2e/__screenshots__/visual-regression.spec.ts/resize-empty-mobile.png and b/tests/e2e/__screenshots__/visual-regression.spec.ts/resize-empty-mobile.png differ diff --git a/tests/integration/content-aware-resize.test.ts b/tests/integration/content-aware-resize.test.ts index 7ef21283..4baf6e14 100644 --- a/tests/integration/content-aware-resize.test.ts +++ b/tests/integration/content-aware-resize.test.ts @@ -1,10 +1,9 @@ /** - * Integration tests for the content-aware resize (seam carving) API endpoint. + * Integration tests for the content-aware resize (seam carving via caire) API endpoint. * - * This tool uses the Python sidecar, so in CI/test environments where Python - * is not available the route will return 422 (Python error). - * Tests gracefully handle both scenarios while still verifying route existence - * and input validation. + * This tool uses the caire Go binary. In environments where caire is not + * installed the route will return 422. Tests gracefully handle both scenarios + * while still verifying route existence and input validation. */ import { readFileSync } from "node:fs"; @@ -49,8 +48,7 @@ describe("Content-Aware Resize", () => { body, }); - // 200 = Python available, 422 = Python error - // Any of these proves the route is registered and reachable + // 200 = caire available, 422 = caire not found expect([200, 422]).toContain(res.statusCode); }, 60_000); @@ -72,6 +70,27 @@ describe("Content-Aware Resize", () => { expect(res.statusCode).toBe(400); }); + it("rejects requests without width, height, or square", async () => { + const { body, contentType } = createMultipartPayload([ + { name: "file", filename: "test.png", contentType: "image/png", content: PNG_200x150 }, + { name: "settings", content: JSON.stringify({ protectFaces: false }) }, + ]); + + const res = await app.inject({ + method: "POST", + url: "/api/v1/tools/content-aware-resize", + headers: { + authorization: `Bearer ${adminToken}`, + "content-type": contentType, + }, + body, + }); + + expect(res.statusCode).toBe(400); + const resBody = JSON.parse(res.body); + expect(resBody.error).toContain("width, height, or square"); + }); + it("processes with only width specified", async () => { const { body, contentType } = createMultipartPayload([ { name: "file", filename: "test.png", contentType: "image/png", content: PNG_200x150 }, @@ -88,7 +107,7 @@ describe("Content-Aware Resize", () => { body, }); - // Accept 200 (Python available) or 422 (Python not available) + // Accept 200 (caire available) or 422 (caire not available) expect([200, 422]).toContain(res.statusCode); if (res.statusCode === 200) { @@ -123,10 +142,10 @@ describe("Content-Aware Resize", () => { } }, 60_000); - it("rejects enlargement beyond source dimensions", async () => { + it("supports enlargement via seam insertion", async () => { const { body, contentType } = createMultipartPayload([ { name: "file", filename: "test.png", contentType: "image/png", content: PNG_200x150 }, - { name: "settings", content: JSON.stringify({ width: 400, protectFaces: false }) }, + { name: "settings", content: JSON.stringify({ width: 300, protectFaces: false }) }, ]); const res = await app.inject({ @@ -139,14 +158,116 @@ describe("Content-Aware Resize", () => { body, }); - // 422 = Python caught the enlargement error - // Should never be 200 since 400 > 200px source width - expect(res.statusCode).not.toBe(200); - expect(res.statusCode).toBe(422); + // 200 = caire enlarged successfully, 422 = caire not available + expect([200, 422]).toContain(res.statusCode); - if (res.statusCode === 422) { + if (res.statusCode === 200) { const resBody = JSON.parse(res.body); - expect(resBody.error || resBody.details).toBeDefined(); + expect(resBody.downloadUrl).toBeDefined(); + expect(resBody.width).toBe(300); } }, 60_000); + + it("accepts blurRadius and sobelThreshold options", async () => { + const { body, contentType } = createMultipartPayload([ + { name: "file", filename: "test.png", contentType: "image/png", content: PNG_200x150 }, + { + name: "settings", + content: JSON.stringify({ + width: 150, + protectFaces: false, + blurRadius: 6, + sobelThreshold: 4, + }), + }, + ]); + + const res = await app.inject({ + method: "POST", + url: "/api/v1/tools/content-aware-resize", + headers: { + authorization: `Bearer ${adminToken}`, + "content-type": contentType, + }, + body, + }); + + expect([200, 422]).toContain(res.statusCode); + + if (res.statusCode === 200) { + const resBody = JSON.parse(res.body); + expect(resBody.downloadUrl).toBeDefined(); + expect(resBody.width).toBe(150); + } + }, 60_000); + + it("supports square mode", async () => { + const { body, contentType } = createMultipartPayload([ + { name: "file", filename: "test.png", contentType: "image/png", content: PNG_200x150 }, + { name: "settings", content: JSON.stringify({ square: true, protectFaces: false }) }, + ]); + + const res = await app.inject({ + method: "POST", + url: "/api/v1/tools/content-aware-resize", + headers: { + authorization: `Bearer ${adminToken}`, + "content-type": contentType, + }, + body, + }); + + expect([200, 422]).toContain(res.statusCode); + + if (res.statusCode === 200) { + const resBody = JSON.parse(res.body); + expect(resBody.downloadUrl).toBeDefined(); + // Square mode produces equal width and height + expect(resBody.width).toBe(resBody.height); + } + }, 60_000); + + it("rejects invalid blurRadius", async () => { + const { body, contentType } = createMultipartPayload([ + { name: "file", filename: "test.png", contentType: "image/png", content: PNG_200x150 }, + { + name: "settings", + content: JSON.stringify({ width: 150, blurRadius: 50 }), + }, + ]); + + const res = await app.inject({ + method: "POST", + url: "/api/v1/tools/content-aware-resize", + headers: { + authorization: `Bearer ${adminToken}`, + "content-type": contentType, + }, + body, + }); + + expect(res.statusCode).toBe(400); + }); + + it("rejects invalid sobelThreshold", async () => { + const { body, contentType } = createMultipartPayload([ + { name: "file", filename: "test.png", contentType: "image/png", content: PNG_200x150 }, + { + name: "settings", + content: JSON.stringify({ width: 150, sobelThreshold: 0 }), + }, + ]); + + const res = await app.inject({ + method: "POST", + url: "/api/v1/tools/content-aware-resize", + headers: { + authorization: `Bearer ${adminToken}`, + "content-type": contentType, + }, + body, + }); + + expect(res.statusCode).toBe(400); + }); });