Merge pull request #42 from stirling-image/feat/caire-content-aware-resize

feat: replace Python seam carving with caire Go binary

apps/api/src/routes/tools/content-aware-resize.ts

@@ -7,10 +7,20 @@ import { z } from "zod";
 import { autoOrient } from "../../lib/auto-orient.js";
 import { validateImageBuffer } from "../../lib/file-validation.js";
 import { createWorkspace } from "../../lib/workspace.js";
-import { updateSingleFileProgress } from "../progress.js";
 import { registerToolProcessFn } from "../tool-factory.js";
 
-/** Content-aware resize (seam carving) route. */
+const settingsSchema = z.object({
+  width: z.number().positive().optional(),
+  height: z.number().positive().optional(),
+  protectFaces: z.boolean().default(false),
+  blurRadius: z.number().min(0).max(20).default(4),
+  sobelThreshold: z.number().min(1).max(20).default(2),
+  square: z.boolean().default(false),
+});
+
+type Settings = z.infer<typeof settingsSchema>;
+
+/** Content-aware resize (seam carving via caire) route. */
 export function registerContentAwareResize(app: FastifyInstance) {
   app.post(
     "/api/v1/tools/content-aware-resize",
@@ -18,7 +28,6 @@ export function registerContentAwareResize(app: FastifyInstance) {
       let fileBuffer: Buffer | null = null;
       let filename = "image";
       let settingsRaw: string | null = null;
-      let clientJobId: string | null = null;
 
       try {
         const parts = request.parts();
@@ -32,8 +41,6 @@ export function registerContentAwareResize(app: FastifyInstance) {
             filename = basename(part.filename ?? "image");
           } else if (part.fieldname === "settings") {
             settingsRaw = part.value as string;
-          } else if (part.fieldname === "clientJobId") {
-            clientJobId = part.value as string;
           }
         }
       } catch (err) {
@@ -52,15 +59,37 @@ export function registerContentAwareResize(app: FastifyInstance) {
         return reply.status(400).send({ error: `Invalid image: ${validation.reason}` });
       }
 
+      // Validate settings
+      let settings: Settings;
+      try {
+        const parsed = settingsRaw ? JSON.parse(settingsRaw) : {};
+        const result = settingsSchema.safeParse(parsed);
+        if (!result.success) {
+          return reply.status(400).send({
+            error: "Invalid settings",
+            details: result.error.issues.map((i) => ({
+              path: i.path.join("."),
+              message: i.message,
+            })),
+          });
+        }
+        settings = result.data;
+      } catch {
+        return reply.status(400).send({ error: "Settings must be valid JSON" });
+      }
+
+      if (!settings.square && !settings.width && !settings.height) {
+        return reply.status(400).send({
+          error: "Either width, height, or square mode must be specified",
+        });
+      }
+
       try {
-        const settings = settingsRaw ? JSON.parse(settingsRaw) : {};
         request.log.info(
           {
             toolId: "content-aware-resize",
             imageSize: fileBuffer.length,
-            width: settings.width,
+            ...settings,
-            height: settings.height,
-            protectFaces: settings.protectFaces,
           },
           "Starting content-aware resize",
         );
@@ -75,43 +104,21 @@ export function registerContentAwareResize(app: FastifyInstance) {
         const inputPath = join(workspacePath, "input", filename);
         await writeFile(inputPath, fileBuffer);
 
-        // Process
+        // Process with caire
-        const jobIdForProgress = clientJobId;
+        const result = await seamCarve(fileBuffer, join(workspacePath, "output"), {
-        const onProgress = jobIdForProgress
+          width: settings.width,
-          ? (percent: number, stage: string) => {
+          height: settings.height,
-              updateSingleFileProgress({
+          protectFaces: settings.protectFaces,
-                jobId: jobIdForProgress,
+          blurRadius: settings.blurRadius,
-                phase: "processing",
+          sobelThreshold: settings.sobelThreshold,
-                stage,
+          square: settings.square,
-                percent,
+        });
-              });
-            }
-          : undefined;
-
-        const result = await seamCarve(
-          fileBuffer,
-          join(workspacePath, "output"),
-          {
-            width: settings.width,
-            height: settings.height,
-            protectFaces: settings.protectFaces ?? true,
-          },
-          onProgress,
-        );
 
         // Save output
         const outputFilename = `${filename.replace(/\.[^.]+$/, "")}_seam.png`;
         const outputPath = join(workspacePath, "output", outputFilename);
         await writeFile(outputPath, result.buffer);
 
-        if (clientJobId) {
-          updateSingleFileProgress({
-            jobId: clientJobId,
-            phase: "complete",
-            percent: 100,
-          });
-        }
-
         return reply.send({
           jobId,
           downloadUrl: `/api/v1/download/${jobId}/${encodeURIComponent(outputFilename)}`,
@@ -130,24 +137,22 @@ export function registerContentAwareResize(app: FastifyInstance) {
     },
   );
 
-  // Register in the pipeline/batch registry so this tool can be used
+  // Register in the pipeline/batch registry
-  // as a step in automation pipelines (without progress callbacks).
   registerToolProcessFn({
     toolId: "content-aware-resize",
-    settingsSchema: z.object({
+    settingsSchema,
-      width: z.number().positive().optional(),
-      height: z.number().positive().optional(),
-      protectFaces: z.boolean().default(true),
-    }),
     process: async (inputBuffer, settings, filename) => {
-      const s = settings as { width?: number; height?: number; protectFaces?: boolean };
+      const s = settings as Settings;
       const orientedBuffer = await autoOrient(inputBuffer);
       const jobId = randomUUID();
       const workspacePath = await createWorkspace(jobId);
       const result = await seamCarve(orientedBuffer, join(workspacePath, "output"), {
         width: s.width,
         height: s.height,
-        protectFaces: s.protectFaces ?? true,
+        protectFaces: s.protectFaces,
+        blurRadius: s.blurRadius,
+        sobelThreshold: s.sobelThreshold,
+        square: s.square,
       });
       const outputFilename = `${filename.replace(/\.[^.]+$/, "")}_seam.png`;
       return { buffer: result.buffer, filename: outputFilename, contentType: "image/png" };

apps/web/src/components/tools/resize-settings.tsx

@@ -1,5 +1,5 @@
 import { SOCIAL_MEDIA_PRESETS } from "@stirling-image/shared";
-import { Download, Link, Unlink } from "lucide-react";
+import { Download, Info, Link, Unlink } from "lucide-react";
 import { useCallback, useEffect, useRef, useState } from "react";
 import { ProgressCard } from "@/components/common/progress-card";
 import { useToolProcessor } from "@/hooks/use-tool-processor";
@@ -17,6 +17,17 @@ const FIT_LABELS: Record<FitMode, string> = {
 // Group presets by platform
 const platforms = [...new Set(SOCIAL_MEDIA_PRESETS.map((p) => p.platform))];
 
+function HintIcon({ text }: { text: string }) {
+  return (
+    <span className="relative group">
+      <Info className="h-3 w-3 text-muted-foreground" />
+      <span className="pointer-events-none absolute left-1/2 -translate-x-1/2 bottom-full mb-1.5 w-48 rounded bg-foreground px-2 py-1.5 text-[11px] leading-tight text-background opacity-0 transition-opacity group-hover:opacity-100 z-10">
+        {text}
+      </span>
+    </span>
+  );
+}
+
 export interface ResizeControlsProps {
   onChange?: (settings: Record<string, unknown>) => void;
 }
@@ -31,7 +42,10 @@ export function ResizeControls({ onChange }: ResizeControlsProps) {
   const [lockAspect, setLockAspect] = useState(true);
   const [withoutEnlargement, setWithoutEnlargement] = useState(false);
   const [contentAware, setContentAware] = useState(false);
-  const [protectFaces, setProtectFaces] = useState(true);
+  const [protectFaces, setProtectFaces] = useState(false);
+  const [blurRadius, setBlurRadius] = useState(4);
+  const [sobelThreshold, setSobelThreshold] = useState(2);
+  const [squareMode, setSquareMode] = useState(false);
 
   const onChangeRef = useRef(onChange);
   useEffect(() => {
@@ -42,9 +56,14 @@ export function ResizeControls({ onChange }: ResizeControlsProps) {
     const settings: Record<string, unknown> = {};
     if (contentAware) {
       settings.contentAware = true;
-      if (width) settings.width = Number(width);
+      if (!squareMode) {
-      if (height) settings.height = Number(height);
+        if (width) settings.width = Number(width);
+        if (height) settings.height = Number(height);
+      }
       settings.protectFaces = protectFaces;
+      settings.blurRadius = blurRadius;
+      settings.sobelThreshold = sobelThreshold;
+      settings.square = squareMode;
     } else if (tab === "scale") {
       settings.percentage = Number(percentage);
     } else {
@@ -54,7 +73,19 @@ export function ResizeControls({ onChange }: ResizeControlsProps) {
       settings.withoutEnlargement = withoutEnlargement;
     }
     onChangeRef.current?.(settings);
-  }, [tab, width, height, percentage, fit, withoutEnlargement, contentAware, protectFaces]);
+  }, [
+    tab,
+    width,
+    height,
+    percentage,
+    fit,
+    withoutEnlargement,
+    contentAware,
+    protectFaces,
+    blurRadius,
+    sobelThreshold,
+    squareMode,
+  ]);
 
   const handlePreset = (preset: (typeof SOCIAL_MEDIA_PRESETS)[number]) => {
     const key = `${preset.platform}-${preset.name}`;
@@ -84,7 +115,8 @@ export function ResizeControls({ onChange }: ResizeControlsProps) {
           value={width}
           onChange={(e) => setWidth(e.target.value)}
           placeholder="Auto"
-          className="w-full mt-0.5 px-2 py-1.5 rounded border border-border bg-background text-sm text-foreground"
+          disabled={squareMode && contentAware}
+          className="w-full mt-0.5 px-2 py-1.5 rounded border border-border bg-background text-sm text-foreground disabled:opacity-50"
         />
       </div>
       <button
@@ -105,52 +137,28 @@ export function ResizeControls({ onChange }: ResizeControlsProps) {
           value={height}
           onChange={(e) => setHeight(e.target.value)}
           placeholder="Auto"
-          className="w-full mt-0.5 px-2 py-1.5 rounded border border-border bg-background text-sm text-foreground"
+          disabled={squareMode && contentAware}
+          className="w-full mt-0.5 px-2 py-1.5 rounded border border-border bg-background text-sm text-foreground disabled:opacity-50"
         />
       </div>
     </div>
   );
 
+  const enlargementCheckbox = (
+    <label className="flex items-center gap-1.5 text-xs text-foreground">
+      <input
+        type="checkbox"
+        checked={withoutEnlargement}
+        onChange={(e) => setWithoutEnlargement(e.target.checked)}
+        className="rounded"
+      />
+      <span>Limit to original size</span>
+      <HintIcon text="If your image is already smaller than the target, keep it as-is instead of scaling it up" />
+    </label>
+  );
+
   return (
     <div className="space-y-4">
-      {/* Content-aware toggle */}
-      <div className="flex items-center justify-between">
-        <span className="text-sm font-medium text-foreground">Content-aware</span>
-        <button
-          type="button"
-          role="switch"
-          aria-checked={contentAware}
-          onClick={() => setContentAware(!contentAware)}
-          className={`relative inline-flex h-5 w-9 items-center rounded-full transition-colors ${
-            contentAware ? "bg-primary" : "bg-muted"
-          }`}
-        >
-          <span
-            className={`inline-block h-3.5 w-3.5 rounded-full bg-white shadow-sm transition-transform ${
-              contentAware ? "translate-x-4" : "translate-x-0.5"
-            }`}
-          />
-        </button>
-      </div>
-
-      {/* Content-aware inputs */}
-      {contentAware && (
-        <div className="space-y-3">
-          {dimensionInputs}
-
-          {/* Protect faces */}
-          <label className="flex items-center gap-2 text-sm text-foreground">
-            <input
-              type="checkbox"
-              checked={protectFaces}
-              onChange={(e) => setProtectFaces(e.target.checked)}
-              className="rounded"
-            />
-            Protect faces
-          </label>
-        </div>
-      )}
-
       {/* Standard resize tabs */}
       {!contentAware && (
         <>
@@ -196,7 +204,7 @@ export function ResizeControls({ onChange }: ResizeControlsProps) {
                         >
                           <span>{preset.name}</span>
                           <span className="text-xs tabular-nums">
-                            {preset.width} × {preset.height}
+                            {preset.width} x {preset.height}
                           </span>
                         </button>
                       );
@@ -205,16 +213,7 @@ export function ResizeControls({ onChange }: ResizeControlsProps) {
                 </div>
               ))}
 
-              {/* Don't enlarge */}
+              {enlargementCheckbox}
-              <label className="flex items-center gap-2 text-sm text-foreground">
-                <input
-                  type="checkbox"
-                  checked={withoutEnlargement}
-                  onChange={(e) => setWithoutEnlargement(e.target.checked)}
-                  className="rounded"
-                />
-                Don&apos;t enlarge
-              </label>
             </div>
           )}
 
@@ -240,16 +239,7 @@ export function ResizeControls({ onChange }: ResizeControlsProps) {
                 </div>
               </div>
 
-              {/* Don't enlarge */}
+              {enlargementCheckbox}
-              <label className="flex items-center gap-2 text-sm text-foreground">
-                <input
-                  type="checkbox"
-                  checked={withoutEnlargement}
-                  onChange={(e) => setWithoutEnlargement(e.target.checked)}
-                  className="rounded"
-                />
-                Don&apos;t enlarge
-              </label>
             </div>
           )}
 
@@ -289,6 +279,98 @@ export function ResizeControls({ onChange }: ResizeControlsProps) {
           )}
         </>
       )}
+
+      {/* Content-aware section - positioned below standard resize */}
+      <div className="border-t border-border pt-3">
+        <div className="flex items-center justify-between">
+          <div>
+            <span className="text-xs font-medium text-muted-foreground">Content-aware</span>
+          </div>
+          <button
+            type="button"
+            role="switch"
+            aria-checked={contentAware}
+            onClick={() => setContentAware(!contentAware)}
+            className={`relative inline-flex h-4 w-7 items-center rounded-full transition-colors ${
+              contentAware ? "bg-primary" : "bg-muted"
+            }`}
+          >
+            <span
+              className={`inline-block h-3 w-3 rounded-full bg-white shadow-sm transition-transform ${
+                contentAware ? "translate-x-3.5" : "translate-x-0.5"
+              }`}
+            />
+          </button>
+        </div>
+
+        {/* Content-aware options (expanded when toggled) */}
+        {contentAware && (
+          <div className="mt-3 space-y-3">
+            {/* Dimensions */}
+            {dimensionInputs}
+
+            {/* Square mode */}
+            <label className="flex items-center gap-2 text-xs text-foreground">
+              <input
+                type="checkbox"
+                checked={squareMode}
+                onChange={(e) => setSquareMode(e.target.checked)}
+                className="rounded"
+              />
+              Resize to square
+            </label>
+
+            {/* Face protection */}
+            <label className="flex items-center gap-2 text-xs text-foreground">
+              <input
+                type="checkbox"
+                checked={protectFaces}
+                onChange={(e) => setProtectFaces(e.target.checked)}
+                className="rounded"
+              />
+              Protect faces
+            </label>
+
+            {/* Blur radius */}
+            <div>
+              <div className="flex items-center justify-between">
+                <label htmlFor="blur-radius" className="text-xs text-muted-foreground">
+                  Smoothing
+                </label>
+                <span className="text-xs tabular-nums text-muted-foreground">{blurRadius}</span>
+              </div>
+              <input
+                id="blur-radius"
+                type="range"
+                min={0}
+                max={20}
+                value={blurRadius}
+                onChange={(e) => setBlurRadius(Number(e.target.value))}
+                className="w-full mt-1 h-1.5 rounded-full appearance-none bg-muted accent-primary"
+              />
+            </div>
+
+            {/* Sobel threshold */}
+            <div>
+              <div className="flex items-center justify-between">
+                <label htmlFor="sobel-threshold" className="text-xs text-muted-foreground">
+                  Edge sensitivity
+                </label>
+                <span className="text-xs tabular-nums text-muted-foreground">{sobelThreshold}</span>
+              </div>
+              <input
+                id="sobel-threshold"
+                type="range"
+                min={1}
+                max={20}
+                value={sobelThreshold}
+                onChange={(e) => setSobelThreshold(Number(e.target.value))}
+                className="w-full mt-1 h-1.5 rounded-full appearance-none bg-muted accent-primary"
+              />
+            </div>
+          </div>
+        )}
+      </div>
     </div>
   );
 }
@@ -323,7 +405,7 @@ export function ResizeSettings() {
     hasFile &&
     !processing &&
     (isContentAware
-      ? Boolean(settings.width) || Boolean(settings.height)
+      ? Boolean(settings.width) || Boolean(settings.height) || Boolean(settings.square)
       : tab === "scale"
         ? Number(settings.percentage) > 0
         : Boolean(settings.width) || Boolean(settings.height));

apps/web/src/hooks/use-tool-processor.ts

@@ -29,6 +29,10 @@ const IDLE_PROGRESS: ToolProgress = {
 // smart-crop is category "ai" but uses Sharp (no Python), so it's excluded.
 const AI_PYTHON_TOOLS = new Set<string>(PYTHON_SIDECAR_TOOLS);
 
+// Tools that take a few seconds (not instant like Sharp, not minutes like AI).
+// Uses a smoother progress: upload 0-40%, then a gradual fill during processing.
+const MEDIUM_TOOLS = new Set(["content-aware-resize"]);
+
 export function useToolProcessor(toolId: string) {
   const {
     processing,
@@ -49,11 +53,14 @@ export function useToolProcessor(toolId: string) {
   const eventSourceRef = useRef<EventSource | null>(null);
 
   const isAiTool = AI_PYTHON_TOOLS.has(toolId);
+  const isMediumTool = MEDIUM_TOOLS.has(toolId);
+  const processingTimerRef = useRef<ReturnType<typeof setInterval> | null>(null);
 
   // Clean up on unmount
   useEffect(() => {
     return () => {
       if (elapsedRef.current) clearInterval(elapsedRef.current);
+      if (processingTimerRef.current) clearInterval(processingTimerRef.current);
       if (eventSourceRef.current) eventSourceRef.current.close();
       if (xhrRef.current) xhrRef.current.abort();
     };
@@ -134,12 +141,13 @@ export function useToolProcessor(toolId: string) {
       const xhr = new XMLHttpRequest();
       xhrRef.current = xhr;
 
-      // Timeout: 60s for fast tools, 5 min for AI tools
+      // Timeout: 60s for fast/medium tools, 5 min for AI tools
       xhr.timeout = isAiTool ? 300_000 : 60_000;
 
-      // For AI tools: upload = 0-15%, processing = 15-100% (continuous, no reset)
+      // For AI tools: upload = 0-15%, processing = 15-100% (SSE-driven)
+      // For medium tools: upload = 0-40%, processing = 40-95% (gradual fill)
       // For fast tools: upload = 0-100%, processing = brief 100% hold
-      const UPLOAD_WEIGHT = isAiTool ? 15 : 100;
+      const UPLOAD_WEIGHT = isAiTool ? 15 : isMediumTool ? 40 : 100;
 
       xhr.upload.onprogress = (event) => {
         if (event.lengthComputable) {
@@ -158,10 +166,25 @@ export function useToolProcessor(toolId: string) {
           percent: UPLOAD_WEIGHT,
           stage: isAiTool ? "Starting..." : "Processing...",
         }));
+
+        // Medium tools: gradually fill from upload weight to 95% over ~15s
+        if (isMediumTool) {
+          const start = UPLOAD_WEIGHT;
+          const target = 95;
+          const step = (target - start) / 30; // 30 ticks over ~15s
+          processingTimerRef.current = setInterval(() => {
+            setProgress((prev) => {
+              if (prev.phase !== "processing") return prev;
+              const next = Math.min(target, prev.percent + step);
+              return { ...prev, percent: next };
+            });
+          }, 500);
+        }
       };
 
       xhr.onload = () => {
         if (elapsedRef.current) clearInterval(elapsedRef.current);
+        if (processingTimerRef.current) clearInterval(processingTimerRef.current);
         if (eventSourceRef.current) {
           eventSourceRef.current.close();
           eventSourceRef.current = null;
@@ -201,6 +224,7 @@ export function useToolProcessor(toolId: string) {
 
       xhr.onerror = () => {
         if (elapsedRef.current) clearInterval(elapsedRef.current);
+        if (processingTimerRef.current) clearInterval(processingTimerRef.current);
         if (eventSourceRef.current) {
           eventSourceRef.current.close();
           eventSourceRef.current = null;
@@ -212,6 +236,7 @@ export function useToolProcessor(toolId: string) {
 
       xhr.ontimeout = () => {
         if (elapsedRef.current) clearInterval(elapsedRef.current);
+        if (processingTimerRef.current) clearInterval(processingTimerRef.current);
         if (eventSourceRef.current) {
           eventSourceRef.current.close();
           eventSourceRef.current = null;
@@ -227,7 +252,7 @@ export function useToolProcessor(toolId: string) {
       });
       xhr.send(formData);
     },
-    [toolId, isAiTool, setProcessing, setError, setProcessedUrl, setSizes, setJobId],
+    [toolId, isAiTool, isMediumTool, setProcessing, setError, setProcessedUrl, setSizes, setJobId],
   );
 
   const processAllFiles = useCallback(

docker/Dockerfile

@@ -36,13 +36,27 @@ RUN --mount=type=cache,id=turbo-cache,target=/app/.turbo \
     pnpm --filter @stirling-image/web build
 
 # ============================================
-# Stage 2: Platform-specific base images
+# Stage 2: Build caire (content-aware resize)
+# ============================================
+FROM golang:1.23-bookworm AS caire-builder
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    libwayland-dev libx11-dev libx11-xcb-dev libxkbcommon-x11-dev \
+    libgles2-mesa-dev libegl1-mesa-dev libffi-dev libxcursor-dev \
+    libxrandr-dev libxinerama-dev libxi-dev libxxf86vm-dev \
+    libvulkan-dev libxfixes-dev pkg-config \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN go install github.com/esimov/caire/cmd/caire@v1.5.0
+
+# ============================================
+# Stage 3: Platform-specific base images
 # ============================================
 FROM node:22-bookworm AS base-linux-arm64
 FROM nvidia/cuda:12.6.3-runtime-ubuntu24.04 AS base-linux-amd64
 
 # ============================================
-# Stage 3: Production runtime
+# Stage 4: Production runtime
 # ============================================
 ARG TARGETOS
 ARG TARGETARCH
@@ -78,8 +92,13 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     tesseract-ocr tesseract-ocr-eng tesseract-ocr-deu tesseract-ocr-fra tesseract-ocr-spa \
     build-essential \
     libgl1 libglib2.0-0 \
+    libegl1 libwayland-egl1 libwayland-client0 libwayland-cursor0 \
+    libxkbcommon-x11-0 libxkbcommon0 libxcursor1 \
     && rm -rf /var/lib/apt/lists/*
 
+# Caire binary (content-aware seam carving)
+COPY --from=caire-builder /go/bin/caire /usr/local/bin/caire
+
 # Python venv - Layer 1: Base packages (rarely change, ~3 GB)
 RUN python3 -m venv /opt/venv && \
     /opt/venv/bin/pip install --upgrade pip && \
@@ -116,8 +135,6 @@ RUN if [ "$TARGETARCH" = "amd64" ]; then \
     /opt/venv/bin/pip install mediapipe==0.10.18 \
 ; fi
 
-RUN /opt/venv/bin/pip install seam-carving==1.1.0
-
 # Pre-download and verify all ML models
 # Note: on amd64, paddlepaddle-gpu can't import without the CUDA driver (only
 # available at runtime). The download script gracefully skips PaddleOCR model

docker/download_models.py

@@ -110,9 +110,8 @@ def smoke_test():
     from PIL import Image
     import cv2
     import numpy
-    import seam_carving
     from rembg import new_session
-    print("  CPU imports OK (Pillow, cv2, numpy, seam_carving, rembg)")
+    print("  CPU imports OK (Pillow, cv2, numpy, rembg)")
 
     # MediaPipe is CPU-only, should always import
     import mediapipe as mp

packages/ai/python/requirements-gpu.txt

@@ -7,4 +7,3 @@ onnxruntime-gpu==1.20.1
 numpy==1.26.4
 Pillow==11.1.0
 opencv-python-headless==4.10.0.84
-seam-carving==1.1.0

packages/ai/python/requirements.txt

@@ -7,4 +7,3 @@ onnxruntime==1.20.1
 numpy==1.26.4
 Pillow==11.1.0
 opencv-python-headless==4.10.0.84
-seam-carving==1.1.0

packages/ai/python/seam_carve.py

@@ -1,161 +0,0 @@
-"""
-Content-aware image resize using seam carving.
-Uses the seam-carving library (li-plus) with optional face protection via MediaPipe.
-
-Args:
-    sys.argv[1]: input image path
-    sys.argv[2]: output image path
-    sys.argv[3]: JSON settings string with keys:
-        - width (int, optional): target width
-        - height (int, optional): target height
-        - protectFaces (bool, optional): enable face detection for protection mask
-"""
-
-import json
-import sys
-
-
-def emit_progress(percent, stage):
-    """Emit structured progress to stderr for bridge.ts to capture."""
-    print(json.dumps({"progress": percent, "stage": stage}), file=sys.stderr, flush=True)
-
-
-def build_face_mask(img_array):
-    """Detect faces with MediaPipe and return a boolean keep_mask."""
-    import numpy as np
-
-    try:
-        import mediapipe as mp
-    except ImportError:
-        emit_progress(20, "MediaPipe not available, skipping face protection")
-        return None
-
-    h, w = img_array.shape[:2]
-    mask = np.zeros((h, w), dtype=bool)
-
-    face_detection = mp.solutions.face_detection
-    detector = face_detection.FaceDetection(model_selection=1, min_detection_confidence=0.5)
-
-    try:
-        results = detector.process(img_array)
-        if not results.detections:
-            emit_progress(20, "No faces detected")
-            return None
-
-        for detection in results.detections:
-            bbox = detection.location_data.relative_bounding_box
-            x = int(bbox.xmin * w)
-            y = int(bbox.ymin * h)
-            bw = int(bbox.width * w)
-            bh = int(bbox.height * h)
-
-            # Add 20% padding around face
-            pad_x = int(bw * 0.2)
-            pad_y = int(bh * 0.2)
-            x1 = max(0, x - pad_x)
-            y1 = max(0, y - pad_y)
-            x2 = min(w, x + bw + pad_x)
-            y2 = min(h, y + bh + pad_y)
-
-            mask[y1:y2, x1:x2] = True
-
-        emit_progress(20, f"Detected {len(results.detections)} face(s)")
-        return mask
-    finally:
-        detector.close()
-
-
-def main():
-    if len(sys.argv) < 4:
-        print(json.dumps({"success": False, "error": "Usage: seam_carve.py <input> <output> <settings>"}))
-        sys.exit(1)
-
-    input_path = sys.argv[1]
-    output_path = sys.argv[2]
-
-    try:
-        settings = json.loads(sys.argv[3])
-    except (json.JSONDecodeError, ValueError):
-        print(json.dumps({"success": False, "error": "Invalid settings JSON"}))
-        sys.exit(1)
-
-    target_width = settings.get("width")
-    target_height = settings.get("height")
-    protect_faces = settings.get("protectFaces", False)
-
-    try:
-        import numpy as np
-        from PIL import Image
-    except ImportError:
-        print(json.dumps({"success": False, "error": "Pillow/numpy not installed"}))
-        sys.exit(1)
-
-    try:
-        import seam_carving
-    except ImportError:
-        print(json.dumps({"success": False, "error": "seam-carving package not installed"}))
-        sys.exit(1)
-
-    try:
-        emit_progress(0, "Loading image")
-        img = Image.open(input_path).convert("RGB")
-        img_array = np.array(img)
-        src_h, src_w = img_array.shape[:2]
-
-        # Default to source dimensions if not specified
-        if target_width is None:
-            target_width = src_w
-        if target_height is None:
-            target_height = src_h
-
-        # Validate: shrink only
-        if target_width > src_w or target_height > src_h:
-            print(json.dumps({
-                "success": False,
-                "error": f"Content-aware resize only supports shrinking. Source is {src_w}x{src_h}, target is {target_width}x{target_height}."
-            }))
-            sys.exit(1)
-
-        # Nothing to do
-        if target_width == src_w and target_height == src_h:
-            img.save(output_path)
-            print(json.dumps({"success": True, "width": src_w, "height": src_h}))
-            return
-
-        # Warn about large images
-        if src_w > 3000 or src_h > 3000:
-            emit_progress(5, "Large image detected, this may take a while")
-
-        # Face protection mask
-        keep_mask = None
-        if protect_faces:
-            emit_progress(10, "Detecting faces")
-            keep_mask = build_face_mask(img_array)
-
-        emit_progress(25, "Starting seam carving")
-
-        # seam_carving.resize takes size as (width, height)
-        result = seam_carving.resize(
-            img_array,
-            (target_width, target_height),
-            energy_mode="backward",
-            order="width-first",
-            keep_mask=keep_mask,
-        )
-
-        emit_progress(90, "Saving result")
-        Image.fromarray(result).save(output_path)
-
-        print(json.dumps({
-            "success": True,
-            "width": result.shape[1],
-            "height": result.shape[0],
-        }))
-
-    except Exception as e:
-        print(json.dumps({"success": False, "error": str(e)}))
-        sys.exit(1)
-
-
-if __name__ == "__main__":
-    main()

packages/ai/src/seam-carving.ts

@@ -1,11 +1,19 @@
-import { readFile, writeFile } from "node:fs/promises";
+import { execFile } from "node:child_process";
+import { randomUUID } from "node:crypto";
+import { readFile, rm, writeFile } from "node:fs/promises";
 import { join } from "node:path";
-import { type ProgressCallback, runPythonWithProgress } from "./bridge.js";
+import { promisify } from "node:util";
+import sharp from "sharp";
+
+const execFileAsync = promisify(execFile);
 
 export interface SeamCarveOptions {
   width?: number;
   height?: number;
   protectFaces?: boolean;
+  blurRadius?: number;
+  sobelThreshold?: number;
+  square?: boolean;
 }
 
 export interface SeamCarveResult {
@@ -14,31 +22,77 @@ export interface SeamCarveResult {
   height: number;
 }
 
+/**
+ * Discover the caire binary. Checks PATH (Docker installs to /usr/local/bin)
+ * and the CAIRE_PATH env var for local development.
+ */
+let cachedCairePath: string | null = null;
+
+async function findCaire(): Promise<string> {
+  if (cachedCairePath) return cachedCairePath;
+
+  const candidates = process.env.CAIRE_PATH ? [process.env.CAIRE_PATH, "caire"] : ["caire"];
+
+  for (const cmd of candidates) {
+    try {
+      await execFileAsync(cmd, ["-help"], { timeout: 5_000 });
+      cachedCairePath = cmd;
+      return cmd;
+    } catch {
+      // try next
+    }
+  }
+  throw new Error(
+    "caire binary not found. Install via: go install github.com/esimov/caire/cmd/caire@v1.5.0",
+  );
+}
+
+/**
+ * Content-aware resize using caire (Go seam carving engine).
+ * Supports both shrinking and enlarging via seam removal/insertion.
+ */
 export async function seamCarve(
   inputBuffer: Buffer,
   outputDir: string,
   options: SeamCarveOptions = {},
-  onProgress?: ProgressCallback,
 ): Promise<SeamCarveResult> {
-  const inputPath = join(outputDir, "input_seam_carve.png");
+  const cairePath = await findCaire();
-  const outputPath = join(outputDir, "output_seam_carve.png");
+  const id = randomUUID();
+  const inputPath = join(outputDir, `caire-in-${id}.png`);
+  const outputPath = join(outputDir, `caire-out-${id}.png`);
 
-  await writeFile(inputPath, inputBuffer);
+  try {
-  const { stdout } = await runPythonWithProgress(
+    await writeFile(inputPath, inputBuffer);
-    "seam_carve.py",
-    [inputPath, outputPath, JSON.stringify(options)],
-    { onProgress },
-  );
 
-  const result = JSON.parse(stdout);
+    // Build caire arguments
-  if (!result.success) {
+    const args = ["-in", inputPath, "-out", outputPath, "-preview=false"];
-    throw new Error(result.error || "Content-aware resize failed");
+
+    if (options.square) {
+      // Caire -square requires -width and -height set to the shortest edge
+      const meta = await sharp(inputBuffer).metadata();
+      const shortest = Math.min(meta.width ?? 0, meta.height ?? 0);
+      args.push("-square", "-width", String(shortest), "-height", String(shortest));
+    } else {
+      if (options.width) args.push("-width", String(options.width));
+      if (options.height) args.push("-height", String(options.height));
+    }
+
+    if (options.protectFaces) args.push("-face");
+    if (options.blurRadius !== undefined) args.push("-blur", String(options.blurRadius));
+    if (options.sobelThreshold !== undefined) args.push("-sobel", String(options.sobelThreshold));
+
+    await execFileAsync(cairePath, args, { timeout: 60_000 });
+
+    const buffer = await readFile(outputPath);
+    const meta = await sharp(buffer).metadata();
+
+    return {
+      buffer,
+      width: meta.width ?? 0,
+      height: meta.height ?? 0,
+    };
+  } finally {
+    await rm(inputPath, { force: true }).catch(() => {});
+    await rm(outputPath, { force: true }).catch(() => {});
   }
-
-  const buffer = await readFile(outputPath);
-  return {
-    buffer,
-    width: result.width,
-    height: result.height,
-  };
 }

packages/shared/src/constants.ts

@@ -390,5 +390,4 @@ export const PYTHON_SIDECAR_TOOLS = [
   "blur-faces",
   "erase-object",
   "ocr",
-  "content-aware-resize",
 ] as const;

tests/integration/content-aware-resize.test.ts

@@ -1,10 +1,9 @@
 /**
- * Integration tests for the content-aware resize (seam carving) API endpoint.
+ * Integration tests for the content-aware resize (seam carving via caire) API endpoint.
  *
- * This tool uses the Python sidecar, so in CI/test environments where Python
+ * This tool uses the caire Go binary. In environments where caire is not
- * is not available the route will return 422 (Python error).
+ * installed the route will return 422. Tests gracefully handle both scenarios
- * Tests gracefully handle both scenarios while still verifying route existence
+ * while still verifying route existence and input validation.
- * and input validation.
  */
 
 import { readFileSync } from "node:fs";
@@ -49,8 +48,7 @@ describe("Content-Aware Resize", () => {
       body,
     });
 
-    // 200 = Python available, 422 = Python error
+    // 200 = caire available, 422 = caire not found
-    // Any of these proves the route is registered and reachable
     expect([200, 422]).toContain(res.statusCode);
   }, 60_000);
 
@@ -72,6 +70,27 @@ describe("Content-Aware Resize", () => {
     expect(res.statusCode).toBe(400);
   });
 
+  it("rejects requests without width, height, or square", async () => {
+    const { body, contentType } = createMultipartPayload([
+      { name: "file", filename: "test.png", contentType: "image/png", content: PNG_200x150 },
+      { name: "settings", content: JSON.stringify({ protectFaces: false }) },
+    ]);
+
+    const res = await app.inject({
+      method: "POST",
+      url: "/api/v1/tools/content-aware-resize",
+      headers: {
+        authorization: `Bearer ${adminToken}`,
+        "content-type": contentType,
+      },
+      body,
+    });
+
+    expect(res.statusCode).toBe(400);
+    const resBody = JSON.parse(res.body);
+    expect(resBody.error).toContain("width, height, or square");
+  });
+
   it("processes with only width specified", async () => {
     const { body, contentType } = createMultipartPayload([
       { name: "file", filename: "test.png", contentType: "image/png", content: PNG_200x150 },
@@ -88,7 +107,7 @@ describe("Content-Aware Resize", () => {
       body,
     });
 
-    // Accept 200 (Python available) or 422 (Python not available)
+    // Accept 200 (caire available) or 422 (caire not available)
     expect([200, 422]).toContain(res.statusCode);
 
     if (res.statusCode === 200) {
@@ -123,10 +142,10 @@ describe("Content-Aware Resize", () => {
     }
   }, 60_000);
 
-  it("rejects enlargement beyond source dimensions", async () => {
+  it("supports enlargement via seam insertion", async () => {
     const { body, contentType } = createMultipartPayload([
       { name: "file", filename: "test.png", contentType: "image/png", content: PNG_200x150 },
-      { name: "settings", content: JSON.stringify({ width: 400, protectFaces: false }) },
+      { name: "settings", content: JSON.stringify({ width: 300, protectFaces: false }) },
     ]);
 
     const res = await app.inject({
@@ -139,14 +158,116 @@ describe("Content-Aware Resize", () => {
       body,
     });
 
-    // 422 = Python caught the enlargement error
+    // 200 = caire enlarged successfully, 422 = caire not available
-    // Should never be 200 since 400 > 200px source width
+    expect([200, 422]).toContain(res.statusCode);
-    expect(res.statusCode).not.toBe(200);
-    expect(res.statusCode).toBe(422);
 
-    if (res.statusCode === 422) {
+    if (res.statusCode === 200) {
       const resBody = JSON.parse(res.body);
-      expect(resBody.error || resBody.details).toBeDefined();
+      expect(resBody.downloadUrl).toBeDefined();
+      expect(resBody.width).toBe(300);
     }
   }, 60_000);
+
+  it("accepts blurRadius and sobelThreshold options", async () => {
+    const { body, contentType } = createMultipartPayload([
+      { name: "file", filename: "test.png", contentType: "image/png", content: PNG_200x150 },
+      {
+        name: "settings",
+        content: JSON.stringify({
+          width: 150,
+          protectFaces: false,
+          blurRadius: 6,
+          sobelThreshold: 4,
+        }),
+      },
+    ]);
+
+    const res = await app.inject({
+      method: "POST",
+      url: "/api/v1/tools/content-aware-resize",
+      headers: {
+        authorization: `Bearer ${adminToken}`,
+        "content-type": contentType,
+      },
+      body,
+    });
+
+    expect([200, 422]).toContain(res.statusCode);
+
+    if (res.statusCode === 200) {
+      const resBody = JSON.parse(res.body);
+      expect(resBody.downloadUrl).toBeDefined();
+      expect(resBody.width).toBe(150);
+    }
+  }, 60_000);
+
+  it("supports square mode", async () => {
+    const { body, contentType } = createMultipartPayload([
+      { name: "file", filename: "test.png", contentType: "image/png", content: PNG_200x150 },
+      { name: "settings", content: JSON.stringify({ square: true, protectFaces: false }) },
+    ]);
+
+    const res = await app.inject({
+      method: "POST",
+      url: "/api/v1/tools/content-aware-resize",
+      headers: {
+        authorization: `Bearer ${adminToken}`,
+        "content-type": contentType,
+      },
+      body,
+    });
+
+    expect([200, 422]).toContain(res.statusCode);
+
+    if (res.statusCode === 200) {
+      const resBody = JSON.parse(res.body);
+      expect(resBody.downloadUrl).toBeDefined();
+      // Square mode produces equal width and height
+      expect(resBody.width).toBe(resBody.height);
+    }
+  }, 60_000);
+
+  it("rejects invalid blurRadius", async () => {
+    const { body, contentType } = createMultipartPayload([
+      { name: "file", filename: "test.png", contentType: "image/png", content: PNG_200x150 },
+      {
+        name: "settings",
+        content: JSON.stringify({ width: 150, blurRadius: 50 }),
+      },
+    ]);
+
+    const res = await app.inject({
+      method: "POST",
+      url: "/api/v1/tools/content-aware-resize",
+      headers: {
+        authorization: `Bearer ${adminToken}`,
+        "content-type": contentType,
+      },
+      body,
+    });
+
+    expect(res.statusCode).toBe(400);
+  });
+
+  it("rejects invalid sobelThreshold", async () => {
+    const { body, contentType } = createMultipartPayload([
+      { name: "file", filename: "test.png", contentType: "image/png", content: PNG_200x150 },
+      {
+        name: "settings",
+        content: JSON.stringify({ width: 150, sobelThreshold: 0 }),
+      },
+    ]);
+
+    const res = await app.inject({
+      method: "POST",
+      url: "/api/v1/tools/content-aware-resize",
+      headers: {
+        authorization: `Bearer ${adminToken}`,
+        "content-type": contentType,
+      },
+      body,
+    });
+
+    expect(res.statusCode).toBe(400);
+  });
 });