Merge pull request #42 from stirling-image/feat/caire-content-aware-resize

feat: replace Python seam carving with caire Go binary
This commit is contained in:
stirling-image
2026-04-11 17:50:35 +08:00
committed by GitHub
13 changed files with 466 additions and 327 deletions
@@ -7,10 +7,20 @@ import { z } from "zod";
import { autoOrient } from "../../lib/auto-orient.js";
import { validateImageBuffer } from "../../lib/file-validation.js";
import { createWorkspace } from "../../lib/workspace.js";
import { updateSingleFileProgress } from "../progress.js";
import { registerToolProcessFn } from "../tool-factory.js";
/** Content-aware resize (seam carving) route. */
const settingsSchema = z.object({
width: z.number().positive().optional(),
height: z.number().positive().optional(),
protectFaces: z.boolean().default(false),
blurRadius: z.number().min(0).max(20).default(4),
sobelThreshold: z.number().min(1).max(20).default(2),
square: z.boolean().default(false),
});
type Settings = z.infer<typeof settingsSchema>;
/** Content-aware resize (seam carving via caire) route. */
export function registerContentAwareResize(app: FastifyInstance) {
app.post(
"/api/v1/tools/content-aware-resize",
@@ -18,7 +28,6 @@ export function registerContentAwareResize(app: FastifyInstance) {
let fileBuffer: Buffer | null = null;
let filename = "image";
let settingsRaw: string | null = null;
let clientJobId: string | null = null;
try {
const parts = request.parts();
@@ -32,8 +41,6 @@ export function registerContentAwareResize(app: FastifyInstance) {
filename = basename(part.filename ?? "image");
} else if (part.fieldname === "settings") {
settingsRaw = part.value as string;
} else if (part.fieldname === "clientJobId") {
clientJobId = part.value as string;
}
}
} catch (err) {
@@ -52,15 +59,37 @@ export function registerContentAwareResize(app: FastifyInstance) {
return reply.status(400).send({ error: `Invalid image: ${validation.reason}` });
}
// Validate settings
let settings: Settings;
try {
const parsed = settingsRaw ? JSON.parse(settingsRaw) : {};
const result = settingsSchema.safeParse(parsed);
if (!result.success) {
return reply.status(400).send({
error: "Invalid settings",
details: result.error.issues.map((i) => ({
path: i.path.join("."),
message: i.message,
})),
});
}
settings = result.data;
} catch {
return reply.status(400).send({ error: "Settings must be valid JSON" });
}
if (!settings.square && !settings.width && !settings.height) {
return reply.status(400).send({
error: "Either width, height, or square mode must be specified",
});
}
try {
const settings = settingsRaw ? JSON.parse(settingsRaw) : {};
request.log.info(
{
toolId: "content-aware-resize",
imageSize: fileBuffer.length,
width: settings.width,
height: settings.height,
protectFaces: settings.protectFaces,
...settings,
},
"Starting content-aware resize",
);
@@ -75,43 +104,21 @@ export function registerContentAwareResize(app: FastifyInstance) {
const inputPath = join(workspacePath, "input", filename);
await writeFile(inputPath, fileBuffer);
// Process
const jobIdForProgress = clientJobId;
const onProgress = jobIdForProgress
? (percent: number, stage: string) => {
updateSingleFileProgress({
jobId: jobIdForProgress,
phase: "processing",
stage,
percent,
});
}
: undefined;
const result = await seamCarve(
fileBuffer,
join(workspacePath, "output"),
{
width: settings.width,
height: settings.height,
protectFaces: settings.protectFaces ?? true,
},
onProgress,
);
// Process with caire
const result = await seamCarve(fileBuffer, join(workspacePath, "output"), {
width: settings.width,
height: settings.height,
protectFaces: settings.protectFaces,
blurRadius: settings.blurRadius,
sobelThreshold: settings.sobelThreshold,
square: settings.square,
});
// Save output
const outputFilename = `${filename.replace(/\.[^.]+$/, "")}_seam.png`;
const outputPath = join(workspacePath, "output", outputFilename);
await writeFile(outputPath, result.buffer);
if (clientJobId) {
updateSingleFileProgress({
jobId: clientJobId,
phase: "complete",
percent: 100,
});
}
return reply.send({
jobId,
downloadUrl: `/api/v1/download/${jobId}/${encodeURIComponent(outputFilename)}`,
@@ -130,24 +137,22 @@ export function registerContentAwareResize(app: FastifyInstance) {
},
);
// Register in the pipeline/batch registry so this tool can be used
// as a step in automation pipelines (without progress callbacks).
// Register in the pipeline/batch registry
registerToolProcessFn({
toolId: "content-aware-resize",
settingsSchema: z.object({
width: z.number().positive().optional(),
height: z.number().positive().optional(),
protectFaces: z.boolean().default(true),
}),
settingsSchema,
process: async (inputBuffer, settings, filename) => {
const s = settings as { width?: number; height?: number; protectFaces?: boolean };
const s = settings as Settings;
const orientedBuffer = await autoOrient(inputBuffer);
const jobId = randomUUID();
const workspacePath = await createWorkspace(jobId);
const result = await seamCarve(orientedBuffer, join(workspacePath, "output"), {
width: s.width,
height: s.height,
protectFaces: s.protectFaces ?? true,
protectFaces: s.protectFaces,
blurRadius: s.blurRadius,
sobelThreshold: s.sobelThreshold,
square: s.square,
});
const outputFilename = `${filename.replace(/\.[^.]+$/, "")}_seam.png`;
return { buffer: result.buffer, filename: outputFilename, contentType: "image/png" };
+149 -67
View File
@@ -1,5 +1,5 @@
import { SOCIAL_MEDIA_PRESETS } from "@stirling-image/shared";
import { Download, Link, Unlink } from "lucide-react";
import { Download, Info, Link, Unlink } from "lucide-react";
import { useCallback, useEffect, useRef, useState } from "react";
import { ProgressCard } from "@/components/common/progress-card";
import { useToolProcessor } from "@/hooks/use-tool-processor";
@@ -17,6 +17,17 @@ const FIT_LABELS: Record<FitMode, string> = {
// Group presets by platform
const platforms = [...new Set(SOCIAL_MEDIA_PRESETS.map((p) => p.platform))];
function HintIcon({ text }: { text: string }) {
return (
<span className="relative group">
<Info className="h-3 w-3 text-muted-foreground" />
<span className="pointer-events-none absolute left-1/2 -translate-x-1/2 bottom-full mb-1.5 w-48 rounded bg-foreground px-2 py-1.5 text-[11px] leading-tight text-background opacity-0 transition-opacity group-hover:opacity-100 z-10">
{text}
</span>
</span>
);
}
export interface ResizeControlsProps {
onChange?: (settings: Record<string, unknown>) => void;
}
@@ -31,7 +42,10 @@ export function ResizeControls({ onChange }: ResizeControlsProps) {
const [lockAspect, setLockAspect] = useState(true);
const [withoutEnlargement, setWithoutEnlargement] = useState(false);
const [contentAware, setContentAware] = useState(false);
const [protectFaces, setProtectFaces] = useState(true);
const [protectFaces, setProtectFaces] = useState(false);
const [blurRadius, setBlurRadius] = useState(4);
const [sobelThreshold, setSobelThreshold] = useState(2);
const [squareMode, setSquareMode] = useState(false);
const onChangeRef = useRef(onChange);
useEffect(() => {
@@ -42,9 +56,14 @@ export function ResizeControls({ onChange }: ResizeControlsProps) {
const settings: Record<string, unknown> = {};
if (contentAware) {
settings.contentAware = true;
if (width) settings.width = Number(width);
if (height) settings.height = Number(height);
if (!squareMode) {
if (width) settings.width = Number(width);
if (height) settings.height = Number(height);
}
settings.protectFaces = protectFaces;
settings.blurRadius = blurRadius;
settings.sobelThreshold = sobelThreshold;
settings.square = squareMode;
} else if (tab === "scale") {
settings.percentage = Number(percentage);
} else {
@@ -54,7 +73,19 @@ export function ResizeControls({ onChange }: ResizeControlsProps) {
settings.withoutEnlargement = withoutEnlargement;
}
onChangeRef.current?.(settings);
}, [tab, width, height, percentage, fit, withoutEnlargement, contentAware, protectFaces]);
}, [
tab,
width,
height,
percentage,
fit,
withoutEnlargement,
contentAware,
protectFaces,
blurRadius,
sobelThreshold,
squareMode,
]);
const handlePreset = (preset: (typeof SOCIAL_MEDIA_PRESETS)[number]) => {
const key = `${preset.platform}-${preset.name}`;
@@ -84,7 +115,8 @@ export function ResizeControls({ onChange }: ResizeControlsProps) {
value={width}
onChange={(e) => setWidth(e.target.value)}
placeholder="Auto"
className="w-full mt-0.5 px-2 py-1.5 rounded border border-border bg-background text-sm text-foreground"
disabled={squareMode && contentAware}
className="w-full mt-0.5 px-2 py-1.5 rounded border border-border bg-background text-sm text-foreground disabled:opacity-50"
/>
</div>
<button
@@ -105,52 +137,28 @@ export function ResizeControls({ onChange }: ResizeControlsProps) {
value={height}
onChange={(e) => setHeight(e.target.value)}
placeholder="Auto"
className="w-full mt-0.5 px-2 py-1.5 rounded border border-border bg-background text-sm text-foreground"
disabled={squareMode && contentAware}
className="w-full mt-0.5 px-2 py-1.5 rounded border border-border bg-background text-sm text-foreground disabled:opacity-50"
/>
</div>
</div>
);
const enlargementCheckbox = (
<label className="flex items-center gap-1.5 text-xs text-foreground">
<input
type="checkbox"
checked={withoutEnlargement}
onChange={(e) => setWithoutEnlargement(e.target.checked)}
className="rounded"
/>
<span>Limit to original size</span>
<HintIcon text="If your image is already smaller than the target, keep it as-is instead of scaling it up" />
</label>
);
return (
<div className="space-y-4">
{/* Content-aware toggle */}
<div className="flex items-center justify-between">
<span className="text-sm font-medium text-foreground">Content-aware</span>
<button
type="button"
role="switch"
aria-checked={contentAware}
onClick={() => setContentAware(!contentAware)}
className={`relative inline-flex h-5 w-9 items-center rounded-full transition-colors ${
contentAware ? "bg-primary" : "bg-muted"
}`}
>
<span
className={`inline-block h-3.5 w-3.5 rounded-full bg-white shadow-sm transition-transform ${
contentAware ? "translate-x-4" : "translate-x-0.5"
}`}
/>
</button>
</div>
{/* Content-aware inputs */}
{contentAware && (
<div className="space-y-3">
{dimensionInputs}
{/* Protect faces */}
<label className="flex items-center gap-2 text-sm text-foreground">
<input
type="checkbox"
checked={protectFaces}
onChange={(e) => setProtectFaces(e.target.checked)}
className="rounded"
/>
Protect faces
</label>
</div>
)}
{/* Standard resize tabs */}
{!contentAware && (
<>
@@ -196,7 +204,7 @@ export function ResizeControls({ onChange }: ResizeControlsProps) {
>
<span>{preset.name}</span>
<span className="text-xs tabular-nums">
{preset.width} × {preset.height}
{preset.width} x {preset.height}
</span>
</button>
);
@@ -205,16 +213,7 @@ export function ResizeControls({ onChange }: ResizeControlsProps) {
</div>
))}
{/* Don't enlarge */}
<label className="flex items-center gap-2 text-sm text-foreground">
<input
type="checkbox"
checked={withoutEnlargement}
onChange={(e) => setWithoutEnlargement(e.target.checked)}
className="rounded"
/>
Don&apos;t enlarge
</label>
{enlargementCheckbox}
</div>
)}
@@ -240,16 +239,7 @@ export function ResizeControls({ onChange }: ResizeControlsProps) {
</div>
</div>
{/* Don't enlarge */}
<label className="flex items-center gap-2 text-sm text-foreground">
<input
type="checkbox"
checked={withoutEnlargement}
onChange={(e) => setWithoutEnlargement(e.target.checked)}
className="rounded"
/>
Don&apos;t enlarge
</label>
{enlargementCheckbox}
</div>
)}
@@ -289,6 +279,98 @@ export function ResizeControls({ onChange }: ResizeControlsProps) {
)}
</>
)}
{/* Content-aware section - positioned below standard resize */}
<div className="border-t border-border pt-3">
<div className="flex items-center justify-between">
<div>
<span className="text-xs font-medium text-muted-foreground">Content-aware</span>
</div>
<button
type="button"
role="switch"
aria-checked={contentAware}
onClick={() => setContentAware(!contentAware)}
className={`relative inline-flex h-4 w-7 items-center rounded-full transition-colors ${
contentAware ? "bg-primary" : "bg-muted"
}`}
>
<span
className={`inline-block h-3 w-3 rounded-full bg-white shadow-sm transition-transform ${
contentAware ? "translate-x-3.5" : "translate-x-0.5"
}`}
/>
</button>
</div>
{/* Content-aware options (expanded when toggled) */}
{contentAware && (
<div className="mt-3 space-y-3">
{/* Dimensions */}
{dimensionInputs}
{/* Square mode */}
<label className="flex items-center gap-2 text-xs text-foreground">
<input
type="checkbox"
checked={squareMode}
onChange={(e) => setSquareMode(e.target.checked)}
className="rounded"
/>
Resize to square
</label>
{/* Face protection */}
<label className="flex items-center gap-2 text-xs text-foreground">
<input
type="checkbox"
checked={protectFaces}
onChange={(e) => setProtectFaces(e.target.checked)}
className="rounded"
/>
Protect faces
</label>
{/* Blur radius */}
<div>
<div className="flex items-center justify-between">
<label htmlFor="blur-radius" className="text-xs text-muted-foreground">
Smoothing
</label>
<span className="text-xs tabular-nums text-muted-foreground">{blurRadius}</span>
</div>
<input
id="blur-radius"
type="range"
min={0}
max={20}
value={blurRadius}
onChange={(e) => setBlurRadius(Number(e.target.value))}
className="w-full mt-1 h-1.5 rounded-full appearance-none bg-muted accent-primary"
/>
</div>
{/* Sobel threshold */}
<div>
<div className="flex items-center justify-between">
<label htmlFor="sobel-threshold" className="text-xs text-muted-foreground">
Edge sensitivity
</label>
<span className="text-xs tabular-nums text-muted-foreground">{sobelThreshold}</span>
</div>
<input
id="sobel-threshold"
type="range"
min={1}
max={20}
value={sobelThreshold}
onChange={(e) => setSobelThreshold(Number(e.target.value))}
className="w-full mt-1 h-1.5 rounded-full appearance-none bg-muted accent-primary"
/>
</div>
</div>
)}
</div>
</div>
);
}
@@ -323,7 +405,7 @@ export function ResizeSettings() {
hasFile &&
!processing &&
(isContentAware
? Boolean(settings.width) || Boolean(settings.height)
? Boolean(settings.width) || Boolean(settings.height) || Boolean(settings.square)
: tab === "scale"
? Number(settings.percentage) > 0
: Boolean(settings.width) || Boolean(settings.height));
+29 -4
View File
@@ -29,6 +29,10 @@ const IDLE_PROGRESS: ToolProgress = {
// smart-crop is category "ai" but uses Sharp (no Python), so it's excluded.
const AI_PYTHON_TOOLS = new Set<string>(PYTHON_SIDECAR_TOOLS);
// Tools that take a few seconds (not instant like Sharp, not minutes like AI).
// Uses a smoother progress: upload 0-40%, then a gradual fill during processing.
const MEDIUM_TOOLS = new Set(["content-aware-resize"]);
export function useToolProcessor(toolId: string) {
const {
processing,
@@ -49,11 +53,14 @@ export function useToolProcessor(toolId: string) {
const eventSourceRef = useRef<EventSource | null>(null);
const isAiTool = AI_PYTHON_TOOLS.has(toolId);
const isMediumTool = MEDIUM_TOOLS.has(toolId);
const processingTimerRef = useRef<ReturnType<typeof setInterval> | null>(null);
// Clean up on unmount
useEffect(() => {
return () => {
if (elapsedRef.current) clearInterval(elapsedRef.current);
if (processingTimerRef.current) clearInterval(processingTimerRef.current);
if (eventSourceRef.current) eventSourceRef.current.close();
if (xhrRef.current) xhrRef.current.abort();
};
@@ -134,12 +141,13 @@ export function useToolProcessor(toolId: string) {
const xhr = new XMLHttpRequest();
xhrRef.current = xhr;
// Timeout: 60s for fast tools, 5 min for AI tools
// Timeout: 60s for fast/medium tools, 5 min for AI tools
xhr.timeout = isAiTool ? 300_000 : 60_000;
// For AI tools: upload = 0-15%, processing = 15-100% (continuous, no reset)
// For AI tools: upload = 0-15%, processing = 15-100% (SSE-driven)
// For medium tools: upload = 0-40%, processing = 40-95% (gradual fill)
// For fast tools: upload = 0-100%, processing = brief 100% hold
const UPLOAD_WEIGHT = isAiTool ? 15 : 100;
const UPLOAD_WEIGHT = isAiTool ? 15 : isMediumTool ? 40 : 100;
xhr.upload.onprogress = (event) => {
if (event.lengthComputable) {
@@ -158,10 +166,25 @@ export function useToolProcessor(toolId: string) {
percent: UPLOAD_WEIGHT,
stage: isAiTool ? "Starting..." : "Processing...",
}));
// Medium tools: gradually fill from upload weight to 95% over ~15s
if (isMediumTool) {
const start = UPLOAD_WEIGHT;
const target = 95;
const step = (target - start) / 30; // 30 ticks over ~15s
processingTimerRef.current = setInterval(() => {
setProgress((prev) => {
if (prev.phase !== "processing") return prev;
const next = Math.min(target, prev.percent + step);
return { ...prev, percent: next };
});
}, 500);
}
};
xhr.onload = () => {
if (elapsedRef.current) clearInterval(elapsedRef.current);
if (processingTimerRef.current) clearInterval(processingTimerRef.current);
if (eventSourceRef.current) {
eventSourceRef.current.close();
eventSourceRef.current = null;
@@ -201,6 +224,7 @@ export function useToolProcessor(toolId: string) {
xhr.onerror = () => {
if (elapsedRef.current) clearInterval(elapsedRef.current);
if (processingTimerRef.current) clearInterval(processingTimerRef.current);
if (eventSourceRef.current) {
eventSourceRef.current.close();
eventSourceRef.current = null;
@@ -212,6 +236,7 @@ export function useToolProcessor(toolId: string) {
xhr.ontimeout = () => {
if (elapsedRef.current) clearInterval(elapsedRef.current);
if (processingTimerRef.current) clearInterval(processingTimerRef.current);
if (eventSourceRef.current) {
eventSourceRef.current.close();
eventSourceRef.current = null;
@@ -227,7 +252,7 @@ export function useToolProcessor(toolId: string) {
});
xhr.send(formData);
},
[toolId, isAiTool, setProcessing, setError, setProcessedUrl, setSizes, setJobId],
[toolId, isAiTool, isMediumTool, setProcessing, setError, setProcessedUrl, setSizes, setJobId],
);
const processAllFiles = useCallback(
+21 -4
View File
@@ -36,13 +36,27 @@ RUN --mount=type=cache,id=turbo-cache,target=/app/.turbo \
pnpm --filter @stirling-image/web build
# ============================================
# Stage 2: Platform-specific base images
# Stage 2: Build caire (content-aware resize)
# ============================================
FROM golang:1.23-bookworm AS caire-builder
RUN apt-get update && apt-get install -y --no-install-recommends \
libwayland-dev libx11-dev libx11-xcb-dev libxkbcommon-x11-dev \
libgles2-mesa-dev libegl1-mesa-dev libffi-dev libxcursor-dev \
libxrandr-dev libxinerama-dev libxi-dev libxxf86vm-dev \
libvulkan-dev libxfixes-dev pkg-config \
&& rm -rf /var/lib/apt/lists/*
RUN go install github.com/esimov/caire/cmd/caire@v1.5.0
# ============================================
# Stage 3: Platform-specific base images
# ============================================
FROM node:22-bookworm AS base-linux-arm64
FROM nvidia/cuda:12.6.3-runtime-ubuntu24.04 AS base-linux-amd64
# ============================================
# Stage 3: Production runtime
# Stage 4: Production runtime
# ============================================
ARG TARGETOS
ARG TARGETARCH
@@ -78,8 +92,13 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
tesseract-ocr tesseract-ocr-eng tesseract-ocr-deu tesseract-ocr-fra tesseract-ocr-spa \
build-essential \
libgl1 libglib2.0-0 \
libegl1 libwayland-egl1 libwayland-client0 libwayland-cursor0 \
libxkbcommon-x11-0 libxkbcommon0 libxcursor1 \
&& rm -rf /var/lib/apt/lists/*
# Caire binary (content-aware seam carving)
COPY --from=caire-builder /go/bin/caire /usr/local/bin/caire
# Python venv - Layer 1: Base packages (rarely change, ~3 GB)
RUN python3 -m venv /opt/venv && \
/opt/venv/bin/pip install --upgrade pip && \
@@ -116,8 +135,6 @@ RUN if [ "$TARGETARCH" = "amd64" ]; then \
/opt/venv/bin/pip install mediapipe==0.10.18 \
; fi
RUN /opt/venv/bin/pip install seam-carving==1.1.0
# Pre-download and verify all ML models
# Note: on amd64, paddlepaddle-gpu can't import without the CUDA driver (only
# available at runtime). The download script gracefully skips PaddleOCR model
+1 -2
View File
@@ -110,9 +110,8 @@ def smoke_test():
from PIL import Image
import cv2
import numpy
import seam_carving
from rembg import new_session
print(" CPU imports OK (Pillow, cv2, numpy, seam_carving, rembg)")
print(" CPU imports OK (Pillow, cv2, numpy, rembg)")
# MediaPipe is CPU-only, should always import
import mediapipe as mp
-1
View File
@@ -7,4 +7,3 @@ onnxruntime-gpu==1.20.1
numpy==1.26.4
Pillow==11.1.0
opencv-python-headless==4.10.0.84
seam-carving==1.1.0
-1
View File
@@ -7,4 +7,3 @@ onnxruntime==1.20.1
numpy==1.26.4
Pillow==11.1.0
opencv-python-headless==4.10.0.84
seam-carving==1.1.0
-161
View File
@@ -1,161 +0,0 @@
"""
Content-aware image resize using seam carving.
Uses the seam-carving library (li-plus) with optional face protection via MediaPipe.
Args:
sys.argv[1]: input image path
sys.argv[2]: output image path
sys.argv[3]: JSON settings string with keys:
- width (int, optional): target width
- height (int, optional): target height
- protectFaces (bool, optional): enable face detection for protection mask
"""
import json
import sys
def emit_progress(percent, stage):
"""Emit structured progress to stderr for bridge.ts to capture."""
print(json.dumps({"progress": percent, "stage": stage}), file=sys.stderr, flush=True)
def build_face_mask(img_array):
"""Detect faces with MediaPipe and return a boolean keep_mask."""
import numpy as np
try:
import mediapipe as mp
except ImportError:
emit_progress(20, "MediaPipe not available, skipping face protection")
return None
h, w = img_array.shape[:2]
mask = np.zeros((h, w), dtype=bool)
face_detection = mp.solutions.face_detection
detector = face_detection.FaceDetection(model_selection=1, min_detection_confidence=0.5)
try:
results = detector.process(img_array)
if not results.detections:
emit_progress(20, "No faces detected")
return None
for detection in results.detections:
bbox = detection.location_data.relative_bounding_box
x = int(bbox.xmin * w)
y = int(bbox.ymin * h)
bw = int(bbox.width * w)
bh = int(bbox.height * h)
# Add 20% padding around face
pad_x = int(bw * 0.2)
pad_y = int(bh * 0.2)
x1 = max(0, x - pad_x)
y1 = max(0, y - pad_y)
x2 = min(w, x + bw + pad_x)
y2 = min(h, y + bh + pad_y)
mask[y1:y2, x1:x2] = True
emit_progress(20, f"Detected {len(results.detections)} face(s)")
return mask
finally:
detector.close()
def main():
if len(sys.argv) < 4:
print(json.dumps({"success": False, "error": "Usage: seam_carve.py <input> <output> <settings>"}))
sys.exit(1)
input_path = sys.argv[1]
output_path = sys.argv[2]
try:
settings = json.loads(sys.argv[3])
except (json.JSONDecodeError, ValueError):
print(json.dumps({"success": False, "error": "Invalid settings JSON"}))
sys.exit(1)
target_width = settings.get("width")
target_height = settings.get("height")
protect_faces = settings.get("protectFaces", False)
try:
import numpy as np
from PIL import Image
except ImportError:
print(json.dumps({"success": False, "error": "Pillow/numpy not installed"}))
sys.exit(1)
try:
import seam_carving
except ImportError:
print(json.dumps({"success": False, "error": "seam-carving package not installed"}))
sys.exit(1)
try:
emit_progress(0, "Loading image")
img = Image.open(input_path).convert("RGB")
img_array = np.array(img)
src_h, src_w = img_array.shape[:2]
# Default to source dimensions if not specified
if target_width is None:
target_width = src_w
if target_height is None:
target_height = src_h
# Validate: shrink only
if target_width > src_w or target_height > src_h:
print(json.dumps({
"success": False,
"error": f"Content-aware resize only supports shrinking. Source is {src_w}x{src_h}, target is {target_width}x{target_height}."
}))
sys.exit(1)
# Nothing to do
if target_width == src_w and target_height == src_h:
img.save(output_path)
print(json.dumps({"success": True, "width": src_w, "height": src_h}))
return
# Warn about large images
if src_w > 3000 or src_h > 3000:
emit_progress(5, "Large image detected, this may take a while")
# Face protection mask
keep_mask = None
if protect_faces:
emit_progress(10, "Detecting faces")
keep_mask = build_face_mask(img_array)
emit_progress(25, "Starting seam carving")
# seam_carving.resize takes size as (width, height)
result = seam_carving.resize(
img_array,
(target_width, target_height),
energy_mode="backward",
order="width-first",
keep_mask=keep_mask,
)
emit_progress(90, "Saving result")
Image.fromarray(result).save(output_path)
print(json.dumps({
"success": True,
"width": result.shape[1],
"height": result.shape[0],
}))
except Exception as e:
print(json.dumps({"success": False, "error": str(e)}))
sys.exit(1)
if __name__ == "__main__":
main()
+75 -21
View File
@@ -1,11 +1,19 @@
import { readFile, writeFile } from "node:fs/promises";
import { execFile } from "node:child_process";
import { randomUUID } from "node:crypto";
import { readFile, rm, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { type ProgressCallback, runPythonWithProgress } from "./bridge.js";
import { promisify } from "node:util";
import sharp from "sharp";
const execFileAsync = promisify(execFile);
export interface SeamCarveOptions {
width?: number;
height?: number;
protectFaces?: boolean;
blurRadius?: number;
sobelThreshold?: number;
square?: boolean;
}
export interface SeamCarveResult {
@@ -14,31 +22,77 @@ export interface SeamCarveResult {
height: number;
}
/**
* Discover the caire binary. Checks PATH (Docker installs to /usr/local/bin)
* and the CAIRE_PATH env var for local development.
*/
let cachedCairePath: string | null = null;
async function findCaire(): Promise<string> {
if (cachedCairePath) return cachedCairePath;
const candidates = process.env.CAIRE_PATH ? [process.env.CAIRE_PATH, "caire"] : ["caire"];
for (const cmd of candidates) {
try {
await execFileAsync(cmd, ["-help"], { timeout: 5_000 });
cachedCairePath = cmd;
return cmd;
} catch {
// try next
}
}
throw new Error(
"caire binary not found. Install via: go install github.com/esimov/caire/cmd/caire@v1.5.0",
);
}
/**
* Content-aware resize using caire (Go seam carving engine).
* Supports both shrinking and enlarging via seam removal/insertion.
*/
export async function seamCarve(
inputBuffer: Buffer,
outputDir: string,
options: SeamCarveOptions = {},
onProgress?: ProgressCallback,
): Promise<SeamCarveResult> {
const inputPath = join(outputDir, "input_seam_carve.png");
const outputPath = join(outputDir, "output_seam_carve.png");
const cairePath = await findCaire();
const id = randomUUID();
const inputPath = join(outputDir, `caire-in-${id}.png`);
const outputPath = join(outputDir, `caire-out-${id}.png`);
await writeFile(inputPath, inputBuffer);
const { stdout } = await runPythonWithProgress(
"seam_carve.py",
[inputPath, outputPath, JSON.stringify(options)],
{ onProgress },
);
try {
await writeFile(inputPath, inputBuffer);
const result = JSON.parse(stdout);
if (!result.success) {
throw new Error(result.error || "Content-aware resize failed");
// Build caire arguments
const args = ["-in", inputPath, "-out", outputPath, "-preview=false"];
if (options.square) {
// Caire -square requires -width and -height set to the shortest edge
const meta = await sharp(inputBuffer).metadata();
const shortest = Math.min(meta.width ?? 0, meta.height ?? 0);
args.push("-square", "-width", String(shortest), "-height", String(shortest));
} else {
if (options.width) args.push("-width", String(options.width));
if (options.height) args.push("-height", String(options.height));
}
if (options.protectFaces) args.push("-face");
if (options.blurRadius !== undefined) args.push("-blur", String(options.blurRadius));
if (options.sobelThreshold !== undefined) args.push("-sobel", String(options.sobelThreshold));
await execFileAsync(cairePath, args, { timeout: 60_000 });
const buffer = await readFile(outputPath);
const meta = await sharp(buffer).metadata();
return {
buffer,
width: meta.width ?? 0,
height: meta.height ?? 0,
};
} finally {
await rm(inputPath, { force: true }).catch(() => {});
await rm(outputPath, { force: true }).catch(() => {});
}
const buffer = await readFile(outputPath);
return {
buffer,
width: result.width,
height: result.height,
};
}
-1
View File
@@ -390,5 +390,4 @@ export const PYTHON_SIDECAR_TOOLS = [
"blur-faces",
"erase-object",
"ocr",
"content-aware-resize",
] as const;
Binary file not shown.

Before

Width:  |  Height:  |  Size: 54 KiB

After

Width:  |  Height:  |  Size: 54 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 38 KiB

After

Width:  |  Height:  |  Size: 38 KiB

+137 -16
View File
@@ -1,10 +1,9 @@
/**
* Integration tests for the content-aware resize (seam carving) API endpoint.
* Integration tests for the content-aware resize (seam carving via caire) API endpoint.
*
* This tool uses the Python sidecar, so in CI/test environments where Python
* is not available the route will return 422 (Python error).
* Tests gracefully handle both scenarios while still verifying route existence
* and input validation.
* This tool uses the caire Go binary. In environments where caire is not
* installed the route will return 422. Tests gracefully handle both scenarios
* while still verifying route existence and input validation.
*/
import { readFileSync } from "node:fs";
@@ -49,8 +48,7 @@ describe("Content-Aware Resize", () => {
body,
});
// 200 = Python available, 422 = Python error
// Any of these proves the route is registered and reachable
// 200 = caire available, 422 = caire not found
expect([200, 422]).toContain(res.statusCode);
}, 60_000);
@@ -72,6 +70,27 @@ describe("Content-Aware Resize", () => {
expect(res.statusCode).toBe(400);
});
it("rejects requests without width, height, or square", async () => {
const { body, contentType } = createMultipartPayload([
{ name: "file", filename: "test.png", contentType: "image/png", content: PNG_200x150 },
{ name: "settings", content: JSON.stringify({ protectFaces: false }) },
]);
const res = await app.inject({
method: "POST",
url: "/api/v1/tools/content-aware-resize",
headers: {
authorization: `Bearer ${adminToken}`,
"content-type": contentType,
},
body,
});
expect(res.statusCode).toBe(400);
const resBody = JSON.parse(res.body);
expect(resBody.error).toContain("width, height, or square");
});
it("processes with only width specified", async () => {
const { body, contentType } = createMultipartPayload([
{ name: "file", filename: "test.png", contentType: "image/png", content: PNG_200x150 },
@@ -88,7 +107,7 @@ describe("Content-Aware Resize", () => {
body,
});
// Accept 200 (Python available) or 422 (Python not available)
// Accept 200 (caire available) or 422 (caire not available)
expect([200, 422]).toContain(res.statusCode);
if (res.statusCode === 200) {
@@ -123,10 +142,10 @@ describe("Content-Aware Resize", () => {
}
}, 60_000);
it("rejects enlargement beyond source dimensions", async () => {
it("supports enlargement via seam insertion", async () => {
const { body, contentType } = createMultipartPayload([
{ name: "file", filename: "test.png", contentType: "image/png", content: PNG_200x150 },
{ name: "settings", content: JSON.stringify({ width: 400, protectFaces: false }) },
{ name: "settings", content: JSON.stringify({ width: 300, protectFaces: false }) },
]);
const res = await app.inject({
@@ -139,14 +158,116 @@ describe("Content-Aware Resize", () => {
body,
});
// 422 = Python caught the enlargement error
// Should never be 200 since 400 > 200px source width
expect(res.statusCode).not.toBe(200);
expect(res.statusCode).toBe(422);
// 200 = caire enlarged successfully, 422 = caire not available
expect([200, 422]).toContain(res.statusCode);
if (res.statusCode === 422) {
if (res.statusCode === 200) {
const resBody = JSON.parse(res.body);
expect(resBody.error || resBody.details).toBeDefined();
expect(resBody.downloadUrl).toBeDefined();
expect(resBody.width).toBe(300);
}
}, 60_000);
it("accepts blurRadius and sobelThreshold options", async () => {
const { body, contentType } = createMultipartPayload([
{ name: "file", filename: "test.png", contentType: "image/png", content: PNG_200x150 },
{
name: "settings",
content: JSON.stringify({
width: 150,
protectFaces: false,
blurRadius: 6,
sobelThreshold: 4,
}),
},
]);
const res = await app.inject({
method: "POST",
url: "/api/v1/tools/content-aware-resize",
headers: {
authorization: `Bearer ${adminToken}`,
"content-type": contentType,
},
body,
});
expect([200, 422]).toContain(res.statusCode);
if (res.statusCode === 200) {
const resBody = JSON.parse(res.body);
expect(resBody.downloadUrl).toBeDefined();
expect(resBody.width).toBe(150);
}
}, 60_000);
it("supports square mode", async () => {
const { body, contentType } = createMultipartPayload([
{ name: "file", filename: "test.png", contentType: "image/png", content: PNG_200x150 },
{ name: "settings", content: JSON.stringify({ square: true, protectFaces: false }) },
]);
const res = await app.inject({
method: "POST",
url: "/api/v1/tools/content-aware-resize",
headers: {
authorization: `Bearer ${adminToken}`,
"content-type": contentType,
},
body,
});
expect([200, 422]).toContain(res.statusCode);
if (res.statusCode === 200) {
const resBody = JSON.parse(res.body);
expect(resBody.downloadUrl).toBeDefined();
// Square mode produces equal width and height
expect(resBody.width).toBe(resBody.height);
}
}, 60_000);
it("rejects invalid blurRadius", async () => {
const { body, contentType } = createMultipartPayload([
{ name: "file", filename: "test.png", contentType: "image/png", content: PNG_200x150 },
{
name: "settings",
content: JSON.stringify({ width: 150, blurRadius: 50 }),
},
]);
const res = await app.inject({
method: "POST",
url: "/api/v1/tools/content-aware-resize",
headers: {
authorization: `Bearer ${adminToken}`,
"content-type": contentType,
},
body,
});
expect(res.statusCode).toBe(400);
});
it("rejects invalid sobelThreshold", async () => {
const { body, contentType } = createMultipartPayload([
{ name: "file", filename: "test.png", contentType: "image/png", content: PNG_200x150 },
{
name: "settings",
content: JSON.stringify({ width: 150, sobelThreshold: 0 }),
},
]);
const res = await app.inject({
method: "POST",
url: "/api/v1/tools/content-aware-resize",
headers: {
authorization: `Bearer ${adminToken}`,
"content-type": contentType,
},
body,
});
expect(res.statusCode).toBe(400);
});
});