From 92d4d2d9c639d86e8d6b42fa0ad37b9df7242928 Mon Sep 17 00:00:00 2001 From: Siddharth Kumar Sah Date: Mon, 13 Apr 2026 00:47:53 +0800 Subject: [PATCH] feat(smart-crop): overhaul with face detection, social presets, and 3 modes Replace the confusing 2-mode smart crop with a clear 3-mode system: - Subject Focus: Sharp attention/entropy saliency crop with social media presets - Face Focus: MediaPipe face detection with headshot framing presets - Auto Trim: Border removal with optional pad-to-square Adds detectFaces() to AI package, face preset constants, backward compatibility for old mode names, and comprehensive integration tests. --- apps/api/src/openapi.yaml | 16 +- apps/api/src/routes/tools/smart-crop.ts | 233 +++++-- .../components/tools/smart-crop-settings.tsx | 627 ++++++++++++------ packages/ai/python/detect_faces.py | 39 +- packages/ai/src/face-detection.ts | 41 +- packages/ai/src/index.ts | 3 +- packages/shared/src/constants.ts | 17 +- packages/shared/src/i18n/en.ts | 7 +- tests/integration/api.test.ts | 92 +++ 9 files changed, 798 insertions(+), 277 deletions(-) diff --git a/apps/api/src/openapi.yaml b/apps/api/src/openapi.yaml index baf25cf8..bcc9617a 100644 --- a/apps/api/src/openapi.yaml +++ b/apps/api/src/openapi.yaml @@ -780,7 +780,7 @@ paths: post: tags: [Tools] summary: Smart crop - description: Automatically crop to the most interesting region at the specified dimensions. + description: Smart crop with three modes - subject focus, face focus, or auto trim. security: - bearerAuth: [] requestBody: @@ -799,8 +799,18 @@ paths: type: string description: | JSON string with options: - - `width` (integer, required) — Target width in pixels - - `height` (integer, required) — Target height in pixels + - `mode` (string) — "subject" (default), "face", or "trim" + - `strategy` (string) — "attention" (default) or "entropy" (subject mode) + - `width` (integer) — Target width in pixels (default 1080) + - `height` (integer) — Target height in pixels (default 1080) + - `padding` (integer 0-50) — Padding percentage around focus area + - `facePreset` (string) — "closeup", "head-shoulders", "upper-body", "half-body" (face mode) + - `sensitivity` (number 0-1) — Face detection sensitivity (face mode) + - `threshold` (integer 0-255) — Trim tolerance (trim mode) + - `padToSquare` (boolean) — Pad to square after trimming (trim mode) + - `padColor` (string) — Hex color for padding (trim mode) + - `targetSize` (integer) — Target size for padded output (trim mode) + - `quality` (integer 1-100) — Output quality responses: "200": description: Processed image diff --git a/apps/api/src/routes/tools/smart-crop.ts b/apps/api/src/routes/tools/smart-crop.ts index 54814026..8e402034 100644 --- a/apps/api/src/routes/tools/smart-crop.ts +++ b/apps/api/src/routes/tools/smart-crop.ts @@ -1,26 +1,178 @@ +import { detectFaces } from "@stirling-image/ai"; +import { SMART_CROP_FACE_PRESETS } from "@stirling-image/shared"; import type { FastifyInstance } from "fastify"; import sharp from "sharp"; import { z } from "zod"; import { resolveOutputFormat } from "../../lib/output-format.js"; import { createToolRoute } from "../tool-factory.js"; -const settingsSchema = z.object({ - mode: z.enum(["attention", "content"]).default("attention"), - width: z.number().int().positive().optional(), - height: z.number().int().positive().optional(), - threshold: z.number().int().min(0).max(255).default(30), - padToSquare: z.boolean().default(false), - padColor: z.string().default("#ffffff"), - targetSize: z.number().int().positive().optional(), - quality: z.number().int().min(1).max(100).optional(), -}); +const settingsSchema = z + .object({ + mode: z + .enum(["subject", "face", "trim", "attention", "content"]) + .default("subject") + .transform((v) => { + if (v === "attention") return "subject" as const; + if (v === "content") return "trim" as const; + return v; + }), + strategy: z.enum(["attention", "entropy"]).default("attention"), + width: z.number().int().positive().optional(), + height: z.number().int().positive().optional(), + padding: z.number().int().min(0).max(50).default(0), + facePreset: z + .enum(["closeup", "head-shoulders", "upper-body", "half-body"]) + .default("head-shoulders"), + sensitivity: z.number().min(0).max(1).default(0.5), + threshold: z.number().int().min(0).max(255).default(30), + padToSquare: z.boolean().default(false), + padColor: z.string().default("#ffffff"), + targetSize: z.number().int().positive().optional(), + quality: z.number().int().min(1).max(100).optional(), + }) + .transform((s) => ({ + ...s, + mode: s.mode as "subject" | "face" | "trim", + })); + +function clampRegion( + left: number, + top: number, + cropW: number, + cropH: number, + imgW: number, + imgH: number, +) { + const w = Math.min(cropW, imgW); + const h = Math.min(cropH, imgH); + let l = left; + let t = top; + + if (l < 0) l = 0; + if (t < 0) t = 0; + if (l + w > imgW) l = imgW - w; + if (t + h > imgH) t = imgH - h; + + return { + left: Math.round(Math.max(0, l)), + top: Math.round(Math.max(0, t)), + width: Math.round(w), + height: Math.round(h), + }; +} + +async function processSubject( + inputBuffer: Buffer, + settings: z.output, +): Promise { + const w = settings.width ?? 1080; + const h = settings.height ?? 1080; + const strategy = + settings.strategy === "entropy" ? sharp.strategy.entropy : sharp.strategy.attention; + + if (settings.padding > 0) { + const scale = 1 + settings.padding / 100; + const oversizeW = Math.round(w * scale); + const oversizeH = Math.round(h * scale); + + const oversize = await sharp(inputBuffer) + .resize(oversizeW, oversizeH, { fit: "cover", position: strategy }) + .toBuffer(); + + const extractLeft = Math.round((oversizeW - w) / 2); + const extractTop = Math.round((oversizeH - h) / 2); + + return sharp(oversize) + .extract({ left: extractLeft, top: extractTop, width: w, height: h }) + .toBuffer(); + } + + return sharp(inputBuffer).resize(w, h, { fit: "cover", position: strategy }).toBuffer(); +} + +async function processFace( + inputBuffer: Buffer, + settings: z.output, +): Promise { + const result = await detectFaces(inputBuffer, { sensitivity: settings.sensitivity }); + + if (result.facesDetected === 0) { + return processSubject(inputBuffer, { ...settings, strategy: "attention" }); + } + + const meta = await sharp(inputBuffer).metadata(); + const imgW = meta.width ?? 1; + const imgH = meta.height ?? 1; + const targetW = settings.width ?? 1080; + const targetH = settings.height ?? 1080; + + const faces = result.faces; + const minX = Math.min(...faces.map((f) => f.x)); + const minY = Math.min(...faces.map((f) => f.y)); + const maxX = Math.max(...faces.map((f) => f.x + f.w)); + const maxY = Math.max(...faces.map((f) => f.y + f.h)); + + const cx = (minX + maxX) / 2; + const cy = (minY + maxY) / 2; + const unionH = maxY - minY; + + const preset = SMART_CROP_FACE_PRESETS.find((p) => p.id === settings.facePreset); + const multiplier = preset?.multiplier ?? 2.8; + + const aspectRatio = targetW / targetH; + let cropH = unionH * multiplier * (1 + settings.padding / 100); + let cropW = cropH * aspectRatio; + + if (cropW > imgW) { + cropW = imgW; + cropH = cropW / aspectRatio; + } + if (cropH > imgH) { + cropH = imgH; + cropW = cropH * aspectRatio; + } + + const left = cx - cropW / 2; + const top = cy - cropH / 2; + const region = clampRegion(left, top, cropW, cropH, imgW, imgH); + + if (region.width < 1 || region.height < 1) { + return processSubject(inputBuffer, { ...settings, strategy: "attention" }); + } + + const extracted = await sharp(inputBuffer).extract(region).toBuffer(); + return sharp(extracted).resize(targetW, targetH, { fit: "fill" }).toBuffer(); +} + +async function processTrim( + inputBuffer: Buffer, + settings: z.output, +): Promise { + if (settings.padToSquare || settings.targetSize) { + const trimmed = await sharp(inputBuffer) + .trim({ threshold: settings.threshold }) + .toBuffer({ resolveWithObject: true }); + + const w = trimmed.info.width; + const h = trimmed.info.height; + const target = settings.targetSize || Math.max(w, h); + const padR = Math.round(Number.parseInt(settings.padColor.slice(1, 3), 16)); + const padG = Math.round(Number.parseInt(settings.padColor.slice(3, 5), 16)); + const padB = Math.round(Number.parseInt(settings.padColor.slice(5, 7), 16)); + + return sharp(trimmed.data) + .resize({ + width: target, + height: target, + fit: "contain", + background: { r: padR, g: padG, b: padB, alpha: 1 }, + }) + .toBuffer(); + } + + return sharp(inputBuffer).trim({ threshold: settings.threshold }).toBuffer(); +} -/** - * Smart crop with two modes: - * - "attention": Sharp's entropy/saliency detection to crop to the most interesting region - * - "content": Trims uniform-color borders (like GIMP's "Crop to Content"), - * optionally pads to a square at a target size - */ export function registerSmartCrop(app: FastifyInstance) { createToolRoute(app, { toolId: "smart-crop", @@ -29,49 +181,18 @@ export function registerSmartCrop(app: FastifyInstance) { const outputFormat = await resolveOutputFormat(inputBuffer, filename, settings.quality); let result: Buffer; - if (settings.mode === "content") { - if (settings.padToSquare || settings.targetSize) { - // Trim first to get dimensions, then pad to square - const trimmed = await sharp(inputBuffer) - .trim({ threshold: settings.threshold }) - .toBuffer({ resolveWithObject: true }); - - const w = trimmed.info.width; - const h = trimmed.info.height; - const target = settings.targetSize || Math.max(w, h); - const padR = Math.round(parseInt(settings.padColor.slice(1, 3), 16)); - const padG = Math.round(parseInt(settings.padColor.slice(3, 5), 16)); - const padB = Math.round(parseInt(settings.padColor.slice(5, 7), 16)); - - const padded = await sharp(trimmed.data) - .resize({ - width: target, - height: target, - fit: "contain", - background: { r: padR, g: padG, b: padB, alpha: 1 }, - }) - .toFormat(outputFormat.format, { quality: outputFormat.quality }) - .toBuffer(); - result = padded; - } else { - // Simple trim + format in one pass (no intermediate encode) - result = await sharp(inputBuffer) - .trim({ threshold: settings.threshold }) - .toFormat(outputFormat.format, { quality: outputFormat.quality }) - .toBuffer(); - } + if (settings.mode === "face") { + result = await processFace(inputBuffer, settings); + } else if (settings.mode === "trim") { + result = await processTrim(inputBuffer, settings); } else { - const w = settings.width ?? 1080; - const h = settings.height ?? 1080; - result = await sharp(inputBuffer) - .resize(w, h, { - fit: "cover", - position: sharp.strategy.attention, - }) - .toFormat(outputFormat.format, { quality: outputFormat.quality }) - .toBuffer(); + result = await processSubject(inputBuffer, settings); } + result = await sharp(result) + .toFormat(outputFormat.format, { quality: outputFormat.quality }) + .toBuffer(); + const stem = filename.replace(/\.[^.]+$/, ""); const outputFilename = `${stem}_smartcrop.${outputFormat.extension}`; return { buffer: result, filename: outputFilename, contentType: outputFormat.contentType }; diff --git a/apps/web/src/components/tools/smart-crop-settings.tsx b/apps/web/src/components/tools/smart-crop-settings.tsx index fa5e5aba..7983c17c 100644 --- a/apps/web/src/components/tools/smart-crop-settings.tsx +++ b/apps/web/src/components/tools/smart-crop-settings.tsx @@ -1,131 +1,465 @@ -import { useState } from "react"; +import { SMART_CROP_FACE_PRESETS, SOCIAL_MEDIA_PRESETS } from "@stirling-image/shared"; +import { ArrowLeftRight, Info } from "lucide-react"; +import { useEffect, useRef, useState } from "react"; import { ProgressCard } from "@/components/common/progress-card"; import { useToolProcessor } from "@/hooks/use-tool-processor"; import { useFileStore } from "@/stores/file-store"; -type Mode = "content" | "attention"; +type CropMode = "subject" | "face" | "trim"; +type SubjectTab = "presets" | "custom"; + +const platforms = [...new Set(SOCIAL_MEDIA_PRESETS.map((p) => p.platform))]; const ASPECT_PRESETS = [ - { label: "1:1 Square", w: 1080, h: 1080 }, - { label: "16:9 Landscape", w: 1920, h: 1080 }, - { label: "9:16 Portrait", w: 1080, h: 1920 }, - { label: "4:3 Standard", w: 1440, h: 1080 }, - { label: "3:2 Photo", w: 1620, h: 1080 }, - { label: "Custom", w: 0, h: 0 }, + { label: "1:1", w: 1080, h: 1080 }, + { label: "4:3", w: 1440, h: 1080 }, + { label: "3:2", w: 1620, h: 1080 }, + { label: "16:9", w: 1920, h: 1080 }, + { label: "4:5", w: 1080, h: 1350 }, + { label: "9:16", w: 1080, h: 1920 }, ]; +function HintIcon({ text }: { text: string }) { + return ( + + + + {text} + + + ); +} + export interface SmartCropControlsProps { onChange?: (settings: Record) => void; } export function SmartCropControls({ onChange }: SmartCropControlsProps) { - const [mode, setMode] = useState("content"); + const [mode, setMode] = useState("subject"); + const [subjectTab, setSubjectTab] = useState("custom"); + const [selectedPreset, setSelectedPreset] = useState(null); - // Attention mode state + // Subject mode state + const [strategy, setStrategy] = useState<"attention" | "entropy">("attention"); + + // Face mode state + const [facePreset, setFacePreset] = useState("head-shoulders"); + const [sensitivity, setSensitivity] = useState(50); + + // Shared subject/face state const [width, setWidth] = useState("1080"); const [height, setHeight] = useState("1080"); - const [preset, setPreset] = useState("1:1 Square"); + const [padding, setPadding] = useState(0); - // Content mode state + // Trim mode state const [threshold, setThreshold] = useState(30); const [padToSquare, setPadToSquare] = useState(false); const [padColor, setPadColor] = useState("#ffffff"); const [targetSize, setTargetSize] = useState("1000"); + + // Shared const [quality, setQuality] = useState(95); - const emit = (overrides: Record = {}) => { - if (mode === "content") { - onChange?.({ - mode: "content", + const onChangeRef = useRef(onChange); + useEffect(() => { + onChangeRef.current = onChange; + }); + + useEffect(() => { + if (mode === "subject") { + onChangeRef.current?.({ + mode: "subject", + strategy, + width: Number(width), + height: Number(height), + padding, + quality, + }); + } else if (mode === "face") { + onChangeRef.current?.({ + mode: "face", + facePreset, + sensitivity: sensitivity / 100, + width: Number(width), + height: Number(height), + padding, + quality, + }); + } else { + onChangeRef.current?.({ + mode: "trim", threshold, padToSquare, padColor, quality, ...(padToSquare ? { targetSize: Number(targetSize) } : {}), - ...overrides, }); + } + }, [ + mode, + strategy, + facePreset, + sensitivity, + width, + height, + padding, + threshold, + padToSquare, + padColor, + targetSize, + quality, + ]); + + const handleSocialPreset = (preset: (typeof SOCIAL_MEDIA_PRESETS)[number]) => { + const key = `${preset.platform}-${preset.name}`; + if (selectedPreset === key) { + setSelectedPreset(null); + setWidth("1080"); + setHeight("1080"); } else { - onChange?.({ - mode: "attention", - width: Number(width), - height: Number(height), - quality, - ...overrides, - }); + setSelectedPreset(key); + setWidth(String(preset.width)); + setHeight(String(preset.height)); } }; - const handleModeChange = (m: Mode) => { - setMode(m); - if (m === "content") { - onChange?.({ mode: "content", threshold, padToSquare, padColor, quality }); - } else { - onChange?.({ mode: "attention", width: Number(width), height: Number(height), quality }); - } + const handleAspectPreset = (p: (typeof ASPECT_PRESETS)[number]) => { + setWidth(String(p.w)); + setHeight(String(p.h)); + setSelectedPreset(null); }; - const handlePreset = (label: string) => { - setPreset(label); - const p = ASPECT_PRESETS.find((a) => a.label === label); - if (p && p.w > 0) { - setWidth(String(p.w)); - setHeight(String(p.h)); - emit({ width: p.w, height: p.h }); - } + const swapDimensions = () => { + const tmp = width; + setWidth(height); + setHeight(tmp); + setSelectedPreset(null); }; + const modeTabClass = (m: CropMode) => + `flex-1 text-xs py-1.5 rounded ${mode === m ? "bg-primary text-primary-foreground" : "bg-muted text-muted-foreground"}`; + + const subTabClass = (t: SubjectTab) => + `flex-1 text-[11px] py-1 rounded ${subjectTab === t ? "bg-primary/15 text-foreground font-medium" : "text-muted-foreground"}`; + + // Shared dimension inputs with swap button + const dimensionInputs = ( +
+
+ + { + setWidth(e.target.value); + setSelectedPreset(null); + }} + min={1} + className="w-full mt-0.5 px-2 py-1.5 rounded border border-border bg-background text-sm text-foreground" + /> +
+ +
+ + { + setHeight(e.target.value); + setSelectedPreset(null); + }} + min={1} + className="w-full mt-0.5 px-2 py-1.5 rounded border border-border bg-background text-sm text-foreground" + /> +
+
+ ); + + // Quick aspect ratio buttons + const aspectButtons = ( +
+ {ASPECT_PRESETS.map((p) => { + const isActive = width === String(p.w) && height === String(p.h); + return ( + + ); + })} +
+ ); + + // Quality slider (shared) + const qualitySlider = ( +
+
+ + {quality}% +
+ setQuality(Number(e.target.value))} + className="w-full mt-1" + /> +

+ For JPEG and WebP outputs. PNG is always lossless. +

+
+ ); + return (
- {/* Mode toggle */} -
- -
- - -
+ {/* Mode tabs */} +
+ + +
- {mode === "content" ? ( - <> + {/* ─── Subject Focus ─── */} + {mode === "subject" && ( +
+ {/* Sub-tabs: Presets / Custom */} +
+ + +
+ + {subjectTab === "presets" ? ( +
+ {platforms.map((platform) => ( +
+

{platform}

+
+ {SOCIAL_MEDIA_PRESETS.filter((p) => p.platform === platform).map((preset) => { + const key = `${preset.platform}-${preset.name}`; + const isSelected = selectedPreset === key; + return ( + + ); + })} +
+
+ ))} +
+ ) : ( + <> + {dimensionInputs} + {aspectButtons} + + )} + + {/* Strategy toggle */} +
+
+ Detection Strategy + +
+
+ + +
+
+ + {/* Padding slider */} +
+
+
+ + +
+ {padding}% +
+ setPadding(Number(e.target.value))} + className="w-full mt-1" + /> +
+ + {qualitySlider} + +

+ Detects the most interesting region using saliency analysis and crops to your target + size. +

+
+ )} + + {/* ─── Face Focus ─── */} + {mode === "face" && ( +
+ {/* Face preset buttons */} +
+ Framing +
+ {SMART_CROP_FACE_PRESETS.map((p) => ( + + ))} +
+
+ + {/* Target size */} + {dimensionInputs} + {aspectButtons} + + {/* Sensitivity slider */} +
+
+ + {sensitivity}% +
+ setSensitivity(Number(e.target.value))} + className="w-full mt-1" + /> +
+ More faces + Fewer false positives +
+
+ + {/* Padding slider */} +
+
+
+ + +
+ {padding}% +
+ setPadding(Number(e.target.value))} + className="w-full mt-1" + /> +
+ + {qualitySlider} + +

+ Uses AI face detection to keep faces properly framed. Falls back to Subject Focus if no + faces are detected. +

+
+ )} + + {/* ─── Auto Trim ─── */} + {mode === "trim" && ( +
{/* Threshold */}
-
{ - const v = Number(e.target.value); - setThreshold(v); - emit({ threshold: v }); - }} + onChange={(e) => setThreshold(Number(e.target.value))} className="w-full mt-1" />

@@ -137,16 +471,13 @@ export function SmartCropControls({ onChange }: SmartCropControlsProps) { {/* Pad to square */}

{ - setPadToSquare(e.target.checked); - emit({ padToSquare: e.target.checked }); - }} + onChange={(e) => setPadToSquare(e.target.checked)} className="rounded border-border" /> -
@@ -154,138 +485,41 @@ export function SmartCropControls({ onChange }: SmartCropControlsProps) { {padToSquare && (
-
-
)} -

- Trims uniform-color borders around the subject, like GIMP's "Crop to Content." Enable - "Pad to square" to produce e-commerce ready images. -

- - ) : ( - <> - {/* Aspect ratio preset */} -
- - -
- - {/* Width / Height */} -
-
- - { - setWidth(e.target.value); - setPreset("Custom"); - emit({ width: Number(e.target.value) }); - }} - min={1} - className="w-full mt-0.5 px-2 py-1.5 rounded border border-border bg-background text-sm text-foreground" - /> -
-
- - { - setHeight(e.target.value); - setPreset("Custom"); - emit({ height: Number(e.target.value) }); - }} - min={1} - className="w-full mt-0.5 px-2 py-1.5 rounded border border-border bg-background text-sm text-foreground" - /> -
-
+ {qualitySlider}

- Uses entropy-based attention detection to find the most interesting region and crops to - it. Good for thumbnails and social media images. + Removes uniform-color borders around your subject. Enable "Pad to square" for e-commerce + ready images.

- - )} - - {/* Quality slider */} -
-
- - {quality}%
- { - const v = Number(e.target.value); - setQuality(v); - emit({ quality: v }); - }} - className="w-full mt-1" - /> -

- For JPEG and WebP outputs. PNG is always lossless. -

-
+ )}
); } @@ -296,10 +530,11 @@ export function SmartCropSettings() { useToolProcessor("smart-crop"); const [settings, setSettings] = useState>({ - mode: "content", - threshold: 30, - padToSquare: false, - padColor: "#ffffff", + mode: "subject", + strategy: "attention", + width: 1080, + height: 1080, + padding: 0, quality: 95, }); @@ -313,14 +548,16 @@ export function SmartCropSettings() { const hasFile = files.length > 0; const mode = settings.mode as string; - const canProcess = - mode === "content" || (Number(settings.width) > 0 && Number(settings.height) > 0); + const canProcess = mode === "trim" || (Number(settings.width) > 0 && Number(settings.height) > 0); const handleSubmit = (e: React.FormEvent) => { e.preventDefault(); if (hasFile && canProcess && !processing) handleProcess(); }; + const buttonLabel = + mode === "face" ? "Face Crop" : mode === "trim" ? "Trim Borders" : "Smart Crop"; + return (
@@ -343,7 +580,7 @@ export function SmartCropSettings() { disabled={!hasFile || !canProcess || processing} className="w-full py-2.5 rounded-lg bg-primary text-primary-foreground font-medium disabled:opacity-50 disabled:cursor-not-allowed flex items-center justify-center gap-2" > - {mode === "content" ? "Crop to Content" : "Smart Crop"} + {files.length > 1 ? `${buttonLabel} (${files.length} files)` : buttonLabel} )} diff --git a/packages/ai/python/detect_faces.py b/packages/ai/python/detect_faces.py index c3c10405..8c297dd1 100644 --- a/packages/ai/python/detect_faces.py +++ b/packages/ai/python/detect_faces.py @@ -15,6 +15,7 @@ def main(): blur_radius = settings.get("blurRadius", 30) sensitivity = settings.get("sensitivity", 0.5) + detect_only = settings.get("detectOnly", False) try: emit_progress(10, "Preparing") @@ -64,26 +65,30 @@ def main(): w = int(bbox.width * iw) h = int(bbox.height * ih) - # Add padding around the face - pad = int(max(w, h) * 0.1) - x1 = max(0, x - pad) - y1 = max(0, y - pad) - x2 = min(img.width, x + w + pad) - y2 = min(img.height, y + h + pad) + if not detect_only: + # Add padding around the face + pad = int(max(w, h) * 0.1) + x1 = max(0, x - pad) + y1 = max(0, y - pad) + x2 = min(img.width, x + w + pad) + y2 = min(img.height, y + h + pad) + + face_region = img.crop((x1, y1, x2, y2)) + blurred = face_region.filter( + ImageFilter.GaussianBlur(blur_radius) + ) + img.paste(blurred, (x1, y1)) + emit_progress( + 50 + int((i + 1) / num_faces * 40), + f"Blurring face {i + 1} of {num_faces}", + ) - face_region = img.crop((x1, y1, x2, y2)) - blurred = face_region.filter( - ImageFilter.GaussianBlur(blur_radius) - ) - img.paste(blurred, (x1, y1)) faces.append({"x": x, "y": y, "w": w, "h": h}) - emit_progress( - 50 + int((i + 1) / num_faces * 40), - f"Blurring face {i + 1} of {num_faces}", - ) - emit_progress(95, "Saving result") - img.save(output_path) + if not detect_only: + emit_progress(95, "Saving result") + img.save(output_path) + print( json.dumps( { diff --git a/packages/ai/src/face-detection.ts b/packages/ai/src/face-detection.ts index 36c552b5..50026087 100644 --- a/packages/ai/src/face-detection.ts +++ b/packages/ai/src/face-detection.ts @@ -1,4 +1,5 @@ -import { readFile, writeFile } from "node:fs/promises"; +import { readFile, unlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; import { join } from "node:path"; import { type ProgressCallback, runPythonWithProgress } from "./bridge.js"; @@ -7,6 +8,10 @@ export interface BlurFacesOptions { sensitivity?: number; } +export interface DetectFacesOptions { + sensitivity?: number; +} + export interface FaceRegion { x: number; y: number; @@ -20,6 +25,11 @@ export interface BlurFacesResult { faces: FaceRegion[]; } +export interface DetectFacesResult { + facesDetected: number; + faces: FaceRegion[]; +} + export async function blurFaces( inputBuffer: Buffer, outputDir: string, @@ -48,3 +58,32 @@ export async function blurFaces( faces: result.faces ?? [], }; } + +export async function detectFaces( + inputBuffer: Buffer, + options: DetectFacesOptions = {}, + onProgress?: ProgressCallback, +): Promise { + const inputPath = join(tmpdir(), `detect_faces_${Date.now()}.png`); + + try { + await writeFile(inputPath, inputBuffer); + const { stdout } = await runPythonWithProgress( + "detect_faces.py", + [inputPath, "unused", JSON.stringify({ ...options, detectOnly: true })], + { onProgress }, + ); + + const result = JSON.parse(stdout); + if (!result.success) { + throw new Error(result.error || "Face detection failed"); + } + + return { + facesDetected: result.facesDetected, + faces: result.faces ?? [], + }; + } finally { + await unlink(inputPath).catch(() => {}); + } +} diff --git a/packages/ai/src/index.ts b/packages/ai/src/index.ts index 64dc8cd0..3924106c 100644 --- a/packages/ai/src/index.ts +++ b/packages/ai/src/index.ts @@ -1,6 +1,7 @@ export { removeBackground } from "./background-removal.js"; export { isGpuAvailable, shutdownDispatcher } from "./bridge.js"; -export { blurFaces } from "./face-detection.js"; +export type { DetectFacesResult, FaceRegion } from "./face-detection.js"; +export { blurFaces, detectFaces } from "./face-detection.js"; export { inpaint } from "./inpainting.js"; export { extractText } from "./ocr.js"; export { seamCarve } from "./seam-carving.js"; diff --git a/packages/shared/src/constants.ts b/packages/shared/src/constants.ts index a3a7d232..5586da55 100644 --- a/packages/shared/src/constants.ts +++ b/packages/shared/src/constants.ts @@ -132,7 +132,7 @@ export const TOOLS: Tool[] = [ { id: "erase-object", name: "Object Eraser", - description: "Paint over unwanted elements", + description: "Remove unwanted objects with AI", category: "ai", icon: "Wand2", route: "/erase-object", @@ -156,7 +156,7 @@ export const TOOLS: Tool[] = [ { id: "smart-crop", name: "Smart Crop", - description: "AI detects subject and crops optimally", + description: "Smart subject, face, or trim-based cropping", category: "ai", icon: "Focus", route: "/smart-crop", @@ -354,6 +354,19 @@ export const SOCIAL_MEDIA_PRESETS: SocialMediaPreset[] = [ { platform: "Threads", name: "Post Image", width: 1080, height: 1080 }, ]; +export interface SmartCropFacePreset { + id: string; + label: string; + multiplier: number; +} + +export const SMART_CROP_FACE_PRESETS: SmartCropFacePreset[] = [ + { id: "closeup", label: "Close-up", multiplier: 1.8 }, + { id: "head-shoulders", label: "Head & Shoulders", multiplier: 2.8 }, + { id: "upper-body", label: "Upper Body", multiplier: 4.5 }, + { id: "half-body", label: "Half Body", multiplier: 7.0 }, +]; + export const APP_VERSION = "1.14.0"; /** diff --git a/packages/shared/src/i18n/en.ts b/packages/shared/src/i18n/en.ts index 65058f53..f1747c91 100644 --- a/packages/shared/src/i18n/en.ts +++ b/packages/shared/src/i18n/en.ts @@ -58,7 +58,7 @@ export const en = { description: "AI-powered background removal", }, upscale: { name: "Image Upscaling", description: "AI super-resolution enhancement" }, - "erase-object": { name: "Object Eraser", description: "Paint over unwanted elements" }, + "erase-object": { name: "Object Eraser", description: "Remove unwanted objects with AI" }, ocr: { name: "OCR / Text Extraction", description: "Extract text from images with AI-powered accuracy", @@ -67,7 +67,10 @@ export const en = { name: "Face / PII Blur", description: "Auto-detect and blur faces and sensitive info", }, - "smart-crop": { name: "Smart Crop", description: "AI detects subject and crops optimally" }, + "smart-crop": { + name: "Smart Crop", + description: "Smart subject, face, or trim-based cropping", + }, "content-aware-resize": { name: "Content-Aware Resize", description: "Intelligently resize images while preserving important content", diff --git a/tests/integration/api.test.ts b/tests/integration/api.test.ts index f5680f0a..899932e0 100644 --- a/tests/integration/api.test.ts +++ b/tests/integration/api.test.ts @@ -3228,6 +3228,98 @@ describe("Smart crop format preservation", () => { }); expect(res.statusCode).toBe(200); }); + + it("subject mode with entropy strategy", async () => { + const { body: payload, contentType } = createMultipartPayload([ + { name: "file", filename: "photo.jpg", contentType: "image/jpeg", content: JPG_100x100 }, + { + name: "settings", + content: JSON.stringify({ mode: "subject", strategy: "entropy", width: 50, height: 50 }), + }, + ]); + + const res = await app.inject({ + method: "POST", + url: "/api/v1/tools/smart-crop", + headers: { + authorization: `Bearer ${adminToken}`, + "content-type": contentType, + }, + payload, + }); + expect(res.statusCode).toBe(200); + const body = JSON.parse(res.body); + expect(body.downloadUrl).toMatch(/_smartcrop\.jpg/); + }); + + it("subject mode with padding", async () => { + const { body: payload, contentType } = createMultipartPayload([ + { name: "file", filename: "photo.jpg", contentType: "image/jpeg", content: JPG_100x100 }, + { + name: "settings", + content: JSON.stringify({ mode: "subject", width: 50, height: 50, padding: 10 }), + }, + ]); + + const res = await app.inject({ + method: "POST", + url: "/api/v1/tools/smart-crop", + headers: { + authorization: `Bearer ${adminToken}`, + "content-type": contentType, + }, + payload, + }); + expect(res.statusCode).toBe(200); + const body = JSON.parse(res.body); + expect(body.downloadUrl).toMatch(/_smartcrop\.jpg/); + }); + + it("trim mode with new mode name", async () => { + const { body: payload, contentType } = createMultipartPayload([ + { name: "file", filename: "photo.png", contentType: "image/png", content: PNG_200x150 }, + { + name: "settings", + content: JSON.stringify({ mode: "trim", threshold: 30 }), + }, + ]); + + const res = await app.inject({ + method: "POST", + url: "/api/v1/tools/smart-crop", + headers: { + authorization: `Bearer ${adminToken}`, + "content-type": contentType, + }, + payload, + }); + expect(res.statusCode).toBe(200); + const body = JSON.parse(res.body); + expect(body.downloadUrl).toMatch(/_smartcrop\.png/); + }); + + it("defaults to subject mode when no mode specified", async () => { + const { body: payload, contentType } = createMultipartPayload([ + { name: "file", filename: "photo.jpg", contentType: "image/jpeg", content: JPG_100x100 }, + { + name: "settings", + content: JSON.stringify({ width: 50, height: 50 }), + }, + ]); + + const res = await app.inject({ + method: "POST", + url: "/api/v1/tools/smart-crop", + headers: { + authorization: `Bearer ${adminToken}`, + "content-type": contentType, + }, + payload, + }); + expect(res.statusCode).toBe(200); + const body = JSON.parse(res.body); + expect(body.downloadUrl).toMatch(/_smartcrop\.jpg/); + }); }); // ═══════════════════════════════════════════════════════════════════════════