feat(smart-crop): overhaul with face detection, social presets, and 3 modes

Replace the confusing 2-mode smart crop with a clear 3-mode system: - Subject Focus: Sharp attention/entropy saliency crop with social media presets - Face Focus: MediaPipe face detection with headshot framing presets - Auto Trim: Border removal with optional pad-to-square Adds detectFaces() to AI package, face preset constants, backward compatibility for old mode names, and comprehensive integration tests.
2026-08-03 07:46:42 +02:00 · 2026-04-13 00:47:53 +08:00
parent 29fafd0722
commit 92d4d2d9c6
9 changed files with 798 additions and 277 deletions
@@ -780,7 +780,7 @@ paths:
    post:
      tags: [Tools]
      summary: Smart crop
-      description: Automatically crop to the most interesting region at the specified dimensions.
+      description: Smart crop with three modes - subject focus, face focus, or auto trim.
      security:
        - bearerAuth: []
      requestBody:
@@ -799,8 +799,18 @@ paths:
                  type: string
                  description: |
                    JSON string with options:
-                    - `width` (integer, required) — Target width in pixels
-                    - `height` (integer, required) — Target height in pixels
+                    - `mode` (string) — "subject" (default), "face", or "trim"
+                    - `strategy` (string) — "attention" (default) or "entropy" (subject mode)
+                    - `width` (integer) — Target width in pixels (default 1080)
+                    - `height` (integer) — Target height in pixels (default 1080)
+                    - `padding` (integer 0-50) — Padding percentage around focus area
+                    - `facePreset` (string) — "closeup", "head-shoulders", "upper-body", "half-body" (face mode)
+                    - `sensitivity` (number 0-1) — Face detection sensitivity (face mode)
+                    - `threshold` (integer 0-255) — Trim tolerance (trim mode)
+                    - `padToSquare` (boolean) — Pad to square after trimming (trim mode)
+                    - `padColor` (string) — Hex color for padding (trim mode)
+                    - `targetSize` (integer) — Target size for padded output (trim mode)
+                    - `quality` (integer 1-100) — Output quality
      responses:
        "200":
          description: Processed image
@@ -1,26 +1,178 @@
+import { detectFaces } from "@stirling-image/ai";
+import { SMART_CROP_FACE_PRESETS } from "@stirling-image/shared";
 import type { FastifyInstance } from "fastify";
 import sharp from "sharp";
 import { z } from "zod";
 import { resolveOutputFormat } from "../../lib/output-format.js";
 import { createToolRoute } from "../tool-factory.js";

-const settingsSchema = z.object({
-  mode: z.enum(["attention", "content"]).default("attention"),
-  width: z.number().int().positive().optional(),
-  height: z.number().int().positive().optional(),
-  threshold: z.number().int().min(0).max(255).default(30),
-  padToSquare: z.boolean().default(false),
-  padColor: z.string().default("#ffffff"),
-  targetSize: z.number().int().positive().optional(),
-  quality: z.number().int().min(1).max(100).optional(),
-});
+const settingsSchema = z
+  .object({
+    mode: z
+      .enum(["subject", "face", "trim", "attention", "content"])
+      .default("subject")
+      .transform((v) => {
+        if (v === "attention") return "subject" as const;
+        if (v === "content") return "trim" as const;
+        return v;
+      }),
+    strategy: z.enum(["attention", "entropy"]).default("attention"),
+    width: z.number().int().positive().optional(),
+    height: z.number().int().positive().optional(),
+    padding: z.number().int().min(0).max(50).default(0),
+    facePreset: z
+      .enum(["closeup", "head-shoulders", "upper-body", "half-body"])
+      .default("head-shoulders"),
+    sensitivity: z.number().min(0).max(1).default(0.5),
+    threshold: z.number().int().min(0).max(255).default(30),
+    padToSquare: z.boolean().default(false),
+    padColor: z.string().default("#ffffff"),
+    targetSize: z.number().int().positive().optional(),
+    quality: z.number().int().min(1).max(100).optional(),
+  })
+  .transform((s) => ({
+    ...s,
+    mode: s.mode as "subject" | "face" | "trim",
+  }));
+
+function clampRegion(
+  left: number,
+  top: number,
+  cropW: number,
+  cropH: number,
+  imgW: number,
+  imgH: number,
+) {
+  const w = Math.min(cropW, imgW);
+  const h = Math.min(cropH, imgH);
+  let l = left;
+  let t = top;
+
+  if (l < 0) l = 0;
+  if (t < 0) t = 0;
+  if (l + w > imgW) l = imgW - w;
+  if (t + h > imgH) t = imgH - h;
+
+  return {
+    left: Math.round(Math.max(0, l)),
+    top: Math.round(Math.max(0, t)),
+    width: Math.round(w),
+    height: Math.round(h),
+  };
+}
+
+async function processSubject(
+  inputBuffer: Buffer,
+  settings: z.output<typeof settingsSchema>,
+): Promise<Buffer> {
+  const w = settings.width ?? 1080;
+  const h = settings.height ?? 1080;
+  const strategy =
+    settings.strategy === "entropy" ? sharp.strategy.entropy : sharp.strategy.attention;
+
+  if (settings.padding > 0) {
+    const scale = 1 + settings.padding / 100;
+    const oversizeW = Math.round(w * scale);
+    const oversizeH = Math.round(h * scale);
+
+    const oversize = await sharp(inputBuffer)
+      .resize(oversizeW, oversizeH, { fit: "cover", position: strategy })
+      .toBuffer();
+
+    const extractLeft = Math.round((oversizeW - w) / 2);
+    const extractTop = Math.round((oversizeH - h) / 2);
+
+    return sharp(oversize)
+      .extract({ left: extractLeft, top: extractTop, width: w, height: h })
+      .toBuffer();
+  }
+
+  return sharp(inputBuffer).resize(w, h, { fit: "cover", position: strategy }).toBuffer();
+}
+
+async function processFace(
+  inputBuffer: Buffer,
+  settings: z.output<typeof settingsSchema>,
+): Promise<Buffer> {
+  const result = await detectFaces(inputBuffer, { sensitivity: settings.sensitivity });
+
+  if (result.facesDetected === 0) {
+    return processSubject(inputBuffer, { ...settings, strategy: "attention" });
+  }
+
+  const meta = await sharp(inputBuffer).metadata();
+  const imgW = meta.width ?? 1;
+  const imgH = meta.height ?? 1;
+  const targetW = settings.width ?? 1080;
+  const targetH = settings.height ?? 1080;
+
+  const faces = result.faces;
+  const minX = Math.min(...faces.map((f) => f.x));
+  const minY = Math.min(...faces.map((f) => f.y));
+  const maxX = Math.max(...faces.map((f) => f.x + f.w));
+  const maxY = Math.max(...faces.map((f) => f.y + f.h));
+
+  const cx = (minX + maxX) / 2;
+  const cy = (minY + maxY) / 2;
+  const unionH = maxY - minY;
+
+  const preset = SMART_CROP_FACE_PRESETS.find((p) => p.id === settings.facePreset);
+  const multiplier = preset?.multiplier ?? 2.8;
+
+  const aspectRatio = targetW / targetH;
+  let cropH = unionH * multiplier * (1 + settings.padding / 100);
+  let cropW = cropH * aspectRatio;
+
+  if (cropW > imgW) {
+    cropW = imgW;
+    cropH = cropW / aspectRatio;
+  }
+  if (cropH > imgH) {
+    cropH = imgH;
+    cropW = cropH * aspectRatio;
+  }
+
+  const left = cx - cropW / 2;
+  const top = cy - cropH / 2;
+  const region = clampRegion(left, top, cropW, cropH, imgW, imgH);
+
+  if (region.width < 1 || region.height < 1) {
+    return processSubject(inputBuffer, { ...settings, strategy: "attention" });
+  }
+
+  const extracted = await sharp(inputBuffer).extract(region).toBuffer();
+  return sharp(extracted).resize(targetW, targetH, { fit: "fill" }).toBuffer();
+}
+
+async function processTrim(
+  inputBuffer: Buffer,
+  settings: z.output<typeof settingsSchema>,
+): Promise<Buffer> {
+  if (settings.padToSquare || settings.targetSize) {
+    const trimmed = await sharp(inputBuffer)
+      .trim({ threshold: settings.threshold })
+      .toBuffer({ resolveWithObject: true });
+
+    const w = trimmed.info.width;
+    const h = trimmed.info.height;
+    const target = settings.targetSize || Math.max(w, h);
+    const padR = Math.round(Number.parseInt(settings.padColor.slice(1, 3), 16));
+    const padG = Math.round(Number.parseInt(settings.padColor.slice(3, 5), 16));
+    const padB = Math.round(Number.parseInt(settings.padColor.slice(5, 7), 16));
+
+    return sharp(trimmed.data)
+      .resize({
+        width: target,
+        height: target,
+        fit: "contain",
+        background: { r: padR, g: padG, b: padB, alpha: 1 },
+      })
+      .toBuffer();
+  }
+
+  return sharp(inputBuffer).trim({ threshold: settings.threshold }).toBuffer();
+}

-/**
- * Smart crop with two modes:
- * - "attention": Sharp's entropy/saliency detection to crop to the most interesting region
- * - "content": Trims uniform-color borders (like GIMP's "Crop to Content"),
- *    optionally pads to a square at a target size
- */
 export function registerSmartCrop(app: FastifyInstance) {
  createToolRoute(app, {
    toolId: "smart-crop",
@@ -29,49 +181,18 @@ export function registerSmartCrop(app: FastifyInstance) {
      const outputFormat = await resolveOutputFormat(inputBuffer, filename, settings.quality);
      let result: Buffer;

-      if (settings.mode === "content") {
-        if (settings.padToSquare || settings.targetSize) {
-          // Trim first to get dimensions, then pad to square
-          const trimmed = await sharp(inputBuffer)
-            .trim({ threshold: settings.threshold })
-            .toBuffer({ resolveWithObject: true });
-
-          const w = trimmed.info.width;
-          const h = trimmed.info.height;
-          const target = settings.targetSize || Math.max(w, h);
-          const padR = Math.round(parseInt(settings.padColor.slice(1, 3), 16));
-          const padG = Math.round(parseInt(settings.padColor.slice(3, 5), 16));
-          const padB = Math.round(parseInt(settings.padColor.slice(5, 7), 16));
-
-          const padded = await sharp(trimmed.data)
-            .resize({
-              width: target,
-              height: target,
-              fit: "contain",
-              background: { r: padR, g: padG, b: padB, alpha: 1 },
-            })
-            .toFormat(outputFormat.format, { quality: outputFormat.quality })
-            .toBuffer();
-          result = padded;
-        } else {
-          // Simple trim + format in one pass (no intermediate encode)
-          result = await sharp(inputBuffer)
-            .trim({ threshold: settings.threshold })
-            .toFormat(outputFormat.format, { quality: outputFormat.quality })
-            .toBuffer();
-        }
+      if (settings.mode === "face") {
+        result = await processFace(inputBuffer, settings);
+      } else if (settings.mode === "trim") {
+        result = await processTrim(inputBuffer, settings);
      } else {
-        const w = settings.width ?? 1080;
-        const h = settings.height ?? 1080;
-        result = await sharp(inputBuffer)
-          .resize(w, h, {
-            fit: "cover",
-            position: sharp.strategy.attention,
-          })
-          .toFormat(outputFormat.format, { quality: outputFormat.quality })
-          .toBuffer();
+        result = await processSubject(inputBuffer, settings);
      }

+      result = await sharp(result)
+        .toFormat(outputFormat.format, { quality: outputFormat.quality })
+        .toBuffer();
+
      const stem = filename.replace(/\.[^.]+$/, "");
      const outputFilename = `${stem}_smartcrop.${outputFormat.extension}`;
      return { buffer: result, filename: outputFilename, contentType: outputFormat.contentType };