From 694cc86aff22c73e7aa37efa4b38cbd109df26a8 Mon Sep 17 00:00:00 2001 From: SnapOtter Date: Mon, 11 May 2026 21:12:00 +0800 Subject: [PATCH] feat: add content-aware-crop API route with async SSE processing --- .../src/routes/tools/content-aware-crop.ts | 349 ++++++++++++++++++ apps/api/src/routes/tools/index.ts | 2 + 2 files changed, 351 insertions(+) create mode 100644 apps/api/src/routes/tools/content-aware-crop.ts diff --git a/apps/api/src/routes/tools/content-aware-crop.ts b/apps/api/src/routes/tools/content-aware-crop.ts new file mode 100644 index 00000000..03efd4fa --- /dev/null +++ b/apps/api/src/routes/tools/content-aware-crop.ts @@ -0,0 +1,349 @@ +import { randomUUID } from "node:crypto"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { outpaint } from "@snapotter/ai"; +import { getBundleForTool, TOOL_BUNDLE_MAP } from "@snapotter/shared"; +import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify"; +import sharp from "sharp"; +import { z } from "zod"; +import { autoOrient } from "../../lib/auto-orient.js"; +import { formatZodErrors } from "../../lib/errors.js"; +import { isToolInstalled } from "../../lib/feature-status.js"; +import { validateImageBuffer } from "../../lib/file-validation.js"; +import { sanitizeFilename } from "../../lib/filename.js"; +import { decodeToSharpCompat, needsCliDecode } from "../../lib/format-decoders.js"; +import { encodeJxl } from "../../lib/format-encoders.js"; +import { decodeHeic, encodeHeic } from "../../lib/heic-converter.js"; +import { resolveOutputFormat } from "../../lib/output-format.js"; +import { createWorkspace } from "../../lib/workspace.js"; +import { updateSingleFileProgress } from "../progress.js"; +import { registerToolProcessFn } from "../tool-factory.js"; + +const EXT_MAP: Record = { + jpeg: "jpg", + jpg: "jpg", + png: "png", + webp: "webp", + tiff: "tiff", + gif: "gif", + avif: "avif", + heic: "heic", + heif: "heif", + jxl: "jxl", +}; + +const BROWSER_PREVIEWABLE = new Set(["png", "jpg", "jpeg", "webp", "gif", "avif", "bmp"]); + +const settingsSchema = z.object({ + extendTop: z.number().int().min(0).default(0), + extendRight: z.number().int().min(0).default(0), + extendBottom: z.number().int().min(0).default(0), + extendLeft: z.number().int().min(0).default(0), + format: z + .enum(["auto", "png", "jpg", "jpeg", "webp", "tiff", "gif", "avif", "heic", "heif", "jxl"]) + .default("auto"), + quality: z.number().int().min(1).max(100).default(95), +}); + +type Settings = z.infer; + +/** + * Content-aware crop (outpainting) route. + * Extends image borders using AI-powered outpainting via LaMa. + */ +export function registerContentAwareCrop(app: FastifyInstance) { + app.post( + "/api/v1/tools/content-aware-crop", + async (request: FastifyRequest, reply: FastifyReply) => { + const toolId = "content-aware-crop"; + if (!isToolInstalled(toolId)) { + const bundle = getBundleForTool(toolId); + return reply.status(501).send({ + error: "Feature not installed", + code: "FEATURE_NOT_INSTALLED", + feature: TOOL_BUNDLE_MAP[toolId], + featureName: bundle?.name ?? toolId, + estimatedSize: bundle?.estimatedSize ?? "unknown", + }); + } + + let fileBuffer: Buffer | null = null; + let filename = "image"; + let settingsRaw: string | null = null; + let clientJobId: string | null = null; + + try { + const parts = request.parts(); + for await (const part of parts) { + if (part.type === "file") { + const chunks: Buffer[] = []; + for await (const chunk of part.file) { + chunks.push(chunk); + } + fileBuffer = Buffer.concat(chunks); + filename = sanitizeFilename(part.filename ?? "image"); + } else if (part.fieldname === "settings") { + settingsRaw = part.value as string; + } else if (part.fieldname === "clientJobId") { + const raw = part.value as string; + if (/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(raw)) { + clientJobId = raw; + } + } + } + } catch (err) { + return reply.status(400).send({ + error: "Failed to parse multipart request", + details: err instanceof Error ? err.message : String(err), + }); + } + + if (!fileBuffer || fileBuffer.length === 0) { + return reply.status(400).send({ error: "No image file provided" }); + } + + const validation = await validateImageBuffer(fileBuffer, filename); + if (!validation.valid) { + return reply.status(400).send({ error: `Invalid image: ${validation.reason}` }); + } + + // Validate settings + let settings: Settings; + try { + const raw = settingsRaw ? JSON.parse(settingsRaw) : {}; + const result = settingsSchema.safeParse(raw); + if (!result.success) { + return reply + .status(400) + .send({ error: "Invalid settings", details: formatZodErrors(result.error.issues) }); + } + settings = result.data; + } catch { + return reply.status(400).send({ error: "Settings must be valid JSON" }); + } + + // At least one extend direction must be > 0 + if ( + settings.extendTop === 0 && + settings.extendRight === 0 && + settings.extendBottom === 0 && + settings.extendLeft === 0 + ) { + return reply.status(400).send({ + error: "At least one extend direction must be greater than 0", + }); + } + + let format: string = settings.format; + let quality = settings.quality; + + if (format === "auto") { + const detected = await resolveOutputFormat(fileBuffer, filename); + format = detected.format === "jpeg" ? "jpg" : detected.format; + quality = detected.quality; + } + + try { + // Decode HEIC/HEIF input + if (validation.format === "heif") { + fileBuffer = await decodeHeic(fileBuffer); + } + + // Decode CLI-decoded formats (RAW, TGA, PSD, EXR, HDR) + if (needsCliDecode(validation.format)) { + fileBuffer = await decodeToSharpCompat(fileBuffer, validation.format); + } + + // Auto-orient to fix EXIF rotation + fileBuffer = await autoOrient(fileBuffer); + } catch (err) { + request.log.error({ err, toolId: "content-aware-crop" }, "Input decoding failed"); + return reply.status(422).send({ + error: "Content-aware crop failed", + details: err instanceof Error ? err.message : "Unknown error", + }); + } + + const originalSize = fileBuffer.length; + const jobId = randomUUID(); + const progressJobId = clientJobId || jobId; + let workspacePath: string; + try { + workspacePath = await createWorkspace(jobId); + const inputPath = join(workspacePath, "input", filename); + await writeFile(inputPath, fileBuffer); + } catch (err) { + request.log.error({ err, toolId: "content-aware-crop" }, "Workspace creation failed"); + return reply.status(422).send({ + error: "Content-aware crop failed", + details: err instanceof Error ? err.message : "Unknown error", + }); + } + + const log = request.log; + log.info( + { + toolId: "content-aware-crop", + imageSize: originalSize, + extendTop: settings.extendTop, + extendRight: settings.extendRight, + extendBottom: settings.extendBottom, + extendLeft: settings.extendLeft, + format, + }, + "Starting content-aware crop", + ); + + // Reply immediately so the HTTP connection closes within proxy timeout limits. + // The result will be delivered via the SSE progress channel. + reply.status(202).send({ jobId: progressJobId, async: true }); + + const onProgress = (percent: number, stage: string) => { + updateSingleFileProgress({ + jobId: progressJobId, + phase: "processing", + stage, + percent, + }); + }; + + // Fire-and-forget: processing happens after the response is sent + (async () => { + const resultBuffer = await outpaint( + fileBuffer, + { + extendTop: settings.extendTop, + extendRight: settings.extendRight, + extendBottom: settings.extendBottom, + extendLeft: settings.extendLeft, + }, + join(workspacePath, "output"), + onProgress, + ); + + // Convert to the requested output format using Sharp + const needsNodeConversion = ["heic", "heif", "avif", "jxl"].includes(format); + let outputBuffer: Buffer; + let finalFormat = format; + + if (needsNodeConversion) { + if (format === "heic" || format === "heif") { + outputBuffer = await encodeHeic(resultBuffer, quality); + finalFormat = format; + } else if (format === "jxl") { + outputBuffer = await encodeJxl(resultBuffer, quality); + finalFormat = "jxl"; + } else { + outputBuffer = await sharp(resultBuffer).avif({ quality }).toBuffer(); + finalFormat = "avif"; + } + } else if (format === "jpg" || format === "jpeg") { + outputBuffer = await sharp(resultBuffer).jpeg({ quality }).toBuffer(); + finalFormat = "jpg"; + } else if (format === "webp") { + outputBuffer = await sharp(resultBuffer).webp({ quality }).toBuffer(); + finalFormat = "webp"; + } else if (format === "tiff") { + outputBuffer = await sharp(resultBuffer).tiff({ quality }).toBuffer(); + finalFormat = "tiff"; + } else if (format === "gif") { + outputBuffer = await sharp(resultBuffer).gif().toBuffer(); + finalFormat = "gif"; + } else { + outputBuffer = resultBuffer; + finalFormat = "png"; + } + + // Save output + const ext = EXT_MAP[finalFormat] || "png"; + const outputFilename = `${filename.replace(/\.[^.]+$/, "")}_extended.${ext}`; + const outputPath = join(workspacePath, "output", outputFilename); + await writeFile(outputPath, outputBuffer); + + // Generate browser-compatible preview for non-previewable formats + let previewUrl: string | undefined; + if (!BROWSER_PREVIEWABLE.has(finalFormat)) { + try { + const previewInput = + finalFormat === "heic" || finalFormat === "heif" + ? await decodeHeic(outputBuffer) + : outputBuffer; + const previewBuffer = await sharp(previewInput).webp({ quality: 80 }).toBuffer(); + const previewPath = join(workspacePath, "output", "preview.webp"); + await writeFile(previewPath, previewBuffer); + previewUrl = `/api/v1/download/${jobId}/preview.webp`; + } catch { + // Non-fatal - frontend will show fallback + } + } + + const downloadUrl = `/api/v1/download/${jobId}/${encodeURIComponent(outputFilename)}`; + updateSingleFileProgress({ + jobId: progressJobId, + phase: "complete", + percent: 100, + result: { + jobId, + downloadUrl, + previewUrl, + originalSize, + processedSize: outputBuffer.length, + }, + }); + + log.info( + { toolId: "content-aware-crop", jobId, downloadUrl }, + "Content-aware crop complete", + ); + })().catch((err) => { + log.error({ err, toolId: "content-aware-crop" }, "Content-aware crop failed"); + updateSingleFileProgress({ + jobId: progressJobId, + phase: "failed", + percent: 0, + error: err instanceof Error ? err.message : "Content-aware crop failed", + }); + }); + }, + ); + + // Register in the pipeline/batch registry so this tool can be used + // as a step in automation pipelines (without progress callbacks). + registerToolProcessFn({ + toolId: "content-aware-crop", + settingsSchema, + process: async (inputBuffer, settings, filename) => { + const s = settings as Settings; + + // Decode HEIC/HEIF for pipeline/batch mode + const ext = filename.split(".").pop()?.toLowerCase() ?? ""; + let buf = inputBuffer; + if (["heic", "heif", "hif"].includes(ext)) { + buf = await decodeHeic(buf); + } + // Decode CLI-decoded formats for pipeline/batch mode + const cliCheck = await validateImageBuffer(inputBuffer, filename); + if (cliCheck.valid && needsCliDecode(cliCheck.format)) { + buf = await decodeToSharpCompat(inputBuffer, cliCheck.format); + } + + const orientedBuffer = await autoOrient(buf); + const jobId = randomUUID(); + const workspacePath = await createWorkspace(jobId); + + const resultBuffer = await outpaint( + orientedBuffer, + { + extendTop: s.extendTop, + extendRight: s.extendRight, + extendBottom: s.extendBottom, + extendLeft: s.extendLeft, + }, + join(workspacePath, "output"), + ); + + const outputFilename = `${filename.replace(/\.[^.]+$/, "")}_extended.png`; + return { buffer: resultBuffer, filename: outputFilename, contentType: "image/png" }; + }, + }); +} diff --git a/apps/api/src/routes/tools/index.ts b/apps/api/src/routes/tools/index.ts index aede7ca1..e934eb54 100644 --- a/apps/api/src/routes/tools/index.ts +++ b/apps/api/src/routes/tools/index.ts @@ -15,6 +15,7 @@ import { registerColorize } from "./colorize.js"; import { registerCompare } from "./compare.js"; import { registerCompose } from "./compose.js"; import { registerCompress } from "./compress.js"; +import { registerContentAwareCrop } from "./content-aware-crop.js"; import { registerContentAwareResize } from "./content-aware-resize.js"; import { registerConvert } from "./convert.js"; import { registerCrop } from "./crop.js"; @@ -147,6 +148,7 @@ export async function registerToolRoutes(app: FastifyInstance): Promise { { id: "smart-crop", register: registerSmartCrop }, { id: "image-enhancement", register: registerImageEnhancement }, { id: "content-aware-resize", register: registerContentAwareResize }, + { id: "content-aware-crop", register: registerContentAwareCrop }, { id: "colorize", register: registerColorize }, { id: "enhance-faces", register: registerEnhanceFaces }, { id: "noise-removal", register: registerNoiseRemoval },