feat(red-eye-removal): SOTA red eye removal with MediaPipe Face Mesh + OpenCV LAB correction (#60)

Uses MediaPipe Face Mesh (refine_landmarks=True) for precise iris localization
and OpenCV LAB color space for accurate red-eye detection and luminance-preserving
correction. Zero new dependencies - leverages existing MediaPipe + OpenCV stack.

- Sensitivity slider (LAB 'a' channel threshold)
- Correction strength slider (pupil darkening factor)
- Output format selector (Original/PNG/JPEG/WebP)
- Before/after preview, progress stages, batch processing
- Pipeline support via Controls/Settings split

Co-authored-by: stirling-image <stirling-image@users.noreply.github.com>
This commit is contained in:
stirling-image
2026-04-13 20:22:30 +08:00
committed by GitHub
co-authored by stirling-image
parent dfffc0a8cc
commit 9ddeac92b6
9 changed files with 705 additions and 0 deletions
+2
View File
@@ -28,6 +28,7 @@ import { registerNoiseRemoval } from "./noise-removal.js";
import { registerOcr } from "./ocr.js";
import { registerPdfToImage } from "./pdf-to-image.js";
import { registerQrGenerate } from "./qr-generate.js";
import { registerRedEyeRemoval } from "./red-eye-removal.js";
import { registerRemoveBackground } from "./remove-background.js";
import { registerReplaceColor } from "./replace-color.js";
import { registerResize } from "./resize.js";
@@ -134,6 +135,7 @@ export async function registerToolRoutes(app: FastifyInstance): Promise<void> {
{ id: "content-aware-resize", register: registerContentAwareResize },
{ id: "colorize", register: registerColorize },
{ id: "noise-removal", register: registerNoiseRemoval },
{ id: "red-eye-removal", register: registerRedEyeRemoval },
];
let skipped = 0;
@@ -0,0 +1,164 @@
import { randomUUID } from "node:crypto";
import { writeFile } from "node:fs/promises";
import { basename, join } from "node:path";
import { removeRedEye } from "@stirling-image/ai";
import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
import { z } from "zod";
import { autoOrient } from "../../lib/auto-orient.js";
import { validateImageBuffer } from "../../lib/file-validation.js";
import { createWorkspace } from "../../lib/workspace.js";
import { updateSingleFileProgress } from "../progress.js";
import { registerToolProcessFn } from "../tool-factory.js";
/** Red eye detection and removal route. */
export function registerRedEyeRemoval(app: FastifyInstance) {
app.post(
"/api/v1/tools/red-eye-removal",
async (request: FastifyRequest, reply: FastifyReply) => {
let fileBuffer: Buffer | null = null;
let filename = "image";
let settingsRaw: string | null = null;
let clientJobId: string | null = null;
try {
const parts = request.parts();
for await (const part of parts) {
if (part.type === "file") {
const chunks: Buffer[] = [];
for await (const chunk of part.file) {
chunks.push(chunk);
}
fileBuffer = Buffer.concat(chunks);
filename = basename(part.filename ?? "image");
} else if (part.fieldname === "settings") {
settingsRaw = part.value as string;
} else if (part.fieldname === "clientJobId") {
clientJobId = part.value as string;
}
}
} catch (err) {
return reply.status(400).send({
error: "Failed to parse multipart request",
details: err instanceof Error ? err.message : String(err),
});
}
if (!fileBuffer || fileBuffer.length === 0) {
return reply.status(400).send({ error: "No image file provided" });
}
const validation = await validateImageBuffer(fileBuffer);
if (!validation.valid) {
return reply.status(400).send({ error: `Invalid image: ${validation.reason}` });
}
try {
const settings = settingsRaw ? JSON.parse(settingsRaw) : {};
request.log.info(
{
toolId: "red-eye-removal",
imageSize: fileBuffer.length,
sensitivity: settings.sensitivity,
strength: settings.strength,
},
"Starting red eye removal",
);
// Auto-orient to fix EXIF rotation before face detection
fileBuffer = await autoOrient(fileBuffer);
const jobId = randomUUID();
const workspacePath = await createWorkspace(jobId);
// Save input
const inputPath = join(workspacePath, "input", filename);
await writeFile(inputPath, fileBuffer);
// Process
const jobIdForProgress = clientJobId;
const onProgress = jobIdForProgress
? (percent: number, stage: string) => {
updateSingleFileProgress({
jobId: jobIdForProgress,
phase: "processing",
stage,
percent,
});
}
: undefined;
const result = await removeRedEye(
fileBuffer,
join(workspacePath, "output"),
{
sensitivity: settings.sensitivity ?? 50,
strength: settings.strength ?? 70,
format: settings.format,
quality: settings.quality ?? 90,
},
onProgress,
);
// Save output
const name = filename.replace(/\.[^.]+$/, "");
const outputFilename = `${name}_redeye_fixed.png`;
const outputPath = join(workspacePath, "output", outputFilename);
await writeFile(outputPath, result.buffer);
if (clientJobId) {
updateSingleFileProgress({
jobId: clientJobId,
phase: "complete",
percent: 100,
});
}
return reply.send({
jobId,
downloadUrl: `/api/v1/download/${jobId}/${encodeURIComponent(outputFilename)}`,
originalSize: fileBuffer.length,
processedSize: result.buffer.length,
facesDetected: result.facesDetected,
eyesCorrected: result.eyesCorrected,
});
} catch (err) {
request.log.error({ err, toolId: "red-eye-removal" }, "Red eye removal failed");
return reply.status(422).send({
error: "Red eye removal failed",
details: err instanceof Error ? err.message : "Unknown error",
});
}
},
);
// Register in the pipeline/batch registry so this tool can be used
// as a step in automation pipelines (without progress callbacks).
registerToolProcessFn({
toolId: "red-eye-removal",
settingsSchema: z.object({
sensitivity: z.number().min(0).max(100).default(50),
strength: z.number().min(0).max(100).default(70),
format: z.string().optional(),
quality: z.number().min(1).max(100).default(90),
}),
process: async (inputBuffer, settings, filename) => {
const s = settings as {
sensitivity?: number;
strength?: number;
format?: string;
quality?: number;
};
const orientedBuffer = await autoOrient(inputBuffer);
const jobId = randomUUID();
const workspacePath = await createWorkspace(jobId);
const result = await removeRedEye(orientedBuffer, join(workspacePath, "output"), {
sensitivity: s.sensitivity ?? 50,
strength: s.strength ?? 70,
format: s.format,
quality: s.quality ?? 90,
});
const outputFilename = `${filename.replace(/\.[^.]+$/, "")}_redeye_fixed.png`;
return { buffer: result.buffer, filename: outputFilename, contentType: "image/png" };
},
});
}
@@ -0,0 +1,215 @@
import { Download } from "lucide-react";
import { useEffect, useRef, useState } from "react";
import { ProgressCard } from "@/components/common/progress-card";
import { useToolProcessor } from "@/hooks/use-tool-processor";
import { useFileStore } from "@/stores/file-store";
const LOSSY_FORMATS = new Set(["jpeg", "webp"]);
export interface RedEyeRemovalControlsProps {
settings?: Record<string, unknown>;
onChange?: (settings: Record<string, unknown>) => void;
}
export function RedEyeRemovalControls({
settings: initialSettings,
onChange,
}: RedEyeRemovalControlsProps) {
const [sensitivity, setSensitivity] = useState(50);
const [strength, setStrength] = useState(70);
const [outputFormat, setOutputFormat] = useState<"original" | "png" | "jpeg" | "webp">(
"original",
);
const [quality, setQuality] = useState(90);
// One-time init from pipeline settings
const initializedRef = useRef(false);
useEffect(() => {
if (!initialSettings || initializedRef.current) return;
initializedRef.current = true;
if (initialSettings.sensitivity != null) setSensitivity(Number(initialSettings.sensitivity));
if (initialSettings.strength != null) setStrength(Number(initialSettings.strength));
if (initialSettings.format != null)
setOutputFormat(initialSettings.format as "original" | "png" | "jpeg" | "webp");
if (initialSettings.quality != null) setQuality(Number(initialSettings.quality));
}, [initialSettings]);
// Emit settings on change
const onChangeRef = useRef(onChange);
useEffect(() => {
onChangeRef.current = onChange;
});
useEffect(() => {
onChangeRef.current?.({
sensitivity,
strength,
format: outputFormat,
quality,
});
}, [sensitivity, strength, outputFormat, quality]);
const tabClass = (active: boolean) =>
`flex-1 text-xs py-1.5 rounded ${active ? "bg-primary text-primary-foreground" : "bg-muted text-muted-foreground hover:bg-muted/80"}`;
return (
<div className="space-y-4">
{/* Sensitivity slider */}
<div>
<div className="flex justify-between items-center">
<p className="text-sm font-medium text-muted-foreground">Sensitivity</p>
<span className="text-sm font-mono tabular-nums font-medium">{sensitivity}</span>
</div>
<input
type="range"
min={0}
max={100}
step={1}
value={sensitivity}
onChange={(e) => setSensitivity(Number(e.target.value))}
className="w-full h-1.5 rounded-full appearance-none bg-muted accent-primary"
/>
<div className="flex justify-between text-[10px] text-muted-foreground mt-0.5">
<span>Strict</span>
<span>Aggressive</span>
</div>
</div>
{/* Correction Strength slider */}
<div>
<div className="flex justify-between items-center">
<p className="text-sm font-medium text-muted-foreground">Correction Strength</p>
<span className="text-sm font-mono tabular-nums font-medium">{strength}</span>
</div>
<input
type="range"
min={0}
max={100}
step={1}
value={strength}
onChange={(e) => setStrength(Number(e.target.value))}
className="w-full h-1.5 rounded-full appearance-none bg-muted accent-primary"
/>
<div className="flex justify-between text-[10px] text-muted-foreground mt-0.5">
<span>Subtle</span>
<span>Dark</span>
</div>
</div>
<div className="border-t border-border pt-3" />
{/* Output format */}
<div>
<p className="text-xs text-muted-foreground mb-1">Output Format</p>
<div className="grid grid-cols-4 gap-1">
{(["original", "png", "jpeg", "webp"] as const).map((f) => (
<button
key={f}
type="button"
onClick={() => setOutputFormat(f)}
className={tabClass(outputFormat === f)}
>
{f === "original" ? "Original" : f.toUpperCase()}
</button>
))}
</div>
</div>
{/* Quality slider (lossy formats only) */}
{LOSSY_FORMATS.has(outputFormat) && (
<div>
<div className="flex justify-between items-center">
<p className="text-sm font-medium text-muted-foreground">Quality</p>
<span className="text-sm font-mono tabular-nums font-medium">{quality}</span>
</div>
<input
type="range"
min={1}
max={100}
step={1}
value={quality}
onChange={(e) => setQuality(Number(e.target.value))}
className="w-full h-1.5 rounded-full appearance-none bg-muted accent-primary"
/>
</div>
)}
</div>
);
}
export function RedEyeRemovalSettings() {
const { files } = useFileStore();
const {
processFiles,
processAllFiles,
processing,
error,
downloadUrl,
originalSize,
processedSize,
progress,
} = useToolProcessor("red-eye-removal");
const [settings, setSettings] = useState<Record<string, unknown>>({});
const handleProcess = () => {
if (files.length > 1) {
processAllFiles(files, settings);
} else {
processFiles(files, settings);
}
};
const hasFile = files.length > 0;
const hasMultiple = files.length > 1;
return (
<div className="space-y-4">
<RedEyeRemovalControls onChange={setSettings} />
{/* Error */}
{error && <p className="text-xs text-red-500">{error}</p>}
{/* Size info */}
{originalSize != null && processedSize != null && (
<div className="text-xs text-muted-foreground space-y-0.5">
<p>Original: {(originalSize / 1024).toFixed(1)} KB</p>
<p>Fixed: {(processedSize / 1024).toFixed(1)} KB</p>
</div>
)}
{/* Process button / progress */}
{processing ? (
<ProgressCard
active={processing}
phase={progress.phase === "idle" ? "uploading" : progress.phase}
label={hasMultiple ? `Fixing red eye in ${files.length} images` : "Fixing red eye"}
percent={progress.percent}
elapsed={progress.elapsed}
/>
) : (
<button
type="button"
data-testid="red-eye-removal-submit"
onClick={handleProcess}
disabled={!hasFile || processing}
className="w-full py-2.5 rounded-lg bg-primary text-primary-foreground font-medium disabled:opacity-50 disabled:cursor-not-allowed flex items-center justify-center gap-2"
>
{hasMultiple ? `Fix Red Eye (${files.length} files)` : "Fix Red Eye"}
</button>
)}
{/* Download (single file - batch uses Download All ZIP in tool-page) */}
{!hasMultiple && downloadUrl && (
<a
href={downloadUrl}
download
data-testid="red-eye-removal-download"
className="w-full py-2.5 rounded-lg border border-primary text-primary font-medium flex items-center justify-center gap-2 hover:bg-primary/5"
>
<Download className="h-4 w-4" />
Download
</a>
)}
</div>
);
}
+6
View File
@@ -254,6 +254,11 @@ const NoiseRemovalSettings = lazy(() =>
default: m.NoiseRemovalSettings,
})),
);
const RedEyeRemovalSettings = lazy(() =>
import("@/components/tools/red-eye-removal-settings").then((m) => ({
default: m.RedEyeRemovalSettings,
})),
);
// ── Color tool wrapper ─────────────────────────────────────────────
// Color tools share a single component but differ by toolId.
@@ -384,6 +389,7 @@ export const toolRegistry = new Map<string, ToolRegistryEntry>([
],
["colorize", { displayMode: "before-after", Settings: ColorizeSettings }],
["noise-removal", { displayMode: "before-after", Settings: NoiseRemovalSettings }],
["red-eye-removal", { displayMode: "before-after", Settings: RedEyeRemovalSettings }],
]);
export function getToolRegistryEntry(toolId: string): ToolRegistryEntry | undefined {
+252
View File
@@ -0,0 +1,252 @@
"""Red-eye removal using MediaPipe Face Mesh."""
import sys
import json
import os
def emit_progress(percent, stage):
"""Emit structured progress to stderr for bridge.ts to capture."""
print(json.dumps({"progress": percent, "stage": stage}), file=sys.stderr, flush=True)
def main():
input_path = sys.argv[1]
output_path = sys.argv[2]
settings = json.loads(sys.argv[3]) if len(sys.argv) > 3 else {}
sensitivity = settings.get("sensitivity", 50)
strength = settings.get("strength", 70)
out_format = settings.get("format", "original")
quality = settings.get("quality", 90)
# Map sensitivity (0-100) to LAB "a" channel threshold.
# Higher sensitivity = lower threshold = more pixels flagged as red.
threshold = 170 - (sensitivity / 100) * 50
# Map strength (0-100) to darken factor.
# Higher strength = darker correction.
darken_factor = 1.0 - (strength / 100) * 0.7
try:
emit_progress(10, "Preparing image")
from PIL import Image
img = Image.open(input_path).convert("RGB")
width, height = img.size
# Determine output format
if out_format == "original":
ext = os.path.splitext(input_path)[1].lower()
if ext in (".heic", ".heif"):
save_format = "PNG"
if not output_path.lower().endswith(".png"):
output_path = os.path.splitext(output_path)[0] + ".png"
elif ext in (".jpg", ".jpeg"):
save_format = "JPEG"
elif ext == ".webp":
save_format = "WEBP"
else:
save_format = "PNG"
elif out_format == "jpeg":
save_format = "JPEG"
if not output_path.lower().endswith((".jpg", ".jpeg")):
output_path = os.path.splitext(output_path)[0] + ".jpg"
elif out_format == "webp":
save_format = "WEBP"
if not output_path.lower().endswith(".webp"):
output_path = os.path.splitext(output_path)[0] + ".webp"
else:
save_format = "PNG"
if not output_path.lower().endswith(".png"):
output_path = os.path.splitext(output_path)[0] + ".png"
format_label = save_format.lower()
if format_label == "jpeg":
format_label = "jpg"
try:
import mediapipe as mp
import numpy as np
import cv2
emit_progress(25, "Detecting faces")
img_array = np.array(img)
mesh = mp.solutions.face_mesh.FaceMesh(
static_image_mode=True,
max_num_faces=10,
refine_landmarks=True,
min_detection_confidence=0.5,
)
results = mesh.process(img_array)
mesh.close()
faces_detected = 0
eyes_corrected = 0
if results.multi_face_landmarks:
faces_detected = len(results.multi_face_landmarks)
emit_progress(50, "Analyzing eyes")
# Iris landmark indices
right_iris = [468, 469, 470, 471, 472] # 468 = center
left_iris = [473, 474, 475, 476, 477] # 473 = center
if faces_detected > 0:
all_eyes = []
for face_landmarks in results.multi_face_landmarks:
landmarks = face_landmarks.landmark
for iris_indices in [right_iris, left_iris]:
center_idx = iris_indices[0]
contour_indices = iris_indices[1:]
cx = int(landmarks[center_idx].x * width)
cy = int(landmarks[center_idx].y * height)
# Compute radius from contour landmarks
radii = []
for idx in contour_indices:
px = int(landmarks[idx].x * width)
py = int(landmarks[idx].y * height)
dist = np.sqrt((px - cx) ** 2 + (py - cy) ** 2)
radii.append(dist)
radius = np.mean(radii) if radii else 5.0
all_eyes.append((cx, cy, radius))
total_eyes = len(all_eyes)
for eye_i, (cx, cy, radius) in enumerate(all_eyes):
progress = 50 + int((eye_i + 1) / total_eyes * 40)
emit_progress(progress, f"Correcting eye {eye_i + 1} of {total_eyes}")
# Padded radius for the circular mask
padded_radius = radius * 1.3
r_int = int(np.ceil(padded_radius))
# Bounding box for the ROI
x1 = max(0, cx - r_int)
y1 = max(0, cy - r_int)
x2 = min(width, cx + r_int)
y2 = min(height, cy + r_int)
if x2 <= x1 or y2 <= y1:
continue
# Create circular mask in ROI space
roi_h = y2 - y1
roi_w = x2 - x1
yy, xx = np.ogrid[:roi_h, :roi_w]
local_cx = cx - x1
local_cy = cy - y1
circle_mask = ((xx - local_cx) ** 2 + (yy - local_cy) ** 2) <= (padded_radius ** 2)
# Extract ROI
roi = img_array[y1:y2, x1:x2].copy()
# Convert to LAB
roi_lab = cv2.cvtColor(roi, cv2.COLOR_RGB2LAB).astype(np.float64)
L_chan = roi_lab[:, :, 0]
a_chan = roi_lab[:, :, 1]
# LAB red detection: a > threshold AND 50 < L < 220
lab_red = (a_chan > threshold) & (L_chan > 50) & (L_chan < 220)
# HSV saturation check
roi_hsv = cv2.cvtColor(roi, cv2.COLOR_RGB2HSV)
S_chan = roi_hsv[:, :, 1]
hsv_saturated = S_chan > 60
# Intersection: LAB-red AND HSV-saturated AND inside circle
red_mask = lab_red & hsv_saturated & circle_mask
# Morphological cleanup
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (3, 3))
red_mask_u8 = red_mask.astype(np.uint8) * 255
red_mask_u8 = cv2.morphologyEx(red_mask_u8, cv2.MORPH_CLOSE, kernel)
red_mask_u8 = cv2.morphologyEx(red_mask_u8, cv2.MORPH_OPEN, kernel)
red_pixel_count = np.count_nonzero(red_mask_u8)
if red_pixel_count < 3:
continue
# Correct red pixels in LAB space
corrected_lab = roi_lab.copy()
mask_bool = red_mask_u8 > 0
corrected_lab[:, :, 0][mask_bool] = corrected_lab[:, :, 0][mask_bool] * darken_factor
corrected_lab[:, :, 1][mask_bool] = 128 # neutral a
corrected_lab[:, :, 2][mask_bool] = 128 # neutral b
corrected_lab = np.clip(corrected_lab, 0, 255).astype(np.uint8)
corrected_rgb = cv2.cvtColor(corrected_lab, cv2.COLOR_LAB2RGB)
# Soft mask for blending (Gaussian blur on mask edges)
soft_mask = cv2.GaussianBlur(
red_mask_u8.astype(np.float32), (5, 5), 1.5
)
soft_mask = soft_mask / 255.0
soft_mask = soft_mask[:, :, np.newaxis]
# Alpha blend corrected with original
blended = (corrected_rgb.astype(np.float32) * soft_mask +
roi.astype(np.float32) * (1.0 - soft_mask))
blended = np.clip(blended, 0, 255).astype(np.uint8)
img_array[y1:y2, x1:x2] = blended
eyes_corrected += 1
# Update the PIL image from the corrected array
img = Image.fromarray(img_array)
emit_progress(95, "Saving result")
save_kwargs = {}
if save_format == "JPEG":
save_kwargs["quality"] = quality
elif save_format == "WEBP":
save_kwargs["quality"] = quality
img.save(output_path, format=save_format, **save_kwargs)
print(
json.dumps(
{
"success": True,
"facesDetected": faces_detected,
"eyesCorrected": eyes_corrected,
"width": width,
"height": height,
"format": format_label,
"output_path": output_path,
}
)
)
except ImportError:
print(
json.dumps(
{
"success": False,
"error": "Red-eye removal requires MediaPipe, NumPy, and OpenCV. Install with: pip install mediapipe numpy opencv-python",
}
)
)
sys.exit(1)
except ImportError:
print(
json.dumps(
{
"success": False,
"error": "Pillow is not installed. Install with: pip install Pillow",
}
)
)
sys.exit(1)
except Exception as e:
print(json.dumps({"success": False, "error": str(e)}))
sys.exit(1)
if __name__ == "__main__":
main()
+1
View File
@@ -6,5 +6,6 @@ export { blurFaces, detectFaces } from "./face-detection.js";
export { inpaint } from "./inpainting.js";
export { noiseRemoval } from "./noise-removal.js";
export { extractText } from "./ocr.js";
export { removeRedEye } from "./red-eye-removal.js";
export { seamCarve } from "./seam-carving.js";
export { upscale } from "./upscaling.js";
+52
View File
@@ -0,0 +1,52 @@
import { readFile, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { type ProgressCallback, runPythonWithProgress } from "./bridge.js";
export interface RedEyeRemovalOptions {
sensitivity?: number;
strength?: number;
format?: string;
quality?: number;
}
export interface RedEyeRemovalResult {
buffer: Buffer;
facesDetected: number;
eyesCorrected: number;
width: number;
height: number;
format: string;
}
export async function removeRedEye(
inputBuffer: Buffer,
outputDir: string,
options: RedEyeRemovalOptions = {},
onProgress?: ProgressCallback,
): Promise<RedEyeRemovalResult> {
const inputPath = join(outputDir, "input_redeye.png");
const outputPath = join(outputDir, "output_redeye.png");
await writeFile(inputPath, inputBuffer);
const { stdout } = await runPythonWithProgress(
"red_eye_removal.py",
[inputPath, outputPath, JSON.stringify(options)],
{ onProgress },
);
const result = JSON.parse(stdout);
if (!result.success) {
throw new Error(result.error || "Red eye removal failed");
}
const actualOutputPath = result.output_path || outputPath;
const buffer = await readFile(actualOutputPath);
return {
buffer,
facesDetected: result.facesDetected ?? 0,
eyesCorrected: result.eyesCorrected ?? 0,
width: result.width,
height: result.height,
format: result.format ?? "png",
};
}
+9
View File
@@ -193,6 +193,14 @@ export const TOOLS: Tool[] = [
icon: "Sparkles",
route: "/noise-removal",
},
{
id: "red-eye-removal",
name: "Red Eye Removal",
description: "AI-detect and fix red eye in flash photos",
category: "ai",
icon: "Eye",
route: "/red-eye-removal",
},
// Watermark & Overlay
{
id: "watermark-text",
@@ -414,4 +422,5 @@ export const PYTHON_SIDECAR_TOOLS = [
"ocr",
"colorize",
"noise-removal",
"red-eye-removal",
] as const;
+4
View File
@@ -84,6 +84,10 @@ export const en = {
name: "Noise Removal",
description: "AI-powered noise and grain removal",
},
"red-eye-removal": {
name: "Red Eye Removal",
description: "AI-powered red eye detection and correction for flash photos",
},
"content-aware-resize": {
name: "Content-Aware Resize",
description: "Intelligently resize images while preserving important content",