mirror of
https://github.com/snapotter-hq/SnapOtter.git
synced 2026-08-03 07:46:42 +02:00
feat: add Phase 4 AI tools with Python bridge and 6 new tools
Add Python bridge (packages/ai/src/bridge.ts) that calls Python scripts via child_process with venv-first fallback to system python3. Implements 6 AI-powered tools: - Remove Background: rembg-based with U2-Net/IS-Net models - Image Upscaling: Real-ESRGAN with Lanczos fallback - OCR/Text Extraction: Tesseract + PaddleOCR engines - Face/PII Blur: MediaPipe face detection with configurable blur - Object Eraser: LaMa inpainting with mask-based input - Smart Crop: Sharp attention-based entropy cropping (no Python needed) Each tool includes: Python script, TypeScript wrapper, API route, and React settings component. All Python scripts handle ImportError gracefully with clear installation messages.
This commit is contained in:
@@ -0,0 +1,86 @@
|
||||
import type { FastifyInstance, FastifyRequest, FastifyReply } from "fastify";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import { join, basename } from "node:path";
|
||||
import { blurFaces } from "@stirling-image/ai";
|
||||
import { createWorkspace } from "../../lib/workspace.js";
|
||||
|
||||
/**
|
||||
* Face detection and blurring route.
|
||||
* Uses MediaPipe for detection, PIL for blurring.
|
||||
*/
|
||||
export function registerBlurFaces(app: FastifyInstance) {
|
||||
app.post(
|
||||
"/api/v1/tools/blur-faces",
|
||||
async (request: FastifyRequest, reply: FastifyReply) => {
|
||||
let fileBuffer: Buffer | null = null;
|
||||
let filename = "image";
|
||||
let settingsRaw: string | null = null;
|
||||
|
||||
try {
|
||||
const parts = request.parts();
|
||||
for await (const part of parts) {
|
||||
if (part.type === "file") {
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of part.file) {
|
||||
chunks.push(chunk);
|
||||
}
|
||||
fileBuffer = Buffer.concat(chunks);
|
||||
filename = basename(part.filename ?? "image");
|
||||
} else if (part.fieldname === "settings") {
|
||||
settingsRaw = part.value as string;
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
return reply.status(400).send({
|
||||
error: "Failed to parse multipart request",
|
||||
details: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
}
|
||||
|
||||
if (!fileBuffer || fileBuffer.length === 0) {
|
||||
return reply.status(400).send({ error: "No image file provided" });
|
||||
}
|
||||
|
||||
try {
|
||||
const settings = settingsRaw ? JSON.parse(settingsRaw) : {};
|
||||
const jobId = randomUUID();
|
||||
const workspacePath = await createWorkspace(jobId);
|
||||
|
||||
// Save input
|
||||
const inputPath = join(workspacePath, "input", filename);
|
||||
await writeFile(inputPath, fileBuffer);
|
||||
|
||||
// Process
|
||||
const result = await blurFaces(
|
||||
fileBuffer,
|
||||
join(workspacePath, "output"),
|
||||
{
|
||||
blurRadius: settings.blurRadius ?? 30,
|
||||
sensitivity: settings.sensitivity ?? 0.5,
|
||||
},
|
||||
);
|
||||
|
||||
// Save output
|
||||
const outputFilename =
|
||||
filename.replace(/\.[^.]+$/, "") + "_blurred.png";
|
||||
const outputPath = join(workspacePath, "output", outputFilename);
|
||||
await writeFile(outputPath, result.buffer);
|
||||
|
||||
return reply.send({
|
||||
jobId,
|
||||
downloadUrl: `/api/v1/download/${jobId}/${encodeURIComponent(outputFilename)}`,
|
||||
originalSize: fileBuffer.length,
|
||||
processedSize: result.buffer.length,
|
||||
facesDetected: result.facesDetected,
|
||||
faces: result.faces,
|
||||
});
|
||||
} catch (err) {
|
||||
return reply.status(422).send({
|
||||
error: "Face blur failed",
|
||||
details: err instanceof Error ? err.message : "Unknown error",
|
||||
});
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
import type { FastifyInstance, FastifyRequest, FastifyReply } from "fastify";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import { join, basename } from "node:path";
|
||||
import { inpaint } from "@stirling-image/ai";
|
||||
import { createWorkspace } from "../../lib/workspace.js";
|
||||
|
||||
/**
|
||||
* Object eraser / inpainting route.
|
||||
* Accepts an image and a mask image, erases masked areas.
|
||||
*/
|
||||
export function registerEraseObject(app: FastifyInstance) {
|
||||
app.post(
|
||||
"/api/v1/tools/erase-object",
|
||||
async (request: FastifyRequest, reply: FastifyReply) => {
|
||||
let imageBuffer: Buffer | null = null;
|
||||
let maskBuffer: Buffer | null = null;
|
||||
let filename = "image";
|
||||
|
||||
try {
|
||||
const parts = request.parts();
|
||||
for await (const part of parts) {
|
||||
if (part.type === "file") {
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of part.file) {
|
||||
chunks.push(chunk);
|
||||
}
|
||||
const buf = Buffer.concat(chunks);
|
||||
if (part.fieldname === "mask") {
|
||||
maskBuffer = buf;
|
||||
} else {
|
||||
imageBuffer = buf;
|
||||
filename = basename(part.filename ?? "image");
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
return reply.status(400).send({
|
||||
error: "Failed to parse multipart request",
|
||||
details: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
}
|
||||
|
||||
if (!imageBuffer || imageBuffer.length === 0) {
|
||||
return reply.status(400).send({ error: "No image file provided" });
|
||||
}
|
||||
if (!maskBuffer || maskBuffer.length === 0) {
|
||||
return reply
|
||||
.status(400)
|
||||
.send({ error: "No mask image provided. Upload a mask as a second file with fieldname 'mask'" });
|
||||
}
|
||||
|
||||
try {
|
||||
const jobId = randomUUID();
|
||||
const workspacePath = await createWorkspace(jobId);
|
||||
|
||||
// Save input
|
||||
const inputPath = join(workspacePath, "input", filename);
|
||||
await writeFile(inputPath, imageBuffer);
|
||||
|
||||
// Process
|
||||
const resultBuffer = await inpaint(
|
||||
imageBuffer,
|
||||
maskBuffer,
|
||||
join(workspacePath, "output"),
|
||||
);
|
||||
|
||||
// Save output
|
||||
const outputFilename =
|
||||
filename.replace(/\.[^.]+$/, "") + "_erased.png";
|
||||
const outputPath = join(workspacePath, "output", outputFilename);
|
||||
await writeFile(outputPath, resultBuffer);
|
||||
|
||||
return reply.send({
|
||||
jobId,
|
||||
downloadUrl: `/api/v1/download/${jobId}/${encodeURIComponent(outputFilename)}`,
|
||||
originalSize: imageBuffer.length,
|
||||
processedSize: resultBuffer.length,
|
||||
});
|
||||
} catch (err) {
|
||||
return reply.status(422).send({
|
||||
error: "Object erasing failed",
|
||||
details: err instanceof Error ? err.message : "Unknown error",
|
||||
});
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
@@ -32,6 +32,13 @@ import { registerFavicon } from "./favicon.js";
|
||||
import { registerImageToPdf } from "./image-to-pdf.js";
|
||||
// Phase 3: Adjustments extra
|
||||
import { registerReplaceColor } from "./replace-color.js";
|
||||
// Phase 4: AI Tools
|
||||
import { registerRemoveBackground } from "./remove-background.js";
|
||||
import { registerUpscale } from "./upscale.js";
|
||||
import { registerOcr } from "./ocr.js";
|
||||
import { registerBlurFaces } from "./blur-faces.js";
|
||||
import { registerEraseObject } from "./erase-object.js";
|
||||
import { registerSmartCrop } from "./smart-crop.js";
|
||||
|
||||
/**
|
||||
* Registry that imports and registers all tool routes.
|
||||
@@ -79,5 +86,13 @@ export async function registerToolRoutes(app: FastifyInstance): Promise<void> {
|
||||
// Phase 3: Adjustments extra
|
||||
registerReplaceColor(app);
|
||||
|
||||
app.log.info("Tool routes registered (26 tools, 29 endpoints)");
|
||||
// Phase 4: AI Tools
|
||||
registerRemoveBackground(app);
|
||||
registerUpscale(app);
|
||||
registerOcr(app);
|
||||
registerBlurFaces(app);
|
||||
registerEraseObject(app);
|
||||
registerSmartCrop(app);
|
||||
|
||||
app.log.info("Tool routes registered (32 tools, 35 endpoints)");
|
||||
}
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
import type { FastifyInstance, FastifyRequest, FastifyReply } from "fastify";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { basename } from "node:path";
|
||||
import { extractText } from "@stirling-image/ai";
|
||||
import { createWorkspace } from "../../lib/workspace.js";
|
||||
|
||||
/**
|
||||
* OCR / text extraction route.
|
||||
* Returns JSON with extracted text rather than an image.
|
||||
*/
|
||||
export function registerOcr(app: FastifyInstance) {
|
||||
app.post(
|
||||
"/api/v1/tools/ocr",
|
||||
async (request: FastifyRequest, reply: FastifyReply) => {
|
||||
let fileBuffer: Buffer | null = null;
|
||||
let filename = "image";
|
||||
let settingsRaw: string | null = null;
|
||||
|
||||
try {
|
||||
const parts = request.parts();
|
||||
for await (const part of parts) {
|
||||
if (part.type === "file") {
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of part.file) {
|
||||
chunks.push(chunk);
|
||||
}
|
||||
fileBuffer = Buffer.concat(chunks);
|
||||
filename = basename(part.filename ?? "image");
|
||||
} else if (part.fieldname === "settings") {
|
||||
settingsRaw = part.value as string;
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
return reply.status(400).send({
|
||||
error: "Failed to parse multipart request",
|
||||
details: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
}
|
||||
|
||||
if (!fileBuffer || fileBuffer.length === 0) {
|
||||
return reply.status(400).send({ error: "No image file provided" });
|
||||
}
|
||||
|
||||
try {
|
||||
const settings = settingsRaw ? JSON.parse(settingsRaw) : {};
|
||||
const jobId = randomUUID();
|
||||
const workspacePath = await createWorkspace(jobId);
|
||||
|
||||
const result = await extractText(fileBuffer, workspacePath, {
|
||||
engine: settings.engine,
|
||||
language: settings.language,
|
||||
});
|
||||
|
||||
return reply.send({
|
||||
jobId,
|
||||
filename,
|
||||
text: result.text,
|
||||
engine: result.engine,
|
||||
});
|
||||
} catch (err) {
|
||||
return reply.status(422).send({
|
||||
error: "OCR failed",
|
||||
details: err instanceof Error ? err.message : "Unknown error",
|
||||
});
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
import type { FastifyInstance, FastifyRequest, FastifyReply } from "fastify";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import { join, basename } from "node:path";
|
||||
import { removeBackground } from "@stirling-image/ai";
|
||||
import { createWorkspace } from "../../lib/workspace.js";
|
||||
|
||||
/**
|
||||
* AI background removal route.
|
||||
* Uses Python + rembg under the hood.
|
||||
*/
|
||||
export function registerRemoveBackground(app: FastifyInstance) {
|
||||
app.post(
|
||||
"/api/v1/tools/remove-background",
|
||||
async (request: FastifyRequest, reply: FastifyReply) => {
|
||||
let fileBuffer: Buffer | null = null;
|
||||
let filename = "image";
|
||||
let settingsRaw: string | null = null;
|
||||
|
||||
try {
|
||||
const parts = request.parts();
|
||||
for await (const part of parts) {
|
||||
if (part.type === "file") {
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of part.file) {
|
||||
chunks.push(chunk);
|
||||
}
|
||||
fileBuffer = Buffer.concat(chunks);
|
||||
filename = basename(part.filename ?? "image");
|
||||
} else if (part.fieldname === "settings") {
|
||||
settingsRaw = part.value as string;
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
return reply.status(400).send({
|
||||
error: "Failed to parse multipart request",
|
||||
details: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
}
|
||||
|
||||
if (!fileBuffer || fileBuffer.length === 0) {
|
||||
return reply.status(400).send({ error: "No image file provided" });
|
||||
}
|
||||
|
||||
try {
|
||||
const settings = settingsRaw ? JSON.parse(settingsRaw) : {};
|
||||
const jobId = randomUUID();
|
||||
const workspacePath = await createWorkspace(jobId);
|
||||
|
||||
// Save input
|
||||
const inputPath = join(workspacePath, "input", filename);
|
||||
await writeFile(inputPath, fileBuffer);
|
||||
|
||||
// Process
|
||||
const resultBuffer = await removeBackground(
|
||||
fileBuffer,
|
||||
join(workspacePath, "output"),
|
||||
{ model: settings.model },
|
||||
);
|
||||
|
||||
// Save output
|
||||
const outputFilename = filename.replace(/\.[^.]+$/, "") + "_nobg.png";
|
||||
const outputPath = join(workspacePath, "output", outputFilename);
|
||||
await writeFile(outputPath, resultBuffer);
|
||||
|
||||
return reply.send({
|
||||
jobId,
|
||||
downloadUrl: `/api/v1/download/${jobId}/${encodeURIComponent(outputFilename)}`,
|
||||
originalSize: fileBuffer.length,
|
||||
processedSize: resultBuffer.length,
|
||||
});
|
||||
} catch (err) {
|
||||
return reply.status(422).send({
|
||||
error: "Background removal failed",
|
||||
details: err instanceof Error ? err.message : "Unknown error",
|
||||
});
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
import { z } from "zod";
|
||||
import sharp from "sharp";
|
||||
import type { FastifyInstance } from "fastify";
|
||||
import { createToolRoute } from "../tool-factory.js";
|
||||
|
||||
const settingsSchema = z.object({
|
||||
width: z.number().int().positive(),
|
||||
height: z.number().int().positive(),
|
||||
});
|
||||
|
||||
/**
|
||||
* Smart crop using Sharp's attention-based strategy.
|
||||
* Uses entropy/saliency detection to find the most interesting region.
|
||||
* No Python needed.
|
||||
*/
|
||||
export function registerSmartCrop(app: FastifyInstance) {
|
||||
createToolRoute(app, {
|
||||
toolId: "smart-crop",
|
||||
settingsSchema,
|
||||
process: async (inputBuffer, settings, filename) => {
|
||||
const result = await sharp(inputBuffer)
|
||||
.resize(settings.width, settings.height, {
|
||||
fit: "cover",
|
||||
position: sharp.strategy.attention,
|
||||
})
|
||||
.png()
|
||||
.toBuffer();
|
||||
|
||||
const outputFilename = filename.replace(/\.[^.]+$/, "") + "_smartcrop.png";
|
||||
return { buffer: result, filename: outputFilename, contentType: "image/png" };
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
import type { FastifyInstance, FastifyRequest, FastifyReply } from "fastify";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import { join, basename } from "node:path";
|
||||
import { upscale } from "@stirling-image/ai";
|
||||
import { createWorkspace } from "../../lib/workspace.js";
|
||||
|
||||
/**
|
||||
* AI image upscaling route.
|
||||
* Uses Real-ESRGAN when available, falls back to Lanczos.
|
||||
*/
|
||||
export function registerUpscale(app: FastifyInstance) {
|
||||
app.post(
|
||||
"/api/v1/tools/upscale",
|
||||
async (request: FastifyRequest, reply: FastifyReply) => {
|
||||
let fileBuffer: Buffer | null = null;
|
||||
let filename = "image";
|
||||
let settingsRaw: string | null = null;
|
||||
|
||||
try {
|
||||
const parts = request.parts();
|
||||
for await (const part of parts) {
|
||||
if (part.type === "file") {
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of part.file) {
|
||||
chunks.push(chunk);
|
||||
}
|
||||
fileBuffer = Buffer.concat(chunks);
|
||||
filename = basename(part.filename ?? "image");
|
||||
} else if (part.fieldname === "settings") {
|
||||
settingsRaw = part.value as string;
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
return reply.status(400).send({
|
||||
error: "Failed to parse multipart request",
|
||||
details: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
}
|
||||
|
||||
if (!fileBuffer || fileBuffer.length === 0) {
|
||||
return reply.status(400).send({ error: "No image file provided" });
|
||||
}
|
||||
|
||||
try {
|
||||
const settings = settingsRaw ? JSON.parse(settingsRaw) : {};
|
||||
const scale = Number(settings.scale) || 2;
|
||||
|
||||
const jobId = randomUUID();
|
||||
const workspacePath = await createWorkspace(jobId);
|
||||
|
||||
// Save input
|
||||
const inputPath = join(workspacePath, "input", filename);
|
||||
await writeFile(inputPath, fileBuffer);
|
||||
|
||||
// Process
|
||||
const result = await upscale(
|
||||
fileBuffer,
|
||||
join(workspacePath, "output"),
|
||||
{ scale },
|
||||
);
|
||||
|
||||
// Save output
|
||||
const outputFilename =
|
||||
filename.replace(/\.[^.]+$/, "") + `_${scale}x.png`;
|
||||
const outputPath = join(workspacePath, "output", outputFilename);
|
||||
await writeFile(outputPath, result.buffer);
|
||||
|
||||
return reply.send({
|
||||
jobId,
|
||||
downloadUrl: `/api/v1/download/${jobId}/${encodeURIComponent(outputFilename)}`,
|
||||
originalSize: fileBuffer.length,
|
||||
processedSize: result.buffer.length,
|
||||
width: result.width,
|
||||
height: result.height,
|
||||
method: result.method,
|
||||
});
|
||||
} catch (err) {
|
||||
return reply.status(422).send({
|
||||
error: "Upscaling failed",
|
||||
details: err instanceof Error ? err.message : "Unknown error",
|
||||
});
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user