mirror of
https://github.com/snapotter-hq/SnapOtter.git
synced 2026-08-03 07:46:42 +02:00
feat: make AI tools pipeline-compatible and add search to tool picker
Register remove-background, upscale, and blur-faces in the pipeline tool registry via registerToolProcessFn(). These tools keep their custom HTTP routes (with progress callbacks) for direct use, but now also provide a simple process function for pipeline/batch execution. Add a search bar to the pipeline tool picker so users can quickly find tools by name or description. Uses the existing SearchBar component and the same filtering pattern as the main tool panel. Update tests to reflect that these 3 AI tools are now pipeline- compatible (moved from excluded to included assertions).
This commit is contained in:
@@ -49,13 +49,21 @@ export function getToolConfig(toolId: string): AnyToolRouteConfig | undefined {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Return the IDs of all tools registered via createToolRoute().
|
* Return the IDs of all tools in the pipeline/batch registry.
|
||||||
* These are the tools that can be used in pipelines and batch processing.
|
|
||||||
*/
|
*/
|
||||||
export function getRegisteredToolIds(): string[] {
|
export function getRegisteredToolIds(): string[] {
|
||||||
return [...toolRegistry.keys()];
|
return [...toolRegistry.keys()];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Register a tool's process function in the pipeline/batch registry
|
||||||
|
* without creating an HTTP route. Use this for tools that have their
|
||||||
|
* own custom HTTP route but should still be usable in pipelines.
|
||||||
|
*/
|
||||||
|
export function registerToolProcessFn(config: AnyToolRouteConfig): void {
|
||||||
|
toolRegistry.set(config.toolId, config);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Factory that registers a POST /api/v1/tools/:toolId route.
|
* Factory that registers a POST /api/v1/tools/:toolId route.
|
||||||
*
|
*
|
||||||
|
|||||||
@@ -3,10 +3,12 @@ import { writeFile } from "node:fs/promises";
|
|||||||
import { basename, join } from "node:path";
|
import { basename, join } from "node:path";
|
||||||
import { blurFaces } from "@stirling-image/ai";
|
import { blurFaces } from "@stirling-image/ai";
|
||||||
import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
|
import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
|
||||||
|
import { z } from "zod";
|
||||||
import { autoOrient } from "../../lib/auto-orient.js";
|
import { autoOrient } from "../../lib/auto-orient.js";
|
||||||
import { validateImageBuffer } from "../../lib/file-validation.js";
|
import { validateImageBuffer } from "../../lib/file-validation.js";
|
||||||
import { createWorkspace } from "../../lib/workspace.js";
|
import { createWorkspace } from "../../lib/workspace.js";
|
||||||
import { updateSingleFileProgress } from "../progress.js";
|
import { updateSingleFileProgress } from "../progress.js";
|
||||||
|
import { registerToolProcessFn } from "../tool-factory.js";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Face detection and blurring route.
|
* Face detection and blurring route.
|
||||||
@@ -115,4 +117,26 @@ export function registerBlurFaces(app: FastifyInstance) {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// Register in the pipeline/batch registry so this tool can be used
|
||||||
|
// as a step in automation pipelines (without progress callbacks).
|
||||||
|
registerToolProcessFn({
|
||||||
|
toolId: "blur-faces",
|
||||||
|
settingsSchema: z.object({
|
||||||
|
blurRadius: z.number().min(1).max(100).default(30),
|
||||||
|
sensitivity: z.number().min(0).max(1).default(0.5),
|
||||||
|
}),
|
||||||
|
process: async (inputBuffer, settings, filename) => {
|
||||||
|
const s = settings as { blurRadius?: number; sensitivity?: number };
|
||||||
|
const orientedBuffer = await autoOrient(inputBuffer);
|
||||||
|
const jobId = randomUUID();
|
||||||
|
const workspacePath = await createWorkspace(jobId);
|
||||||
|
const result = await blurFaces(orientedBuffer, join(workspacePath, "output"), {
|
||||||
|
blurRadius: s.blurRadius ?? 30,
|
||||||
|
sensitivity: s.sensitivity ?? 0.5,
|
||||||
|
});
|
||||||
|
const outputFilename = `${filename.replace(/\.[^.]+$/, "")}_blurred.png`;
|
||||||
|
return { buffer: result.buffer, filename: outputFilename, contentType: "image/png" };
|
||||||
|
},
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,9 +3,11 @@ import { writeFile } from "node:fs/promises";
|
|||||||
import { basename, join } from "node:path";
|
import { basename, join } from "node:path";
|
||||||
import { removeBackground } from "@stirling-image/ai";
|
import { removeBackground } from "@stirling-image/ai";
|
||||||
import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
|
import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
|
||||||
|
import { z } from "zod";
|
||||||
import { validateImageBuffer } from "../../lib/file-validation.js";
|
import { validateImageBuffer } from "../../lib/file-validation.js";
|
||||||
import { createWorkspace } from "../../lib/workspace.js";
|
import { createWorkspace } from "../../lib/workspace.js";
|
||||||
import { updateSingleFileProgress } from "../progress.js";
|
import { updateSingleFileProgress } from "../progress.js";
|
||||||
|
import { registerToolProcessFn } from "../tool-factory.js";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* AI background removal route.
|
* AI background removal route.
|
||||||
@@ -108,4 +110,25 @@ export function registerRemoveBackground(app: FastifyInstance) {
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
|
|
||||||
|
// Register in the pipeline/batch registry so this tool can be used
|
||||||
|
// as a step in automation pipelines (without progress callbacks).
|
||||||
|
registerToolProcessFn({
|
||||||
|
toolId: "remove-background",
|
||||||
|
settingsSchema: z.object({
|
||||||
|
model: z.string().optional(),
|
||||||
|
backgroundColor: z.string().optional(),
|
||||||
|
}),
|
||||||
|
process: async (inputBuffer, settings, filename) => {
|
||||||
|
const s = settings as { model?: string; backgroundColor?: string };
|
||||||
|
const jobId = randomUUID();
|
||||||
|
const workspacePath = await createWorkspace(jobId);
|
||||||
|
const resultBuffer = await removeBackground(inputBuffer, join(workspacePath, "output"), {
|
||||||
|
model: s.model,
|
||||||
|
backgroundColor: s.backgroundColor,
|
||||||
|
});
|
||||||
|
const outputFilename = `${filename.replace(/\.[^.]+$/, "")}_nobg.png`;
|
||||||
|
return { buffer: resultBuffer, filename: outputFilename, contentType: "image/png" };
|
||||||
|
},
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,10 +3,12 @@ import { writeFile } from "node:fs/promises";
|
|||||||
import { basename, join } from "node:path";
|
import { basename, join } from "node:path";
|
||||||
import { upscale } from "@stirling-image/ai";
|
import { upscale } from "@stirling-image/ai";
|
||||||
import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
|
import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
|
||||||
|
import { z } from "zod";
|
||||||
import { autoOrient } from "../../lib/auto-orient.js";
|
import { autoOrient } from "../../lib/auto-orient.js";
|
||||||
import { validateImageBuffer } from "../../lib/file-validation.js";
|
import { validateImageBuffer } from "../../lib/file-validation.js";
|
||||||
import { createWorkspace } from "../../lib/workspace.js";
|
import { createWorkspace } from "../../lib/workspace.js";
|
||||||
import { updateSingleFileProgress } from "../progress.js";
|
import { updateSingleFileProgress } from "../progress.js";
|
||||||
|
import { registerToolProcessFn } from "../tool-factory.js";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* AI image upscaling route.
|
* AI image upscaling route.
|
||||||
@@ -114,4 +116,22 @@ export function registerUpscale(app: FastifyInstance) {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// Register in the pipeline/batch registry so this tool can be used
|
||||||
|
// as a step in automation pipelines (without progress callbacks).
|
||||||
|
registerToolProcessFn({
|
||||||
|
toolId: "upscale",
|
||||||
|
settingsSchema: z.object({
|
||||||
|
scale: z.union([z.number(), z.string()]).transform(Number).default(2),
|
||||||
|
}),
|
||||||
|
process: async (inputBuffer, settings, filename) => {
|
||||||
|
const scale = Number((settings as { scale?: number }).scale) || 2;
|
||||||
|
const orientedBuffer = await autoOrient(inputBuffer);
|
||||||
|
const jobId = randomUUID();
|
||||||
|
const workspacePath = await createWorkspace(jobId);
|
||||||
|
const result = await upscale(orientedBuffer, join(workspacePath, "output"), { scale });
|
||||||
|
const outputFilename = `${filename.replace(/\.[^.]+$/, "")}_${scale}x.png`;
|
||||||
|
return { buffer: result.buffer, filename: outputFilename, contentType: "image/png" };
|
||||||
|
},
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ import {
|
|||||||
X,
|
X,
|
||||||
} from "lucide-react";
|
} from "lucide-react";
|
||||||
import { useCallback, useEffect, useMemo, useState } from "react";
|
import { useCallback, useEffect, useMemo, useState } from "react";
|
||||||
|
import { SearchBar } from "@/components/common/search-bar";
|
||||||
import { apiGet } from "@/lib/api";
|
import { apiGet } from "@/lib/api";
|
||||||
import { cn } from "@/lib/utils";
|
import { cn } from "@/lib/utils";
|
||||||
import { PipelineStepSettings } from "./pipeline-step-settings";
|
import { PipelineStepSettings } from "./pipeline-step-settings";
|
||||||
@@ -64,6 +65,7 @@ export function PipelineBuilder({
|
|||||||
const [disabledTools, setDisabledTools] = useState<string[]>([]);
|
const [disabledTools, setDisabledTools] = useState<string[]>([]);
|
||||||
const [experimentalEnabled, setExperimentalEnabled] = useState(false);
|
const [experimentalEnabled, setExperimentalEnabled] = useState(false);
|
||||||
const [pipelineToolIds, setPipelineToolIds] = useState<string[] | null>(null);
|
const [pipelineToolIds, setPipelineToolIds] = useState<string[] | null>(null);
|
||||||
|
const [toolSearch, setToolSearch] = useState("");
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
apiGet<{ settings: Record<string, string> }>("/v1/settings")
|
apiGet<{ settings: Record<string, string> }>("/v1/settings")
|
||||||
@@ -82,14 +84,19 @@ export function PipelineBuilder({
|
|||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
const PIPELINE_TOOLS = useMemo(() => {
|
const PIPELINE_TOOLS = useMemo(() => {
|
||||||
|
const q = toolSearch.toLowerCase();
|
||||||
return PIPELINE_TOOLS_BASE.filter((t) => {
|
return PIPELINE_TOOLS_BASE.filter((t) => {
|
||||||
if (disabledTools.includes(t.id)) return false;
|
if (disabledTools.includes(t.id)) return false;
|
||||||
if (t.experimental && !experimentalEnabled) return false;
|
if (t.experimental && !experimentalEnabled) return false;
|
||||||
// Only show tools that are registered in the pipeline-compatible tool registry
|
// Only show tools that are registered in the pipeline-compatible tool registry
|
||||||
if (pipelineToolIds && !pipelineToolIds.includes(t.id)) return false;
|
if (pipelineToolIds && !pipelineToolIds.includes(t.id)) return false;
|
||||||
|
// Search filter
|
||||||
|
if (q && !t.name.toLowerCase().includes(q) && !t.description.toLowerCase().includes(q)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
return true;
|
return true;
|
||||||
});
|
});
|
||||||
}, [disabledTools, experimentalEnabled, pipelineToolIds]);
|
}, [disabledTools, experimentalEnabled, pipelineToolIds, toolSearch]);
|
||||||
|
|
||||||
const addStep = useCallback(
|
const addStep = useCallback(
|
||||||
(toolId: string) => {
|
(toolId: string) => {
|
||||||
@@ -296,39 +303,50 @@ export function PipelineBuilder({
|
|||||||
|
|
||||||
{/* Add Step */}
|
{/* Add Step */}
|
||||||
{showToolPicker ? (
|
{showToolPicker ? (
|
||||||
<div className="rounded-lg border border-border bg-background p-3 space-y-2 max-h-64 overflow-y-auto">
|
<div className="rounded-lg border border-border bg-background p-3 space-y-2 max-h-80 overflow-y-auto">
|
||||||
<div className="flex items-center justify-between mb-2">
|
<div className="flex items-center justify-between mb-2">
|
||||||
<span className="text-sm font-medium text-foreground">Add a step</span>
|
<span className="text-sm font-medium text-foreground">Add a step</span>
|
||||||
<button
|
<button
|
||||||
type="button"
|
type="button"
|
||||||
onClick={() => setShowToolPicker(false)}
|
onClick={() => {
|
||||||
|
setShowToolPicker(false);
|
||||||
|
setToolSearch("");
|
||||||
|
}}
|
||||||
className="p-1 rounded hover:bg-muted text-muted-foreground"
|
className="p-1 rounded hover:bg-muted text-muted-foreground"
|
||||||
>
|
>
|
||||||
<X className="h-4 w-4" />
|
<X className="h-4 w-4" />
|
||||||
</button>
|
</button>
|
||||||
</div>
|
</div>
|
||||||
{PIPELINE_TOOLS.map((tool) => {
|
<SearchBar value={toolSearch} onChange={setToolSearch} placeholder="Search tools..." />
|
||||||
const Icon = iconsMap[tool.icon] || icons.FileImage;
|
{PIPELINE_TOOLS.length === 0 ? (
|
||||||
return (
|
<p className="text-sm text-muted-foreground text-center py-4">No tools found</p>
|
||||||
<button
|
) : (
|
||||||
key={tool.id}
|
PIPELINE_TOOLS.map((tool) => {
|
||||||
type="button"
|
const Icon = iconsMap[tool.icon] || icons.FileImage;
|
||||||
onClick={() => addStep(tool.id)}
|
return (
|
||||||
className="flex items-center gap-2 w-full px-3 py-2 rounded-lg hover:bg-muted text-sm text-left transition-colors"
|
<button
|
||||||
>
|
key={tool.id}
|
||||||
<Icon className="h-4 w-4 text-muted-foreground shrink-0" />
|
type="button"
|
||||||
<div className="flex-1 min-w-0">
|
onClick={() => addStep(tool.id)}
|
||||||
<div className="font-medium text-foreground">{tool.name}</div>
|
className="flex items-center gap-2 w-full px-3 py-2 rounded-lg hover:bg-muted text-sm text-left transition-colors"
|
||||||
<div className="text-xs text-muted-foreground truncate">{tool.description}</div>
|
>
|
||||||
</div>
|
<Icon className="h-4 w-4 text-muted-foreground shrink-0" />
|
||||||
</button>
|
<div className="flex-1 min-w-0">
|
||||||
);
|
<div className="font-medium text-foreground">{tool.name}</div>
|
||||||
})}
|
<div className="text-xs text-muted-foreground truncate">{tool.description}</div>
|
||||||
|
</div>
|
||||||
|
</button>
|
||||||
|
);
|
||||||
|
})
|
||||||
|
)}
|
||||||
</div>
|
</div>
|
||||||
) : (
|
) : (
|
||||||
<button
|
<button
|
||||||
type="button"
|
type="button"
|
||||||
onClick={() => setShowToolPicker(true)}
|
onClick={() => {
|
||||||
|
setShowToolPicker(true);
|
||||||
|
setToolSearch("");
|
||||||
|
}}
|
||||||
className="flex items-center gap-2 w-full justify-center px-4 py-2.5 rounded-lg border border-dashed border-border text-sm text-muted-foreground hover:border-primary hover:text-primary transition-colors"
|
className="flex items-center gap-2 w-full justify-center px-4 py-2.5 rounded-lg border border-dashed border-border text-sm text-muted-foreground hover:border-primary hover:text-primary transition-colors"
|
||||||
>
|
>
|
||||||
<Plus className="h-4 w-4" />
|
<Plus className="h-4 w-4" />
|
||||||
|
|||||||
@@ -2342,18 +2342,26 @@ describe("Pipeline", () => {
|
|||||||
expect(toolIds).toContain("rotate");
|
expect(toolIds).toContain("rotate");
|
||||||
});
|
});
|
||||||
|
|
||||||
it("excludes custom-route tools that are not pipeline-compatible", async () => {
|
it("includes AI tools registered for pipeline use", async () => {
|
||||||
|
const res = await app.inject({
|
||||||
|
method: "GET",
|
||||||
|
url: "/api/v1/pipeline/tools",
|
||||||
|
headers: { authorization: `Bearer ${adminToken}` },
|
||||||
|
});
|
||||||
|
const { toolIds } = JSON.parse(res.body);
|
||||||
|
expect(toolIds).toContain("remove-background");
|
||||||
|
expect(toolIds).toContain("upscale");
|
||||||
|
expect(toolIds).toContain("blur-faces");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("excludes tools that are not pipeline-compatible", async () => {
|
||||||
const res = await app.inject({
|
const res = await app.inject({
|
||||||
method: "GET",
|
method: "GET",
|
||||||
url: "/api/v1/pipeline/tools",
|
url: "/api/v1/pipeline/tools",
|
||||||
headers: { authorization: `Bearer ${adminToken}` },
|
headers: { authorization: `Bearer ${adminToken}` },
|
||||||
});
|
});
|
||||||
const { toolIds } = JSON.parse(res.body);
|
const { toolIds } = JSON.parse(res.body);
|
||||||
// These tools use custom routes and are not in the tool registry
|
|
||||||
expect(toolIds).not.toContain("remove-background");
|
|
||||||
expect(toolIds).not.toContain("upscale");
|
|
||||||
expect(toolIds).not.toContain("ocr");
|
expect(toolIds).not.toContain("ocr");
|
||||||
expect(toolIds).not.toContain("blur-faces");
|
|
||||||
expect(toolIds).not.toContain("erase-object");
|
expect(toolIds).not.toContain("erase-object");
|
||||||
expect(toolIds).not.toContain("info");
|
expect(toolIds).not.toContain("info");
|
||||||
expect(toolIds).not.toContain("collage");
|
expect(toolIds).not.toContain("collage");
|
||||||
@@ -2361,8 +2369,8 @@ describe("Pipeline", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe("Pipeline rejects custom-route tools", () => {
|
describe("Pipeline rejects incompatible tools", () => {
|
||||||
const customRouteTools = ["remove-background", "upscale", "ocr", "blur-faces", "erase-object"];
|
const customRouteTools = ["ocr", "erase-object"];
|
||||||
|
|
||||||
for (const toolId of customRouteTools) {
|
for (const toolId of customRouteTools) {
|
||||||
it(`returns 400 when pipeline uses "${toolId}" (custom-route tool)`, async () => {
|
it(`returns 400 when pipeline uses "${toolId}" (custom-route tool)`, async () => {
|
||||||
@@ -2669,8 +2677,8 @@ describe("Batch processing", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe("Batch rejects custom-route tools", () => {
|
describe("Batch rejects incompatible tools", () => {
|
||||||
const customRouteTools = ["remove-background", "upscale", "ocr", "blur-faces", "erase-object"];
|
const customRouteTools = ["ocr", "erase-object"];
|
||||||
|
|
||||||
for (const toolId of customRouteTools) {
|
for (const toolId of customRouteTools) {
|
||||||
it(`returns 404 for batch "${toolId}" (custom-route tool)`, async () => {
|
it(`returns 404 for batch "${toolId}" (custom-route tool)`, async () => {
|
||||||
|
|||||||
Reference in New Issue
Block a user