feat(api,web): add batch processing with ZIP download and SSE progress

Backend: POST /api/v1/tools/:toolId/batch accepts multiple files +
settings, processes via p-queue with CONCURRENT_JOBS concurrency limit,
streams ZIP response using archiver. Tool registry in tool-factory
enables batch to reuse any registered tool's process function. SSE
endpoint at GET /api/v1/jobs/:jobId/progress provides real-time updates.
Handles partial failures gracefully, preserves filenames, deduplicates
collisions. Frontend: use-batch-processor hook handles upload, SSE
progress tracking, and automatic ZIP download.
This commit is contained in:
Siddharth Kumar Sah
2026-03-22 04:03:38 +08:00
parent 1a22522349
commit ce641168c1
7 changed files with 1141 additions and 11 deletions
+18
View File
@@ -19,6 +19,21 @@ export interface ToolRouteConfig<T> {
) => Promise<{ buffer: Buffer; filename: string; contentType: string }>;
}
/**
* In-memory registry of all tool configs, keyed by toolId.
* Populated by createToolRoute() calls; used by batch processing.
*/
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const toolRegistry = new Map<string, ToolRouteConfig<any>>();
/**
* Retrieve a registered tool config by its ID.
*/
// eslint-disable-next-line @typescript-eslint/no-explicit-any
export function getToolConfig(toolId: string): ToolRouteConfig<any> | undefined {
return toolRegistry.get(toolId);
}
/**
* Sanitize a filename to prevent path traversal attacks.
*/
@@ -51,6 +66,9 @@ export function createToolRoute<T>(
app: FastifyInstance,
config: ToolRouteConfig<T>,
): void {
// Register in the tool registry for batch processing
toolRegistry.set(config.toolId, config);
app.post(
`/api/v1/tools/${config.toolId}`,
async (request: FastifyRequest, reply: FastifyReply) => {