mirror of
https://github.com/snapotter-hq/SnapOtter.git
synced 2026-08-03 07:46:42 +02:00
fix: resolve Sharp 0.35.1 and BullMQ type incompatibilities after dep bumps
Sharp 0.35.1 moved FormatEnum to a namespace export and removed "avif" from FormatEnum (now a separate literal in toFormat). BullMQ 5.78.1 bundles ioredis 5.10.1 while we have 5.11.1, causing structural type mismatch. Also fixes new Biome 1.9 lint rules.
This commit is contained in:
@@ -16,7 +16,7 @@ import type Redis from "ioredis";
|
||||
import { db, schema } from "../db/index.js";
|
||||
import { createRedisConnection, sharedRedis } from "./connection.js";
|
||||
import { getQueue } from "./queues.js";
|
||||
import { bullPrefix, POOLS, type Pool } from "./types.js";
|
||||
import { bullPrefix, POOLS } from "./types.js";
|
||||
|
||||
// ── Per-worker cancel registry ──────────────────────────────────
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
* Uses ioredis with settings compatible with BullMQ's requirements
|
||||
* (maxRetriesPerRequest: null for blocking commands).
|
||||
*/
|
||||
import type { ConnectionOptions } from "bullmq";
|
||||
import Redis from "ioredis";
|
||||
import { env } from "../config.js";
|
||||
|
||||
@@ -19,6 +20,11 @@ export function createRedisConnection(): Redis {
|
||||
});
|
||||
}
|
||||
|
||||
// ioredis 5.11 vs BullMQ's bundled 5.10 type mismatch
|
||||
export function createBullMQConnection(): ConnectionOptions {
|
||||
return createRedisConnection() as unknown as ConnectionOptions;
|
||||
}
|
||||
|
||||
let _shared: Redis | null = null;
|
||||
|
||||
/**
|
||||
|
||||
@@ -10,9 +10,9 @@ import { FlowProducer, type Job, QueueEvents } from "bullmq";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { env } from "../config.js";
|
||||
import { db, schema } from "../db/index.js";
|
||||
import { createRedisConnection } from "./connection.js";
|
||||
import { createBullMQConnection } from "./connection.js";
|
||||
import { getQueue } from "./queues.js";
|
||||
import { POOLS, type Pool, queueName, type ToolJobData, type ToolJobResult } from "./types.js";
|
||||
import { type Pool, queueName, type ToolJobData, type ToolJobResult } from "./types.js";
|
||||
|
||||
// ── QueueEvents (one per pool, lazy) ────────────────────────────
|
||||
|
||||
@@ -22,7 +22,7 @@ function getQueueEvents(pool: Pool): QueueEvents {
|
||||
let qe = queueEventsMap.get(pool);
|
||||
if (!qe) {
|
||||
qe = new QueueEvents(queueName(pool), {
|
||||
connection: createRedisConnection(),
|
||||
connection: createBullMQConnection(),
|
||||
});
|
||||
queueEventsMap.set(pool, qe);
|
||||
}
|
||||
@@ -42,7 +42,7 @@ let _flowProducer: FlowProducer | null = null;
|
||||
export function getFlowProducer(): FlowProducer {
|
||||
if (!_flowProducer) {
|
||||
_flowProducer = new FlowProducer({
|
||||
connection: createRedisConnection(),
|
||||
connection: createBullMQConnection(),
|
||||
});
|
||||
}
|
||||
return _flowProducer;
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
* default job options (retry policy, TTL-based cleanup).
|
||||
*/
|
||||
import { Queue } from "bullmq";
|
||||
import { createRedisConnection } from "./connection.js";
|
||||
import { createBullMQConnection } from "./connection.js";
|
||||
import { POOLS, type Pool, queueName, type ToolJobData, type ToolJobResult } from "./types.js";
|
||||
|
||||
const queues = new Map<Pool, Queue<ToolJobData, ToolJobResult>>();
|
||||
@@ -15,7 +15,7 @@ export function getQueue(pool: Pool): Queue<ToolJobData, ToolJobResult> {
|
||||
let q = queues.get(pool);
|
||||
if (!q) {
|
||||
q = new Queue<ToolJobData, ToolJobResult>(queueName(pool), {
|
||||
connection: createRedisConnection(),
|
||||
connection: createBullMQConnection(),
|
||||
defaultJobOptions: {
|
||||
attempts: pool === "ai" ? 1 : 2,
|
||||
backoff: { type: "exponential", delay: 1000 },
|
||||
|
||||
@@ -35,10 +35,10 @@ async function readSettingValue(key: string): Promise<string | null> {
|
||||
return row?.value ?? null;
|
||||
}
|
||||
|
||||
export async function runSiemForward(): Promise<{ forwarded: number } | void> {
|
||||
export async function runSiemForward(): Promise<{ forwarded: number } | undefined> {
|
||||
// 1. Read SIEM config
|
||||
const config = await readSiemConfig();
|
||||
if (!config || !config.enabled || !config.webhookUrl) {
|
||||
if (!config?.enabled || !config.webhookUrl) {
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
@@ -43,8 +43,8 @@ import {
|
||||
import { hasAiJobHandler, runAiToolJob } from "./ai-handlers.js";
|
||||
import { recordChildOutcome } from "./batch-progress.js";
|
||||
import { registerCancelable, unregisterCancelable } from "./cancel.js";
|
||||
import { createRedisConnection } from "./connection.js";
|
||||
import { autoSaveToLibrary, buildOutputName, generatePreview } from "./postprocess.js";
|
||||
import { createBullMQConnection } from "./connection.js";
|
||||
import { buildOutputName, generatePreview } from "./postprocess.js";
|
||||
import { runSystemJob } from "./system-jobs.js";
|
||||
import { POOLS, type Pool, queueName, type ToolJobData, type ToolJobResult } from "./types.js";
|
||||
|
||||
@@ -578,7 +578,7 @@ async function processPipelineFinalize(job: Job<ToolJobData>): Promise<ToolJobRe
|
||||
// Copy last step's output to outputs/<pipelineJobId>/<filename> so
|
||||
// the legacy download URL /api/v1/download/<pipelineJobId>/... works.
|
||||
const lastOutputBuffer = await getObjectBuffer(lastOutputRef);
|
||||
const outFilename = lastOutputRef.split("/").pop()!;
|
||||
const outFilename = lastOutputRef.split("/").pop() ?? "output";
|
||||
const parentKey = `outputs/${data.jobId}/${outFilename}`;
|
||||
await putObject(parentKey, lastOutputBuffer);
|
||||
|
||||
@@ -636,13 +636,15 @@ async function processPipelineFinalize(job: Job<ToolJobData>): Promise<ToolJobRe
|
||||
* failures would skip the DB write and leave the row "processing".
|
||||
*/
|
||||
async function processBatchChild(job: Job<ToolJobData>): Promise<ToolJobResult> {
|
||||
const parentId = job.data.parentId ?? "";
|
||||
const totalFiles = job.data.totalFiles ?? 0;
|
||||
try {
|
||||
const result = await processToolJob(job);
|
||||
await recordChildOutcome(job.data.parentId!, job.data.totalFiles!, job.data.filename);
|
||||
await recordChildOutcome(parentId, totalFiles, job.data.filename);
|
||||
return result;
|
||||
} catch (err) {
|
||||
const error = stripInternalPaths(err instanceof Error ? err.message : String(err));
|
||||
await recordChildOutcome(job.data.parentId!, job.data.totalFiles!, job.data.filename, error);
|
||||
await recordChildOutcome(parentId, totalFiles, job.data.filename, error);
|
||||
// Return a completed job with a failure marker so the parent runs.
|
||||
return {
|
||||
outputRefs: [],
|
||||
@@ -687,7 +689,7 @@ async function processBatchFinalize(job: Job<ToolJobData>): Promise<ToolJobResul
|
||||
}
|
||||
|
||||
if (row.status === "completed" && row.outputRefs?.[0]) {
|
||||
const outFilename = row.outputRefs[0].split("/").pop()!;
|
||||
const outFilename = row.outputRefs[0].split("/").pop() ?? "output";
|
||||
manifest.push({ index: i, filename: outFilename, outputRef: row.outputRefs[0] });
|
||||
} else {
|
||||
const errorMsg = (row.error as { message?: string } | null)?.message ?? "Processing failed";
|
||||
@@ -732,7 +734,7 @@ export function startWorkers(): void {
|
||||
};
|
||||
|
||||
const worker = new Worker<ToolJobData, unknown>(queueName(pool), systemProcessor, {
|
||||
connection: createRedisConnection(),
|
||||
connection: createBullMQConnection(),
|
||||
concurrency: workerConcurrency,
|
||||
stalledInterval: 30_000,
|
||||
});
|
||||
@@ -754,7 +756,7 @@ export function startWorkers(): void {
|
||||
};
|
||||
|
||||
const worker = new Worker<ToolJobData, ToolJobResult>(queueName(pool), processor, {
|
||||
connection: createRedisConnection(),
|
||||
connection: createBullMQConnection(),
|
||||
concurrency: workerConcurrency,
|
||||
stalledInterval: 30_000,
|
||||
});
|
||||
|
||||
@@ -1,25 +1,25 @@
|
||||
import sharp from "sharp";
|
||||
|
||||
type SharpFormat = keyof sharp.FormatEnum | "avif";
|
||||
|
||||
export interface OutputFormat {
|
||||
format: keyof sharp.FormatEnum;
|
||||
format: SharpFormat;
|
||||
extension: string;
|
||||
contentType: string;
|
||||
quality: number;
|
||||
}
|
||||
|
||||
const FORMAT_MAP: Record<
|
||||
string,
|
||||
{ format: keyof sharp.FormatEnum; extension: string; contentType: string }
|
||||
> = {
|
||||
jpeg: { format: "jpeg", extension: "jpg", contentType: "image/jpeg" },
|
||||
png: { format: "png", extension: "png", contentType: "image/png" },
|
||||
webp: { format: "webp", extension: "webp", contentType: "image/webp" },
|
||||
gif: { format: "gif", extension: "gif", contentType: "image/gif" },
|
||||
tiff: { format: "tiff", extension: "tiff", contentType: "image/tiff" },
|
||||
avif: { format: "avif", extension: "avif", contentType: "image/avif" },
|
||||
heif: { format: "avif", extension: "avif", contentType: "image/avif" },
|
||||
jxl: { format: "jxl" as keyof sharp.FormatEnum, extension: "jxl", contentType: "image/jxl" },
|
||||
};
|
||||
const FORMAT_MAP: Record<string, { format: SharpFormat; extension: string; contentType: string }> =
|
||||
{
|
||||
jpeg: { format: "jpeg", extension: "jpg", contentType: "image/jpeg" },
|
||||
png: { format: "png", extension: "png", contentType: "image/png" },
|
||||
webp: { format: "webp", extension: "webp", contentType: "image/webp" },
|
||||
gif: { format: "gif", extension: "gif", contentType: "image/gif" },
|
||||
tiff: { format: "tiff", extension: "tiff", contentType: "image/tiff" },
|
||||
avif: { format: "avif", extension: "avif", contentType: "image/avif" },
|
||||
heif: { format: "avif", extension: "avif", contentType: "image/avif" },
|
||||
jxl: { format: "jxl", extension: "jxl", contentType: "image/jxl" },
|
||||
};
|
||||
|
||||
const DEFAULT_QUALITY = 95;
|
||||
const PNG_FALLBACK = FORMAT_MAP.png;
|
||||
|
||||
@@ -26,7 +26,7 @@ export async function deliverWebhook(
|
||||
});
|
||||
|
||||
const headers: Record<string, string> = { "Content-Type": "application/json" };
|
||||
if (authHeader) headers["Authorization"] = authHeader;
|
||||
if (authHeader) headers.Authorization = authHeader;
|
||||
|
||||
let lastError: string | undefined;
|
||||
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
||||
|
||||
@@ -142,7 +142,7 @@ export function registerCompose(app: FastifyInstance) {
|
||||
input: processedOverlay,
|
||||
top: settings.y,
|
||||
left: settings.x,
|
||||
blend: settings.blendMode as import("sharp").Blend,
|
||||
blend: settings.blendMode as sharp.Blend,
|
||||
},
|
||||
])
|
||||
.toBuffer();
|
||||
|
||||
@@ -120,9 +120,7 @@ export function registerConvert(app: FastifyInstance) {
|
||||
}
|
||||
|
||||
const inputExt = extname(filename).toLowerCase().replace(".", "");
|
||||
const sharpOpts: import("sharp").SharpOptions = isSvgBuffer(inputBuffer)
|
||||
? { density: 300 }
|
||||
: {};
|
||||
const sharpOpts: sharp.SharpOptions = isSvgBuffer(inputBuffer) ? { density: 300 } : {};
|
||||
// Preserve animation frames when both input and output are animatable formats
|
||||
if (ANIMATABLE_FORMATS.has(inputExt) && ANIMATABLE_FORMATS.has(settings.format)) {
|
||||
sharpOpts.animated = true;
|
||||
|
||||
@@ -24,14 +24,16 @@ const settingsSchema = z.object({
|
||||
quality: z.number().min(1).max(100).default(90),
|
||||
});
|
||||
|
||||
type SharpFormat = keyof sharp.FormatEnum | "avif";
|
||||
|
||||
function resolveOutputFormat(
|
||||
outputFormat: string,
|
||||
originalExt: string,
|
||||
): { sharpFormat: keyof sharp.FormatEnum | null; ext: string } {
|
||||
): { sharpFormat: SharpFormat | null; ext: string } {
|
||||
if (outputFormat === "original") {
|
||||
return { sharpFormat: null, ext: originalExt };
|
||||
}
|
||||
const map: Record<string, { sharpFormat: keyof sharp.FormatEnum; ext: string }> = {
|
||||
const map: Record<string, { sharpFormat: SharpFormat; ext: string }> = {
|
||||
png: { sharpFormat: "png", ext: ".png" },
|
||||
jpg: { sharpFormat: "jpeg", ext: ".jpg" },
|
||||
webp: { sharpFormat: "webp", ext: ".webp" },
|
||||
|
||||
@@ -209,7 +209,7 @@ export function registerStripMetadata(app: FastifyInstance) {
|
||||
case "webp":
|
||||
result.webp({ quality: 85 });
|
||||
break;
|
||||
case "avif":
|
||||
case "heif":
|
||||
result.avif({ quality: 50 });
|
||||
break;
|
||||
case "tiff":
|
||||
|
||||
Reference in New Issue
Block a user