mirror of
https://github.com/snapotter-hq/SnapOtter.git
synced 2026-08-03 07:46:42 +02:00
Merge pull request #80 from ashim-hq/feat/unlimited-by-default
feat: Unlimited by Default — remove all artificial limits
This commit is contained in:
+29
-8
@@ -1,3 +1,4 @@
|
||||
import { availableParallelism } from "node:os";
|
||||
import { z } from "zod";
|
||||
|
||||
const envSchema = z.object({
|
||||
@@ -13,13 +14,13 @@ const envSchema = z.object({
|
||||
.default("false")
|
||||
.transform((v) => v === "true"),
|
||||
STORAGE_MODE: z.enum(["local", "s3"]).default("local"),
|
||||
FILE_MAX_AGE_HOURS: z.coerce.number().default(24),
|
||||
CLEANUP_INTERVAL_MINUTES: z.coerce.number().default(30),
|
||||
MAX_UPLOAD_SIZE_MB: z.coerce.number().default(100),
|
||||
MAX_BATCH_SIZE: z.coerce.number().default(200),
|
||||
CONCURRENT_JOBS: z.coerce.number().default(3),
|
||||
MAX_MEGAPIXELS: z.coerce.number().default(100),
|
||||
RATE_LIMIT_PER_MIN: z.coerce.number().default(100),
|
||||
FILE_MAX_AGE_HOURS: z.coerce.number().default(72),
|
||||
CLEANUP_INTERVAL_MINUTES: z.coerce.number().default(60),
|
||||
MAX_UPLOAD_SIZE_MB: z.coerce.number().default(0),
|
||||
MAX_BATCH_SIZE: z.coerce.number().default(0),
|
||||
CONCURRENT_JOBS: z.coerce.number().default(0),
|
||||
MAX_MEGAPIXELS: z.coerce.number().default(0),
|
||||
RATE_LIMIT_PER_MIN: z.coerce.number().default(0),
|
||||
DB_PATH: z.string().default("./data/ashim.db"),
|
||||
FILES_STORAGE_PATH: z.string().default("./data/files"),
|
||||
WORKSPACE_PATH: z.string().default("./tmp/workspace"),
|
||||
@@ -27,8 +28,18 @@ const envSchema = z.object({
|
||||
DEFAULT_LOCALE: z.string().default("en"),
|
||||
APP_NAME: z.string().default("ashim"),
|
||||
CORS_ORIGIN: z.string().default(""),
|
||||
MAX_USERS: z.coerce.number().default(5),
|
||||
MAX_USERS: z.coerce.number().default(0),
|
||||
LOG_LEVEL: z.enum(["fatal", "error", "warn", "info", "debug", "trace"]).default("info"),
|
||||
MAX_WORKER_THREADS: z.coerce.number().default(0),
|
||||
PROCESSING_TIMEOUT_S: z.coerce.number().default(0),
|
||||
MAX_PIPELINE_STEPS: z.coerce.number().default(0),
|
||||
MAX_CANVAS_PIXELS: z.coerce.number().default(0),
|
||||
MAX_SVG_SIZE_MB: z.coerce.number().default(0),
|
||||
MAX_LOGO_SIZE_KB: z.coerce.number().default(2048),
|
||||
MAX_SPLIT_GRID: z.coerce.number().default(100),
|
||||
MAX_PDF_PAGES: z.coerce.number().default(0),
|
||||
SESSION_DURATION_HOURS: z.coerce.number().default(168),
|
||||
LOGIN_ATTEMPT_LIMIT: z.coerce.number().default(10),
|
||||
});
|
||||
|
||||
export type Env = z.infer<typeof envSchema>;
|
||||
@@ -36,3 +47,13 @@ export type Env = z.infer<typeof envSchema>;
|
||||
export function loadEnv(): Env {
|
||||
return envSchema.parse(process.env);
|
||||
}
|
||||
|
||||
export function resolveConcurrency(env: Env): number {
|
||||
if (env.CONCURRENT_JOBS > 0) return env.CONCURRENT_JOBS;
|
||||
return Math.max(2, availableParallelism() - 1);
|
||||
}
|
||||
|
||||
export function resolveWorkerThreads(env: Env): number {
|
||||
if (env.MAX_WORKER_THREADS > 0) return env.MAX_WORKER_THREADS;
|
||||
return Math.max(2, availableParallelism() - 1);
|
||||
}
|
||||
|
||||
@@ -51,7 +51,7 @@ export async function inspectMetadata(buffer: Buffer, filename: string): Promise
|
||||
try {
|
||||
await writeFile(tempPath, buffer);
|
||||
const { stdout } = await execFileAsync(bin, ["-json", "-G", "-struct", "-n", tempPath], {
|
||||
timeout: 30_000,
|
||||
timeout: 60_000,
|
||||
maxBuffer: 10 * 1024 * 1024,
|
||||
});
|
||||
|
||||
@@ -126,7 +126,7 @@ export async function writeMetadata(
|
||||
try {
|
||||
await writeFile(tempPath, buffer);
|
||||
await execFileAsync(bin, ["-overwrite_original", ...tags, tempPath], {
|
||||
timeout: 30_000,
|
||||
timeout: 60_000,
|
||||
maxBuffer: 10 * 1024 * 1024,
|
||||
});
|
||||
return await readFile(tempPath);
|
||||
|
||||
@@ -90,7 +90,7 @@ export async function validateImageBuffer(
|
||||
const height = metadata.height ?? 0;
|
||||
const megapixels = (width * height) / 1_000_000;
|
||||
|
||||
if (megapixels > env.MAX_MEGAPIXELS) {
|
||||
if (env.MAX_MEGAPIXELS > 0 && megapixels > env.MAX_MEGAPIXELS) {
|
||||
return {
|
||||
valid: false,
|
||||
reason: `Image exceeds maximum size: ${megapixels.toFixed(1)}MP (limit: ${env.MAX_MEGAPIXELS}MP)`,
|
||||
|
||||
@@ -45,7 +45,7 @@ export async function decodeHeic(buffer: Buffer): Promise<Buffer> {
|
||||
|
||||
try {
|
||||
await writeFile(inputPath, buffer);
|
||||
await execFileAsync(cmd, [inputPath, outputPath], { timeout: 30_000 });
|
||||
await execFileAsync(cmd, [inputPath, outputPath], { timeout: 120_000 });
|
||||
|
||||
// Single-image HEIF: exact filename. Multi-image: -1 suffix on first image.
|
||||
try {
|
||||
@@ -94,7 +94,7 @@ export async function encodeHeic(buffer: Buffer, quality = 80): Promise<Buffer>
|
||||
try {
|
||||
await writeFile(inputPath, buffer);
|
||||
await execFileAsync("heif-enc", ["-q", String(quality), "-o", outputPath, inputPath], {
|
||||
timeout: 30_000,
|
||||
timeout: 120_000,
|
||||
});
|
||||
return await readFile(outputPath);
|
||||
} finally {
|
||||
|
||||
@@ -1,12 +1,13 @@
|
||||
const MAX_SVG_SIZE = 10 * 1024 * 1024; // 10MB
|
||||
import { env } from "../config.js";
|
||||
|
||||
/**
|
||||
* Sanitize an SVG buffer to prevent XXE, SSRF, and script injection.
|
||||
* Throws if the SVG exceeds the maximum allowed size.
|
||||
*/
|
||||
export function sanitizeSvg(buffer: Buffer): Buffer {
|
||||
if (buffer.length > MAX_SVG_SIZE) {
|
||||
throw new Error(`SVG exceeds maximum size of ${MAX_SVG_SIZE / 1024 / 1024}MB`);
|
||||
const maxSvgSize = env.MAX_SVG_SIZE_MB > 0 ? env.MAX_SVG_SIZE_MB * 1024 * 1024 : Infinity;
|
||||
if (buffer.length > maxSvgSize) {
|
||||
throw new Error(`SVG exceeds maximum size of ${env.MAX_SVG_SIZE_MB}MB`);
|
||||
}
|
||||
let svg = buffer.toString("utf-8");
|
||||
// Remove DOCTYPE (XXE prevention, including internal subsets)
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
import { env } from "../config.js";
|
||||
|
||||
type ToolCategory = "sharp" | "ai_cpu" | "ai_gpu" | "external" | "python";
|
||||
|
||||
const TIMEOUT_RATES: Record<ToolCategory, number> = {
|
||||
sharp: 2,
|
||||
ai_cpu: 30,
|
||||
ai_gpu: 5,
|
||||
external: 10,
|
||||
python: 15,
|
||||
};
|
||||
|
||||
export function computeTimeout(megapixels: number, category: ToolCategory, fileCount = 1): number {
|
||||
if (env.PROCESSING_TIMEOUT_S > 0) {
|
||||
return env.PROCESSING_TIMEOUT_S * 1000;
|
||||
}
|
||||
const perFile = Math.max(60_000, megapixels * TIMEOUT_RATES[category] * 1000);
|
||||
return perFile * fileCount;
|
||||
}
|
||||
|
||||
export function computeExternalToolTimeout(megapixels: number): number {
|
||||
if (env.PROCESSING_TIMEOUT_S > 0) {
|
||||
return env.PROCESSING_TIMEOUT_S * 1000;
|
||||
}
|
||||
return Math.max(60_000, megapixels * TIMEOUT_RATES.external * 1000);
|
||||
}
|
||||
@@ -4,15 +4,14 @@
|
||||
* Uses Piscina (backed by worker_threads) so Sharp operations don't block
|
||||
* HTTP request handling, SSE streams, or health checks.
|
||||
*/
|
||||
import { availableParallelism } from "node:os";
|
||||
import { dirname, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import Piscina from "piscina";
|
||||
import { loadEnv, resolveWorkerThreads } from "./env.js";
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
// Size the pool: leave 1 thread for the event loop, min 1 worker
|
||||
const maxThreads = Math.max(1, Math.min(availableParallelism() - 1, 4));
|
||||
const maxThreads = resolveWorkerThreads(loadEnv());
|
||||
|
||||
let pool: Piscina | null = null;
|
||||
|
||||
|
||||
Reference in New Issue
Block a user