feat: pipeline templates, analytics opt-out, 83 conversion presets, positioning + e2e modernization

Lands five integrated branches: pipeline templates (#355), analytics opt-out (#354), 83 conversion presets bringing the catalog to 240 tools (#356), self-hosted positioning (#353), and e2e modernization (#351).

Integration fixes: aligned stale web analytics tests with the opt-out/allow-list model, closed 3 CodeQL incomplete-sanitization alerts in the i18n generator, resolved settings/index/docs/format-matrix conflicts, and corrected tool counts to 240.
This commit is contained in:
SnapOtter
2026-06-28 18:57:53 +08:00
committed by GitHub
parent 88af8d46fb
commit 63a03d26f2
421 changed files with 17308 additions and 2540 deletions
+12
View File
@@ -33,6 +33,7 @@ import {
ensureAnonymousUser,
ensureBuiltinRoles,
ensureDefaultAdmin,
ensureDefaultTeam,
getAuthUser,
} from "./plugins/auth.js";
import { registerMfa } from "./plugins/mfa.js";
@@ -54,6 +55,7 @@ import { filePreviewRoutes } from "./routes/file-preview.js";
import { fileRoutes } from "./routes/files.js";
import { registerMemeTemplates } from "./routes/meme-templates.js";
import { registerPipelineRoutes } from "./routes/pipeline.js";
import { preferencesRoutes } from "./routes/preferences.js";
import { registerProgressRoutes } from "./routes/progress.js";
import { rolesRoutes } from "./routes/roles.js";
import { settingsRoutes } from "./routes/settings.js";
@@ -123,6 +125,7 @@ if (env.SQLITE_MIGRATE_PATH) {
// inserted via data statements. The pg baseline is DDL-only, so roles are
// seeded here at boot time. onConflictDoNothing makes this idempotent.
await ensureBuiltinRoles();
await ensureDefaultTeam();
if (env.AUTH_ENABLED) {
await ensureDefaultAdmin();
@@ -173,6 +176,8 @@ if (!env.COOKIE_SECRET) {
}
await initAnalytics();
const { primeAnalyticsGate } = await import("./lib/analytics-gate.js");
await primeAnalyticsGate();
// Enterprise features (license-gated)
let enterpriseLicense: { org: string; plan: string } | null = null;
@@ -211,6 +216,8 @@ if (env.STORAGE_MODE === "s3") {
// Start the cooperative cancellation listener (Redis pub/sub)
await startCancelListener();
const { startAnalyticsGateListener } = await import("./lib/analytics-gate.js");
await startAnalyticsGateListener();
// Set up AI feature directories and recover from interrupted installs. Both are
// best-effort and must never block boot: ensureAiDirs swallows its own errors,
@@ -362,6 +369,9 @@ await registerIpAllowlist(app);
// Public config routes (no auth required)
await configRoutes(app);
// Per-user preferences (any authenticated user)
await preferencesRoutes(app);
// Auth middleware (must be registered before routes it protects)
await authMiddleware(app);
@@ -761,6 +771,8 @@ async function shutdown(signal: string) {
await closeQueueEvents();
await closeQueues();
await stopCancelListener();
const { stopAnalyticsGateListener } = await import("./lib/analytics-gate.js");
await stopAnalyticsGateListener();
await closeRedis();
console.log("Redis connections closed");
} catch (err) {
+35 -36
View File
@@ -1,10 +1,19 @@
import { ANALYTICS_BAKED } from "@snapotter/shared";
import { analyticsEnabled, gatePrimed } from "./lib/analytics-gate.js";
const FILE_EXT_PATTERN =
/\.(jpe?g|png|pdf|webp|gif|tiff?|bmp|svg|hei[cf]?|avif|raw|cr2|nef|arw|dng|psd|tga|exr|hdr)\b/gi;
const FILE_PATH_PATTERN = /\/(tmp\/workspace|data\/files|data\/ai)\//g;
// Sentry inits at process load, before the gate cache is primed. Until the
// first successful read, stay silent rather than emit on the default-ON cache,
// so an opted-out instance never reports even a boot-window crash.
const sentryActive = () => gatePrimed() && analyticsEnabled();
if (ANALYTICS_BAKED.enabled && ANALYTICS_BAKED.sentryDsn) {
// Collapse any absolute path in a stack frame filename to its basename, so
// even our own source paths never carry a workspace or job directory.
function basename(p: string): string {
const i = Math.max(p.lastIndexOf("/"), p.lastIndexOf("\\"));
return i >= 0 ? p.slice(i + 1) : p;
}
if (ANALYTICS_BAKED.sentryDsn) {
try {
const Sentry = await import("@sentry/node");
const { APP_VERSION } = await import("@snapotter/shared");
@@ -15,52 +24,42 @@ if (ANALYTICS_BAKED.enabled && ANALYTICS_BAKED.sentryDsn) {
environment: process.env.NODE_ENV || "production",
tracesSampleRate: ANALYTICS_BAKED.sampleRate,
sendDefaultPii: false,
// Runtime opt-out: drop the whole transaction when analytics is off.
tracesSampler: () => (sentryActive() ? ANALYTICS_BAKED.sampleRate : 0),
beforeSend(event) {
if (event.user) {
delete event.user.email;
delete event.user.username;
}
if (!sentryActive()) return null; // kill switch (covers auto-captured errors)
// Allow-list: emit only error type + a basename-collapsed stack.
event.message = undefined;
event.logentry = undefined; // structured twin of message (captureMessage path)
event.server_name = undefined; // hostname is not anonymous
event.request = undefined;
event.extra = undefined;
event.contexts = undefined;
event.breadcrumbs = undefined;
event.user = undefined;
if (event.exception?.values) {
for (const ex of event.exception.values) {
if (
ex.value &&
(ex.value.includes("Rate limit exceeded") ||
ex.value.includes("Body cannot be empty") ||
ex.value.includes("Unsupported Media Type") ||
ex.value.includes("Request body size did not match") ||
ex.value.includes("Premature close"))
) {
return null;
}
if (ex.value) {
ex.value = ex.value
.replace(FILE_EXT_PATTERN, ".[REDACTED]")
.replace(FILE_PATH_PATTERN, "/[REDACTED]/");
}
ex.value = ex.type; // never the raw message body
if (ex.stacktrace?.frames) {
for (const frame of ex.stacktrace.frames) {
if (frame.filename) {
frame.filename = frame.filename
.replace(FILE_EXT_PATTERN, ".[REDACTED]")
.replace(FILE_PATH_PATTERN, "/[REDACTED]/");
}
if (frame.filename) frame.filename = basename(frame.filename);
frame.abs_path = undefined;
frame.vars = undefined;
}
}
}
}
return event;
},
beforeBreadcrumb(breadcrumb) {
if (breadcrumb.message) {
breadcrumb.message = breadcrumb.message
.replace(FILE_EXT_PATTERN, ".[REDACTED]")
.replace(FILE_PATH_PATTERN, "/[REDACTED]/");
}
return breadcrumb;
beforeBreadcrumb() {
return null; // breadcrumbs can carry URLs/messages with content; drop them
},
beforeSendTransaction(event) {
return sentryActive() ? event : null;
},
});
console.log("[sentry] initialized with performance tracing, release:", APP_VERSION);
console.log("[sentry] initialized, release:", APP_VERSION);
} catch {
// @sentry/node not available
}
+8 -7
View File
@@ -25,12 +25,13 @@ import { mkdir, readFile, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { context, propagation, ROOT_CONTEXT, SpanStatusCode, trace } from "@opentelemetry/api";
import { ANALYTICS_BAKED, ANALYTICS_EVENTS, getBundleForTool, TOOLS } from "@snapotter/shared";
import { ANALYTICS_EVENTS, getBundleForTool, TOOLS } from "@snapotter/shared";
import { type Job, UnrecoverableError, Worker } from "bullmq";
import { eq } from "drizzle-orm";
import { env } from "../config.js";
import { db, schema } from "../db/index.js";
import { captureException, trackEvent } from "../lib/analytics.js";
import { analyticsEnabled } from "../lib/analytics-gate.js";
import { resolveConcurrency } from "../lib/env.js";
import { friendlyError } from "../lib/errors.js";
import { logger } from "../lib/logger.js";
@@ -315,7 +316,7 @@ async function processToolJob(job: Job<ToolJobData>): Promise<ToolJobResult> {
jobDuration.observe({ pool: data.pool }, durationMs / 1000);
// Analytics: emit tool_used on success
if (ANALYTICS_BAKED.enabled) {
if (analyticsEnabled()) {
const tool = TOOLS.find((t) => t.id === data.toolId);
void trackEvent(
ANALYTICS_EVENTS.TOOL_USED,
@@ -420,7 +421,7 @@ async function processToolJob(job: Job<ToolJobData>): Promise<ToolJobResult> {
}
// Analytics: emit tool_used on failure
if (ANALYTICS_BAKED.enabled) {
if (analyticsEnabled()) {
const tool = TOOLS.find((t) => t.id === data.toolId);
void trackEvent(
ANALYTICS_EVENTS.TOOL_USED,
@@ -648,7 +649,7 @@ async function processPipelineFinalize(job: Job<ToolJobData>): Promise<ToolJobRe
}
// Analytics: emit pipeline_executed on failure
if (ANALYTICS_BAKED.enabled) {
if (analyticsEnabled()) {
void trackEvent(
ANALYTICS_EVENTS.PIPELINE_EXECUTED,
{
@@ -732,7 +733,7 @@ async function processPipelineFinalize(job: Job<ToolJobData>): Promise<ToolJobRe
}
// Analytics: emit pipeline_executed on success
if (ANALYTICS_BAKED.enabled) {
if (analyticsEnabled()) {
void trackEvent(
ANALYTICS_EVENTS.PIPELINE_EXECUTED,
{
@@ -875,7 +876,7 @@ export function startWorkers(): void {
});
worker.on("failed", (job, err) => {
if (ANALYTICS_BAKED.enabled && job) {
if (analyticsEnabled() && job) {
void captureException(err instanceof Error ? err : new Error(String(err)));
}
});
@@ -903,7 +904,7 @@ export function startWorkers(): void {
});
worker.on("failed", (job, err) => {
if (ANALYTICS_BAKED.enabled && job) {
if (analyticsEnabled() && job) {
void captureException(err instanceof Error ? err : new Error(String(err)));
}
});
+36
View File
@@ -0,0 +1,36 @@
// Defense in depth: only these keys may leave the server per event, and only as
// primitives. Free-text fields (error_message, params, search query) are never
// allow-listed, so tool settings and filenames cannot reach PostHog.
const ALLOWED: Record<string, ReadonlySet<string>> = {
tool_used: new Set(["tool_id", "status", "duration_ms", "category", "is_ai_tool", "error_code"]),
pipeline_executed: new Set([
"step_count",
"tool_ids",
"is_batch",
"file_count",
"duration_ms",
"status",
]),
ai_bundle_action: new Set(["bundle_id", "action", "duration_ms"]),
};
function isAllowedValue(value: unknown): boolean {
if (value === null) return false;
const t = typeof value;
if (t === "string" || t === "number" || t === "boolean") return true;
// tool_ids is an array of strings (low-cardinality ids); allow that one shape.
return Array.isArray(value) && value.every((v) => typeof v === "string");
}
export function sanitizeEventProperties(
event: string,
properties: Record<string, unknown>,
): Record<string, unknown> {
const allow = ALLOWED[event];
if (!allow) return {};
const out: Record<string, unknown> = {};
for (const [key, value] of Object.entries(properties)) {
if (allow.has(key) && isAllowedValue(value)) out[key] = value;
}
return out;
}
+118
View File
@@ -0,0 +1,118 @@
import { ANALYTICS_BAKED } from "@snapotter/shared";
import type Redis from "ioredis";
const TTL_MS = 30_000;
const SETTING_KEY = "analyticsEnabled";
// `undefined` from a reader means the key is absent (default ON).
type GateReader = () => Promise<boolean | undefined>;
let cachedEnabled = true; // last known toggle value (default ON)
let knownDisabled = false; // have we positively read "disabled"? fail-closed anchor
let primed = false; // has a successful read happened yet? gates Sentry at cold start
let fetchedAt = 0; // Date.now() of the last read attempt
let reader: GateReader = defaultReader;
async function defaultReader(): Promise<boolean | undefined> {
const { db, schema } = await import("../db/index.js");
const { eq } = await import("drizzle-orm");
const rows = await db
.select({ value: schema.settings.value })
.from(schema.settings)
.where(eq(schema.settings.key, SETTING_KEY))
.limit(1);
if (rows.length === 0) return undefined;
return rows[0].value !== "false";
}
/** Compile-time bake, with a NON-PRODUCTION-only override so tests can force it on. */
export function bakedEnabled(): boolean {
if (process.env.NODE_ENV !== "production") {
const o = process.env.ANALYTICS_BAKED_OVERRIDE;
if (o === "on") return true;
if (o === "off") return false;
}
return ANALYTICS_BAKED.enabled;
}
/** Synchronous effective gate. Safe to call from Sentry beforeSend. Never blocks. */
export function analyticsEnabled(): boolean {
if (!bakedEnabled()) return false;
if (Date.now() - fetchedAt > TTL_MS) {
void refreshAnalyticsGate(); // background refresh; serve the cached value now
}
return cachedEnabled;
}
/** Read the toggle and update the cache. Fails closed on read error. */
export async function refreshAnalyticsGate(): Promise<void> {
try {
const v = await reader();
const on = v === undefined ? true : v;
cachedEnabled = on;
knownDisabled = !on;
primed = true;
fetchedAt = Date.now();
} catch {
// DB read failed. If we ever positively saw "disabled", keep serving disabled
// rather than reverting to the ON default. Otherwise keep the last value.
if (knownDisabled) cachedEnabled = false;
fetchedAt = Date.now(); // do not hammer the DB on repeated errors
}
}
/** Warm the cache at boot before traffic is served. */
export async function primeAnalyticsGate(): Promise<void> {
await refreshAnalyticsGate();
}
/**
* True once a successful read has populated the cache. Backend Sentry inits at
* process load (before the cache is primed), so its hooks check this to stay
* silent during the boot window rather than emitting on the default-ON cache.
*/
export function gatePrimed(): boolean {
return primed;
}
// Test seams (no-ops in production paths).
export function __setReaderForTests(r: GateReader | null): void {
reader = r ?? defaultReader;
}
export function __resetGateForTests(): void {
cachedEnabled = true;
knownDisabled = false;
primed = false;
fetchedAt = 0;
reader = defaultReader;
}
let gateSubscriber: Redis | null = null;
const CHANNEL = async () => {
const { bullPrefix } = await import("../jobs/types.js");
return `${bullPrefix()}:analytics-gate`;
};
/** Subscribe so a setting change on any replica refreshes this process's cache. */
export async function startAnalyticsGateListener(): Promise<void> {
const { createRedisConnection } = await import("../jobs/connection.js");
gateSubscriber = createRedisConnection();
gateSubscriber.on("error", (err) => console.error("Analytics gate subscriber error", err));
await gateSubscriber.subscribe(await CHANNEL());
gateSubscriber.on("message", () => {
void refreshAnalyticsGate();
});
}
/** Publish so every replica drops its cache after a toggle write. */
export async function publishAnalyticsGateInvalidation(): Promise<void> {
const { sharedRedis } = await import("../jobs/connection.js");
await sharedRedis().publish(await CHANNEL(), "1");
}
export async function stopAnalyticsGateListener(): Promise<void> {
if (gateSubscriber) {
await gateSubscriber.quit();
gateSubscriber = null;
}
}
+6 -4
View File
@@ -2,11 +2,13 @@ import { ANALYTICS_BAKED } from "@snapotter/shared";
import { eq } from "drizzle-orm";
import type { PostHog } from "posthog-node";
import { db, schema } from "../db/index.js";
import { sanitizeEventProperties } from "./analytics-allowlist.js";
import { analyticsEnabled, bakedEnabled } from "./analytics-gate.js";
let posthogClient: PostHog | null = null;
export async function initAnalytics(): Promise<void> {
if (!ANALYTICS_BAKED.enabled) return;
if (!bakedEnabled()) return;
if (ANALYTICS_BAKED.posthogApiKey) {
try {
@@ -24,7 +26,7 @@ export async function initAnalytics(): Promise<void> {
export async function captureException(error: unknown): Promise<void> {
try {
if (!ANALYTICS_BAKED.enabled) return;
if (!analyticsEnabled()) return;
const Sentry = await import("@sentry/node");
Sentry.captureException(error);
} catch {
@@ -53,14 +55,14 @@ export async function trackEvent(
distinctId?: string,
): Promise<void> {
try {
if (!ANALYTICS_BAKED.enabled || !posthogClient) return;
if (!analyticsEnabled() || !posthogClient) return;
if (ANALYTICS_BAKED.sampleRate < 1.0) {
if (ANALYTICS_BAKED.sampleRate <= 0.0 || Math.random() >= ANALYTICS_BAKED.sampleRate) return;
}
posthogClient.capture({
distinctId: distinctId ?? (await getInstanceId()),
event,
properties,
properties: sanitizeEventProperties(event, properties),
});
} catch {
// analytics must never throw
+1
View File
@@ -33,6 +33,7 @@ const envSchema = z
CONCURRENT_JOBS: z.coerce.number().default(0),
MAX_MEGAPIXELS: z.coerce.number().default(0),
RATE_LIMIT_PER_MIN: z.coerce.number().default(300),
API_KEYS_RATE_LIMIT_PER_MIN: z.coerce.number().default(30),
DATABASE_URL: z.string().default("postgres://snapotter:snapotter@localhost:5432/snapotter"),
SQLITE_MIGRATE_PATH: z.string().default(""),
FILES_STORAGE_PATH: z.string().default("./data/files"),
+4 -2
View File
@@ -52,7 +52,9 @@ async function findDecodeCmd(): Promise<string> {
*/
export async function decodeHeic(buffer: Buffer): Promise<Buffer> {
const cmd = await findDecodeCmd();
const id = randomUUID();
// Include the PID so concurrent processes (and test workers) write to
// distinct, attributable temp paths in the shared tmpdir.
const id = `${process.pid}-${randomUUID()}`;
const inputPath = join(tmpdir(), `heic-in-${id}.heic`);
const outputPath = join(tmpdir(), `heic-out-${id}.png`);
const suffixedPath = outputPath.replace(/\.png$/, "-1.png");
@@ -101,7 +103,7 @@ export async function ensureSharpCompat(buffer: Buffer): Promise<Buffer> {
}
export async function encodeHeic(buffer: Buffer, quality = 80): Promise<Buffer> {
const id = randomUUID();
const id = `${process.pid}-${randomUUID()}`;
const inputPath = join(tmpdir(), `heic-in-${id}.png`);
const outputPath = join(tmpdir(), `heic-out-${id}.heic`);
+10 -21
View File
@@ -3,7 +3,7 @@ info:
title: SnapOtter API
version: 1.17.1
description: |
REST API for SnapOtter, a self-hosted file processing suite with 157 tools across image, video, audio, document, and data.
REST API for SnapOtter, a self-hosted file processing suite with 240 tools across image, video, audio, document, and data.
## Authentication
@@ -5429,15 +5429,6 @@ paths:
type: string
teamName:
type: string
analyticsEnabled:
type: boolean
nullable: true
analyticsConsentShownAt:
type: integer
nullable: true
analyticsConsentRemindAt:
type: integer
nullable: true
expiresAt:
type: string
format: date-time
@@ -5509,15 +5500,6 @@ paths:
type: array
items:
type: string
analyticsEnabled:
type: boolean
nullable: true
analyticsConsentShownAt:
type: integer
nullable: true
analyticsConsentRemindAt:
type: integer
nullable: true
expiresAt:
type: string
format: date-time
@@ -6416,8 +6398,10 @@ paths:
tags: [Analytics]
summary: Get analytics configuration
description: |
Returns analytics provider configuration. When ANALYTICS_ENABLED is
false on the server, all values are empty. Public endpoint.
Returns the effective analytics provider configuration. Analytics is
enabled only when both the compile-time bake and the instance
analyticsEnabled setting allow it. When disabled, the keys, DSN, and
instanceId are blank. Public endpoint.
security: []
responses:
"200":
@@ -6429,6 +6413,11 @@ paths:
properties:
enabled:
type: boolean
description: |
Effective analytics state. True only when both the
compile-time bake and the instance analyticsEnabled
setting are on. When false, the keys, DSN, and instanceId
are blank.
posthogApiKey:
type: string
posthogHost:
+18 -1
View File
@@ -146,7 +146,24 @@ export function createSessionToken(): string {
return randomUUID();
}
// ── Default admin creation ─────────────────────────────────────────
// ── Default team + admin creation ──────────────────────────────────
/**
* Seed the "Default" team that the rest of the platform assumes exists.
*
* The frontend People form always submits team "Default", and register,
* external-auth-resolver, and SCIM all fall back to a "default-team-00000000"
* placeholder ID for it. None of those create the row, so a fresh install has
* no Default team and adding a member via the UI fails with "Team not found".
* Seeding it here (idempotent) fixes that. The ID matches the placeholder the
* fallback paths use so a real row and a fallback reference line up.
*/
export async function ensureDefaultTeam(): Promise<void> {
await db
.insert(schema.teams)
.values({ id: "default-team-00000000", name: "Default" })
.onConflictDoNothing();
}
export async function ensureAnonymousUser(): Promise<void> {
const [existing] = await db.select().from(schema.users).where(eq(schema.users.id, "anonymous"));
+14 -1
View File
@@ -2,16 +2,29 @@ import { ANALYTICS_BAKED } from "@snapotter/shared";
import { eq } from "drizzle-orm";
import type { FastifyInstance } from "fastify";
import { db, schema } from "../db/index.js";
import { analyticsEnabled } from "../lib/analytics-gate.js";
export async function analyticsRoutes(app: FastifyInstance): Promise<void> {
app.get("/api/v1/config/analytics", async () => {
// Effective state: compile-time bake AND the runtime instance opt-out.
if (!analyticsEnabled()) {
return {
enabled: false,
posthogApiKey: "",
posthogHost: "",
sentryDsn: "",
sampleRate: 0,
instanceId: "",
};
}
const [row] = await db
.select()
.from(schema.settings)
.where(eq(schema.settings.key, "instance_id"));
return {
enabled: ANALYTICS_BAKED.enabled,
enabled: true,
posthogApiKey: ANALYTICS_BAKED.posthogApiKey,
posthogHost: ANALYTICS_BAKED.posthogHost,
sentryDsn: ANALYTICS_BAKED.sentryDsn,
+11 -3
View File
@@ -9,11 +9,17 @@ import { randomBytes, randomUUID } from "node:crypto";
import { and, eq } from "drizzle-orm";
import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
import { z } from "zod";
import { env } from "../config.js";
import { db, schema } from "../db/index.js";
import { auditFromRequest } from "../lib/audit.js";
import { getPermissions, hasEffectivePermission } from "../permissions.js";
import { computeKeyPrefix, hashPassword, requireAuth } from "../plugins/auth.js";
// Per-route cap on the API-key management endpoints. Defaults to 30/min as an
// anti-abuse guard; raised via env in the e2e suite, where many api-keys specs
// hit the list endpoint in quick succession on a shared IP.
const API_KEYS_RATE_LIMIT = { max: env.API_KEYS_RATE_LIMIT_PER_MIN, timeWindow: "1 minute" };
const createApiKeySchema = z.object({
name: z.string().max(100, "Key name must be 100 characters or fewer").optional(),
permissions: z.array(z.string()).optional(),
@@ -24,7 +30,7 @@ export async function apiKeyRoutes(app: FastifyInstance): Promise<void> {
// POST /api/v1/api-keys — Generate a new API key
app.post(
"/api/v1/api-keys",
{ config: { rateLimit: { max: 30, timeWindow: "1 minute" } } },
{ config: { rateLimit: API_KEYS_RATE_LIMIT } },
async (request: FastifyRequest, reply: FastifyReply) => {
const user = requireAuth(request, reply);
if (!user) return;
@@ -110,7 +116,7 @@ export async function apiKeyRoutes(app: FastifyInstance): Promise<void> {
// GET /api/v1/api-keys — List user's API keys (never returns the key itself)
app.get(
"/api/v1/api-keys",
{ config: { rateLimit: { max: 30, timeWindow: "1 minute" } } },
{ config: { rateLimit: API_KEYS_RATE_LIMIT } },
async (request: FastifyRequest, reply: FastifyReply) => {
const user = requireAuth(request, reply);
if (!user) return;
@@ -118,6 +124,7 @@ export async function apiKeyRoutes(app: FastifyInstance): Promise<void> {
const selectFields = {
id: schema.apiKeys.id,
name: schema.apiKeys.name,
keyPrefix: schema.apiKeys.keyPrefix,
permissions: schema.apiKeys.permissions,
createdAt: schema.apiKeys.createdAt,
lastUsedAt: schema.apiKeys.lastUsedAt,
@@ -134,6 +141,7 @@ export async function apiKeyRoutes(app: FastifyInstance): Promise<void> {
apiKeys: keys.map((k) => ({
id: k.id,
name: k.name,
prefix: k.keyPrefix ?? "",
permissions: k.permissions ?? null,
createdAt: k.createdAt.toISOString(),
lastUsedAt: k.lastUsedAt?.toISOString() ?? null,
@@ -146,7 +154,7 @@ export async function apiKeyRoutes(app: FastifyInstance): Promise<void> {
// DELETE /api/v1/api-keys/:id — Delete an API key
app.delete(
"/api/v1/api-keys/:id",
{ config: { rateLimit: { max: 30, timeWindow: "1 minute" } } },
{ config: { rateLimit: API_KEYS_RATE_LIMIT } },
async (request: FastifyRequest<{ Params: { id: string } }>, reply: FastifyReply) => {
const user = requireAuth(request, reply);
if (!user) return;
+1 -1
View File
@@ -39,7 +39,7 @@ function generateLlmsTxt(spec: OpenAPISpec): string {
lines.push(`# ${spec.info.title}`);
lines.push("");
lines.push(
"> Self-hosted file processing API with 157 tools across image, video, audio, document, and data. Convert, compress, edit, transcribe, OCR, and more.",
"> Self-hosted file processing API with 240 tools across image, video, audio, document, and data. Convert, compress, edit, transcribe, OCR, and more.",
);
lines.push("");
lines.push("## Docs");
+64
View File
@@ -0,0 +1,64 @@
/**
* Per-user preferences.
*
* Lets any authenticated user read and write their own preferences (e.g. the
* default home view). Distinct from /v1/settings, which is the admin-only
* instance configuration. Values are stored per (userId, key) in the
* user_preferences table.
*/
import { eq } from "drizzle-orm";
import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
import { z } from "zod";
import { db, schema } from "../db/index.js";
import { requireAuth } from "../plugins/auth.js";
const putSchema = z.record(z.string(), z.unknown());
export async function preferencesRoutes(app: FastifyInstance): Promise<void> {
// GET /api/v1/preferences - the current user's preferences as a key->value map
app.get(
"/api/v1/preferences",
{ config: { rateLimit: { max: 300, timeWindow: "1 minute" } } },
async (request: FastifyRequest, reply: FastifyReply) => {
const user = requireAuth(request, reply);
if (!user) return;
const rows = await db
.select()
.from(schema.userPreferences)
.where(eq(schema.userPreferences.userId, user.id));
const preferences: Record<string, unknown> = {};
for (const row of rows) {
preferences[row.key] = (row.value as { value: unknown }).value;
}
return reply.send({ preferences });
},
);
// PUT /api/v1/preferences - upsert one or more of the current user's preferences
app.put(
"/api/v1/preferences",
{ config: { rateLimit: { max: 300, timeWindow: "1 minute" } } },
async (request: FastifyRequest, reply: FastifyReply) => {
const user = requireAuth(request, reply);
if (!user) return;
const parsed = putSchema.safeParse(request.body ?? {});
if (!parsed.success) {
return reply.status(400).send({ error: "Invalid preferences body" });
}
for (const [key, value] of Object.entries(parsed.data)) {
await db
.insert(schema.userPreferences)
.values({ userId: user.id, key, value: { value } })
.onConflictDoUpdate({
target: [schema.userPreferences.userId, schema.userPreferences.key],
set: { value: { value } },
});
}
return reply.send({ ok: true });
},
);
}
+14
View File
@@ -144,6 +144,20 @@ export async function settingsRoutes(app: FastifyInstance): Promise<void> {
});
}
if (entries.some((e) => e.key === "analyticsEnabled")) {
// The setting is already persisted. A Redis hiccup here must not turn a
// successful save into a 500; the TTL refresh converges replicas anyway.
try {
const { refreshAnalyticsGate, publishAnalyticsGateInvalidation } = await import(
"../lib/analytics-gate.js"
);
await refreshAnalyticsGate(); // this replica, immediately
await publishAnalyticsGateInvalidation(); // all other replicas
} catch (err) {
request.log.warn({ err }, "analytics gate invalidation failed (save still applied)");
}
}
return reply.send({ ok: true, updatedCount: entries.length });
});
@@ -0,0 +1,75 @@
import { BASE_CONFIG, CONVERSION_PRESETS } from "@snapotter/shared";
import type { FastifyInstance } from "fastify";
import { z } from "zod";
import { createToolRoute, getToolConfig } from "../tool-factory.js";
import { registerImageToPdfPreset } from "./image-to-pdf.js";
import { registerPdfToImagePreset } from "./pdf-to-image.js";
import { registerSvgToRasterPreset } from "./svg-to-raster.js";
/**
* Narrow settings schema per preset base. Presets lock the output format, so
* they expose only an optional quality/video knob (or nothing). The locked
* format is merged in before the base's own processV2 re-validates.
*/
function presetSchema(base: string) {
if (base === "convert" || base === "svg-to-raster") {
return z.object({ quality: z.number().min(1).max(100).optional() });
}
if (base === "convert-video") {
return z.object({ quality: z.enum(["high", "balanced", "small"]).optional() });
}
if (base === "image-to-pdf") {
return z.object({
pageSize: z.enum(["A4", "Letter", "A3", "A5"]).optional(),
orientation: z.enum(["portrait", "landscape"]).optional(),
});
}
return z.object({});
}
/**
* Register all conversion-preset routes. Registry-group presets delegate to the
* base tool's registered processV2 with their locked settings merged in, so they
* MUST be registered after the base tools (ensured by index.ts ordering). The
* three custom ZIP bases (image-to-pdf, pdf-to-image, svg-to-raster) reuse their
* own parameterized registrars.
*/
export function registerConversionPresets(app: FastifyInstance) {
for (const preset of CONVERSION_PRESETS) {
const cfg = BASE_CONFIG[preset.base];
if (cfg.group === "image-to-pdf") {
registerImageToPdfPreset(app, preset.id, preset.sourceInputs);
continue;
}
if (cfg.group === "pdf-to-image") {
registerPdfToImagePreset(app, preset.id, preset.locked.format as string);
continue;
}
if (cfg.group === "svg-to-raster") {
registerSvgToRasterPreset(app, preset.id, preset.locked.outputFormat as string);
continue;
}
// group "registry": delegate to the base tool's processV2 with locked settings merged in.
const baseConfig = getToolConfig(preset.base);
if (!baseConfig) {
throw new Error(
`Preset "${preset.id}" base "${preset.base}" is not registered in the tool registry`,
);
}
createToolRoute(app, {
toolId: preset.id,
settingsSchema: presetSchema(preset.base),
process: async () => {
throw new Error(`${preset.id} is v2-only`);
},
processV2: async (ctx) => {
const merged = { ...preset.locked, ...(ctx.settings as Record<string, unknown>) };
if (!baseConfig.processV2) {
throw new Error(`Base "${preset.base}" has no processV2 for preset "${preset.id}"`);
}
return baseConfig.processV2({ ...ctx, settings: merged });
},
});
}
}
+42 -1
View File
@@ -5,7 +5,7 @@ import { runMediaTool } from "../../lib/media-tool.js";
import { createToolRoute } from "../tool-factory.js";
const settingsSchema = z.object({
format: z.enum(["mp4", "mov", "webm"]).default("mp4"),
format: z.enum(["mp4", "mov", "webm", "avi", "mkv"]).default("mp4"),
quality: z.enum(["high", "balanced", "small"]).default("balanced"),
});
@@ -19,6 +19,8 @@ const CONTENT_TYPES: Record<string, string> = {
mp4: "video/mp4",
mov: "video/quicktime",
webm: "video/webm",
avi: "video/x-msvideo",
mkv: "video/x-matroska",
};
export function registerConvertVideo(app: FastifyInstance) {
@@ -48,6 +50,45 @@ export function registerConvertVideo(app: FastifyInstance) {
out,
];
}
if (settings.format === "avi") {
return [
"-i",
inPath,
"-c:v",
resolveEncoder("h264"),
"-crf",
CRF[settings.quality].h264,
"-preset",
"medium",
"-pix_fmt",
"yuv420p",
"-c:a",
"libmp3lame",
"-b:a",
"192k",
out,
];
}
if (settings.format === "mkv") {
return [
"-i",
inPath,
"-c:v",
resolveEncoder("h264"),
"-crf",
CRF[settings.quality].h264,
"-preset",
"medium",
"-pix_fmt",
"yuv420p",
"-c:a",
resolveEncoder("aac"),
"-b:a",
"128k",
out,
];
}
// mp4 and mov
return [
"-i",
inPath,
+4 -1
View File
@@ -4,13 +4,14 @@ import { runMediaTool } from "../../lib/media-tool.js";
import { createToolRoute } from "../tool-factory.js";
const settingsSchema = z.object({
format: z.enum(["mp3", "wav", "m4a"]).default("mp3"),
format: z.enum(["mp3", "wav", "m4a", "ogg"]).default("mp3"),
});
const CONTENT_TYPES: Record<string, string> = {
mp3: "audio/mpeg",
wav: "audio/wav",
m4a: "audio/mp4",
ogg: "audio/ogg",
};
export function registerExtractAudio(app: FastifyInstance) {
@@ -33,6 +34,8 @@ export function registerExtractAudio(app: FastifyInstance) {
return ["-i", inPath, "-vn", "-c:a", "pcm_s16le", out];
case "m4a":
return ["-i", inPath, "-vn", "-c:a", "aac", "-b:a", "192k", out];
case "ogg":
return ["-i", inPath, "-vn", "-c:a", "libvorbis", "-q:a", "5", out];
default:
return ["-i", inPath, "-vn", "-c:a", "libmp3lame", "-b:a", "192k", out];
}
+8 -2
View File
@@ -5,9 +5,15 @@ import { runMediaTool } from "../../lib/media-tool.js";
import { createToolRoute } from "../tool-factory.js";
const settingsSchema = z.object({
format: z.enum(["mp4", "webm"]).default("mp4"),
format: z.enum(["mp4", "webm", "mov"]).default("mp4"),
});
const CONTENT_TYPES: Record<string, string> = {
mp4: "video/mp4",
webm: "video/webm",
mov: "video/quicktime",
};
export function registerGifToVideo(app: FastifyInstance) {
createToolRoute(app, {
toolId: "gif-to-video",
@@ -19,7 +25,7 @@ export function registerGifToVideo(app: FastifyInstance) {
const settings = settingsSchema.parse(ctx.settings);
const base = ctx.inputs[0].filename.replace(/\.[^.]+$/, "");
const outName = `${base}.${settings.format}`;
const contentType = settings.format === "mp4" ? "video/mp4" : "video/webm";
const contentType = CONTENT_TYPES[settings.format];
const { outPath } = await runMediaTool(ctx, outName, (inPath, out) => {
if (settings.format === "webm") {
+34 -2
View File
@@ -1,4 +1,5 @@
import { randomUUID } from "node:crypto";
import { apiToolPath } from "@snapotter/shared";
import archiver from "archiver";
import type { FastifyInstance } from "fastify";
import PDFDocument from "pdfkit";
@@ -86,6 +87,12 @@ async function compressImagesForTarget(
return { buffers: bestBuffers, quality: bestQuality, targetMet: finalSize <= budget };
}
/** Case-insensitive check that a filename ends with one of the accepted extensions. */
function matchesAccept(filename: string, accept: string[]): boolean {
const lower = filename.toLowerCase();
return accept.some((ext) => lower.endsWith(ext.toLowerCase()));
}
async function flattenAlpha(buf: Buffer): Promise<Buffer> {
const meta = await sharp(buf).metadata();
if (meta.hasAlpha) {
@@ -96,8 +103,11 @@ async function flattenAlpha(buf: Buffer): Promise<Buffer> {
return buf;
}
export function registerImageToPdf(app: FastifyInstance) {
app.post("/api/v1/tools/image/image-to-pdf", async (request, reply) => {
export function registerImageToPdfRoute(
app: FastifyInstance,
opts: { toolId: string; accept?: string[] },
) {
app.post(apiToolPath(opts.toolId), async (request, reply) => {
const files: Array<{ buffer: Buffer; filename: string }> = [];
let settingsRaw: string | null = null;
@@ -131,6 +141,16 @@ export function registerImageToPdf(app: FastifyInstance) {
return reply.status(400).send({ error: "No image files provided" });
}
if (opts.accept) {
const accept = opts.accept;
const invalid = files.find((file) => !matchesAccept(file.filename, accept));
if (invalid) {
return reply.status(400).send({
error: `Invalid image "${invalid.filename}": this converter only accepts ${accept.join(", ")} files`,
});
}
}
let settings: z.infer<typeof settingsSchema>;
try {
const parsed = settingsRaw ? JSON.parse(settingsRaw) : {};
@@ -338,3 +358,15 @@ export function registerImageToPdf(app: FastifyInstance) {
}
});
}
export function registerImageToPdf(app: FastifyInstance) {
registerImageToPdfRoute(app, { toolId: "image-to-pdf" });
}
/**
* Register an "<image format> to PDF" conversion preset that reuses the
* image-to-pdf route logic with inputs narrowed to the given extensions.
*/
export function registerImageToPdfPreset(app: FastifyInstance, toolId: string, accept: string[]) {
registerImageToPdfRoute(app, { toolId, accept });
}
+5
View File
@@ -33,6 +33,7 @@ import { registerCompress } from "./compress.js";
import { registerCompressPdf } from "./compress-pdf.js";
import { registerCompressVideo } from "./compress-video.js";
import { registerContentAwareResize } from "./content-aware-resize.js";
import { registerConversionPresets } from "./conversion-presets.js";
import { registerConvert } from "./convert.js";
import { registerConvertAudio } from "./convert-audio.js";
import { registerConvertDocument } from "./convert-document.js";
@@ -393,4 +394,8 @@ export async function registerToolRoutes(app: FastifyInstance): Promise<void> {
app.log.info(
`Tool routes: ${registered} active, ${skipped} skipped (${toolRegistrations.length} total)`,
);
// Conversion presets delegate to base tools' registered processV2, so they
// must be registered after the base loop above has populated the registry.
registerConversionPresets(app);
}
+30 -4
View File
@@ -1,4 +1,5 @@
import { randomUUID } from "node:crypto";
import { apiToolPath } from "@snapotter/shared";
import archiver from "archiver";
import type { FastifyInstance } from "fastify";
import * as mupdf from "mupdf";
@@ -179,9 +180,14 @@ function isPdfBuffer(buf: Buffer): boolean {
}
// ── Route registration ───────────────────────────────────────────
export function registerPdfToImage(app: FastifyInstance) {
export function registerPdfToImageRoute(
app: FastifyInstance,
opts: { toolId: string; lockedFormat?: string },
) {
const basePath = apiToolPath(opts.toolId);
// ── Info endpoint ────────────────────────────────────────────
app.post("/api/v1/tools/pdf/pdf-to-image/info", async (request, reply) => {
app.post(`${basePath}/info`, async (request, reply) => {
let fileBuffer: Buffer | null = null;
try {
const result = await readPdfFromParts(request);
@@ -223,7 +229,7 @@ export function registerPdfToImage(app: FastifyInstance) {
});
// ── Preview endpoint (thumbnails) ─────────────────────────────
app.post("/api/v1/tools/pdf/pdf-to-image/preview", async (request, reply) => {
app.post(`${basePath}/preview`, async (request, reply) => {
let fileBuffer: Buffer | null = null;
try {
const result = await readPdfFromParts(request);
@@ -288,7 +294,7 @@ export function registerPdfToImage(app: FastifyInstance) {
});
// ── Main processing endpoint ─────────────────────────────────
app.post("/api/v1/tools/pdf/pdf-to-image", async (request, reply) => {
app.post(basePath, async (request, reply) => {
let fileBuffer: Buffer | null = null;
let settingsRaw: string | null = null;
@@ -325,6 +331,10 @@ export function registerPdfToImage(app: FastifyInstance) {
return reply.status(400).send({ error: "Settings must be valid JSON" });
}
if (opts.lockedFormat) {
settings.format = opts.lockedFormat as typeof settings.format;
}
let doc: mupdf.Document | null = null;
try {
doc = mupdf.Document.openDocument(fileBuffer, "application/pdf");
@@ -405,3 +415,19 @@ export function registerPdfToImage(app: FastifyInstance) {
}
});
}
export function registerPdfToImage(app: FastifyInstance) {
registerPdfToImageRoute(app, { toolId: "pdf-to-image" });
}
/**
* Register a "PDF to <format>" conversion preset that reuses the pdf-to-image
* route logic (ZIP output, info/preview endpoints) with the format locked.
*/
export function registerPdfToImagePreset(
app: FastifyInstance,
toolId: string,
lockedFormat: string,
) {
registerPdfToImageRoute(app, { toolId, lockedFormat });
}
+62 -4
View File
@@ -1,4 +1,5 @@
import { randomUUID } from "node:crypto";
import { apiToolPath } from "@snapotter/shared";
import archiver from "archiver";
import type { FastifyInstance } from "fastify";
import PQueue from "p-queue";
@@ -17,6 +18,12 @@ import { updateJobProgress } from "../progress.js";
const NON_PREVIEWABLE = new Set(["tiff", "heif"]);
/** Case-insensitive check that a filename ends with one of the accepted extensions. */
function matchesAccept(filename: string, accept: string[]): boolean {
const lower = filename.toLowerCase();
return accept.some((ext) => lower.endsWith(ext.toLowerCase()));
}
const settingsSchema = z.object({
width: z.number().min(1).max(65536).optional(),
height: z.number().min(1).max(65536).optional(),
@@ -103,9 +110,14 @@ async function convertSvg(
* SVG to raster conversion.
* Custom route since input is SVG (not validated as image by magic bytes).
*/
export function registerSvgToRaster(app: FastifyInstance) {
export function registerSvgToRasterRoute(
app: FastifyInstance,
opts: { toolId: string; accept?: string[]; lockedFormat?: string },
) {
const basePath = apiToolPath(opts.toolId);
// --- Batch endpoint (registered first for route priority) ---
app.post("/api/v1/tools/image/svg-to-raster/batch", async (request, reply) => {
app.post(`${basePath}/batch`, async (request, reply) => {
const files: ParsedSvgFile[] = [];
let settingsRaw: string | null = null;
let clientJobId: string | null = null;
@@ -151,6 +163,16 @@ export function registerSvgToRaster(app: FastifyInstance) {
});
}
if (opts.accept) {
const accept = opts.accept;
const invalid = files.find((file) => !matchesAccept(file.filename, accept));
if (invalid) {
return reply.status(400).send({
error: "File is not a valid SVG. This tool only accepts SVG files.",
});
}
}
let settings: z.infer<typeof settingsSchema>;
try {
const parsed = settingsRaw ? JSON.parse(settingsRaw) : {};
@@ -166,6 +188,10 @@ export function registerSvgToRaster(app: FastifyInstance) {
return reply.status(400).send({ error: "Settings must be valid JSON" });
}
if (opts.lockedFormat) {
settings.outputFormat = opts.lockedFormat as typeof settings.outputFormat;
}
const jobId = clientJobId || randomUUID();
const queue = new PQueue({ concurrency: resolveConcurrency(env) });
const results: ({ buffer: Buffer; filename: string } | null)[] = new Array(files.length).fill(
@@ -347,9 +373,10 @@ export function registerSvgToRaster(app: FastifyInstance) {
});
// --- Single-file endpoint ---
app.post("/api/v1/tools/image/svg-to-raster", async (request, reply) => {
app.post(basePath, async (request, reply) => {
let fileBuffer: Buffer | null = null;
let filename = "output";
let uploadFilename = "output";
let settingsRaw: string | null = null;
try {
@@ -361,7 +388,8 @@ export function registerSvgToRaster(app: FastifyInstance) {
chunks.push(chunk);
}
fileBuffer = Buffer.concat(chunks);
filename = sanitizeFilename(part.filename ?? "output").replace(/\.svgz?$/i, "");
uploadFilename = sanitizeFilename(part.filename ?? "output");
filename = uploadFilename.replace(/\.svgz?$/i, "");
} else if (part.fieldname === "settings") {
settingsRaw = part.value as string;
}
@@ -377,6 +405,12 @@ export function registerSvgToRaster(app: FastifyInstance) {
return reply.status(400).send({ error: "No SVG file provided" });
}
if (opts.accept && !matchesAccept(uploadFilename, opts.accept)) {
return reply.status(400).send({
error: "File is not a valid SVG. This tool only accepts SVG files.",
});
}
try {
fileBuffer = decompressSvgz(fileBuffer);
} catch (err) {
@@ -413,6 +447,10 @@ export function registerSvgToRaster(app: FastifyInstance) {
return reply.status(400).send({ error: "Settings must be valid JSON" });
}
if (opts.lockedFormat) {
settings.outputFormat = opts.lockedFormat as typeof settings.outputFormat;
}
try {
const {
buffer,
@@ -453,3 +491,23 @@ export function registerSvgToRaster(app: FastifyInstance) {
}
});
}
/**
* SVG to raster conversion.
* Custom route since input is SVG (not validated as image by magic bytes).
*/
export function registerSvgToRaster(app: FastifyInstance) {
registerSvgToRasterRoute(app, { toolId: "svg-to-raster" });
}
/**
* Register an "SVG to <format>" conversion preset that reuses the svg-to-raster
* route logic with the output format locked and inputs narrowed to SVG.
*/
export function registerSvgToRasterPreset(
app: FastifyInstance,
toolId: string,
lockedFormat: string,
) {
registerSvgToRasterRoute(app, { toolId, lockedFormat, accept: [".svg", ".svgz"] });
}