diff --git a/.dockerignore b/.dockerignore index f2703e6c..b390e366 100644 --- a/.dockerignore +++ b/.dockerignore @@ -34,7 +34,12 @@ docs .github .husky .releaserc.json -scripts +# Exclude scripts/ from the build context EXCEPT bake-analytics.mjs, which +# docker/Dockerfile COPYs to bake the analytics config. A blanket `scripts` +# ignore breaks the production image build (COPY scripts/bake-analytics.mjs -> +# "not found"); use dir/* + negation so the one needed file is re-included. +scripts/* +!scripts/bake-analytics.mjs # IDE .vscode diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index 1f846c44..f8e4dbdf 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -188,6 +188,27 @@ try { // Enterprise package not available } +// S3 storage is a licensed feature (s3_storage). packages/enterprise now ships in +// every image, so STORAGE_MODE=s3 would otherwise function without any license check. +// Enforce the gate at boot so an unlicensed deploy fails fast rather than silently +// writing data to S3 it isn't entitled to use. +if (env.STORAGE_MODE === "s3") { + let s3Licensed = false; + try { + const { isFeatureEnabled } = await import("@snapotter/enterprise"); + s3Licensed = isFeatureEnabled("s3_storage"); + } catch { + s3Licensed = false; + } + if (!s3Licensed) { + console.error( + "[FATAL] STORAGE_MODE=s3 requires a license that includes the s3_storage feature. " + + "Set a valid SNAPOTTER_LICENSE_KEY (team or enterprise plan) or use STORAGE_MODE=local.", + ); + process.exit(1); + } +} + // Start the cooperative cancellation listener (Redis pub/sub) await startCancelListener(); @@ -625,6 +646,22 @@ if (await shouldRunStartupCleanup()) { // Start BullMQ worker pools (after route registration so the tool registry is full) startWorkers(); +// Reconcile orphaned job rows. A jobs row created without a tool_id/pool (e.g. an +// SSE-progress placeholder for a clientJobId whose client then disconnected) is +// never enqueued to BullMQ, so unlike a genuinely interrupted job (which BullMQ's +// stalled-detection requeues) it would sit in 'processing'/'queued' forever -- +// inflating the per-user concurrent-job count and the upgrade-check in-flight gate. +// Only rows with an empty tool_id are touched, so real jobs are never affected. +void db + .execute( + sql`UPDATE jobs SET status = 'failed', error = '{"message":"Orphaned job reconciled at startup"}'::jsonb, completed_at = now() WHERE status IN ('processing','queued') AND (tool_id IS NULL OR tool_id = '')`, + ) + .then((r) => { + const n = (r as { rowCount?: number }).rowCount ?? 0; + if (n > 0) app.log.info({ count: n }, "Reconciled orphaned job rows at startup"); + }) + .catch((err) => app.log.warn({ err }, "Orphaned-job reconciliation failed")); + // Warm the per-pool QueueEvents consumers so the first synchronous tool request // after boot does not pay the lazy-connect cost (and cannot miss a fast job's // completion event). Non-blocking: a slow/unreachable Redis must not stall boot; diff --git a/apps/api/src/jobs/system-jobs.ts b/apps/api/src/jobs/system-jobs.ts index 9a21f110..4e5c30d9 100644 --- a/apps/api/src/jobs/system-jobs.ts +++ b/apps/api/src/jobs/system-jobs.ts @@ -15,6 +15,7 @@ import { env } from "../config.js"; import { db, schema } from "../db/index.js"; import { getMaxAgeMs } from "../lib/cleanup.js"; import { deletePrefix, listJobDirs, type ObjectInfo } from "../lib/object-storage.js"; +import { getSettingNumber } from "../lib/settings-helpers.js"; import { runAuditArchive } from "./audit-archive.js"; import { getQueue } from "./queues.js"; import { runSiemForward } from "./siem-forward.js"; @@ -275,15 +276,20 @@ async function retentionSweep(): Promise { WHERE u.legal_hold = true OR t.legal_hold = true )`; - if (env.JOBS_RETENTION_DAYS > 0) { + // The Data Retention settings UI writes jobsRetentionDays / auditRetentionDays to + // the DB. Honor those (the env vars are the fallback default), mirroring how the + // temp-file sweep reads tempFileMaxAgeHours; otherwise the UI controls are no-ops. + const jobsRetentionDays = await getSettingNumber("jobsRetentionDays", env.JOBS_RETENTION_DAYS); + if (jobsRetentionDays > 0) { await db.execute( sql`DELETE FROM jobs - WHERE created_at < now() - ${env.JOBS_RETENTION_DAYS} * interval '1 day' + WHERE created_at < now() - ${jobsRetentionDays} * interval '1 day' AND status IN ('completed', 'failed', 'canceled') AND (user_id IS NULL OR user_id NOT IN ${heldUsersSubquery})`, ); } - if (env.AUDIT_RETENTION_DAYS > 0) { + const auditRetentionDays = await getSettingNumber("auditRetentionDays", env.AUDIT_RETENTION_DAYS); + if (auditRetentionDays > 0) { const tamperResult = await db .select({ value: schema.settings.value }) .from(schema.settings) @@ -296,7 +302,7 @@ async function retentionSweep(): Promise { if (!isTamperResistant) { await db.execute( sql`DELETE FROM audit_log - WHERE created_at < now() - ${env.AUDIT_RETENTION_DAYS} * interval '1 day' + WHERE created_at < now() - ${auditRetentionDays} * interval '1 day' AND (actor_id IS NULL OR actor_id NOT IN ${heldUsersSubquery})`, ); } diff --git a/apps/api/src/lib/metrics.ts b/apps/api/src/lib/metrics.ts index 5e7320c0..d483bbdc 100644 --- a/apps/api/src/lib/metrics.ts +++ b/apps/api/src/lib/metrics.ts @@ -5,7 +5,7 @@ * and a metricsText() function that appends live queue-depth gauges * from BullMQ before returning the scrape payload. */ -import { Counter, collectDefaultMetrics, Gauge, Histogram, Registry } from "prom-client"; +import { Counter, collectDefaultMetrics, Histogram, Registry } from "prom-client"; import { perPoolCounts } from "../jobs/queues.js"; export const registry = new Registry(); @@ -34,13 +34,6 @@ export const requestDuration = new Histogram({ registers: [registry], }); -export const storageUsage = new Gauge({ - name: "snapotter_storage_bytes", - help: "Storage usage in bytes", - labelNames: ["category"] as const, - registers: [registry], -}); - export const authAttempts = new Counter({ name: "snapotter_auth_attempts_total", help: "Authentication attempts", diff --git a/apps/api/src/routes/roles.ts b/apps/api/src/routes/roles.ts index c78cb37c..8cac646a 100644 --- a/apps/api/src/routes/roles.ts +++ b/apps/api/src/routes/roles.ts @@ -22,6 +22,9 @@ const ALL_PERMISSIONS: Permission[] = [ "features:manage", "system:health", "audit:read", + "compliance:manage", + "webhooks:manage", + "security:manage", ]; const roleNameField = z diff --git a/apps/api/src/tracing.ts b/apps/api/src/tracing.ts index 3843398d..946a45df 100644 --- a/apps/api/src/tracing.ts +++ b/apps/api/src/tracing.ts @@ -79,7 +79,10 @@ const endpoint = process.env.OTEL_EXPORTER_OTLP_ENDPOINT; if (endpoint) { try { const enterprise = await import("@snapotter/enterprise"); - const licenseKey = process.env.LICENSE_KEY ?? ""; + // The rest of the app reads SNAPOTTER_LICENSE_KEY (env.ts / index.ts). Accept + // either here so distributed_tracing activates with the same key as every other + // enterprise feature; LICENSE_KEY kept as an override for preload-only setups. + const licenseKey = process.env.LICENSE_KEY ?? process.env.SNAPOTTER_LICENSE_KEY ?? ""; if (licenseKey) { enterprise.initEnterprise(licenseKey); } diff --git a/apps/docs/guide/configuration.md b/apps/docs/guide/configuration.md index 8e1a34f2..ff2d55bb 100644 --- a/apps/docs/guide/configuration.md +++ b/apps/docs/guide/configuration.md @@ -33,7 +33,7 @@ All configuration is done through environment variables. Every variable has a se | Variable | Default | Description | |---|---|---| -| `STORAGE_MODE` | `local` | `local` or `s3`. Only local storage is currently implemented. | +| `STORAGE_MODE` | `local` | `local` or `s3`. S3/MinIO requires a license with the s3_storage feature. | | `DATABASE_URL` | `postgres://snapotter:snapotter@postgres:5432/snapotter` | PostgreSQL connection string. | | `REDIS_URL` | `redis://redis:6379` | Redis connection string (used for BullMQ job queues). | | `WORKSPACE_PATH` | `./tmp/workspace` | Directory for temporary files during processing. Cleaned up automatically. | diff --git a/apps/web/src/App.tsx b/apps/web/src/App.tsx index d675a802..e1d5b7ac 100644 --- a/apps/web/src/App.tsx +++ b/apps/web/src/App.tsx @@ -1,4 +1,4 @@ -import { en } from "@snapotter/shared"; +import { ANALYTICS_EVENTS, en } from "@snapotter/shared"; import { Component, type ErrorInfo, lazy, type ReactNode, Suspense, useEffect } from "react"; import { BrowserRouter, Navigate, Route, Routes, useLocation } from "react-router-dom"; import { Toaster, toast } from "sonner"; @@ -8,7 +8,7 @@ import { RouteAnnouncer } from "./components/common/route-announcer"; import { I18nProvider } from "./contexts/i18n-context"; import { useAuth } from "./hooks/use-auth"; import { useMobile } from "./hooks/use-mobile"; -import { initAnalytics } from "./lib/analytics"; +import { initAnalytics, track } from "./lib/analytics"; import { useAnalyticsStore } from "./stores/analytics-store"; // Lazy-load all pages so each page's JS (and its icons/deps) is only @@ -48,6 +48,8 @@ class ErrorBoundary extends Component< componentDidCatch(error: Error, info: ErrorInfo) { console.error("Uncaught render error:", error, info.componentStack); + // Mirror the crash to PostHog (error class only, no PII). track() is best-effort. + track(ANALYTICS_EVENTS.TOOL_CLIENT_ERROR, { error_name: error.name }); import("@sentry/react") .then((Sentry) => { Sentry.captureException(error, { diff --git a/apps/web/src/components/settings/settings-dialog.tsx b/apps/web/src/components/settings/settings-dialog.tsx index ca682c55..d61315ed 100644 --- a/apps/web/src/components/settings/settings-dialog.tsx +++ b/apps/web/src/components/settings/settings-dialog.tsx @@ -690,20 +690,10 @@ function SystemSection() { {t.settings.dataRetention.title} - - updateSetting("tempFileMaxAgeHours", e.target.value)} - aria-label={t.settings.dataRetention.fileMaxAgeHours} - className="px-3 py-1.5 rounded-lg border border-border bg-background text-sm text-foreground w-24" - min={1} - max={8760} - /> - + {/* NOTE: the temp-file TTL (tempFileMaxAgeHours) is configured once under + File Management above. A second control here bound the same key with a + different default, so editing either silently overwrote the other. + Data Retention keeps only the DB-row retention controls below. */} +