feat: expand Prometheus metrics with request duration, storage, and auth counters

apps/api/src/index.ts

@@ -22,6 +22,7 @@ import { shouldRunStartupCleanup } from "./lib/cleanup.js";
 import { buildCsp } from "./lib/csp.js";
 import { ensureAiDirs, recoverInterruptedInstalls } from "./lib/feature-status.js";
 
+import { requestDuration } from "./lib/metrics.js";
 import { getSettingString } from "./lib/settings-helpers.js";
 import { requirePermission } from "./permissions.js";
 import {
@@ -270,6 +271,26 @@ app.addHook("onSend", async (_request, reply) => {
   reply.header("Content-Security-Policy", buildCsp(_request.url.startsWith("/api/docs")));
 });
 
+// Record HTTP request duration for Prometheus (bounded cardinality: 5 route groups * 5 status classes)
+app.addHook("onResponse", (request, reply, done) => {
+  const duration = reply.elapsedTime / 1000;
+
+  const url = request.url;
+  let routeGroup = "other";
+  if (url.startsWith("/api/v1/tools/") || url.startsWith("/api/v1/jobs/")) routeGroup = "tools";
+  else if (url.startsWith("/api/auth/") || url.startsWith("/api/v1/enterprise/"))
+    routeGroup = "auth";
+  else if (url.startsWith("/api/v1/admin/") || url.startsWith("/api/v1/settings"))
+    routeGroup = "admin";
+  else if (url.startsWith("/api/v1/files")) routeGroup = "files";
+  else if (url.startsWith("/api/v1/scim/")) routeGroup = "scim";
+
+  const statusClass = `${Math.floor(reply.statusCode / 100)}xx`;
+
+  requestDuration.observe({ route_group: routeGroup, status_class: statusClass }, duration);
+  done();
+});
+
 // Always register rate-limit plugin so per-route limits (login brute-force protection) work.
 // max=0 means "unlimited" (50k/min) -- @fastify/rate-limit treats literal 0 as "block all".
 await app.register(rateLimit, {

apps/api/src/lib/metrics.ts

@@ -5,7 +5,7 @@
  * and a metricsText() function that appends live queue-depth gauges
  * from BullMQ before returning the scrape payload.
  */
-import { Counter, collectDefaultMetrics, Histogram, Registry } from "prom-client";
+import { Counter, collectDefaultMetrics, Gauge, Histogram, Registry } from "prom-client";
 import { perPoolCounts } from "../jobs/queues.js";
 
 export const registry = new Registry();
@@ -26,6 +26,28 @@ export const jobDuration = new Histogram({
   registers: [registry],
 });
 
+export const requestDuration = new Histogram({
+  name: "snapotter_http_request_duration_seconds",
+  help: "HTTP request duration in seconds",
+  labelNames: ["route_group", "status_class"] as const,
+  buckets: [0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10],
+  registers: [registry],
+});
+
+export const storageUsage = new Gauge({
+  name: "snapotter_storage_bytes",
+  help: "Storage usage in bytes",
+  labelNames: ["category"] as const,
+  registers: [registry],
+});
+
+export const authAttempts = new Counter({
+  name: "snapotter_auth_attempts_total",
+  help: "Authentication attempts",
+  labelNames: ["method", "result"] as const,
+  registers: [registry],
+});
+
 export async function metricsText(): Promise<string> {
   const counts = await perPoolCounts();
   const lines: string[] = [

apps/api/src/plugins/auth.ts

@@ -7,6 +7,7 @@ import { env } from "../config.js";
 import { db, schema } from "../db/index.js";
 import { sharedRedis } from "../jobs/connection.js";
 import { auditFromRequest, sanitizeAuditInput } from "../lib/audit.js";
+import { authAttempts } from "../lib/metrics.js";
 import { getSettingNumber, getSettingString } from "../lib/settings-helpers.js";
 import { getPermissions, requirePermission } from "../permissions.js";
 
@@ -326,6 +327,7 @@ export async function authRoutes(app: FastifyInstance): Promise<void> {
       const audit = auditFromRequest(request);
 
       if (!user || !user.passwordHash) {
+        authAttempts.inc({ method: "password", result: "failure" });
         await audit("LOGIN_FAILED", {
           username: sanitizeAuditInput(body.username),
           reason: "unknown_user",
@@ -335,6 +337,7 @@ export async function authRoutes(app: FastifyInstance): Promise<void> {
 
       const valid = await verifyPassword(body.password, user.passwordHash);
       if (!valid) {
+        authAttempts.inc({ method: "password", result: "failure" });
         await audit("LOGIN_FAILED", {
           username: sanitizeAuditInput(body.username),
           reason: "bad_password",
@@ -395,6 +398,7 @@ export async function authRoutes(app: FastifyInstance): Promise<void> {
         }
       }
 
+      authAttempts.inc({ method: "password", result: "success" });
       await audit("LOGIN_SUCCESS", { userId: user.id, username: user.username });
 
       const [teamRow] = await db.select().from(schema.teams).where(eq(schema.teams.id, user.team));
@@ -1091,6 +1095,7 @@ export async function authMiddleware(app: FastifyInstance): Promise<void> {
               .from(schema.users)
               .where(eq(schema.users.id, key.userId));
             if (apiUser) {
+              authAttempts.inc({ method: "apikey", result: "success" });
               const keyPermissions = key.permissions ?? undefined;
               (request as FastifyRequest & { user?: AuthUser }).user = {
                 id: apiUser.id,
@@ -1102,6 +1107,7 @@ export async function authMiddleware(app: FastifyInstance): Promise<void> {
             }
           }
         }
+        authAttempts.inc({ method: "apikey", result: "failure" });
       }
 
       // Public routes can proceed without a valid session

apps/api/src/plugins/oidc.ts

@@ -5,6 +5,7 @@ import { env } from "../config.js";
 import { db, schema } from "../db/index.js";
 import { auditFromRequest, sanitizeAuditInput } from "../lib/audit.js";
 import { resolveExternalUser, sanitizeUsername } from "../lib/external-auth-resolver.js";
+import { authAttempts } from "../lib/metrics.js";
 import { createSessionToken } from "./auth.js";
 
 // ── Types ─────────────────────────────────────────────────────────
@@ -189,6 +190,7 @@ export async function oidcRoutes(app: FastifyInstance): Promise<void> {
         { error: query.error, description: query.error_description },
         "OIDC IdP returned error",
       );
+      authAttempts.inc({ method: "oidc", result: "failure" });
       await audit("OIDC_LOGIN_FAILED", {
         reason: sanitizeAuditInput(String(query.error)),
       });
@@ -219,6 +221,7 @@ export async function oidcRoutes(app: FastifyInstance): Promise<void> {
       });
     } catch (err) {
       request.log.error({ err }, "OIDC token exchange failed");
+      authAttempts.inc({ method: "oidc", result: "failure" });
       await audit("OIDC_LOGIN_FAILED", { reason: "token_exchange_failed" });
       return redirectToLogin(reply, "oidc_auth_failed");
     }
@@ -227,6 +230,7 @@ export async function oidcRoutes(app: FastifyInstance): Promise<void> {
     const claims = tokenResponse.claims();
     if (!claims) {
       request.log.error("OIDC callback: no ID token claims");
+      authAttempts.inc({ method: "oidc", result: "failure" });
       await audit("OIDC_LOGIN_FAILED", { reason: "no_id_token" });
       return redirectToLogin(reply, "oidc_auth_failed");
     }
@@ -254,6 +258,7 @@ export async function oidcRoutes(app: FastifyInstance): Promise<void> {
     });
 
     if (result.action === "denied" || !result.user) {
+      authAttempts.inc({ method: "oidc", result: "failure" });
       if (result.deniedReason === "user_limit_reached") {
         return redirectToLogin(reply, "oidc_user_limit_reached");
       }
@@ -273,6 +278,7 @@ export async function oidcRoutes(app: FastifyInstance): Promise<void> {
       idToken,
     });
 
+    authAttempts.inc({ method: "oidc", result: "success" });
     await audit("OIDC_LOGIN_SUCCESS", {
       userId: resolvedUser.id,
       username: resolvedUser.username,

apps/api/src/plugins/saml.ts

@@ -10,6 +10,7 @@ import {
   resolveExternalUser,
   sanitizeUsername,
 } from "../lib/external-auth-resolver.js";
+import { authAttempts } from "../lib/metrics.js";
 import { createSessionToken } from "./auth.js";
 
 // -- SAML instance factory ----------------------------------------------------
@@ -100,6 +101,7 @@ export async function registerSaml(app: FastifyInstance): Promise<void> {
       profile = result.profile;
     } catch (err) {
       request.log.error({ err }, "SAML assertion validation failed");
+      authAttempts.inc({ method: "saml", result: "failure" });
       await audit("SAML_LOGIN_FAILED", {
         error: err instanceof Error ? err.message : "Unknown error",
       });
@@ -108,6 +110,7 @@ export async function registerSaml(app: FastifyInstance): Promise<void> {
 
     if (!profile || !profile.nameID) {
       request.log.warn("SAML callback: no profile or nameID in assertion");
+      authAttempts.inc({ method: "saml", result: "failure" });
       await audit("SAML_LOGIN_FAILED", { reason: "missing_profile" });
       return redirectToLogin(reply, "saml_auth_failed");
     }
@@ -140,6 +143,7 @@ export async function registerSaml(app: FastifyInstance): Promise<void> {
     });
 
     if (result.action === "denied" || !result.user) {
+      authAttempts.inc({ method: "saml", result: "failure" });
       const errorParam =
         result.deniedReason === "user_limit_reached"
           ? "saml_user_limit_reached"
@@ -159,6 +163,7 @@ export async function registerSaml(app: FastifyInstance): Promise<void> {
       expiresAt,
     });
 
+    authAttempts.inc({ method: "saml", result: "success" });
     await audit("SAML_LOGIN_SUCCESS", {
       userId: resolvedUser.id,
       username: resolvedUser.username,