fix(api): wait for Postgres and Redis at startup instead of crash-looping (#537)

Briefly retry Postgres and Redis connectivity at boot (waitForService, DB_STARTUP_TIMEOUT_MS) so an ordered-but-not-yet-ready dependency recovers cleanly instead of crash-looping.

apps/api/src/index.ts

@@ -38,6 +38,7 @@ import { purgeOcrRuntimeDownloads, runOcrRuntimeMaintenance } from "./lib/ocr-ru
 import { getSettingString } from "./lib/settings-helpers.js";
 import { assertStorageWritable } from "./lib/storage-writable.js";
 import { gatherSystemProperties } from "./lib/system-info.js";
+import { waitForService } from "./lib/wait-for-service.js";
 import { requirePermission } from "./permissions.js";
 import {
   authMiddleware,
@@ -77,8 +78,22 @@ import { registerToolRoutes } from "./routes/tools/index.js";
 import { userFileRoutes } from "./routes/user-files.js";
 import { shutdownTracing } from "./tracing.js";
 
-// Run before anything else
+// Run before anything else. Wait briefly for Postgres to accept connections: on a
+// fresh boot it may still be starting (Compose without a healthcheck gate, or a
+// systemd unit ordered after Debian's no-op `postgresql.service` umbrella rather
+// than the real cluster), and a short retry turns a crash-loop into a clean start.
 try {
+  await waitForService(() => db.execute(sql`SELECT 1`).then(() => undefined), {
+    timeoutMs: env.DB_STARTUP_TIMEOUT_MS,
+    intervalMs: 1_000,
+    onRetry: (attempt) => {
+      if (attempt === 1) {
+        console.log(
+          `Waiting up to ${Math.round(env.DB_STARTUP_TIMEOUT_MS / 1000)}s for Postgres to accept connections...`,
+        );
+      }
+    },
+  });
   await runMigrations();
 } catch (err) {
   const safeUrl = env.DATABASE_URL.replace(/:\/\/[^@]*@/, "://***@");
@@ -90,9 +105,25 @@ try {
 }
 console.log("Database initialized");
 
-// Verify Redis is reachable (required for BullMQ job queues)
+// Verify Redis is reachable (required for BullMQ job queues). Same brief wait as
+// Postgres so an ordered-but-not-ready Redis recovers instead of crash-looping.
 try {
-  await pingRedis();
+  await waitForService(
+    async () => {
+      if (!(await pingRedis())) throw new Error("Redis did not answer PONG");
+    },
+    {
+      timeoutMs: env.DB_STARTUP_TIMEOUT_MS,
+      intervalMs: 1_000,
+      onRetry: (attempt) => {
+        if (attempt === 1) {
+          console.log(
+            `Waiting up to ${Math.round(env.DB_STARTUP_TIMEOUT_MS / 1000)}s for Redis to accept connections...`,
+          );
+        }
+      },
+    },
+  );
 } catch (err) {
   const safeUrl = env.REDIS_URL.replace(/:\/\/[^@]*@/, "://***@");
   console.error(

apps/api/src/lib/env.ts

@@ -36,6 +36,10 @@ const envSchema = z
     RATE_LIMIT_PER_MIN: z.coerce.number().default(1000),
     API_KEYS_RATE_LIMIT_PER_MIN: z.coerce.number().default(30),
     DATABASE_URL: z.string().default("postgres://snapotter:snapotter@localhost:5432/snapotter"),
+    // How long to wait for Postgres/Redis to accept connections at startup before
+    // giving up. A dependency ordered but not yet ready (fresh boot) recovers within
+    // this window instead of crash-looping. 0 = try once, fail fast.
+    DB_STARTUP_TIMEOUT_MS: z.coerce.number().default(30_000),
     SQLITE_MIGRATE_PATH: z.string().default(""),
     DATA_DIR: z.string().default("./data"),
     FILES_STORAGE_PATH: z.string().default("./data/files"),

apps/api/src/lib/wait-for-service.ts

@@ -0,0 +1,46 @@
+export interface WaitForServiceOptions {
+  /** Give up (and rethrow the last probe error) after this many milliseconds. */
+  timeoutMs: number;
+  /** Delay between probe attempts. */
+  intervalMs: number;
+  /** Called after each failed attempt, before the next sleep. */
+  onRetry?: (attempt: number, err: unknown) => void;
+  /** Injectable clock (tests); defaults to Date.now. */
+  now?: () => number;
+  /** Injectable sleep (tests); defaults to a real timer. */
+  sleep?: (ms: number) => Promise<void>;
+}
+
+const realSleep = (ms: number): Promise<void> => new Promise((resolve) => setTimeout(resolve, ms));
+
+/**
+ * Poll `probe` until it resolves without throwing, or `timeoutMs` elapses.
+ *
+ * A startup dependency (Postgres, Redis) is often reachable only a few seconds
+ * after the app process starts: Docker Compose without a healthcheck gate, or a
+ * native systemd unit ordered after Debian's no-op `postgresql.service` umbrella
+ * rather than the real cluster. Retrying briefly turns a crash-loop into a clean
+ * boot; a genuinely-absent dependency still surfaces the real error on timeout.
+ */
+export async function waitForService(
+  probe: () => Promise<void>,
+  options: WaitForServiceOptions,
+): Promise<void> {
+  const { timeoutMs, intervalMs, onRetry } = options;
+  const now = options.now ?? Date.now;
+  const sleep = options.sleep ?? realSleep;
+
+  const deadline = now() + timeoutMs;
+  let attempt = 0;
+  for (;;) {
+    attempt++;
+    try {
+      await probe();
+      return;
+    } catch (err) {
+      if (now() >= deadline) throw err;
+      onRetry?.(attempt, err);
+      await sleep(intervalMs);
+    }
+  }
+}