fix(api): wait for Postgres and Redis at startup instead of crash-looping (#537)

Briefly retry Postgres and Redis connectivity at boot (waitForService, DB_STARTUP_TIMEOUT_MS) so an ordered-but-not-yet-ready dependency recovers cleanly instead of crash-looping.

.env.example

@@ -67,6 +67,10 @@ APP_NAME=snapotter
 DATABASE_URL=postgres://snapotter:snapotter@localhost:5432/snapotter
 # Redis connection (required; this default matches docker-compose.dev.yml)
 REDIS_URL=redis://localhost:6379
+# Startup grace: wait this long (ms) for Postgres/Redis to accept connections
+# before failing. Lets a dependency that is still booting recover instead of
+# crash-looping. 0 = try once, fail fast.
+DB_STARTUP_TIMEOUT_MS=30000
 
 # Job spine tuning
 SYNC_WAIT_MS=8000              # sync-response window (ms) before returning 202

apps/api/src/index.ts

@@ -38,6 +38,7 @@ import { purgeOcrRuntimeDownloads, runOcrRuntimeMaintenance } from "./lib/ocr-ru
 import { getSettingString } from "./lib/settings-helpers.js";
 import { assertStorageWritable } from "./lib/storage-writable.js";
 import { gatherSystemProperties } from "./lib/system-info.js";
+import { waitForService } from "./lib/wait-for-service.js";
 import { requirePermission } from "./permissions.js";
 import {
   authMiddleware,
@@ -77,8 +78,22 @@ import { registerToolRoutes } from "./routes/tools/index.js";
 import { userFileRoutes } from "./routes/user-files.js";
 import { shutdownTracing } from "./tracing.js";
 
-// Run before anything else
+// Run before anything else. Wait briefly for Postgres to accept connections: on a
+// fresh boot it may still be starting (Compose without a healthcheck gate, or a
+// systemd unit ordered after Debian's no-op `postgresql.service` umbrella rather
+// than the real cluster), and a short retry turns a crash-loop into a clean start.
 try {
+  await waitForService(() => db.execute(sql`SELECT 1`).then(() => undefined), {
+    timeoutMs: env.DB_STARTUP_TIMEOUT_MS,
+    intervalMs: 1_000,
+    onRetry: (attempt) => {
+      if (attempt === 1) {
+        console.log(
+          `Waiting up to ${Math.round(env.DB_STARTUP_TIMEOUT_MS / 1000)}s for Postgres to accept connections...`,
+        );
+      }
+    },
+  });
   await runMigrations();
 } catch (err) {
   const safeUrl = env.DATABASE_URL.replace(/:\/\/[^@]*@/, "://***@");
@@ -90,9 +105,25 @@ try {
 }
 console.log("Database initialized");
 
-// Verify Redis is reachable (required for BullMQ job queues)
+// Verify Redis is reachable (required for BullMQ job queues). Same brief wait as
+// Postgres so an ordered-but-not-ready Redis recovers instead of crash-looping.
 try {
-  await pingRedis();
+  await waitForService(
+    async () => {
+      if (!(await pingRedis())) throw new Error("Redis did not answer PONG");
+    },
+    {
+      timeoutMs: env.DB_STARTUP_TIMEOUT_MS,
+      intervalMs: 1_000,
+      onRetry: (attempt) => {
+        if (attempt === 1) {
+          console.log(
+            `Waiting up to ${Math.round(env.DB_STARTUP_TIMEOUT_MS / 1000)}s for Redis to accept connections...`,
+          );
+        }
+      },
+    },
+  );
 } catch (err) {
   const safeUrl = env.REDIS_URL.replace(/:\/\/[^@]*@/, "://***@");
   console.error(

apps/api/src/lib/env.ts

@@ -36,6 +36,10 @@ const envSchema = z
     RATE_LIMIT_PER_MIN: z.coerce.number().default(1000),
     API_KEYS_RATE_LIMIT_PER_MIN: z.coerce.number().default(30),
     DATABASE_URL: z.string().default("postgres://snapotter:snapotter@localhost:5432/snapotter"),
+    // How long to wait for Postgres/Redis to accept connections at startup before
+    // giving up. A dependency ordered but not yet ready (fresh boot) recovers within
+    // this window instead of crash-looping. 0 = try once, fail fast.
+    DB_STARTUP_TIMEOUT_MS: z.coerce.number().default(30_000),
     SQLITE_MIGRATE_PATH: z.string().default(""),
     DATA_DIR: z.string().default("./data"),
     FILES_STORAGE_PATH: z.string().default("./data/files"),

apps/api/src/lib/wait-for-service.ts

@@ -0,0 +1,46 @@
+export interface WaitForServiceOptions {
+  /** Give up (and rethrow the last probe error) after this many milliseconds. */
+  timeoutMs: number;
+  /** Delay between probe attempts. */
+  intervalMs: number;
+  /** Called after each failed attempt, before the next sleep. */
+  onRetry?: (attempt: number, err: unknown) => void;
+  /** Injectable clock (tests); defaults to Date.now. */
+  now?: () => number;
+  /** Injectable sleep (tests); defaults to a real timer. */
+  sleep?: (ms: number) => Promise<void>;
+}
+
+const realSleep = (ms: number): Promise<void> => new Promise((resolve) => setTimeout(resolve, ms));
+
+/**
+ * Poll `probe` until it resolves without throwing, or `timeoutMs` elapses.
+ *
+ * A startup dependency (Postgres, Redis) is often reachable only a few seconds
+ * after the app process starts: Docker Compose without a healthcheck gate, or a
+ * native systemd unit ordered after Debian's no-op `postgresql.service` umbrella
+ * rather than the real cluster. Retrying briefly turns a crash-loop into a clean
+ * boot; a genuinely-absent dependency still surfaces the real error on timeout.
+ */
+export async function waitForService(
+  probe: () => Promise<void>,
+  options: WaitForServiceOptions,
+): Promise<void> {
+  const { timeoutMs, intervalMs, onRetry } = options;
+  const now = options.now ?? Date.now;
+  const sleep = options.sleep ?? realSleep;
+
+  const deadline = now() + timeoutMs;
+  let attempt = 0;
+  for (;;) {
+    attempt++;
+    try {
+      await probe();
+      return;
+    } catch (err) {
+      if (now() >= deadline) throw err;
+      onRetry?.(attempt, err);
+      await sleep(intervalMs);
+    }
+  }
+}

tests/unit/wait-for-service.test.ts

@@ -0,0 +1,79 @@
+import { describe, expect, it } from "vitest";
+import { waitForService } from "../../apps/api/src/lib/wait-for-service.js";
+
+// Deterministic fake clock: `sleep` advances virtual time and `now` reads it,
+// so we can assert retry timing without real timers slowing the suite down.
+function fakeClock() {
+  let ms = 0;
+  return {
+    now: () => ms,
+    sleep: async (delta: number) => {
+      ms += delta;
+    },
+    elapsed: () => ms,
+  };
+}
+
+describe("waitForService", () => {
+  it("resolves immediately when the probe succeeds on the first attempt", async () => {
+    const clock = fakeClock();
+    let attempts = 0;
+    await waitForService(
+      async () => {
+        attempts++;
+      },
+      { timeoutMs: 30_000, intervalMs: 1_000, now: clock.now, sleep: clock.sleep },
+    );
+    expect(attempts).toBe(1);
+    expect(clock.elapsed()).toBe(0); // never slept
+  });
+
+  it("retries until the probe succeeds, then resolves", async () => {
+    const clock = fakeClock();
+    let attempts = 0;
+    await waitForService(
+      async () => {
+        attempts++;
+        if (attempts < 3) throw new Error("not ready");
+      },
+      { timeoutMs: 30_000, intervalMs: 1_000, now: clock.now, sleep: clock.sleep },
+    );
+    expect(attempts).toBe(3);
+    expect(clock.elapsed()).toBe(2_000); // slept twice between three attempts
+  });
+
+  it("rejects with the last probe error once the timeout elapses", async () => {
+    const clock = fakeClock();
+    let attempts = 0;
+    await expect(
+      waitForService(
+        async () => {
+          attempts++;
+          throw new Error(`refused #${attempts}`);
+        },
+        { timeoutMs: 3_000, intervalMs: 1_000, now: clock.now, sleep: clock.sleep },
+      ),
+    ).rejects.toThrow("refused #4"); // attempts fire at t=0,1000,2000,3000
+    expect(attempts).toBe(4);
+  });
+
+  it("reports each failed attempt through the onRetry callback", async () => {
+    const clock = fakeClock();
+    let attempts = 0;
+    const retried: number[] = [];
+    await waitForService(
+      async () => {
+        attempts++;
+        if (attempts < 3) throw new Error("not ready");
+      },
+      {
+        timeoutMs: 30_000,
+        intervalMs: 1_000,
+        now: clock.now,
+        sleep: clock.sleep,
+        onRetry: (attempt) => retried.push(attempt),
+      },
+    );
+    expect(retried).toEqual([1, 2]); // two failures before the third-attempt success
+  });
+});