fix(docker): make storage writable under non-root/foreign UIDs (TrueNAS, OpenShift) (#299)

The entrypoint only fixed volume permissions when started as root (chown +
gosu-drop to snapotter). Launched under a non-root/foreign UID (TrueNAS app
user, Kubernetes runAsUser, OpenShift) it did no permission setup, so /data and
/tmp/workspace -- owned by uid 999 from the image -- were not writable by the
running user. Uploads and processing then failed with a cryptic EACCES
("workspace folder is not writable") and AI bundle installs failed the same way,
while health checks still reported the container healthy.

- entrypoint: source new entrypoint-lib.sh; verify writability up front when
  non-root, and as snapotter after chown when root (catches root-squashed
  mounts), failing fast with an actionable message (which dir, uid/gid, how to
  fix) instead of a late, cryptic EACCES
- Dockerfile: own /data and /tmp/workspace as snapotter:0, group-writable with
  setgid, so an arbitrary UID with the root supplementary group (OpenShift /
  Kubernetes fsGroup) can write; keep /opt/venv world-readable for the AI venv
  bootstrap under arbitrary UIDs
- api: assert storage writability at boot (lib/storage-writable.ts), failing
  fast with the same guidance even when the entrypoint is bypassed
- docs: add a Storage permissions section (named volumes, bind mounts, TrueNAS,
  Kubernetes/OpenShift) and cross-link it from the security guide

Fixes #230

apps/api/src/index.ts

@@ -25,6 +25,7 @@ import { ensureAiDirs, recoverInterruptedInstalls } from "./lib/feature-status.j
 import { logger } from "./lib/logger.js";
 import { requestDuration } from "./lib/metrics.js";
 import { getSettingString } from "./lib/settings-helpers.js";
+import { assertStorageWritable } from "./lib/storage-writable.js";
 import { requirePermission } from "./permissions.js";
 import {
   authMiddleware,
@@ -87,6 +88,18 @@ try {
 }
 console.log("Redis connected");
 
+// Verify the local storage directories are writable before serving. A non-root
+// container launched against a volume it cannot write (TrueNAS, Kubernetes
+// runAsUser / OpenShift, or a bind mount owned by another user) would otherwise
+// boot "healthy" and fail with a cryptic EACCES on the first file operation.
+try {
+  await assertStorageWritable();
+  console.log("Storage directories writable");
+} catch (err) {
+  console.error(`FATAL: ${(err as Error).message}`);
+  process.exit(1);
+}
+
 // Auto-import 1.x SQLite database on first boot (before default user creation)
 if (env.SQLITE_MIGRATE_PATH) {
   const { rows } = await db.execute(sql`SELECT count(*)::int AS n FROM users`);

apps/api/src/lib/storage-writable.ts

@@ -0,0 +1,73 @@
+import { randomUUID } from "node:crypto";
+import { mkdir, rm, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { env } from "../config.js";
+
+// Error codes that specifically mean "the running user may not write here".
+// Distinct from capacity (ENOSPC) or other I/O faults, which are handled
+// elsewhere and must not be misreported as a permissions problem.
+const NOT_WRITABLE_CODES = new Set(["EACCES", "EPERM", "EROFS"]);
+
+/**
+ * Returns true if `dir` can be created (when missing) and written to by the
+ * current process. Creates the directory recursively, writes a short-lived
+ * probe file, then removes it. Permission/read-only failures (EACCES, EPERM,
+ * EROFS) resolve to false; any other error is rethrown so genuine faults are
+ * not silently swallowed.
+ */
+export async function isDirWritable(dir: string): Promise<boolean> {
+  try {
+    await mkdir(dir, { recursive: true });
+    const probe = join(dir, `.snapotter-write-probe-${randomUUID()}`);
+    await writeFile(probe, "");
+    await rm(probe, { force: true });
+    return true;
+  } catch (err) {
+    const code = (err as NodeJS.ErrnoException).code;
+    if (code && NOT_WRITABLE_CODES.has(code)) return false;
+    throw err;
+  }
+}
+
+/** Best-effort current uid/gid as strings ("?" where getuid is unavailable). */
+function currentIds(): { uid: string; gid: string } {
+  const uid = typeof process.getuid === "function" ? String(process.getuid()) : "?";
+  const gid = typeof process.getgid === "function" ? String(process.getgid()) : "?";
+  return { uid, gid };
+}
+
+/**
+ * Actionable error text for a storage directory the process cannot write to.
+ * Names the directory and the running uid/gid, then lists the supported fixes
+ * for the common "container runs under a foreign/non-root UID" deployments
+ * (TrueNAS, Kubernetes runAsUser, OpenShift, bind mounts).
+ */
+export function storagePermissionMessage(dir: string): string {
+  const { uid, gid } = currentIds();
+  return [
+    `Storage directory "${dir}" is not writable by the current user (uid=${uid} gid=${gid}).`,
+    "SnapOtter cannot upload, process, or store files until this is fixed. Common fixes:",
+    `  - Host volume owned by another user: on the host run "chown -R ${uid}:${gid} <host-path>"` +
+      " (or set the container user to match the volume's owner).",
+    "  - Running as a non-root user (TrueNAS, Kubernetes runAsUser, OpenShift): run the container" +
+      " as root (the default entrypoint self-corrects), set PUID/PGID to match the volume, or grant" +
+      " the process supplementary group 0 (Kubernetes fsGroup: 0).",
+    "  See https://docs.snapotter.com/guide/deployment#storage-permissions",
+  ].join("\n");
+}
+
+/**
+ * Verifies the local storage directories (WORKSPACE_PATH for processing,
+ * FILES_STORAGE_PATH for the saved library) are writable, throwing an Error
+ * with actionable remediation if not. No-op in S3 storage mode. Called at boot
+ * so a permissions misconfiguration fails fast with a clear message instead of
+ * surfacing as a cryptic EACCES on the first file operation.
+ */
+export async function assertStorageWritable(): Promise<void> {
+  if (env.STORAGE_MODE === "s3") return;
+  for (const dir of [env.WORKSPACE_PATH, env.FILES_STORAGE_PATH]) {
+    if (!(await isDirWritable(dir))) {
+      throw new Error(storagePermissionMessage(dir));
+    }
+  }
+}

apps/docs/guide/deployment.md

@@ -365,6 +365,38 @@ environment:
   - PGID=1000    # Your host GID (run: id -g)
 ```
 
+### Storage permissions
+
+SnapOtter writes to two locations at runtime: `/data` (user files, logs, AI models and the Python venv) and `/tmp/workspace` (temporary processing scratch). Both must be writable by the user the container runs as. If either is not, the container **fails fast at startup** with a message naming the directory, the running UID/GID, and how to fix it — instead of booting "healthy" and then failing on the first upload with a cryptic error.
+
+How permissions are handled depends on how the container is launched:
+
+**Default (starts as root, drops to `snapotter`)** — the entrypoint starts as root, fixes ownership of the mounted volumes, then drops to the unprivileged `snapotter` user via `gosu`. Named volumes work with no configuration. For bind mounts, set `PUID`/`PGID` to your host user (above) so the files it writes are owned by you.
+
+**Kubernetes / OpenShift (non-root via `runAsUser`)** — launched directly as a non-root user, the container cannot chown the volumes itself, so the orchestrator must make them writable. Set `fsGroup`:
+
+```yaml
+securityContext:
+  runAsUser: 999
+  runAsGroup: 999
+  fsGroup: 999        # makes mounted volumes writable by the pod
+```
+
+The image's writable directories are group-owned by GID 0 and group-writable, so a pod running with an **arbitrary UID** plus the root supplementary group (the OpenShift default) can write with no `chown`.
+
+**TrueNAS Scale (and other "foreign UID" setups)** — TrueNAS runs apps as a non-root user (often `568:568`) and mounts host datasets owned by a different user, so neither the entrypoint nor `fsGroup` makes them writable on its own. Choose one:
+
+- **Run the app as root** (recommended) — leave the app's user unset or set it to `0`, and let the default entrypoint fix permissions and drop to `snapotter`.
+- **Run as UID `999`** — set the app's user/group to `999:999` (SnapOtter's built-in `snapotter` user) so it matches the image's ownership.
+- **`chown` the host dataset** to the UID the container runs as, from the TrueNAS shell:
+
+  ```bash
+  # Use the UID from the startup error (or run `id` inside the container)
+  chown -R 568:568 /mnt/<pool>/<dataset>
+  ```
+
+The startup error names the exact UID to use, so the quickest path is to start the app once, read the message, then `chown` (or adjust the user) accordingly.
+
 ## Environment Variables
 
 | Variable | Default | Description |

apps/docs/guide/security.md

@@ -177,7 +177,7 @@ Docker Compose secrets (without Swarm) require Compose v2.23 or later.
 
 ## Kubernetes Deployment
 
-The entrypoint detects when the container is already running as non-root (e.g., via Kubernetes `runAsUser`) and skips the gosu privilege drop automatically.
+The entrypoint detects when the container is already running as non-root (e.g., via Kubernetes `runAsUser`) and skips the gosu privilege drop automatically. In that case it cannot chown the mounted volumes itself, so it verifies they are writable and exits early with actionable guidance if they are not — see [Storage permissions](/guide/deployment#storage-permissions) for `fsGroup` and foreign-UID setups (TrueNAS, OpenShift).
 
 **Recommended Pod SecurityContext:**
 

docker/Dockerfile

@@ -393,10 +393,22 @@ ENV PYTHONWARNINGS=default \
 
 # Create non-root user for runtime
 RUN groupadd -r snapotter && useradd -r -g snapotter -d /app -s /sbin/nologin snapotter
-RUN chown -R snapotter:snapotter /app /data /tmp/workspace /opt/venv
+# /app and /opt/venv are read-only at runtime -> owned by snapotter.
+# /data and /tmp/workspace are written at runtime: make them group-0 (root group)
+# owned and group-writable with the setgid bit so the app can still write when the
+# container is launched under an arbitrary/foreign UID (Kubernetes runAsUser,
+# OpenShift, TrueNAS), which always lands in the root (GID 0) supplementary group.
+# The root entrypoint re-chowns these to snapotter for the default gosu path.
+RUN chown -R snapotter:snapotter /app /opt/venv && \
+    chmod -R a+rX /opt/venv && \
+    chown -R snapotter:0 /data /tmp/workspace && \
+    chmod -R g+rwX /data /tmp/workspace && \
+    find /data /tmp/workspace -type d -exec chmod g+s {} +
 
-# Entrypoint fixes volume permissions then drops to snapotter via gosu
+# Entrypoint fixes volume permissions then drops to snapotter via gosu.
+# entrypoint-lib.sh holds the writability helpers it sources at startup.
 COPY docker/entrypoint.sh /usr/local/bin/entrypoint.sh
+COPY docker/entrypoint-lib.sh /usr/local/bin/entrypoint-lib.sh
 COPY docker/wait-for-postgres.mjs /app/docker/wait-for-postgres.mjs
 RUN chmod +x /usr/local/bin/entrypoint.sh
 

docker/entrypoint-lib.sh

@@ -0,0 +1,57 @@
+#!/bin/sh
+# Shared helpers for the SnapOtter container entrypoint. Sourced by
+# docker/entrypoint.sh and kept in its own file so the permission logic can be
+# unit-tested directly (tests/unit/security/entrypoint-permissions.test.ts)
+# rather than mirrored. Sourcing has no side effects -- only function defs.
+#
+# Functions use _-prefixed variables (sh has no portable `local`) to avoid
+# clobbering the caller's variables.
+
+# dir_writable <dir>
+# Creates <dir> (best-effort, recursive) and returns 0 if the current user can
+# write inside it, 1 otherwise. Probes by creating then removing a temp file:
+# an actual write is the only reliable check across ACLs, NFS root-squash, and
+# read-only mounts, which ownership/mode arithmetic alone would miss.
+dir_writable() {
+  _dw_dir="$1"
+  mkdir -p "$_dw_dir" 2>/dev/null || true
+  _dw_probe="$_dw_dir/.snapotter-write-probe.$$"
+  if touch "$_dw_probe" 2>/dev/null; then
+    rm -f "$_dw_probe" 2>/dev/null || true
+    return 0
+  fi
+  return 1
+}
+
+# print_storage_permission_error <dir>
+# Emits an actionable remediation message to stderr. Mirrors the wording of
+# storagePermissionMessage() in apps/api/src/lib/storage-writable.ts.
+print_storage_permission_error() {
+  _pe_dir="$1"
+  _pe_uid="$(id -u 2>/dev/null || echo '?')"
+  _pe_gid="$(id -g 2>/dev/null || echo '?')"
+  {
+    echo "FATAL: Storage directory \"$_pe_dir\" is not writable by the current user (uid=$_pe_uid gid=$_pe_gid)."
+    echo "SnapOtter cannot upload, process, or store files until this is fixed. Common fixes:"
+    echo "  - Host volume owned by another user: on the host run \"chown -R $_pe_uid:$_pe_gid <host-path>\""
+    echo "    (or set the container user to match the volume's owner)."
+    echo "  - Running as a non-root user (TrueNAS, Kubernetes runAsUser, OpenShift): run the container as"
+    echo "    root (the default entrypoint self-corrects), set PUID/PGID to match the volume, or grant the"
+    echo "    process supplementary group 0 (Kubernetes fsGroup: 0)."
+    echo "  See https://docs.snapotter.com/guide/deployment#storage-permissions"
+  } >&2
+}
+
+# ensure_writable <dir>...
+# Verifies every directory is writable, printing an actionable error for each
+# that is not. Returns 0 only when all are writable, 1 otherwise.
+ensure_writable() {
+  _ew_failed=0
+  for _ew_dir in "$@"; do
+    if ! dir_writable "$_ew_dir"; then
+      print_storage_permission_error "$_ew_dir"
+      _ew_failed=1
+    fi
+  done
+  return "$_ew_failed"
+}

docker/entrypoint.sh

@@ -1,6 +1,10 @@
 #!/bin/sh
 set -e
 
+# Shared permission helpers (dir_writable, ensure_writable). Lives beside this
+# script in the image; sourcing only defines functions (no side effects).
+. /usr/local/bin/entrypoint-lib.sh
+
 # --- Docker secret file convention (_FILE suffix) ---
 # For each supported var, if VAR_FILE is set, read the secret from that file
 # path into VAR. This lets users mount Docker/Kubernetes secrets instead of
@@ -45,6 +49,20 @@ export AUTH_ENABLED="${AUTH_ENABLED:-true}"
 export DEFAULT_USERNAME="${DEFAULT_USERNAME:-admin}"
 export DEFAULT_PASSWORD="${DEFAULT_PASSWORD:-admin}"
 
+# Writable directories the runtime needs: WS = processing scratch, DD = data
+# root (holds files, logs, and the AI venv/models). Honor env overrides.
+WS="${WORKSPACE_PATH:-/tmp/workspace}"
+DD="${DATA_DIR:-/data}"
+
+# When NOT starting as root we cannot chown mounted volumes (TrueNAS, Kubernetes
+# runAsUser / OpenShift). Verify up front that the volumes are writable by this
+# user and fail fast with actionable guidance -- otherwise the venv bootstrap
+# and the app below would die later with a cryptic EACCES.
+if [ "$(id -u)" != "0" ]; then
+  mkdir -p "$DD/files" "$DD/logs" "$DD/ai/models" "$DD/ai/pip-cache" "$DD/ai/venv" "$WS" 2>/dev/null || true
+  ensure_writable "$WS" "$DD" || exit 1
+fi
+
 # Clean up any interrupted bootstrap from a previous start
 AI_VENV="/data/ai/venv"
 AI_VENV_TMP="/data/ai/venv.bootstrapping"
@@ -175,6 +193,13 @@ if [ "$(id -u)" = "0" ]; then
   chown -R snapotter:snapotter /data /tmp/workspace 2>&1 || \
     echo "WARNING: Could not fix volume permissions. Use named volumes (not Windows bind mounts) to avoid this. See docs for details." >&2
 
+  # Root can write anywhere, so verify as the unprivileged snapotter user that
+  # actually runs the app. This catches root-squashed or foreign-owned mounts
+  # where the chown above silently failed, and fails fast with guidance.
+  if ! gosu snapotter sh -c '. /usr/local/bin/entrypoint-lib.sh; ensure_writable "$@"' _ "$WS" "$DD"; then
+    exit 1
+  fi
+
   print_banner
   exec gosu snapotter "$@"
 fi

tests/unit/api/storage-writable.test.ts

@@ -0,0 +1,103 @@
+import { chmodSync, mkdirSync, mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
+
+// storage-writable reads the configured storage paths from config.js. Mock it
+// with a mutable env object so each test can point the paths at a temp dir.
+// vi.hoisted keeps the object available to the hoisted vi.mock factory.
+const mockEnv = vi.hoisted(() => ({
+  STORAGE_MODE: "local",
+  WORKSPACE_PATH: "",
+  FILES_STORAGE_PATH: "",
+}));
+
+vi.mock("../../../apps/api/src/config.js", () => ({ env: mockEnv }));
+
+import {
+  assertStorageWritable,
+  isDirWritable,
+  storagePermissionMessage,
+} from "../../../apps/api/src/lib/storage-writable.js";
+
+// A read-only directory does not block writes for root (DAC_OVERRIDE), so the
+// "not writable" assertions only hold for an unprivileged user.
+const isRoot = typeof process.getuid === "function" && process.getuid() === 0;
+
+let root: string;
+
+beforeAll(() => {
+  root = mkdtempSync(join(tmpdir(), "storage-writable-"));
+});
+
+afterAll(() => {
+  // Restore perms so cleanup can recurse into the read-only dir.
+  try {
+    chmodSync(join(root, "readonly"), 0o755);
+  } catch {
+    /* may not exist */
+  }
+  rmSync(root, { recursive: true, force: true });
+});
+
+describe("isDirWritable", () => {
+  it("returns true for an existing writable directory", async () => {
+    const dir = join(root, "writable");
+    mkdirSync(dir, { recursive: true });
+    expect(await isDirWritable(dir)).toBe(true);
+  });
+
+  it("returns true for a missing directory whose parent is writable (creates it)", async () => {
+    const dir = join(root, "nested", "deep");
+    expect(await isDirWritable(dir)).toBe(true);
+  });
+
+  it.skipIf(isRoot)("returns false for a read-only directory", async () => {
+    const dir = join(root, "readonly");
+    mkdirSync(dir, { recursive: true });
+    chmodSync(dir, 0o555);
+    expect(await isDirWritable(dir)).toBe(false);
+  });
+});
+
+describe("storagePermissionMessage", () => {
+  it("names the directory and gives an actionable chown remediation", () => {
+    const msg = storagePermissionMessage("/tmp/workspace");
+    expect(msg).toContain("/tmp/workspace");
+    expect(msg.toLowerCase()).toContain("not writable");
+    expect(msg).toContain("chown");
+    // Includes the running uid so the operator knows what to chown to.
+    expect(msg).toMatch(/uid=/);
+  });
+});
+
+describe("assertStorageWritable", () => {
+  it("resolves when both storage paths are writable", async () => {
+    mockEnv.STORAGE_MODE = "local";
+    mockEnv.WORKSPACE_PATH = join(root, "ws-ok");
+    mockEnv.FILES_STORAGE_PATH = join(root, "files-ok");
+    await expect(assertStorageWritable()).resolves.toBeUndefined();
+  });
+
+  it.skipIf(isRoot)(
+    "rejects with an actionable message when the workspace is not writable",
+    async () => {
+      const ws = join(root, "ws-ro");
+      mkdirSync(ws, { recursive: true });
+      chmodSync(ws, 0o555);
+      mockEnv.STORAGE_MODE = "local";
+      mockEnv.WORKSPACE_PATH = ws;
+      mockEnv.FILES_STORAGE_PATH = join(root, "files-ok2");
+      await expect(assertStorageWritable()).rejects.toThrow(/not writable/i);
+      await expect(assertStorageWritable()).rejects.toThrow(ws);
+      chmodSync(ws, 0o755);
+    },
+  );
+
+  it("is a no-op in S3 storage mode (does not touch the filesystem)", async () => {
+    mockEnv.STORAGE_MODE = "s3";
+    mockEnv.WORKSPACE_PATH = "/nonexistent/should-not-be-touched";
+    mockEnv.FILES_STORAGE_PATH = "/nonexistent/should-not-be-touched";
+    await expect(assertStorageWritable()).resolves.toBeUndefined();
+  });
+});

tests/unit/security/entrypoint-permissions.test.ts

@@ -0,0 +1,74 @@
+import { spawnSync } from "node:child_process";
+import { chmodSync, mkdirSync, mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { dirname, join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+
+// Exercises the REAL docker/entrypoint-lib.sh functions (sourced, not mirrored)
+// so the test cannot drift from what ships in the image. Mirrors the approach
+// in docker-file-secrets.test.ts.
+const here = dirname(fileURLToPath(import.meta.url));
+const LIB = resolve(here, "../../../docker/entrypoint-lib.sh");
+
+// A read-only directory does not block writes for root (DAC_OVERRIDE), so the
+// "not writable" assertions only hold for an unprivileged user.
+const isRoot = typeof process.getuid === "function" && process.getuid() === 0;
+
+let root: string;
+let writable: string;
+let readonly: string;
+
+beforeAll(() => {
+  root = mkdtempSync(join(tmpdir(), "entrypoint-perms-"));
+  writable = join(root, "writable");
+  mkdirSync(writable, { recursive: true });
+  readonly = join(root, "readonly");
+  mkdirSync(readonly, { recursive: true });
+  chmodSync(readonly, 0o555);
+});
+
+afterAll(() => {
+  try {
+    chmodSync(readonly, 0o755);
+  } catch {
+    /* may not exist */
+  }
+  rmSync(root, { recursive: true, force: true });
+});
+
+// Sources the lib, runs `snippet`, and returns its exit code + captured output
+// without throwing on non-zero exit.
+function runLib(snippet: string): { status: number; stdout: string; stderr: string } {
+  const res = spawnSync("/bin/sh", ["-c", `. "${LIB}"\n${snippet}`], { encoding: "utf-8" });
+  return { status: res.status ?? 1, stdout: res.stdout ?? "", stderr: res.stderr ?? "" };
+}
+
+describe("entrypoint-lib.sh dir_writable", () => {
+  it("succeeds for a writable directory", () => {
+    expect(runLib(`dir_writable '${writable}'`).status).toBe(0);
+  });
+
+  it("creates and succeeds for a missing directory under a writable parent", () => {
+    const fresh = join(root, "fresh", "nested");
+    expect(runLib(`dir_writable '${fresh}'`).status).toBe(0);
+  });
+
+  it.skipIf(isRoot)("fails for a read-only directory", () => {
+    expect(runLib(`dir_writable '${readonly}'`).status).not.toBe(0);
+  });
+});
+
+describe("entrypoint-lib.sh ensure_writable", () => {
+  it("succeeds when all directories are writable", () => {
+    expect(runLib(`ensure_writable '${writable}'`).status).toBe(0);
+  });
+
+  it.skipIf(isRoot)("fails with actionable guidance for a read-only directory", () => {
+    const { status, stderr } = runLib(`ensure_writable '${readonly}'`);
+    expect(status).not.toBe(0);
+    expect(stderr).toContain(readonly);
+    expect(stderr.toLowerCase()).toContain("not writable");
+    expect(stderr).toContain("chown");
+  });
+});