From 98a96bcd2101c9c0d2aca3a0f7434fe26e8af19f Mon Sep 17 00:00:00 2001 From: Renzo F <45401804+rennf93@users.noreply.github.com> Date: Wed, 22 Jul 2026 20:29:50 +0200 Subject: [PATCH] chore(backup): env-gated off-disk mirror + restore drill doc (#645) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pg_dump sidecar wrote its dumps to the same disk it protects — one disk failure lost both. Setting ROBOCO_BACKUP_MIRROR_DIR in .env to a path on a different disk (external/remote mount) arms a mirror step after every successful dump: tmp+rename copy, mirror pruned to the same BACKUP_KEEP, unwritable mirror logs-and-skips without blocking the primary. Unset, the script never attempts a copy — no fake off-disk copies on the same disk. Docs gain the mirror setup and a quarterly restore drill (throwaway pgvector container, pg_restore, row-count sanity check). Co-authored-by: Renn F --- .env.example | 5 ++++ docker-compose.registry.yml | 8 ++++++ docker-compose.yaml | 8 ++++++ docker-compose.yml | 8 ++++++ docker/scripts/backup-entrypoint.sh | 34 +++++++++++++++++++++++ docs/backend/ops/database-backups.md | 40 ++++++++++++++++++++++++++-- 6 files changed, 101 insertions(+), 2 deletions(-) diff --git a/.env.example b/.env.example index 8c21523b..786399d0 100644 --- a/.env.example +++ b/.env.example @@ -59,6 +59,11 @@ # Path to data directory on the host (MUST be absolute for Docker-in-Docker) # ROBOCO_DATA_DIR=/volume1/roboco/data +# Off-disk mirror for the daily pg_dump backups. Point at a path on a +# DIFFERENT disk (external USB, or a mounted remote/cloud share) — a +# same-disk mirror protects nothing. Unset = mirroring off. +# ROBOCO_BACKUP_MIRROR_DIR=/mnt/external/roboco-backups + # ============================================================================= # Application # ============================================================================= diff --git a/docker-compose.registry.yml b/docker-compose.registry.yml index 1767617a..813b7db5 100644 --- a/docker-compose.registry.yml +++ b/docker-compose.registry.yml @@ -110,10 +110,18 @@ services: POSTGRES_USER: roboco POSTGRES_PASSWORD: roboco POSTGRES_DB: roboco + # Armed only when ROBOCO_BACKUP_MIRROR_DIR is set in .env; that host + # path must live on a DIFFERENT disk (external/remote mount) — a + # same-disk mirror protects nothing. Unset → the script skips mirroring. + BACKUP_MIRROR_DIR: ${ROBOCO_BACKUP_MIRROR_DIR:+/backups-mirror} entrypoint: ["/bin/bash", "/scripts/backup-entrypoint.sh"] volumes: - ./docker/scripts/backup-entrypoint.sh:/scripts/backup-entrypoint.sh:ro - ${ROBOCO_DATA_DIR:-./data}/backups:/backups + # Unarmed default deliberately re-mounts the primary backups dir (it + # already exists, so no stray root-owned dir is auto-created) — the + # script never writes there while BACKUP_MIRROR_DIR is empty. + - ${ROBOCO_BACKUP_MIRROR_DIR:-${ROBOCO_DATA_DIR:-./data}/backups}:/backups-mirror depends_on: postgres: condition: service_healthy diff --git a/docker-compose.yaml b/docker-compose.yaml index be834369..8a124046 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -64,10 +64,18 @@ services: POSTGRES_USER: roboco POSTGRES_PASSWORD: roboco POSTGRES_DB: roboco + # Armed only when ROBOCO_BACKUP_MIRROR_DIR is set in .env; that host + # path must live on a DIFFERENT disk (external/remote mount) — a + # same-disk mirror protects nothing. Unset → the script skips mirroring. + BACKUP_MIRROR_DIR: ${ROBOCO_BACKUP_MIRROR_DIR:+/backups-mirror} entrypoint: ["/bin/bash", "/scripts/backup-entrypoint.sh"] volumes: - ./docker/scripts/backup-entrypoint.sh:/scripts/backup-entrypoint.sh:ro - ${ROBOCO_DATA_DIR:-./data}/backups:/backups + # Unarmed default deliberately re-mounts the primary backups dir (it + # already exists, so no stray root-owned dir is auto-created) — the + # script never writes there while BACKUP_MIRROR_DIR is empty. + - ${ROBOCO_BACKUP_MIRROR_DIR:-${ROBOCO_DATA_DIR:-./data}/backups}:/backups-mirror depends_on: postgres: condition: service_healthy diff --git a/docker-compose.yml b/docker-compose.yml index be834369..8a124046 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -64,10 +64,18 @@ services: POSTGRES_USER: roboco POSTGRES_PASSWORD: roboco POSTGRES_DB: roboco + # Armed only when ROBOCO_BACKUP_MIRROR_DIR is set in .env; that host + # path must live on a DIFFERENT disk (external/remote mount) — a + # same-disk mirror protects nothing. Unset → the script skips mirroring. + BACKUP_MIRROR_DIR: ${ROBOCO_BACKUP_MIRROR_DIR:+/backups-mirror} entrypoint: ["/bin/bash", "/scripts/backup-entrypoint.sh"] volumes: - ./docker/scripts/backup-entrypoint.sh:/scripts/backup-entrypoint.sh:ro - ${ROBOCO_DATA_DIR:-./data}/backups:/backups + # Unarmed default deliberately re-mounts the primary backups dir (it + # already exists, so no stray root-owned dir is auto-created) — the + # script never writes there while BACKUP_MIRROR_DIR is empty. + - ${ROBOCO_BACKUP_MIRROR_DIR:-${ROBOCO_DATA_DIR:-./data}/backups}:/backups-mirror depends_on: postgres: condition: service_healthy diff --git a/docker/scripts/backup-entrypoint.sh b/docker/scripts/backup-entrypoint.sh index 678b6ce3..a1184886 100755 --- a/docker/scripts/backup-entrypoint.sh +++ b/docker/scripts/backup-entrypoint.sh @@ -9,6 +9,11 @@ set -u BACKUP_DIR="${BACKUP_DIR:-/backups}" KEEP="${BACKUP_KEEP:-14}" INTERVAL_SECONDS="${BACKUP_INTERVAL_SECONDS:-86400}" +# Off-disk mirror: unset (the default) disables it. Compose sets this only +# when ROBOCO_BACKUP_MIRROR_DIR is set in .env, and that host path should +# live on a DIFFERENT disk (external/remote mount) — a same-disk mirror +# protects nothing. +MIRROR_DIR="${BACKUP_MIRROR_DIR:-}" # pg_dump reads these natively — no need to pass -h/-p/-U/-d by hand. export PGHOST="${POSTGRES_HOST:-roboco-postgres}" @@ -19,14 +24,43 @@ export PGDATABASE="${POSTGRES_DB:-roboco}" mkdir -p "$BACKUP_DIR" +mirror_backup() { + # Copy a fresh dump to the off-disk mirror (tmp+rename, same crash safety + # as the primary write) and prune the mirror to the same $KEEP. Best-effort: + # an unmounted/unwritable mirror logs and skips — never blocks the primary. + local dump="$1" name + [ -n "$MIRROR_DIR" ] || return 0 + name="$(basename "$dump")" + if ! mkdir -p "$MIRROR_DIR" 2>/dev/null || [ ! -w "$MIRROR_DIR" ]; then + echo "[backup] $(date -u -Iseconds) mirror dir ${MIRROR_DIR} not writable — skipping" >&2 + return 0 + fi + rm -f "${MIRROR_DIR}"/roboco-*.dump.tmp + if cp "$dump" "${MIRROR_DIR}/${name}.tmp" && mv "${MIRROR_DIR}/${name}.tmp" "${MIRROR_DIR}/${name}"; then + echo "[backup] $(date -u -Iseconds) mirrored: ${MIRROR_DIR}/${name}" + else + rm -f "${MIRROR_DIR}/${name}.tmp" + echo "[backup] $(date -u -Iseconds) mirror FAILED — this dump stays unmirrored (each cycle mirrors only its own dump)" >&2 + return 0 + fi + # shellcheck disable=SC2012 + ls -1t "${MIRROR_DIR}"/roboco-*.dump 2>/dev/null | tail -n "+$((KEEP + 1))" | while IFS= read -r old; do + rm -f "$old" + done +} + run_backup() { local ts dest + # Any .tmp here is a crash orphan (one dump at a time, unique names, and + # the prune glob never matches them) — sweep before starting. + rm -f "${BACKUP_DIR}"/roboco-*.dump.tmp ts="$(date -u +%Y%m%dT%H%M%SZ)" dest="${BACKUP_DIR}/roboco-${ts}.dump" echo "[backup] $(date -u -Iseconds) starting pg_dump -> ${dest}" if pg_dump -Fc -f "${dest}.tmp"; then mv "${dest}.tmp" "${dest}" echo "[backup] $(date -u -Iseconds) OK: ${dest}" + mirror_backup "${dest}" else rm -f "${dest}.tmp" echo "[backup] $(date -u -Iseconds) FAILED — will retry next cycle" >&2 diff --git a/docs/backend/ops/database-backups.md b/docs/backend/ops/database-backups.md index c0990778..0ea164ef 100644 --- a/docs/backend/ops/database-backups.md +++ b/docs/backend/ops/database-backups.md @@ -8,7 +8,13 @@ On container start, and then every `BACKUP_INTERVAL_SECONDS` (default `86400`, i ## Retention -After each attempt the script prunes `${ROBOCO_DATA_DIR:-./data}/backups` down to the newest `BACKUP_KEEP` dumps (default `14`, i.e. roughly two weeks at the default 24h cadence) by mtime, deleting the rest. There is no offsite copy and no WAL/PITR archiving — this is a point-in-time `pg_dump` snapshot only, taken once a day. +After each attempt the script prunes `${ROBOCO_DATA_DIR:-./data}/backups` down to the newest `BACKUP_KEEP` dumps (default `14`, i.e. roughly two weeks at the default 24h cadence) by mtime, deleting the rest. There is no WAL/PITR archiving — this is a point-in-time `pg_dump` snapshot only, taken once a day. + +## Off-disk mirror + +By default the dumps live on the same disk as the database they protect, so a single disk failure loses both. Setting `ROBOCO_BACKUP_MIRROR_DIR` in `.env` to a host path on a **different** disk (an external USB disk, or a remote share the NAS OS mounts — SMB/NFS/cloud sync target) arms a mirror step: after every successful dump the script copies it into that path (same tmp+rename crash safety) and prunes the mirror to the same `BACKUP_KEEP`. Unset, the mirror is a structural no-op — the script never even attempts a copy (the compose mount then just re-points at the primary backups dir so no stray directory is auto-created). An unmounted or read-only mirror path logs a warning and skips; the primary dump is never blocked by mirror trouble. Note the semantics: each cycle mirrors only its own fresh dump — a dump whose mirror copy failed stays absent from the mirror (there is no backfill), and crash-orphaned `.tmp` files in either directory are swept at the next cycle. + +Point it at a path that actually leaves the machine (e.g. a mounted cloud-synced share) if whole-host loss is in your threat model, not just disk loss. ## Failure behavior @@ -18,6 +24,36 @@ A failed `pg_dump` (network hiccup, postgres briefly unhealthy, disk full) logs Stop anything writing to the database, then restore into a running (empty or throwaway) `roboco` database with `pg_restore`, for example: `docker exec -i roboco-postgres pg_restore -U roboco -d roboco --clean --if-exists < ./data/backups/roboco-20260711T030000Z.dump` (drop the `.tmp` files if any are present — they are in-progress dumps, not backups). Use `pg_restore -l ` first if you want to inspect or selectively restore a subset of objects rather than the whole database. For a fresh empty database instead of `--clean`, create it first (`createdb -U roboco roboco_restore`) and restore into that. +## Restore drill + +A backup that has never been restored is a hope, not a backup. Run this quarterly (takes ~2 minutes, touches nothing in production — it restores into a throwaway container): + +```bash +# 1. Newest dump (skip any .tmp files — those are in-progress, not backups) +DUMP=$(ls -1t ./data/backups/roboco-*.dump | head -1) && echo "$DUMP" + +# 2. Throwaway postgres with the same image the stack pins +docker run -d --name roboco-restore-drill -e POSTGRES_PASSWORD=drill \ + -e POSTGRES_USER=roboco -e POSTGRES_DB=roboco pgvector/pgvector:pg16 +until docker exec roboco-restore-drill pg_isready -U roboco -q; do sleep 1; done + +# 3. Restore the dump into it +docker exec -i roboco-restore-drill pg_restore -U roboco -d roboco \ + --no-owner < "$DUMP" + +# 4. Sanity-check: key tables non-empty and recent +docker exec roboco-restore-drill psql -U roboco -d roboco -c \ + "SELECT (SELECT count(*) FROM tasks) AS tasks, + (SELECT count(*) FROM agents) AS agents, + (SELECT count(*) FROM projects) AS projects, + (SELECT max(created_at) FROM tasks) AS newest_task;" + +# 5. Tear down +docker rm -f roboco-restore-drill +``` + +The drill passes when step 4 shows non-zero counts and a `newest_task` within the last backup interval. A restore error in step 3 or empty counts in step 4 means the backups are not trustworthy — investigate before you need them. + ## Known ceiling -This is an interim measure to close the "zero backups" gap, not a full disaster-recovery story: a single daily snapshot on the same host as the database it's backing up is vulnerable to whole-host loss (disk failure, NAS failure). Copying `${ROBOCO_DATA_DIR:-./data}/backups` offsite periodically, or moving to WAL-based continuous archiving, is the natural next step if that risk matters more than the current simplicity. +This is an interim measure, not a full disaster-recovery story: daily `pg_dump` snapshots mean up to 24h of data loss on a total failure, and there is no WAL-based continuous archiving. The off-disk mirror above covers disk loss; whole-host loss needs the mirror pointed at a path that leaves the machine. Moving to WAL/PITR archiving is the natural next step if the 24h window ever matters more than the current simplicity.