Files
BC-bak/web.py

718 lines
28 KiB
Python
Raw Normal View History

#!/usr/bin/env python3
"""BackupBC Web Interface — Flask app with APScheduler and live SSE log streaming."""
import datetime
import io
import json
import logging
import os
import queue
import shutil
import sqlite3
import threading
import zipfile
from pathlib import Path
from apscheduler.schedulers.background import BackgroundScheduler
from apscheduler.triggers.cron import CronTrigger
from flask import (Flask, Response, abort, flash, jsonify, redirect,
render_template, request, send_file, url_for)
from backup_bc import chain_restore, list_remote_backups, restore_backup, run_backup
# ---------------------------------------------------------------------------
# Paths
# ---------------------------------------------------------------------------
DATA_DIR = Path(os.environ.get("BCBAK_DATA", Path(__file__).parent / "data"))
DATA_DIR.mkdir(parents=True, exist_ok=True)
DB_PATH = DATA_DIR / "bcbak.db"
LOG_DIR = DATA_DIR / "logs"
RESTORE_DIR = DATA_DIR / "restores"
LOG_DIR.mkdir(parents=True, exist_ok=True)
RESTORE_DIR.mkdir(parents=True, exist_ok=True)
# ---------------------------------------------------------------------------
# App
# ---------------------------------------------------------------------------
app = Flask(__name__)
app.secret_key = os.environ.get("SECRET_KEY", os.urandom(24).hex())
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)-8s %(message)s")
scheduler = BackgroundScheduler(timezone="UTC")
_active: dict[int, dict] = {}
_active_lock = threading.Lock()
_active_restores: dict[int, dict] = {}
_restore_lock = threading.Lock()
# ---------------------------------------------------------------------------
# Database
# ---------------------------------------------------------------------------
def get_db() -> sqlite3.Connection:
conn = sqlite3.connect(DB_PATH, check_same_thread=False)
conn.row_factory = sqlite3.Row
conn.execute("PRAGMA journal_mode=WAL")
return conn
def init_db() -> None:
with get_db() as conn:
conn.executescript("""
CREATE TABLE IF NOT EXISTS config (
key TEXT PRIMARY KEY,
value TEXT
);
CREATE TABLE IF NOT EXISTS runs (
id INTEGER PRIMARY KEY AUTOINCREMENT,
started_at TEXT NOT NULL,
completed_at TEXT,
status TEXT NOT NULL DEFAULT 'running',
backup_type TEXT NOT NULL DEFAULT 'full',
triggered_by TEXT NOT NULL DEFAULT 'scheduled',
companies TEXT,
entities_ok INTEGER DEFAULT 0,
entities_error INTEGER DEFAULT 0,
total_records INTEGER DEFAULT 0,
archive_size_mb REAL,
s3_key TEXT,
parent_full_s3_key TEXT,
run_start_time TEXT,
error_message TEXT
);
CREATE TABLE IF NOT EXISTS restores (
id INTEGER PRIMARY KEY AUTOINCREMENT,
started_at TEXT NOT NULL,
completed_at TEXT,
status TEXT NOT NULL DEFAULT 'running',
s3_key TEXT NOT NULL,
output_dir TEXT,
file_count INTEGER DEFAULT 0,
is_chain INTEGER DEFAULT 0,
chain_length INTEGER DEFAULT 1,
error_message TEXT
);
""")
# Migrations for any missing columns
runs_cols = {row[1] for row in conn.execute("PRAGMA table_info(runs)").fetchall()}
restore_cols = {row[1] for row in conn.execute("PRAGMA table_info(restores)").fetchall()}
for col, defn in [
("parent_full_s3_key", "TEXT"),
("run_start_time", "TEXT"),
("total_records", "INTEGER DEFAULT 0"),
]:
if col not in runs_cols:
conn.execute(f"ALTER TABLE runs ADD COLUMN {col} {defn}")
for col, defn in [
("is_chain", "INTEGER DEFAULT 0"),
("chain_length", "INTEGER DEFAULT 1"),
]:
if col not in restore_cols:
conn.execute(f"ALTER TABLE restores ADD COLUMN {col} {defn}")
def cfg_get(key: str, default: str = "") -> str:
with get_db() as conn:
row = conn.execute("SELECT value FROM config WHERE key=?", (key,)).fetchone()
return row["value"] if row else default
def cfg_all() -> dict:
with get_db() as conn:
rows = conn.execute("SELECT key, value FROM config").fetchall()
return {r["key"]: r["value"] for r in rows}
def cfg_set(data: dict) -> None:
with get_db() as conn:
for k, v in data.items():
conn.execute("INSERT OR REPLACE INTO config(key,value) VALUES(?,?)", (k, str(v)))
# ---------------------------------------------------------------------------
# Config builder
# ---------------------------------------------------------------------------
def _config_for_backup() -> dict:
c = cfg_all()
return {
"azure_tenant_id": c.get("azure_tenant_id", ""),
"azure_client_id": c.get("azure_client_id", ""),
"azure_client_secret": c.get("azure_client_secret", ""),
"bc_environment_name": c.get("bc_environment_name", "Production"),
"bc_company_name": c.get("bc_company_name", ""),
"s3_endpoint": c.get("s3_endpoint", ""),
"s3_access_key": c.get("s3_access_key", ""),
"s3_secret_key": c.get("s3_secret_key", ""),
"s3_bucket": c.get("s3_bucket", "bcbak"),
"s3_region": c.get("s3_region", "us-east-1"),
"encryption_passphrase": c.get("encryption_passphrase", ""),
"log_dir": str(LOG_DIR),
"last_full_s3_key": c.get("last_full_s3_key", ""),
}
# ---------------------------------------------------------------------------
# Backup runner
# ---------------------------------------------------------------------------
def _launch_backup(triggered_by: str = "scheduled", force_full: bool = False) -> int:
with _active_lock:
if any(i["status"] == "running" for i in _active.values()):
raise RuntimeError("A backup is already in progress")
backup_type = "full" if force_full else "incremental"
with get_db() as conn:
cur = conn.execute(
"INSERT INTO runs(started_at,status,backup_type,triggered_by) VALUES(?,?,?,?)",
(datetime.datetime.now().isoformat(timespec="seconds"), "running",
backup_type, triggered_by),
)
run_id = cur.lastrowid
log_q: queue.Queue = queue.Queue()
with _active_lock:
_active[run_id] = {"queue": log_q, "status": "running"}
def worker() -> None:
try:
last_run_time = None
if not force_full:
last_run_time = cfg_get("last_incr_run_time") or None
result = run_backup(
_config_for_backup(),
log_callback=lambda m: log_q.put(m),
force_full=force_full,
last_run_time=last_run_time,
)
status = result.get("status", "failed")
if status == "skipped":
status = "success" # skipped = no changes = not a failure
if status == "success" and result.get("s3_key"):
if force_full:
cfg_set({"last_full_s3_key": result["s3_key"]})
if result.get("run_start_time"):
cfg_set({"last_incr_run_time": result["run_start_time"]})
with get_db() as conn:
conn.execute(
"""UPDATE runs
SET status=?, completed_at=?, companies=?,
entities_ok=?, entities_error=?, total_records=?,
archive_size_mb=?, s3_key=?, parent_full_s3_key=?,
run_start_time=?, error_message=?
WHERE id=?""",
(status,
datetime.datetime.now().isoformat(timespec="seconds"),
json.dumps(result.get("companies", [])),
result.get("entities_ok", 0),
result.get("entities_error", 0),
result.get("total_records", 0),
result.get("archive_size_mb"),
result.get("s3_key"),
result.get("parent_full_s3_key"),
result.get("run_start_time"),
result.get("error"),
run_id),
)
with _active_lock:
_active[run_id]["status"] = status
except Exception as exc:
app.logger.error(f"Backup worker error: {exc}", exc_info=True)
with get_db() as conn:
conn.execute(
"UPDATE runs SET status='failed', completed_at=?, error_message=? WHERE id=?",
(datetime.datetime.now().isoformat(timespec="seconds"), str(exc), run_id),
)
log_q.put(f"CRITICAL: {exc}")
with _active_lock:
_active[run_id]["status"] = "failed"
finally:
log_q.put(None)
t = threading.Thread(target=worker, daemon=True, name=f"backup-{run_id}")
t.start()
with _active_lock:
_active[run_id]["thread"] = t
return run_id
# ---------------------------------------------------------------------------
# Restore runner
# ---------------------------------------------------------------------------
def _get_restore_chain(target_s3_key: str) -> list[str] | None:
"""Return ordered [full_key, incr_1, ..., target_key] or None if unavailable."""
with get_db() as conn:
target = conn.execute(
"SELECT * FROM runs WHERE s3_key=? AND status='success'",
(target_s3_key,),
).fetchone()
if not target or target["backup_type"] == "full":
return None
parent_key = target["parent_full_s3_key"]
if not parent_key:
return None
full_run = conn.execute(
"SELECT started_at FROM runs WHERE s3_key=? AND status='success'",
(parent_key,),
).fetchone()
if not full_run:
return None
incrementals = conn.execute(
"""SELECT s3_key FROM runs
WHERE backup_type='incremental' AND parent_full_s3_key=?
AND started_at >= ? AND started_at <= ? AND status='success'
ORDER BY started_at ASC""",
(parent_key, full_run["started_at"], target["started_at"]),
).fetchall()
return [parent_key] + [r["s3_key"] for r in incrementals]
def _launch_restore(s3_key: str, entity_filter: str | None = None, chain: bool = False) -> int:
chain_keys: list[str] | None = None
if chain:
chain_keys = _get_restore_chain(s3_key)
if not chain_keys:
chain = False
is_chain = chain and chain_keys is not None
chain_len = len(chain_keys) if chain_keys else 1
with get_db() as conn:
cur = conn.execute(
"INSERT INTO restores(started_at,status,s3_key,is_chain,chain_length) VALUES(?,?,?,?,?)",
(datetime.datetime.now().isoformat(timespec="seconds"), "running",
s3_key, int(is_chain), chain_len),
)
restore_id = cur.lastrowid
out_dir = RESTORE_DIR / str(restore_id)
log_q: queue.Queue = queue.Queue()
with _restore_lock:
_active_restores[restore_id] = {"queue": log_q, "status": "running"}
def worker() -> None:
try:
cfg = _config_for_backup()
if is_chain:
result = chain_restore(chain_keys, cfg, out_dir,
log_callback=lambda m: log_q.put(m),
entity_filter=entity_filter)
else:
result = restore_backup(s3_key, cfg, out_dir,
log_callback=lambda m: log_q.put(m),
entity_filter=entity_filter)
status = result["status"]
with get_db() as conn:
conn.execute(
"""UPDATE restores
SET status=?,completed_at=?,output_dir=?,file_count=?,error_message=?
WHERE id=?""",
(status,
datetime.datetime.now().isoformat(timespec="seconds"),
str(out_dir),
result.get("file_count", 0),
result.get("error"),
restore_id),
)
with _restore_lock:
_active_restores[restore_id]["status"] = status
except Exception as exc:
app.logger.error(f"Restore worker error: {exc}", exc_info=True)
with get_db() as conn:
conn.execute(
"UPDATE restores SET status='failed',completed_at=?,error_message=? WHERE id=?",
(datetime.datetime.now().isoformat(timespec="seconds"), str(exc), restore_id),
)
log_q.put(f"CRITICAL: {exc}")
with _restore_lock:
_active_restores[restore_id]["status"] = "failed"
finally:
log_q.put(None)
t = threading.Thread(target=worker, daemon=True, name=f"restore-{restore_id}")
t.start()
with _restore_lock:
_active_restores[restore_id]["thread"] = t
return restore_id
# ---------------------------------------------------------------------------
# SSE helper
# ---------------------------------------------------------------------------
def _sse_stream(run_id: int, active_dict: dict, lock: threading.Lock):
with lock:
info = active_dict.get(run_id)
if info is None:
yield f"data: {json.dumps({'done': True, 'status': 'unknown'})}\n\n"
return
log_q = info["queue"]
while True:
try:
msg = log_q.get(timeout=25)
if msg is None:
with lock:
status = active_dict[run_id]["status"]
yield f"data: {json.dumps({'done': True, 'status': status})}\n\n"
return
yield f"data: {json.dumps({'log': msg})}\n\n"
except Exception:
yield f"data: {json.dumps({'keepalive': True})}\n\n"
# ---------------------------------------------------------------------------
# Scheduler
# ---------------------------------------------------------------------------
def _cron_trigger(expr: str) -> CronTrigger:
parts = expr.strip().split()
return CronTrigger(
minute=parts[0], hour=parts[1], day=parts[2],
month=parts[3], day_of_week=parts[4],
)
def _apply_schedule() -> None:
scheduler.remove_all_jobs()
if cfg_get("schedule_incr_enabled", "false").lower() == "true":
cron = cfg_get("schedule_incr_cron", "0 * * * *").strip()
try:
scheduler.add_job(_launch_backup, _cron_trigger(cron), id="incr_backup",
kwargs={"triggered_by": "scheduled", "force_full": False},
misfire_grace_time=3600)
app.logger.info(f"Incremental backup scheduled: {cron}")
except Exception as exc:
app.logger.error(f"Invalid incremental cron '{cron}': {exc}")
if cfg_get("schedule_full_enabled", "false").lower() == "true":
cron = cfg_get("schedule_full_cron", "0 1 * * 0").strip()
try:
scheduler.add_job(_launch_backup, _cron_trigger(cron), id="full_backup",
kwargs={"triggered_by": "scheduled", "force_full": True},
misfire_grace_time=3600)
app.logger.info(f"Full backup scheduled: {cron}")
except Exception as exc:
app.logger.error(f"Invalid full backup cron '{cron}': {exc}")
# ---------------------------------------------------------------------------
# Template filters
# ---------------------------------------------------------------------------
@app.template_filter("duration")
def duration_filter(started: str, completed: str | None) -> str:
if not started or not completed:
return ""
try:
secs = int((datetime.datetime.fromisoformat(completed) -
datetime.datetime.fromisoformat(started)).total_seconds())
return f"{secs // 60}m {secs % 60}s" if secs >= 60 else f"{secs}s"
except Exception:
return ""
@app.template_filter("fmtdt")
def fmtdt_filter(dt_str: str | None) -> str:
return dt_str[:16].replace("T", " ") if dt_str else ""
# ---------------------------------------------------------------------------
# Routes — pages
# ---------------------------------------------------------------------------
@app.route("/")
def dashboard():
with get_db() as conn:
recent = conn.execute("SELECT * FROM runs ORDER BY started_at DESC LIMIT 5").fetchall()
total = conn.execute("SELECT COUNT(*) FROM runs").fetchone()[0]
last_ok = conn.execute("SELECT * FROM runs WHERE status='success' ORDER BY started_at DESC LIMIT 1").fetchone()
last_full = conn.execute("SELECT * FROM runs WHERE status='success' AND backup_type='full' ORDER BY started_at DESC LIMIT 1").fetchone()
last_failed = conn.execute("SELECT * FROM runs WHERE status='failed' ORDER BY started_at DESC LIMIT 1").fetchone()
with _active_lock:
running_id = next((rid for rid, i in _active.items() if i["status"] == "running"), None)
job_incr = scheduler.get_job("incr_backup")
job_full = scheduler.get_job("full_backup")
next_incr = job_incr.next_run_time.strftime("%Y-%m-%d %H:%M UTC") if job_incr and job_incr.next_run_time else None
next_full_run = job_full.next_run_time.strftime("%Y-%m-%d %H:%M UTC") if job_full and job_full.next_run_time else None
return render_template("dashboard.html", recent=recent, total=total,
last_ok=last_ok, last_full=last_full, last_failed=last_failed,
running_id=running_id, next_incr=next_incr, next_full_run=next_full_run)
@app.route("/history")
def history():
page = max(1, request.args.get("page", 1, type=int))
per_page = 25
offset = (page - 1) * per_page
with get_db() as conn:
runs = conn.execute("SELECT * FROM runs ORDER BY started_at DESC LIMIT ? OFFSET ?",
(per_page, offset)).fetchall()
total = conn.execute("SELECT COUNT(*) FROM runs").fetchone()[0]
pages = max(1, (total + per_page - 1) // per_page)
return render_template("history.html", runs=runs, page=page, pages=pages, total=total)
@app.route("/restore")
def restore_list():
with get_db() as conn:
restores = conn.execute("SELECT * FROM restores ORDER BY started_at DESC").fetchall()
remote_backups = []
try:
remote_backups = list_remote_backups(_config_for_backup())
with get_db() as conn:
for b in remote_backups:
run = conn.execute(
"SELECT backup_type, parent_full_s3_key, started_at FROM runs WHERE s3_key=? AND status='success'",
(b["key"],),
).fetchone()
if run:
b["backup_type"] = run["backup_type"]
parent = run["parent_full_s3_key"]
if run["backup_type"] == "incremental" and parent:
full_run = conn.execute(
"SELECT started_at FROM runs WHERE s3_key=? AND status='success'",
(parent,),
).fetchone()
if full_run:
cnt = conn.execute(
"""SELECT COUNT(*) FROM runs
WHERE backup_type='incremental' AND parent_full_s3_key=?
AND started_at>=? AND started_at<=? AND status='success'""",
(parent, full_run["started_at"], run["started_at"]),
).fetchone()[0]
b["chain_available"] = True
b["chain_length"] = cnt + 1
else:
b["chain_available"] = False
else:
b["chain_available"] = False
except Exception as exc:
app.logger.warning(f"Could not list remote backups: {exc}")
return render_template("restore_list.html", restores=restores, remote_backups=remote_backups)
@app.route("/restore/<int:restore_id>")
def restore_view(restore_id: int):
with get_db() as conn:
restore = conn.execute("SELECT * FROM restores WHERE id=?", (restore_id,)).fetchone()
if not restore:
abort(404)
by_company: dict[str, list] = {}
restore_info: dict = {}
SKIP = {"manifest.json", "chain_manifest.json", "_company.json"}
if restore["status"] == "success" and restore["output_dir"]:
output_dir = Path(restore["output_dir"])
# Read manifest
chain_manifest = output_dir / "chain" / "chain_manifest.json"
if chain_manifest.exists():
try:
restore_info = json.loads(chain_manifest.read_text())
restore_info["restore_mode"] = "chain"
except Exception:
pass
else:
for mf in output_dir.rglob("manifest.json"):
try:
restore_info = json.loads(mf.read_text())
restore_info["restore_mode"] = restore_info.get("backup_type", "full")
except Exception:
pass
break
for f in sorted(list(output_dir.rglob("*.json")) + list(output_dir.rglob("*.jsonl"))):
if f.name in SKIP:
continue
rel = f.relative_to(output_dir)
parts = rel.parts
# chain: chain/{company}/{entity}.json → parts[1]=company
# regular: {archive_name}/{company}/{entity}.json → parts[1]=company
company = parts[1] if len(parts) >= 3 else "root"
by_company.setdefault(company, []).append({
"entity": f.stem,
"ext": f.suffix,
"path": str(rel),
"size_kb": round(f.stat().st_size / 1024, 1),
})
with _restore_lock:
streaming_id = (restore_id
if restore_id in _active_restores
and _active_restores[restore_id]["status"] == "running"
else None)
return render_template("restore_view.html", restore=restore, restore_info=restore_info,
by_company=by_company, streaming_id=streaming_id)
@app.route("/settings", methods=["GET", "POST"])
def settings():
if request.method == "POST":
data = dict(request.form)
for checkbox in ("schedule_incr_enabled", "schedule_full_enabled"):
data[checkbox] = "true" if checkbox in request.form else "false"
cfg_set(data)
_apply_schedule()
flash("Settings saved.", "success")
return redirect(url_for("settings"))
return render_template("settings.html", cfg=cfg_all())
# ---------------------------------------------------------------------------
# Routes — API (backup)
# ---------------------------------------------------------------------------
@app.route("/api/run", methods=["POST"])
def api_run():
data = request.get_json(silent=True) or {}
force_full = bool(data.get("force_full", True))
try:
run_id = _launch_backup(triggered_by="manual", force_full=force_full)
return jsonify({"run_id": run_id, "backup_type": "full" if force_full else "incremental"})
except RuntimeError as exc:
return jsonify({"error": str(exc)}), 409
except Exception as exc:
return jsonify({"error": str(exc)}), 500
@app.route("/api/run/<int:run_id>/stream")
def api_stream(run_id: int):
return Response(
_sse_stream(run_id, _active, _active_lock),
mimetype="text/event-stream",
headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
)
@app.route("/api/status")
def api_status():
with _active_lock:
running_id = next((rid for rid, i in _active.items() if i["status"] == "running"), None)
return jsonify({"running": running_id is not None, "run_id": running_id})
# ---------------------------------------------------------------------------
# Routes — API (restore)
# ---------------------------------------------------------------------------
@app.route("/api/restore/chain-info")
def api_restore_chain_info():
s3_key = request.args.get("s3_key", "").strip()
if not s3_key:
return jsonify({"error": "s3_key required"}), 400
chain_keys = _get_restore_chain(s3_key)
return jsonify({
"chain_available": chain_keys is not None,
"chain_length": len(chain_keys) if chain_keys else 1,
"chain_keys": chain_keys or [],
})
@app.route("/api/restore", methods=["POST"])
def api_restore():
data = request.get_json(silent=True) or {}
s3_key = data.get("s3_key", "").strip()
if not s3_key:
return jsonify({"error": "s3_key is required"}), 400
entity_filter = data.get("entities", "").strip() or None
chain = bool(data.get("chain", False))
try:
restore_id = _launch_restore(s3_key, entity_filter=entity_filter, chain=chain)
return jsonify({"restore_id": restore_id})
except Exception as exc:
return jsonify({"error": str(exc)}), 500
@app.route("/api/restore/<int:restore_id>/stream")
def api_restore_stream(restore_id: int):
return Response(
_sse_stream(restore_id, _active_restores, _restore_lock),
mimetype="text/event-stream",
headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
)
@app.route("/restore/<int:restore_id>/download")
def restore_download_file(restore_id: int):
with get_db() as conn:
restore = conn.execute("SELECT * FROM restores WHERE id=?", (restore_id,)).fetchone()
if not restore or not restore["output_dir"]:
abort(404)
rel_path = request.args.get("path", "")
if not rel_path:
abort(400)
output_dir = Path(restore["output_dir"]).resolve()
target = (output_dir / rel_path).resolve()
if not str(target).startswith(str(output_dir)):
abort(403)
if not target.exists():
abort(404)
return send_file(target, as_attachment=True, download_name=target.name)
@app.route("/restore/<int:restore_id>/download-zip")
def restore_download_zip(restore_id: int):
with get_db() as conn:
restore = conn.execute("SELECT * FROM restores WHERE id=?", (restore_id,)).fetchone()
if not restore or restore["status"] != "success" or not restore["output_dir"]:
abort(404)
output_dir = Path(restore["output_dir"])
archive_name = Path(restore["s3_key"]).name.replace(".tar.gz.gpg", "") + "_restored.zip"
buf = io.BytesIO()
with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED, compresslevel=6) as zf:
for f in sorted(output_dir.rglob("*.json")):
zf.write(f, arcname=str(f.relative_to(output_dir)))
for f in sorted(output_dir.rglob("*.jsonl")):
zf.write(f, arcname=str(f.relative_to(output_dir)))
buf.seek(0)
return send_file(buf, mimetype="application/zip",
as_attachment=True, download_name=archive_name)
@app.route("/api/restore/<int:restore_id>", methods=["DELETE"])
def api_restore_delete(restore_id: int):
with get_db() as conn:
restore = conn.execute("SELECT * FROM restores WHERE id=?", (restore_id,)).fetchone()
if not restore:
abort(404)
with _restore_lock:
if restore_id in _active_restores and _active_restores[restore_id]["status"] == "running":
return jsonify({"error": "Restore is still running"}), 409
if restore["output_dir"]:
out = Path(restore["output_dir"])
if out.exists():
shutil.rmtree(out, ignore_errors=True)
with get_db() as conn:
conn.execute("DELETE FROM restores WHERE id=?", (restore_id,))
return jsonify({"ok": True})
# ---------------------------------------------------------------------------
# Bootstrap
# ---------------------------------------------------------------------------
if __name__ == "__main__":
init_db()
_apply_schedule()
scheduler.start()
port = int(os.environ.get("PORT", 1890))
app.logger.info(f"BackupBC web UI starting on port {port}")
app.run(host="0.0.0.0", port=port, debug=False, threaded=True)