Files
joulenap/backend/app/notify/messages.py
T
Catubba 0ad459e14c fix(backend): resolve the nine findings of the backend-block review
The three silent ones first. A 0.9 config that fails to convert used to boot an
empty config that looks exactly like a fresh install: the reason now reaches the
UI (GET /api/status.config_error), the activity log and an ERROR line, and the
.bak parachute is written on the failure branch too, so a later save from the
Advanced tab cannot destroy the original.

"PBS left powered on" was wrong in both directions -- every successful run
against an always-on PBS warned, and a sync route that left its target awake did
not. lease.release() returned one False for four situations, only two of which
cost power; it now returns a ReleaseOutcome that names the reason, which becomes
both the POWEROFF step's detail and RunContext.left_on. The interrupted-run path
keeps a step-derived rule, now paired per device and filtered by managed_power.

422 bodies echoed the whole config, secrets included: a config-level validator
raises at loc=(), so pydantic attached every token, the secret key, the password
hash, the SMTP and bot tokens as the error's input. One helper with
include_input=False now serves all three config-shaped 422 sites.

Also: a redaction placeholder with nothing to resolve against is rejected instead
of silently clearing the credential (a renamed device id, or a create from a
copied body); the ad-hoc "Run verify" asks for outdated_after=0, since None meant
"only never-verified" and skipped exactly the snapshots the button exists for;
the manual power-off holds the single-run lock so it cannot cut a vzdump that
started in the check-then-act gap; _current_run_id is cleared when a run ends, so
a stop landing between two runs cannot hit the wrong one; and the pre-migration
.bak is chmod 0600 like every other secret-bearing file.

Tests: 617 passed, 2 skipped. Every finding was reproduced against the real code
before the fix, and each new test confirmed failing on the pre-fix code.
2026-08-03 00:56:40 +02:00

504 lines
22 KiB
Python

"""Server-side, localized notification text.
Notification bodies are built and translated **on the backend** before sending — they
never pass through the frontend i18n. This is a small dictionary keyed by the
``app.language`` config, with an English fallback, mirroring the per-language approach of
the UI locales but kept deliberately tiny (only the strings that ship in a notification).
"""
from __future__ import annotations
from dataclasses import dataclass, field
from datetime import datetime, tzinfo
from typing import TYPE_CHECKING
from ..config import Config, Route
from ..db.models import Run, RunStatus, RunStep, StepName, StepStatus
if TYPE_CHECKING:
from ..connectors.pbs import DatastoreStatus
@dataclass
class GuestSummary:
"""What the vzdump task did per guest, for the notification's ``Guests`` line.
Deliberately **not** persisted: ``runs`` has no migration path (``init_db`` only calls
``create_all``, which never adds a column to an existing table), so this rides along to
the notifier as an argument exactly like :class:`DatastoreStatus` does.
``failed`` holds display names (the vmid when the guest's name is unknown). Only guests
vzdump actually reported on can land here, so a guest the user excluded from the
selection is never counted or named.
"""
total: int = 0 # guests the run set out to back up
ok: int = 0
failed: list[str] = field(default_factory=list)
@dataclass
class RunContext:
"""Everything a finished run's notification can say about itself.
One object instead of the positional ``(config, run, datastore, guests, next_at)``
tuple that used to be threaded through ``CycleDeps.notify`` ->
``NotificationService.send_run_result`` -> ``build_run_message``: adding a field there
meant editing every call site and every fake, and a caller passing its arguments in
the wrong order got a silently wrong message instead of a TypeError. Everything past
the first two is optional, so a cycle fills in only what it actually knows.
"""
config: Config
run: Run
#: The route this run belongs to, if any — an ad-hoc PBS GC/verify has none. Carries
#: the per-route ``notify`` filter and names the route in the body.
route: Route | None = None
datastore: DatastoreStatus | None = None
guests: GuestSummary | None = None
#: When this route next fires, for the "Next scheduled run" line.
next_at: datetime | None = None
#: PBS ids this run left awake and burning power, with nothing queued to shut them
#: down. Filled in by ``JobService`` after the leases are released: it is the only
#: place that knows *why* a box stayed on, and only some of the reasons cost energy
#: (an always-on box, or one another run still holds, cost nothing).
left_on: list[str] = field(default_factory=list)
# event keys: success | failure | aborted | test
_MESSAGES: dict[str, dict[str, dict[str, str]]] = {
"en": {
"success": {"title": "✅ Joulenap — backup succeeded"},
"failure": {"title": "❌ Joulenap — backup failed"},
"aborted": {"title": "⚠️ Joulenap — backup aborted"},
# Per-kind titles for the non-backup cycles (see _title_for). Full strings rather
# than a "{noun} succeeded" template: Italian needs gender agreement (backup
# riuscito / verifica riuscita), which a noun slot can't express.
"gc": {
"success": "✅ Joulenap — garbage collection succeeded",
"failure": "❌ Joulenap — garbage collection failed",
"aborted": "⚠️ Joulenap — garbage collection aborted",
},
"verify": {
"success": "✅ Joulenap — verification succeeded",
"failure": "❌ Joulenap — verification failed",
"aborted": "⚠️ Joulenap — verification aborted",
},
# External-schedules mode: the watch cycle (wake -> watch PVE/PBS's own jobs ->
# power off).
"monitor": {
"success": "✅ Joulenap — PBS jobs finished",
"failure": "❌ Joulenap — watch cycle failed",
"aborted": "⚠️ Joulenap — watch cycle aborted",
},
"sync": {
"success": "✅ Joulenap — sync succeeded",
"failure": "❌ Joulenap — sync failed",
"aborted": "⚠️ Joulenap — sync aborted",
},
"missed": {
"title": "⚠️ Joulenap — missed scheduled backup",
"intro": "A scheduled backup was skipped because Joulenap was offline when it "
"was due.",
},
"interrupted": {
"title": "⚠️ Joulenap — run interrupted by a restart",
"intro": "Joulenap restarted while a run was in progress; it was marked failed.",
},
"test": {
"title": "🔔 Joulenap — test notification",
"body": "If you can read this, notifications are configured correctly.",
},
"_labels": {
"route": "Route",
"trigger": "Trigger",
"trigger_scheduled": "scheduled",
"trigger_manual": "manual",
"duration": "Duration",
"guests": "Guests",
"failed": "failed",
"datastore": "Datastore",
"used": "used",
"free": "free",
"error": "Error",
"pbs_left_on": "⚠️ PBS left powered on — check it",
"awake_for": "PBS awake for",
"missed_run": "Missed run",
"last_run": "Last backup run",
"next_run": "Next scheduled run",
"run_no": "Run",
# Names for the duration breakdown. Only the phases that do the actual work: the
# wake packet is instant, and wait/power-off are near-constant overhead that would
# just crowd the line.
"phase_backup": "backup",
"phase_gc": "GC",
"phase_verify": "verify",
"phase_monitor": "watch",
"tasks_observed": "PBS jobs observed",
"no_tasks": "⚠️ No PBS job ran — check the schedules on PVE/PBS",
},
},
"it": {
"success": {"title": "✅ Joulenap — backup riuscito"},
"failure": {"title": "❌ Joulenap — backup fallito"},
"aborted": {"title": "⚠️ Joulenap — backup interrotto"},
"gc": {
"success": "✅ Joulenap — garbage collection riuscita",
"failure": "❌ Joulenap — garbage collection fallita",
"aborted": "⚠️ Joulenap — garbage collection interrotta",
},
"verify": {
"success": "✅ Joulenap — verifica riuscita",
"failure": "❌ Joulenap — verifica fallita",
"aborted": "⚠️ Joulenap — verifica interrotta",
},
"monitor": {
"success": "✅ Joulenap — job PBS completati",
"failure": "❌ Joulenap — ciclo di controllo fallito",
"aborted": "⚠️ Joulenap — ciclo di controllo interrotto",
},
"sync": {
"success": "✅ Joulenap — sincronizzazione riuscita",
"failure": "❌ Joulenap — sincronizzazione fallita",
"aborted": "⚠️ Joulenap — sincronizzazione interrotta",
},
"missed": {
"title": "⚠️ Joulenap — backup pianificato mancato",
"intro": "Un backup pianificato è stato saltato perché Joulenap era offline "
"al momento previsto.",
},
"interrupted": {
"title": "⚠️ Joulenap — esecuzione interrotta da un riavvio",
"intro": "Joulenap si è riavviato mentre un'esecuzione era in corso; "
"è stata contrassegnata come fallita.",
},
"test": {
"title": "🔔 Joulenap — notifica di prova",
"body": "Se leggi questo messaggio, le notifiche sono configurate correttamente.",
},
"_labels": {
"route": "Route",
"trigger": "Avvio",
"trigger_scheduled": "pianificato",
"trigger_manual": "manuale",
"duration": "Durata",
"guests": "Guest",
"failed": "falliti",
"datastore": "Datastore",
"used": "usato",
"free": "liberi",
"error": "Errore",
"pbs_left_on": "⚠️ PBS lasciato acceso — controllalo",
"awake_for": "PBS sveglio da",
"missed_run": "Esecuzione mancata",
"last_run": "Ultimo backup eseguito",
"next_run": "Prossima esecuzione pianificata",
"run_no": "Run",
"phase_backup": "backup",
"phase_gc": "GC",
"phase_verify": "verifica",
"phase_monitor": "controllo",
"tasks_observed": "Job PBS osservati",
"no_tasks": "⚠️ Nessun job PBS eseguito — controlla le pianificazioni su PVE/PBS",
},
},
}
_STATUS_EVENT = {
RunStatus.SUCCESS: "success",
RunStatus.FAILURE: "failure",
RunStatus.ABORTED: "aborted",
}
def _pack(language: str) -> dict[str, dict[str, str]]:
return _MESSAGES.get(language, _MESSAGES["en"])
def _title_for(pack: dict[str, dict[str, str]], kind: str, event: str) -> str:
"""Title for a finished run, worded for the kind of cycle it was.
A GC or verify cycle reports its own outcome instead of borrowing the backup wording (a
scheduled verify failure used to notify "backup failed"). Anything without its own block
— a normal backup cycle, or a kind added later — falls back to the backup title, so a new
``RunKind`` degrades to today's behaviour instead of raising.
"""
return pack.get(kind, {}).get(event) or pack[event]["title"]
def _format_duration(seconds: float) -> str:
total = int(seconds)
minutes, secs = divmod(total, 60)
hours, minutes = divmod(minutes, 60)
if hours:
return f"{hours}h {minutes}m {secs}s"
if minutes:
return f"{minutes}m {secs}s"
return f"{secs}s"
#: Steps that get a slot in the duration breakdown, keyed to their ``_labels`` entry. Wake,
#: wait, precheck and power-off are left out on purpose — see the label block's comment.
_PHASE_LABEL = {
StepName.BACKUP: "phase_backup",
StepName.MONITOR: "phase_monitor",
StepName.GC: "phase_gc",
StepName.VERIFY: "phase_verify",
}
def _phase_breakdown(labels: dict[str, str], run: Run) -> str:
"""``backup 7m 10s · GC 1m 6s`` — where the run's time actually went, in step order.
Skipped steps (GC turned off) and steps still running contribute nothing, so the
parentheses never advertise work that didn't happen. A ``StepName`` added later simply
doesn't appear rather than raising.
"""
parts = []
for step in run.steps: # the relationship is ordered by started_at
key = _PHASE_LABEL.get(step.name)
if key is None or step.status == StepStatus.SKIPPED or not step.finished_at:
continue
seconds = (step.finished_at - step.started_at).total_seconds()
parts.append(f"{labels[key]} {_format_duration(seconds)}")
return " · ".join(parts)
def human_bytes(n: int) -> str:
"""Binary-unit size, e.g. ``4.6 TiB`` (PBS reports datastore sizes in bytes)."""
size = float(n)
for unit in ("B", "KiB", "MiB", "GiB", "TiB"):
if size < 1024 or unit == "TiB":
return f"{int(size)} B" if unit == "B" else f"{size:.1f} {unit}"
size /= 1024
return f"{size:.1f} PiB"
def _step_is(step: RunStep, name: StepName) -> bool:
"""Whether ``step`` is an instance of ``name``, labelled or not.
A run touching several devices records ``wait:pbs-01`` / ``poweroff:pbs-02`` (see
``RunRecorder.step``'s ``label``), so an equality test would silently stop matching the
moment a route had more than one target.
"""
return step.name.split(":", 1)[0] == name.value
def _step_label(step: RunStep) -> str | None:
"""The device a step names (``poweroff:pbs-02`` -> ``pbs-02``), or None when unlabelled
— a single-device run doesn't repeat which box it means."""
_, _, label = step.name.partition(":")
return label or None
def _pbs_left_on(config: Config, run: Run) -> bool:
"""True if a PBS came up and nothing ever powered it back off — a box still burning
energy that the user should go and check.
Only for a run a restart interrupted: there the run row is all there is, and no
POWEROFF step was ever reached. A run that *finished* knows the answer exactly and
reports it through ``RunContext.left_on``, because "was it left on?" depends on facts
the timeline doesn't carry (whether another queued route still needs the box).
Two things the steps alone get wrong, both introduced by this same release:
* an **unmanaged** box (``managed_power: false``) is never Joulenap's to power down,
so a run against one must not warn — hence taking ``config``;
* a run holding **several** leases needs the WAIT and POWEROFF steps paired *per
device*, or one box's successful power-off hides another's that stayed up.
An abort *before* the box came up leaves the WAIT step non-SUCCESS, so the PBS is off
and this correctly returns False — hence why it keys on WAIT, not on the run status.
"""
managed = {p.id for p in config.pbss if p.managed_power}
route = next((r for r in config.routes if r.id == run.route_id), None)
# An unlabelled step belongs to the run's only box — a route's target (a sync route
# labels both sides). An ad-hoc GC/verify records neither a label nor a route, so its
# box cannot be named: warn rather than stay silent about a box that may be awake.
default = route.target if route else None
powered_off = {
_step_label(s)
for s in run.steps
if _step_is(s, StepName.POWEROFF) and s.status == StepStatus.SUCCESS
}
for step in run.steps:
if not _step_is(step, StepName.WAIT) or step.status != StepStatus.SUCCESS:
continue
pbs_id = _step_label(step) or default
if pbs_id is not None and pbs_id not in managed:
continue
if _step_label(step) not in powered_off:
return True
return False
#: Route kinds, for the body's ``Route:`` line. Localized because the kind is a user-facing
#: word in the UI too; an unknown kind falls through to its raw value.
_KIND_LABEL = {
"en": {"backup": "backup", "sync": "sync", "external": "external", "verify": "verify"},
"it": {
"backup": "backup",
"sync": "sincronizzazione",
"external": "esterna",
"verify": "verifica",
},
}
def build_run_message(ctx: RunContext) -> tuple[str, str]:
"""``(title, body)`` describing a finished run, in the configured language.
One field per line, in a fixed order; a field whose data is missing drops out entirely
rather than rendering a placeholder. ``ctx.datastore`` (read while the PBS was still
awake) adds the usage line, ``ctx.guests`` the per-guest tally of a backup cycle,
``ctx.next_at`` the schedule's following fire.
"""
config, run = ctx.config, ctx.run
pack = _pack(config.app.language)
labels = pack["_labels"]
event = _STATUS_EVENT.get(run.status, "failure") # RUNNING shouldn't reach here
# The title already conveys success/failure/aborted, so we don't repeat the (untranslated)
# status enum in the body.
lines: list[str] = []
if ctx.route is not None:
# The route's *colour* is deliberately absent: it is a UI affordance, and a hex
# string in a push notification is noise on every channel that could render it.
kinds = _KIND_LABEL.get(config.app.language, _KIND_LABEL["en"])
kind = kinds.get(ctx.route.kind, ctx.route.kind)
lines.append(f"{labels['route']}: {ctx.route.name or ctx.route.id} ({kind})")
lines.append(f"{labels['trigger']}: {labels.get(f'trigger_{run.trigger}', run.trigger)}")
datastore, guests, next_at = ctx.datastore, ctx.guests, ctx.next_at
if run.started_at and run.finished_at:
duration = (run.finished_at - run.started_at).total_seconds()
breakdown = _phase_breakdown(labels, run)
line = f"{labels['duration']}: {_format_duration(duration)}"
lines.append(f"{line} ({breakdown})" if breakdown else line)
# No summary at all (a GC or verify cycle, or an abort before the guests were picked)
# means there is nothing truthful to say about guests — better silent than "0".
if guests is not None and guests.total:
line = f"{labels['guests']}: {guests.ok}/{guests.total}"
if guests.failed:
line += f" ({labels['failed']}: {', '.join(guests.failed)})"
lines.append(line)
# An External route's watch: the MONITOR step's detail is either "N task(s) observed"
# or "no tasks observed" (see route_cycle._external_body) — the count line for the
# former, the your-schedule-didn't-fire warning for the latter.
monitor = next((s for s in run.steps if s.name == StepName.MONITOR), None)
if monitor is not None and monitor.detail:
try:
observed = int(monitor.detail.split()[0])
except ValueError:
lines.append(labels["no_tasks"])
else:
lines.append(f"{labels['tasks_observed']}: {observed}")
if datastore is not None:
lines.append(
f"{labels['datastore']}: {datastore.used_pct}% {labels['used']}, "
f"{human_bytes(datastore.avail)} {labels['free']}"
)
if run.error:
lines.append(f"{labels['error']}: {run.error}")
# From the service, not from the steps: a finished run knows exactly which boxes it
# left burning power, and only those warrant the warning.
if ctx.left_on:
lines.append(labels["pbs_left_on"])
if next_at is not None:
lines.append(f"{labels['next_run']}: {_format_dt(next_at, _timezone(config))}")
if run.id is not None:
lines.append(f"{labels['run_no']} #{run.id}")
return _title_for(pack, run.kind, event), "\n".join(lines)
def _timezone(config: Config) -> tzinfo:
"""The zone every timestamp in a notification is rendered in.
Imported here rather than at module scope because ``core.scheduler`` pulls in ``jobs``,
which pulls in this package — a top-level import would be circular."""
from ..core.scheduler import resolve_timezone
return resolve_timezone(config.app.timezone)
def _format_dt(dt: datetime | None, tz: tzinfo) -> str:
"""A short absolute timestamp for notifications, e.g. ``2026-07-11 04:00 CEST``.
Everything is converted into the configured zone first. Cron-derived times already
arrive in it, but anything read back from the database is UTC (see ``UtcDateTime``) —
without the conversion one line of the same message would silently be hours off from
the ones around it."""
if dt is None:
return "—"
return dt.astimezone(tz).strftime("%Y-%m-%d %H:%M %Z").rstrip()
def build_missed_backup_message(
config: Config,
route: Route,
missed_at: datetime,
last_run_at: datetime | None,
next_at: datetime | None,
) -> tuple[str, str]:
"""``(title, body)`` for a scheduled route that didn't run because the process was down
over its window (BE-R1), in the configured language.
Names the route: with per-route schedules, "a scheduled backup was skipped" no longer
identifies which one.
"""
pack = _pack(config.app.language)
labels = pack["_labels"]
tz = _timezone(config)
lines = [
pack["missed"]["intro"],
"",
f"{labels['route']}: {route.name or route.id}",
f"{labels['missed_run']}: {_format_dt(missed_at, tz)}",
f"{labels['last_run']}: {_format_dt(last_run_at, tz)}",
f"{labels['next_run']}: {_format_dt(next_at, tz)}",
]
return pack["missed"]["title"], "\n".join(lines)
def build_interrupted_message(config: Config, run: Run) -> tuple[str, str]:
"""``(title, body)`` for a run that a restart interrupted (swept to FAILURE at startup,
BE-R2), in the configured language.
Adds the "PBS left powered on" warning when a box Joulenap powers had actually woken
before the crash (WAIT succeeded, no matching POWEROFF) — the whole point of the alert:
a normally-off box that a crash left awake and burning power."""
pack = _pack(config.app.language)
labels = pack["_labels"]
lines = [pack["interrupted"]["intro"]]
if run.error:
lines.append(f"{labels['error']}: {run.error}")
if _pbs_left_on(config, run):
lines.append(labels["pbs_left_on"])
# This alert has no Duration line (the run's own span would span the whole downtime,
# not the work), so the one interval worth reporting is how long the box has been
# awake: from the moment it came up to the restart — and, since nothing powered it
# off, counting still.
awake_since = _awake_since(run)
if awake_since and run.finished_at:
awake = (run.finished_at - awake_since).total_seconds()
lines.append(f"{labels['awake_for']}: {_format_duration(awake)}")
if run.id is not None:
lines.append(f"{labels['run_no']} #{run.id}")
return pack["interrupted"]["title"], "\n".join(lines)
def _awake_since(run: Run) -> datetime | None:
"""When the PBS finished coming up — the WAIT step's finish, or None if it never did."""
for step in run.steps:
if _step_is(step, StepName.WAIT) and step.status == StepStatus.SUCCESS:
return step.finished_at
return None
def build_test_message(config: Config) -> tuple[str, str]:
"""``(title, body)`` for the manual 'send test notification' action."""
pack = _pack(config.app.language)
return pack["test"]["title"], pack["test"]["body"]