mirror of
https://github.com/Joulenap/joulenap.git
synced 2026-08-11 13:21:43 +02:00
The three silent ones first. A 0.9 config that fails to convert used to boot an empty config that looks exactly like a fresh install: the reason now reaches the UI (GET /api/status.config_error), the activity log and an ERROR line, and the .bak parachute is written on the failure branch too, so a later save from the Advanced tab cannot destroy the original. "PBS left powered on" was wrong in both directions -- every successful run against an always-on PBS warned, and a sync route that left its target awake did not. lease.release() returned one False for four situations, only two of which cost power; it now returns a ReleaseOutcome that names the reason, which becomes both the POWEROFF step's detail and RunContext.left_on. The interrupted-run path keeps a step-derived rule, now paired per device and filtered by managed_power. 422 bodies echoed the whole config, secrets included: a config-level validator raises at loc=(), so pydantic attached every token, the secret key, the password hash, the SMTP and bot tokens as the error's input. One helper with include_input=False now serves all three config-shaped 422 sites. Also: a redaction placeholder with nothing to resolve against is rejected instead of silently clearing the credential (a renamed device id, or a create from a copied body); the ad-hoc "Run verify" asks for outdated_after=0, since None meant "only never-verified" and skipped exactly the snapshots the button exists for; the manual power-off holds the single-run lock so it cannot cut a vzdump that started in the check-then-act gap; _current_run_id is cleared when a run ends, so a stop landing between two runs cannot hit the wrong one; and the pre-migration .bak is chmod 0600 like every other secret-bearing file. Tests: 617 passed, 2 skipped. Every finding was reproduced against the real code before the fix, and each new test confirmed failing on the pre-fix code.
504 lines
22 KiB
Python
504 lines
22 KiB
Python
"""Server-side, localized notification text.
|
|
|
|
Notification bodies are built and translated **on the backend** before sending — they
|
|
never pass through the frontend i18n. This is a small dictionary keyed by the
|
|
``app.language`` config, with an English fallback, mirroring the per-language approach of
|
|
the UI locales but kept deliberately tiny (only the strings that ship in a notification).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime, tzinfo
|
|
from typing import TYPE_CHECKING
|
|
|
|
from ..config import Config, Route
|
|
from ..db.models import Run, RunStatus, RunStep, StepName, StepStatus
|
|
|
|
if TYPE_CHECKING:
|
|
from ..connectors.pbs import DatastoreStatus
|
|
|
|
|
|
@dataclass
|
|
class GuestSummary:
|
|
"""What the vzdump task did per guest, for the notification's ``Guests`` line.
|
|
|
|
Deliberately **not** persisted: ``runs`` has no migration path (``init_db`` only calls
|
|
``create_all``, which never adds a column to an existing table), so this rides along to
|
|
the notifier as an argument exactly like :class:`DatastoreStatus` does.
|
|
|
|
``failed`` holds display names (the vmid when the guest's name is unknown). Only guests
|
|
vzdump actually reported on can land here, so a guest the user excluded from the
|
|
selection is never counted or named.
|
|
"""
|
|
|
|
total: int = 0 # guests the run set out to back up
|
|
ok: int = 0
|
|
failed: list[str] = field(default_factory=list)
|
|
|
|
|
|
@dataclass
|
|
class RunContext:
|
|
"""Everything a finished run's notification can say about itself.
|
|
|
|
One object instead of the positional ``(config, run, datastore, guests, next_at)``
|
|
tuple that used to be threaded through ``CycleDeps.notify`` ->
|
|
``NotificationService.send_run_result`` -> ``build_run_message``: adding a field there
|
|
meant editing every call site and every fake, and a caller passing its arguments in
|
|
the wrong order got a silently wrong message instead of a TypeError. Everything past
|
|
the first two is optional, so a cycle fills in only what it actually knows.
|
|
"""
|
|
|
|
config: Config
|
|
run: Run
|
|
#: The route this run belongs to, if any — an ad-hoc PBS GC/verify has none. Carries
|
|
#: the per-route ``notify`` filter and names the route in the body.
|
|
route: Route | None = None
|
|
datastore: DatastoreStatus | None = None
|
|
guests: GuestSummary | None = None
|
|
#: When this route next fires, for the "Next scheduled run" line.
|
|
next_at: datetime | None = None
|
|
#: PBS ids this run left awake and burning power, with nothing queued to shut them
|
|
#: down. Filled in by ``JobService`` after the leases are released: it is the only
|
|
#: place that knows *why* a box stayed on, and only some of the reasons cost energy
|
|
#: (an always-on box, or one another run still holds, cost nothing).
|
|
left_on: list[str] = field(default_factory=list)
|
|
|
|
|
|
# event keys: success | failure | aborted | test
|
|
_MESSAGES: dict[str, dict[str, dict[str, str]]] = {
|
|
"en": {
|
|
"success": {"title": "✅ Joulenap — backup succeeded"},
|
|
"failure": {"title": "❌ Joulenap — backup failed"},
|
|
"aborted": {"title": "⚠️ Joulenap — backup aborted"},
|
|
# Per-kind titles for the non-backup cycles (see _title_for). Full strings rather
|
|
# than a "{noun} succeeded" template: Italian needs gender agreement (backup
|
|
# riuscito / verifica riuscita), which a noun slot can't express.
|
|
"gc": {
|
|
"success": "✅ Joulenap — garbage collection succeeded",
|
|
"failure": "❌ Joulenap — garbage collection failed",
|
|
"aborted": "⚠️ Joulenap — garbage collection aborted",
|
|
},
|
|
"verify": {
|
|
"success": "✅ Joulenap — verification succeeded",
|
|
"failure": "❌ Joulenap — verification failed",
|
|
"aborted": "⚠️ Joulenap — verification aborted",
|
|
},
|
|
# External-schedules mode: the watch cycle (wake -> watch PVE/PBS's own jobs ->
|
|
# power off).
|
|
"monitor": {
|
|
"success": "✅ Joulenap — PBS jobs finished",
|
|
"failure": "❌ Joulenap — watch cycle failed",
|
|
"aborted": "⚠️ Joulenap — watch cycle aborted",
|
|
},
|
|
"sync": {
|
|
"success": "✅ Joulenap — sync succeeded",
|
|
"failure": "❌ Joulenap — sync failed",
|
|
"aborted": "⚠️ Joulenap — sync aborted",
|
|
},
|
|
"missed": {
|
|
"title": "⚠️ Joulenap — missed scheduled backup",
|
|
"intro": "A scheduled backup was skipped because Joulenap was offline when it "
|
|
"was due.",
|
|
},
|
|
"interrupted": {
|
|
"title": "⚠️ Joulenap — run interrupted by a restart",
|
|
"intro": "Joulenap restarted while a run was in progress; it was marked failed.",
|
|
},
|
|
"test": {
|
|
"title": "🔔 Joulenap — test notification",
|
|
"body": "If you can read this, notifications are configured correctly.",
|
|
},
|
|
"_labels": {
|
|
"route": "Route",
|
|
"trigger": "Trigger",
|
|
"trigger_scheduled": "scheduled",
|
|
"trigger_manual": "manual",
|
|
"duration": "Duration",
|
|
"guests": "Guests",
|
|
"failed": "failed",
|
|
"datastore": "Datastore",
|
|
"used": "used",
|
|
"free": "free",
|
|
"error": "Error",
|
|
"pbs_left_on": "⚠️ PBS left powered on — check it",
|
|
"awake_for": "PBS awake for",
|
|
"missed_run": "Missed run",
|
|
"last_run": "Last backup run",
|
|
"next_run": "Next scheduled run",
|
|
"run_no": "Run",
|
|
# Names for the duration breakdown. Only the phases that do the actual work: the
|
|
# wake packet is instant, and wait/power-off are near-constant overhead that would
|
|
# just crowd the line.
|
|
"phase_backup": "backup",
|
|
"phase_gc": "GC",
|
|
"phase_verify": "verify",
|
|
"phase_monitor": "watch",
|
|
"tasks_observed": "PBS jobs observed",
|
|
"no_tasks": "⚠️ No PBS job ran — check the schedules on PVE/PBS",
|
|
},
|
|
},
|
|
"it": {
|
|
"success": {"title": "✅ Joulenap — backup riuscito"},
|
|
"failure": {"title": "❌ Joulenap — backup fallito"},
|
|
"aborted": {"title": "⚠️ Joulenap — backup interrotto"},
|
|
"gc": {
|
|
"success": "✅ Joulenap — garbage collection riuscita",
|
|
"failure": "❌ Joulenap — garbage collection fallita",
|
|
"aborted": "⚠️ Joulenap — garbage collection interrotta",
|
|
},
|
|
"verify": {
|
|
"success": "✅ Joulenap — verifica riuscita",
|
|
"failure": "❌ Joulenap — verifica fallita",
|
|
"aborted": "⚠️ Joulenap — verifica interrotta",
|
|
},
|
|
"monitor": {
|
|
"success": "✅ Joulenap — job PBS completati",
|
|
"failure": "❌ Joulenap — ciclo di controllo fallito",
|
|
"aborted": "⚠️ Joulenap — ciclo di controllo interrotto",
|
|
},
|
|
"sync": {
|
|
"success": "✅ Joulenap — sincronizzazione riuscita",
|
|
"failure": "❌ Joulenap — sincronizzazione fallita",
|
|
"aborted": "⚠️ Joulenap — sincronizzazione interrotta",
|
|
},
|
|
"missed": {
|
|
"title": "⚠️ Joulenap — backup pianificato mancato",
|
|
"intro": "Un backup pianificato è stato saltato perché Joulenap era offline "
|
|
"al momento previsto.",
|
|
},
|
|
"interrupted": {
|
|
"title": "⚠️ Joulenap — esecuzione interrotta da un riavvio",
|
|
"intro": "Joulenap si è riavviato mentre un'esecuzione era in corso; "
|
|
"è stata contrassegnata come fallita.",
|
|
},
|
|
"test": {
|
|
"title": "🔔 Joulenap — notifica di prova",
|
|
"body": "Se leggi questo messaggio, le notifiche sono configurate correttamente.",
|
|
},
|
|
"_labels": {
|
|
"route": "Route",
|
|
"trigger": "Avvio",
|
|
"trigger_scheduled": "pianificato",
|
|
"trigger_manual": "manuale",
|
|
"duration": "Durata",
|
|
"guests": "Guest",
|
|
"failed": "falliti",
|
|
"datastore": "Datastore",
|
|
"used": "usato",
|
|
"free": "liberi",
|
|
"error": "Errore",
|
|
"pbs_left_on": "⚠️ PBS lasciato acceso — controllalo",
|
|
"awake_for": "PBS sveglio da",
|
|
"missed_run": "Esecuzione mancata",
|
|
"last_run": "Ultimo backup eseguito",
|
|
"next_run": "Prossima esecuzione pianificata",
|
|
"run_no": "Run",
|
|
"phase_backup": "backup",
|
|
"phase_gc": "GC",
|
|
"phase_verify": "verifica",
|
|
"phase_monitor": "controllo",
|
|
"tasks_observed": "Job PBS osservati",
|
|
"no_tasks": "⚠️ Nessun job PBS eseguito — controlla le pianificazioni su PVE/PBS",
|
|
},
|
|
},
|
|
}
|
|
|
|
_STATUS_EVENT = {
|
|
RunStatus.SUCCESS: "success",
|
|
RunStatus.FAILURE: "failure",
|
|
RunStatus.ABORTED: "aborted",
|
|
}
|
|
|
|
|
|
def _pack(language: str) -> dict[str, dict[str, str]]:
|
|
return _MESSAGES.get(language, _MESSAGES["en"])
|
|
|
|
|
|
def _title_for(pack: dict[str, dict[str, str]], kind: str, event: str) -> str:
|
|
"""Title for a finished run, worded for the kind of cycle it was.
|
|
|
|
A GC or verify cycle reports its own outcome instead of borrowing the backup wording (a
|
|
scheduled verify failure used to notify "backup failed"). Anything without its own block
|
|
— a normal backup cycle, or a kind added later — falls back to the backup title, so a new
|
|
``RunKind`` degrades to today's behaviour instead of raising.
|
|
"""
|
|
return pack.get(kind, {}).get(event) or pack[event]["title"]
|
|
|
|
|
|
def _format_duration(seconds: float) -> str:
|
|
total = int(seconds)
|
|
minutes, secs = divmod(total, 60)
|
|
hours, minutes = divmod(minutes, 60)
|
|
if hours:
|
|
return f"{hours}h {minutes}m {secs}s"
|
|
if minutes:
|
|
return f"{minutes}m {secs}s"
|
|
return f"{secs}s"
|
|
|
|
|
|
#: Steps that get a slot in the duration breakdown, keyed to their ``_labels`` entry. Wake,
|
|
#: wait, precheck and power-off are left out on purpose — see the label block's comment.
|
|
_PHASE_LABEL = {
|
|
StepName.BACKUP: "phase_backup",
|
|
StepName.MONITOR: "phase_monitor",
|
|
StepName.GC: "phase_gc",
|
|
StepName.VERIFY: "phase_verify",
|
|
}
|
|
|
|
|
|
def _phase_breakdown(labels: dict[str, str], run: Run) -> str:
|
|
"""``backup 7m 10s · GC 1m 6s`` — where the run's time actually went, in step order.
|
|
|
|
Skipped steps (GC turned off) and steps still running contribute nothing, so the
|
|
parentheses never advertise work that didn't happen. A ``StepName`` added later simply
|
|
doesn't appear rather than raising.
|
|
"""
|
|
parts = []
|
|
for step in run.steps: # the relationship is ordered by started_at
|
|
key = _PHASE_LABEL.get(step.name)
|
|
if key is None or step.status == StepStatus.SKIPPED or not step.finished_at:
|
|
continue
|
|
seconds = (step.finished_at - step.started_at).total_seconds()
|
|
parts.append(f"{labels[key]} {_format_duration(seconds)}")
|
|
return " · ".join(parts)
|
|
|
|
|
|
def human_bytes(n: int) -> str:
|
|
"""Binary-unit size, e.g. ``4.6 TiB`` (PBS reports datastore sizes in bytes)."""
|
|
size = float(n)
|
|
for unit in ("B", "KiB", "MiB", "GiB", "TiB"):
|
|
if size < 1024 or unit == "TiB":
|
|
return f"{int(size)} B" if unit == "B" else f"{size:.1f} {unit}"
|
|
size /= 1024
|
|
return f"{size:.1f} PiB"
|
|
|
|
|
|
def _step_is(step: RunStep, name: StepName) -> bool:
|
|
"""Whether ``step`` is an instance of ``name``, labelled or not.
|
|
|
|
A run touching several devices records ``wait:pbs-01`` / ``poweroff:pbs-02`` (see
|
|
``RunRecorder.step``'s ``label``), so an equality test would silently stop matching the
|
|
moment a route had more than one target.
|
|
"""
|
|
return step.name.split(":", 1)[0] == name.value
|
|
|
|
|
|
def _step_label(step: RunStep) -> str | None:
|
|
"""The device a step names (``poweroff:pbs-02`` -> ``pbs-02``), or None when unlabelled
|
|
— a single-device run doesn't repeat which box it means."""
|
|
_, _, label = step.name.partition(":")
|
|
return label or None
|
|
|
|
|
|
def _pbs_left_on(config: Config, run: Run) -> bool:
|
|
"""True if a PBS came up and nothing ever powered it back off — a box still burning
|
|
energy that the user should go and check.
|
|
|
|
Only for a run a restart interrupted: there the run row is all there is, and no
|
|
POWEROFF step was ever reached. A run that *finished* knows the answer exactly and
|
|
reports it through ``RunContext.left_on``, because "was it left on?" depends on facts
|
|
the timeline doesn't carry (whether another queued route still needs the box).
|
|
|
|
Two things the steps alone get wrong, both introduced by this same release:
|
|
|
|
* an **unmanaged** box (``managed_power: false``) is never Joulenap's to power down,
|
|
so a run against one must not warn — hence taking ``config``;
|
|
* a run holding **several** leases needs the WAIT and POWEROFF steps paired *per
|
|
device*, or one box's successful power-off hides another's that stayed up.
|
|
|
|
An abort *before* the box came up leaves the WAIT step non-SUCCESS, so the PBS is off
|
|
and this correctly returns False — hence why it keys on WAIT, not on the run status.
|
|
"""
|
|
managed = {p.id for p in config.pbss if p.managed_power}
|
|
route = next((r for r in config.routes if r.id == run.route_id), None)
|
|
# An unlabelled step belongs to the run's only box — a route's target (a sync route
|
|
# labels both sides). An ad-hoc GC/verify records neither a label nor a route, so its
|
|
# box cannot be named: warn rather than stay silent about a box that may be awake.
|
|
default = route.target if route else None
|
|
powered_off = {
|
|
_step_label(s)
|
|
for s in run.steps
|
|
if _step_is(s, StepName.POWEROFF) and s.status == StepStatus.SUCCESS
|
|
}
|
|
for step in run.steps:
|
|
if not _step_is(step, StepName.WAIT) or step.status != StepStatus.SUCCESS:
|
|
continue
|
|
pbs_id = _step_label(step) or default
|
|
if pbs_id is not None and pbs_id not in managed:
|
|
continue
|
|
if _step_label(step) not in powered_off:
|
|
return True
|
|
return False
|
|
|
|
|
|
#: Route kinds, for the body's ``Route:`` line. Localized because the kind is a user-facing
|
|
#: word in the UI too; an unknown kind falls through to its raw value.
|
|
_KIND_LABEL = {
|
|
"en": {"backup": "backup", "sync": "sync", "external": "external", "verify": "verify"},
|
|
"it": {
|
|
"backup": "backup",
|
|
"sync": "sincronizzazione",
|
|
"external": "esterna",
|
|
"verify": "verifica",
|
|
},
|
|
}
|
|
|
|
|
|
def build_run_message(ctx: RunContext) -> tuple[str, str]:
|
|
"""``(title, body)`` describing a finished run, in the configured language.
|
|
|
|
One field per line, in a fixed order; a field whose data is missing drops out entirely
|
|
rather than rendering a placeholder. ``ctx.datastore`` (read while the PBS was still
|
|
awake) adds the usage line, ``ctx.guests`` the per-guest tally of a backup cycle,
|
|
``ctx.next_at`` the schedule's following fire.
|
|
"""
|
|
config, run = ctx.config, ctx.run
|
|
pack = _pack(config.app.language)
|
|
labels = pack["_labels"]
|
|
event = _STATUS_EVENT.get(run.status, "failure") # RUNNING shouldn't reach here
|
|
|
|
# The title already conveys success/failure/aborted, so we don't repeat the (untranslated)
|
|
# status enum in the body.
|
|
lines: list[str] = []
|
|
if ctx.route is not None:
|
|
# The route's *colour* is deliberately absent: it is a UI affordance, and a hex
|
|
# string in a push notification is noise on every channel that could render it.
|
|
kinds = _KIND_LABEL.get(config.app.language, _KIND_LABEL["en"])
|
|
kind = kinds.get(ctx.route.kind, ctx.route.kind)
|
|
lines.append(f"{labels['route']}: {ctx.route.name or ctx.route.id} ({kind})")
|
|
lines.append(f"{labels['trigger']}: {labels.get(f'trigger_{run.trigger}', run.trigger)}")
|
|
datastore, guests, next_at = ctx.datastore, ctx.guests, ctx.next_at
|
|
if run.started_at and run.finished_at:
|
|
duration = (run.finished_at - run.started_at).total_seconds()
|
|
breakdown = _phase_breakdown(labels, run)
|
|
line = f"{labels['duration']}: {_format_duration(duration)}"
|
|
lines.append(f"{line} ({breakdown})" if breakdown else line)
|
|
# No summary at all (a GC or verify cycle, or an abort before the guests were picked)
|
|
# means there is nothing truthful to say about guests — better silent than "0".
|
|
if guests is not None and guests.total:
|
|
line = f"{labels['guests']}: {guests.ok}/{guests.total}"
|
|
if guests.failed:
|
|
line += f" ({labels['failed']}: {', '.join(guests.failed)})"
|
|
lines.append(line)
|
|
# An External route's watch: the MONITOR step's detail is either "N task(s) observed"
|
|
# or "no tasks observed" (see route_cycle._external_body) — the count line for the
|
|
# former, the your-schedule-didn't-fire warning for the latter.
|
|
monitor = next((s for s in run.steps if s.name == StepName.MONITOR), None)
|
|
if monitor is not None and monitor.detail:
|
|
try:
|
|
observed = int(monitor.detail.split()[0])
|
|
except ValueError:
|
|
lines.append(labels["no_tasks"])
|
|
else:
|
|
lines.append(f"{labels['tasks_observed']}: {observed}")
|
|
if datastore is not None:
|
|
lines.append(
|
|
f"{labels['datastore']}: {datastore.used_pct}% {labels['used']}, "
|
|
f"{human_bytes(datastore.avail)} {labels['free']}"
|
|
)
|
|
if run.error:
|
|
lines.append(f"{labels['error']}: {run.error}")
|
|
|
|
# From the service, not from the steps: a finished run knows exactly which boxes it
|
|
# left burning power, and only those warrant the warning.
|
|
if ctx.left_on:
|
|
lines.append(labels["pbs_left_on"])
|
|
|
|
if next_at is not None:
|
|
lines.append(f"{labels['next_run']}: {_format_dt(next_at, _timezone(config))}")
|
|
if run.id is not None:
|
|
lines.append(f"{labels['run_no']} #{run.id}")
|
|
|
|
return _title_for(pack, run.kind, event), "\n".join(lines)
|
|
|
|
|
|
def _timezone(config: Config) -> tzinfo:
|
|
"""The zone every timestamp in a notification is rendered in.
|
|
|
|
Imported here rather than at module scope because ``core.scheduler`` pulls in ``jobs``,
|
|
which pulls in this package — a top-level import would be circular."""
|
|
from ..core.scheduler import resolve_timezone
|
|
|
|
return resolve_timezone(config.app.timezone)
|
|
|
|
|
|
def _format_dt(dt: datetime | None, tz: tzinfo) -> str:
|
|
"""A short absolute timestamp for notifications, e.g. ``2026-07-11 04:00 CEST``.
|
|
|
|
Everything is converted into the configured zone first. Cron-derived times already
|
|
arrive in it, but anything read back from the database is UTC (see ``UtcDateTime``) —
|
|
without the conversion one line of the same message would silently be hours off from
|
|
the ones around it."""
|
|
if dt is None:
|
|
return "—"
|
|
return dt.astimezone(tz).strftime("%Y-%m-%d %H:%M %Z").rstrip()
|
|
|
|
|
|
def build_missed_backup_message(
|
|
config: Config,
|
|
route: Route,
|
|
missed_at: datetime,
|
|
last_run_at: datetime | None,
|
|
next_at: datetime | None,
|
|
) -> tuple[str, str]:
|
|
"""``(title, body)`` for a scheduled route that didn't run because the process was down
|
|
over its window (BE-R1), in the configured language.
|
|
|
|
Names the route: with per-route schedules, "a scheduled backup was skipped" no longer
|
|
identifies which one.
|
|
"""
|
|
pack = _pack(config.app.language)
|
|
labels = pack["_labels"]
|
|
tz = _timezone(config)
|
|
lines = [
|
|
pack["missed"]["intro"],
|
|
"",
|
|
f"{labels['route']}: {route.name or route.id}",
|
|
f"{labels['missed_run']}: {_format_dt(missed_at, tz)}",
|
|
f"{labels['last_run']}: {_format_dt(last_run_at, tz)}",
|
|
f"{labels['next_run']}: {_format_dt(next_at, tz)}",
|
|
]
|
|
return pack["missed"]["title"], "\n".join(lines)
|
|
|
|
|
|
def build_interrupted_message(config: Config, run: Run) -> tuple[str, str]:
|
|
"""``(title, body)`` for a run that a restart interrupted (swept to FAILURE at startup,
|
|
BE-R2), in the configured language.
|
|
|
|
Adds the "PBS left powered on" warning when a box Joulenap powers had actually woken
|
|
before the crash (WAIT succeeded, no matching POWEROFF) — the whole point of the alert:
|
|
a normally-off box that a crash left awake and burning power."""
|
|
pack = _pack(config.app.language)
|
|
labels = pack["_labels"]
|
|
lines = [pack["interrupted"]["intro"]]
|
|
if run.error:
|
|
lines.append(f"{labels['error']}: {run.error}")
|
|
if _pbs_left_on(config, run):
|
|
lines.append(labels["pbs_left_on"])
|
|
# This alert has no Duration line (the run's own span would span the whole downtime,
|
|
# not the work), so the one interval worth reporting is how long the box has been
|
|
# awake: from the moment it came up to the restart — and, since nothing powered it
|
|
# off, counting still.
|
|
awake_since = _awake_since(run)
|
|
if awake_since and run.finished_at:
|
|
awake = (run.finished_at - awake_since).total_seconds()
|
|
lines.append(f"{labels['awake_for']}: {_format_duration(awake)}")
|
|
if run.id is not None:
|
|
lines.append(f"{labels['run_no']} #{run.id}")
|
|
return pack["interrupted"]["title"], "\n".join(lines)
|
|
|
|
|
|
def _awake_since(run: Run) -> datetime | None:
|
|
"""When the PBS finished coming up — the WAIT step's finish, or None if it never did."""
|
|
for step in run.steps:
|
|
if _step_is(step, StepName.WAIT) and step.status == StepStatus.SUCCESS:
|
|
return step.finished_at
|
|
return None
|
|
|
|
|
|
def build_test_message(config: Config) -> tuple[str, str]:
|
|
"""``(title, body)`` for the manual 'send test notification' action."""
|
|
pack = _pack(config.app.language)
|
|
return pack["test"]["title"], pack["test"]["body"]
|