Files
Catubba f8ce39155b fix(power): lease a backup server per machine, not per configured device
Two datastores on one box are two devices by design, but one power switch. The
lease refcounted per device id, so a single sync route between them held two
independent leases on one machine: releasing the first shut the box down, and
releasing the second reached a machine already going down -- an SSL EOF from the
idle check, then a closed port 22 -- and recorded LEFT_ON. The run's
notification therefore warned "PBS left powered on" about a box that had gone to
sleep exactly as intended. Acquire was already correct by accident, because
_bring_up probes the host; only release, which never probes, was wrong.

lease_key() is the host, normalised the way discovery normalises it, falling
back to the device id when there is no host yet -- half-configured entries are
legal mid-wizard and would otherwise all collide on "". state() takes the device
rather than an id, and _pending_pbs_ids became _pending_pbs_keys: the queue's
answer and the lease's key must be the same space, or a queued run on a box's
other datastore stops holding it.

Re-keying alone was not enough. A same-machine sync then acquired one lease
twice and rendered "left on: still needed by another run" -- true only in the
sense that the same run holds it, so one wrong sentence for another. A run's
devices are now deduplicated by lease key: one machine, one wake, one power-off,
and the multi-device step labels fall away on their own.

Every device on a held machine now reports holders > 0, so the power button is
disabled on the sibling too. That is the point: an SSH poweroff takes down every
PBS instance on the box, including the one a run is using. Port is deliberately
not part of the key for the same reason.
2026-08-07 12:16:56 +02:00

164 lines
5.1 KiB
Python

"""GET /api/status — everything the homepage polls: the state pill, the queue, the
upcoming runs, and one entry per device for the topology.
Describes *Joulenap's* state, not a PBS's: with several backup servers "on/off" is no
longer a single fact, so the pill says what the app is doing and the per-device rows carry
the rest.
"""
from __future__ import annotations
from datetime import datetime
from fastapi import APIRouter, Depends
from pydantic import BaseModel
from sqlalchemy.orm import Session
from .. import config as config_mod
from ..core.config_store import ConfigStore
from ..db import get_session
from . import _probe
from .deps import (
JobService,
Scheduler,
get_config_store,
get_job_service,
get_scheduler,
require_auth,
)
from .schemas import RunSummary
router = APIRouter(dependencies=[Depends(require_auth)], tags=["status"])
class DatastoreInfo(BaseModel):
used: int # bytes
total: int
used_pct: float
class LoadInfo(BaseModel):
cpu: int # percent, 0-100
mem: int # percent, 0-100
uptime: int # seconds since the PBS booted
class PveState(BaseModel):
id: str
online: bool
class PbsState(BaseModel):
id: str
online: bool
managed_power: bool
#: How many runs currently hold this box's power lease. > 0 means the ⏻ button must be
#: disabled: a run is using it.
holders: int
datastore: DatastoreInfo | None = None
load: LoadInfo | None = None
class RunningRun(BaseModel):
run_id: int
kind: str
started_at: datetime
route_id: str | None = None
route_name: str | None = None
class QueuedRunInfo(BaseModel):
#: The queue key: a route id, or ``pbs:<id>:gc`` for an ad-hoc maintenance run.
key: str
route_id: str | None = None
pbs_id: str | None = None
class NextRun(BaseModel):
route_id: str
route_name: str
at: datetime
class StatusResponse(BaseModel):
#: The header pill: "running" (a run is in flight), "paused" (kill-switch off) or
#: "idle". Deliberately a small closed set — the frontend localizes it, never shows it.
state: str
scheduler_enabled: bool
running: RunningRun | None = None
queued: list[QueuedRunInfo] = []
#: Every armed route's next fire, soonest first.
next_runs: list[NextRun] = []
pves: list[PveState] = []
pbss: list[PbsState] = []
last_run: RunSummary | None = None
#: Set when the 0.9 -> 1.0 config migration was refused at startup: the config in use
#: has no devices and no routes, so the UI must say why instead of looking like a fresh
#: install. Rendered as a persistent banner under the header.
config_error: str | None = None
@router.get("/status", response_model=StatusResponse)
def get_status(
store: ConfigStore = Depends(get_config_store),
scheduler: Scheduler = Depends(get_scheduler),
job_service: JobService = Depends(get_job_service),
session: Session = Depends(get_session),
) -> StatusResponse:
config = store.config
running = _probe.running_run(session)
last = _probe.latest_finished_run(session)
pbs_probes = _probe.probe_pbss(config, job_service.deps.connect_pbs)
pve_online = _probe.probe_pves(config)
names = {r.id: (r.name or r.id) for r in config.routes}
if running is not None:
state = "running"
elif not config.app.scheduler_enabled:
state = "paused"
else:
state = "idle"
return StatusResponse(
state=state,
scheduler_enabled=config.app.scheduler_enabled,
running=(
RunningRun(
run_id=running.id,
kind=running.kind,
started_at=running.started_at,
route_id=running.route_id,
route_name=running.route_name,
)
if running
else None
),
queued=[
QueuedRunInfo(key=item.key, route_id=item.route_id, pbs_id=item.pbs_id)
for item in job_service.pending()
],
next_runs=[
NextRun(route_id=route_id, route_name=names.get(route_id, route_id), at=when)
for route_id, when in scheduler.next_runs()
],
pves=[PveState(id=pve.id, online=pve_online.get(pve.id, False)) for pve in config.pves],
pbss=[_pbs_state(pbs, pbs_probes, job_service) for pbs in config.pbss],
last_run=RunSummary.of(last, config.app.language) if last else None,
# Read live, not captured at import: a reload after the user fixes the file clears it.
config_error=config_mod.MIGRATION_ERROR,
)
def _pbs_state(pbs, probes: dict[str, _probe.PbsProbe], job_service: JobService) -> PbsState:
probe = probes.get(pbs.id)
ds = probe.datastore if probe else None
load = probe.load if probe else None
return PbsState(
id=pbs.id,
online=bool(probe and probe.online),
managed_power=pbs.managed_power,
holders=job_service.lease.state(pbs).holders,
datastore=DatastoreInfo(used=ds.used, total=ds.total, used_pct=ds.used_pct) if ds else None,
load=LoadInfo(cpu=load.cpu, mem=load.mem, uptime=load.uptime) if load else None,
)