feat(api): serve cached datastore usage when the PBS is offline

backend/app/api/_probe.py

@@ -5,6 +5,7 @@ reachability + datastore/load probe."""
 from __future__ import annotations
 
 from collections.abc import Callable
+from typing import NamedTuple
 
 from sqlalchemy import select
 from sqlalchemy.orm import Session
@@ -13,6 +14,8 @@ from ..config import Config
 from ..connectors import net
 from ..connectors.errors import ConnectorError
 from ..connectors.pbs import DatastoreStatus, NodeLoad, PbsClient
+from ..db import session_scope
+from ..db.datastore_stats import get_datastore_stat, upsert_datastore_stat
 from ..db.models import Run, RunKind, RunStatus
 
 # Keep the reachability probe snappy — dashboards poll and the PBS is usually off.
@@ -59,3 +62,23 @@ def probe_pbs(
         except ConnectorError:
             pass
     return online, datastore, load
+
+
+class DatastoreView(NamedTuple):
+    total: int
+    used: int
+    used_pct: float
+
+
+def resolve_datastore(datastore: str, live: DatastoreStatus | None) -> DatastoreView | None:
+    """Live-or-cache datastore usage. When ``live`` is present (PBS online) persist it and
+    return it; otherwise return the cached row; otherwise None. Opens its own transaction, so
+    it is safe to call from a request handler and the values are detached (no lazy load)."""
+    with session_scope() as session:
+        if live is not None:
+            upsert_datastore_stat(session, datastore, live.total, live.used)
+            return DatastoreView(live.total, live.used, live.used_pct)
+        row = get_datastore_stat(session, datastore)
+        if row is None:
+            return None
+        return DatastoreView(row.total, row.used, row.used_pct)

backend/app/api/dashboard.py

@@ -61,7 +61,8 @@ def get_dashboard(
 
     config = store.config
     last = _probe.latest_finished_cycle_run(session)
-    pbs_online, ds, _load = _probe.probe_pbs(config, job_service.deps.build_pbs)
+    pbs_online, live_ds, _load = _probe.probe_pbs(config, job_service.deps.build_pbs)
+    ds = _probe.resolve_datastore(config.pbs.datastore, live_ds)
 
     if job_service.is_running:
         pbs_state = "backing_up"

backend/app/api/status.py

@@ -56,7 +56,8 @@ def get_status(
 ) -> StatusResponse:
     config = store.config
     last = _probe.latest_cycle_run(session)
-    pbs_online, ds, nl = _probe.probe_pbs(config, job_service.deps.build_pbs)
+    pbs_online, live_ds, nl = _probe.probe_pbs(config, job_service.deps.build_pbs)
+    ds = _probe.resolve_datastore(config.pbs.datastore, live_ds)
 
     datastore = (
         DatastoreInfo(used=ds.used, total=ds.total, used_pct=ds.used_pct) if ds else None

backend/tests/test_api.py

@@ -409,6 +409,50 @@ def test_probe_pbs_offline_returns_no_datastore():
     assert load is None
 
 
+def test_resolve_datastore_live_upserts_and_returns_live(temp_db):
+    from app.api._probe import resolve_datastore
+    from app.connectors.pbs import DatastoreStatus
+    from app.db import session_scope
+    from app.db.datastore_stats import get_datastore_stat
+
+    view = resolve_datastore("backup", DatastoreStatus(total=10, used=4, avail=6))
+    assert (view.total, view.used) == (10, 4)
+    with session_scope() as s:
+        row = get_datastore_stat(s, "backup")
+    assert row is not None and row.used == 4  # live reading was persisted
+
+
+def test_resolve_datastore_offline_uses_cache(temp_db):
+    from app.api._probe import resolve_datastore
+    from app.db import session_scope
+    from app.db.datastore_stats import upsert_datastore_stat
+
+    with session_scope() as s:
+        upsert_datastore_stat(s, "backup", 8, 2)
+    view = resolve_datastore("backup", None)
+    assert (view.total, view.used, view.used_pct) == (8, 2, 25.0)
+
+
+def test_resolve_datastore_none_when_no_live_no_cache(temp_db):
+    from app.api._probe import resolve_datastore
+
+    assert resolve_datastore("backup", None) is None
+
+
+def test_status_datastore_from_cache_when_offline(app_ctx):
+    client, _app = app_ctx
+    with session_scope() as s:
+        from app.db.datastore_stats import upsert_datastore_stat
+        upsert_datastore_stat(s, "backup", 8_000_000_000, 2_000_000_000)
+
+    body = client.get("/api/status").json()
+    assert body["datastore"] is not None
+    assert body["datastore"]["used_pct"] == 25.0
+    assert body["datastore"]["used"] == 2_000_000_000
+    assert body["datastore"]["total"] == 8_000_000_000
+    assert body["load"] is None  # live-only, stays null when PBS offline
+
+
 def test_account_update_changes_username_and_password(app_ctx, temp_config):
     client, _app = app_ctx
     r = client.put("/api/account", json={"username": "newadmin", "password": "freshpass"})
@@ -510,6 +554,19 @@ def test_dashboard_200_with_header_key(app_ctx):
     assert body["datastore_total_bytes"] is None
 
 
+def test_dashboard_datastore_from_cache_when_offline(app_ctx):
+    client, app = app_ctx
+    key = _enable_api_key(app)
+    with session_scope() as s:  # session_scope already imported at top of test_api.py
+        from app.db.datastore_stats import upsert_datastore_stat
+        upsert_datastore_stat(s, "backup", 8_000_000_000, 2_000_000_000)
+
+    body = client.get("/api/dashboard", headers={"X-API-Key": key}).json()
+    assert body["datastore_used_pct"] == 25.0
+    assert body["datastore_used_bytes"] == 2_000_000_000
+    assert body["datastore_total_bytes"] == 8_000_000_000
+
+
 def test_dashboard_200_with_query_param_key(app_ctx):
     client, app = app_ctx
     key = _enable_api_key(app)