[F034] orchestrator: re-stamp respawn last_check at restore

_pm_made_rule_following_retry bounds its tracing_gap audit lookup with
since = record.get('last_check'). A stale persisted last_check from before
the restart matched pre-restart tracing_gap rows, falsely resetting the
breaker on the very first post-restart spawn — exactly when a fresh strike
count should be evaluating current state.

_partition_respawn_rows now re-stamps last_check to the restore time on
every restorable entry, bounding the lookup to post-restart gaps only.
This commit is contained in:
Renn F
2026-06-28 11:00:00 +02:00
parent fa8e567edd
commit e3011ec145
2 changed files with 32 additions and 2 deletions
+12 -2
View File
@@ -4999,7 +4999,9 @@ class AgentOrchestrator:
@staticmethod
def _partition_respawn_rows(
rows: "Iterable[Any]", status_by_id: dict[Any, Any]
rows: "Iterable[Any]",
status_by_id: dict[Any, Any],
now: datetime | None = None,
) -> tuple[dict[tuple[str, str], dict[str, Any]], list[tuple[str, Any]]]:
"""Split persisted respawn rows into (restorable entries, stale keys).
@@ -5008,9 +5010,17 @@ class AgentOrchestrator:
against a fixed/deleted task. Restorable entries are keyed
``(agent_slug, str(task_id))`` to match the in-memory dict; stale keys
carry the raw ``task_id`` for deletion.
F034: ``last_check`` is re-stamped to ``now`` (the restore time) on
every restorable entry. ``_pm_made_rule_following_retry`` reads
``since = record.get("last_check")`` to bound its tracing_gap audit
lookup; a stale pre-restart ``last_check`` would match a pre-restart
tracing_gap row and falsely reset the breaker on the first post-restart
spawn. Re-stamping bounds the lookup to post-restart gaps only.
"""
from roboco.models.base import TaskStatus
restore_now = now or datetime.now(UTC)
terminal = {TaskStatus.COMPLETED.value, TaskStatus.CANCELLED.value}
restored: dict[tuple[str, str], dict[str, Any]] = {}
stale: list[tuple[str, Any]] = []
@@ -5023,7 +5033,7 @@ class AgentOrchestrator:
restored[(r.agent_slug, str(r.task_id))] = {
"count": r.count,
"last_status": r.last_status,
"last_check": r.last_check,
"last_check": restore_now,
"tracing_resets": r.tracing_resets,
"notified": r.notified,
}
@@ -80,6 +80,26 @@ def test_partition_drops_terminal_and_missing_rows() -> None:
}
def test_partition_restamps_last_check_to_now_to_avoid_stale_tracing_gap() -> None:
# F034: a persisted last_check from BEFORE the restart would make the first
# post-restart ``_pm_made_rule_following_retry`` audit lookup
# (``since = record.get("last_check")``) match a PRE-restart tracing_gap
# row, falsely resetting the breaker on the very first post-restart spawn —
# exactly when a fresh strike count should be evaluating current state.
# Restore must re-stamp last_check to the restore time so only post-restart
# tracing gaps can reset the counter.
tid = uuid4()
stale_check = datetime(2026, 6, 20, tzinfo=UTC)
rows = [_row(tid, last_check=stale_check)]
restore_now = datetime(2026, 6, 28, 12, 0, tzinfo=UTC)
restored, stale = AgentOrchestrator._partition_respawn_rows(
rows, {tid: "in_progress"}, now=restore_now
)
assert stale == []
assert restored[("be-pm", str(tid))]["last_check"] == restore_now
assert restored[("be-pm", str(tid))]["last_check"] != stale_check
# --------------------------------------------------------------------------- #
# Startup loader
# --------------------------------------------------------------------------- #