mirror of
https://github.com/rennf93/roboco.git
synced 2026-08-03 07:23:24 +02:00
[F034] orchestrator: re-stamp respawn last_check at restore
_pm_made_rule_following_retry bounds its tracing_gap audit lookup with
since = record.get('last_check'). A stale persisted last_check from before
the restart matched pre-restart tracing_gap rows, falsely resetting the
breaker on the very first post-restart spawn — exactly when a fresh strike
count should be evaluating current state.
_partition_respawn_rows now re-stamps last_check to the restore time on
every restorable entry, bounding the lookup to post-restart gaps only.
This commit is contained in:
@@ -4999,7 +4999,9 @@ class AgentOrchestrator:
|
|||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _partition_respawn_rows(
|
def _partition_respawn_rows(
|
||||||
rows: "Iterable[Any]", status_by_id: dict[Any, Any]
|
rows: "Iterable[Any]",
|
||||||
|
status_by_id: dict[Any, Any],
|
||||||
|
now: datetime | None = None,
|
||||||
) -> tuple[dict[tuple[str, str], dict[str, Any]], list[tuple[str, Any]]]:
|
) -> tuple[dict[tuple[str, str], dict[str, Any]], list[tuple[str, Any]]]:
|
||||||
"""Split persisted respawn rows into (restorable entries, stale keys).
|
"""Split persisted respawn rows into (restorable entries, stale keys).
|
||||||
|
|
||||||
@@ -5008,9 +5010,17 @@ class AgentOrchestrator:
|
|||||||
against a fixed/deleted task. Restorable entries are keyed
|
against a fixed/deleted task. Restorable entries are keyed
|
||||||
``(agent_slug, str(task_id))`` to match the in-memory dict; stale keys
|
``(agent_slug, str(task_id))`` to match the in-memory dict; stale keys
|
||||||
carry the raw ``task_id`` for deletion.
|
carry the raw ``task_id`` for deletion.
|
||||||
|
|
||||||
|
F034: ``last_check`` is re-stamped to ``now`` (the restore time) on
|
||||||
|
every restorable entry. ``_pm_made_rule_following_retry`` reads
|
||||||
|
``since = record.get("last_check")`` to bound its tracing_gap audit
|
||||||
|
lookup; a stale pre-restart ``last_check`` would match a pre-restart
|
||||||
|
tracing_gap row and falsely reset the breaker on the first post-restart
|
||||||
|
spawn. Re-stamping bounds the lookup to post-restart gaps only.
|
||||||
"""
|
"""
|
||||||
from roboco.models.base import TaskStatus
|
from roboco.models.base import TaskStatus
|
||||||
|
|
||||||
|
restore_now = now or datetime.now(UTC)
|
||||||
terminal = {TaskStatus.COMPLETED.value, TaskStatus.CANCELLED.value}
|
terminal = {TaskStatus.COMPLETED.value, TaskStatus.CANCELLED.value}
|
||||||
restored: dict[tuple[str, str], dict[str, Any]] = {}
|
restored: dict[tuple[str, str], dict[str, Any]] = {}
|
||||||
stale: list[tuple[str, Any]] = []
|
stale: list[tuple[str, Any]] = []
|
||||||
@@ -5023,7 +5033,7 @@ class AgentOrchestrator:
|
|||||||
restored[(r.agent_slug, str(r.task_id))] = {
|
restored[(r.agent_slug, str(r.task_id))] = {
|
||||||
"count": r.count,
|
"count": r.count,
|
||||||
"last_status": r.last_status,
|
"last_status": r.last_status,
|
||||||
"last_check": r.last_check,
|
"last_check": restore_now,
|
||||||
"tracing_resets": r.tracing_resets,
|
"tracing_resets": r.tracing_resets,
|
||||||
"notified": r.notified,
|
"notified": r.notified,
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -80,6 +80,26 @@ def test_partition_drops_terminal_and_missing_rows() -> None:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_partition_restamps_last_check_to_now_to_avoid_stale_tracing_gap() -> None:
|
||||||
|
# F034: a persisted last_check from BEFORE the restart would make the first
|
||||||
|
# post-restart ``_pm_made_rule_following_retry`` audit lookup
|
||||||
|
# (``since = record.get("last_check")``) match a PRE-restart tracing_gap
|
||||||
|
# row, falsely resetting the breaker on the very first post-restart spawn —
|
||||||
|
# exactly when a fresh strike count should be evaluating current state.
|
||||||
|
# Restore must re-stamp last_check to the restore time so only post-restart
|
||||||
|
# tracing gaps can reset the counter.
|
||||||
|
tid = uuid4()
|
||||||
|
stale_check = datetime(2026, 6, 20, tzinfo=UTC)
|
||||||
|
rows = [_row(tid, last_check=stale_check)]
|
||||||
|
restore_now = datetime(2026, 6, 28, 12, 0, tzinfo=UTC)
|
||||||
|
restored, stale = AgentOrchestrator._partition_respawn_rows(
|
||||||
|
rows, {tid: "in_progress"}, now=restore_now
|
||||||
|
)
|
||||||
|
assert stale == []
|
||||||
|
assert restored[("be-pm", str(tid))]["last_check"] == restore_now
|
||||||
|
assert restored[("be-pm", str(tid))]["last_check"] != stale_check
|
||||||
|
|
||||||
|
|
||||||
# --------------------------------------------------------------------------- #
|
# --------------------------------------------------------------------------- #
|
||||||
# Startup loader
|
# Startup loader
|
||||||
# --------------------------------------------------------------------------- #
|
# --------------------------------------------------------------------------- #
|
||||||
|
|||||||
Reference in New Issue
Block a user