mirror of
https://github.com/rennf93/roboco.git
synced 2026-08-03 07:23:24 +02:00
fix(orchestrator): task-scoped oscillation breaker for escalate/unblock ping-pong (#685)
* fix(orchestrator): task-scoped oscillation breaker for escalate/unblock ping-pong An escalation ping-pong oscillates a task between two agents (cell PM escalate_up -> BLOCKED -> main PM unblock -> restored -> respawn -> escalate again). The per-(agent, task) respawn gate never trips on it: the restored side is dispatched by _dispatch_claimed_without_agent, which consults no respawn counter at all, so one side of the round trip always has fuel regardless of the other's strikes — and even a tripped main-PM counter only stalls the task silently at blocked instead of surfacing the oscillation. - Strikes are counted task-scoped at the unblock() chokepoint (agent-agnostic; legitimate needs_revision rework never calls unblock, so it structurally cannot trip this), durable in the existing orchestration_markers column — no migration. - Progress between round-trips (commits / revision_count advancing) resets the count: real forward motion is not an oscillation. - On trip: the task is blocked with a HUMAN resolver (the budget-breach posture), both dispatchers stop respawning onto it, further unblock() refuses until an admin override clears the marker, and the CEO notification names both agents and the cycle count. - _notification_has_live_work now treats a HITL-blocked related task as no live work, closing the same loop for the admin-route escalation path. * fix(orchestrator): wire the oscillation trip to the dispatchers and make recovery reachable - TaskResponse serializes blocker_resolver_type: the dispatchers' HITL-blocked skip and the notification-path live-work check now actually fire over the wire instead of only against in-process rows. - The oscillation marker clears on every human transition out of BLOCKED (snapshot or not), and the human unblock route treats a tripped task as the requested intervention: clears the marker and proceeds, while the agent gateway verb keeps refusing. - The progress fingerprint includes the terminal-children count, so a coordination root whose children advanced between escalations resets instead of accruing toward a false trip. --------- Co-authored-by: Renn F <rennf93@users.noreply.github.com>
This commit is contained in:
@@ -99,6 +99,63 @@ def test_blocked_task_non_cell_team_unassigned_is_unroutable() -> None:
|
||||
assert orch._blocker_resolver_slug(task) is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _dispatch_blocker_work — wire-shaped HITL skip. `_fetch_tasks` hands this
|
||||
# dispatcher plain dicts decoded straight from GET /tasks JSON — this pins
|
||||
# that shape (blocker_resolver_type as the lowercase enum-value string
|
||||
# TaskResponse now serializes) rather than an in-process TaskTable/enum.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dispatch_blocker_work_skips_wire_shaped_hitl_blocked_task(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
orch = _orch()
|
||||
task: dict[str, Any] = {
|
||||
"id": "t1",
|
||||
"status": "blocked",
|
||||
"blocker_resolver_type": "human",
|
||||
"team": "backend",
|
||||
"assigned_to": AGENT_UUIDS["be-dev-1"],
|
||||
}
|
||||
monkeypatch.setattr(orch, "_fetch_tasks", AsyncMock(return_value=[task]))
|
||||
spawn = AsyncMock()
|
||||
monkeypatch.setattr(orch, "spawn_agent", spawn)
|
||||
|
||||
await orch._dispatch_blocker_work(client=MagicMock())
|
||||
|
||||
spawn.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dispatch_blocker_work_spawns_non_hitl_blocked_task(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Control case: an agent-resolvable block (no HITL marker) still
|
||||
dispatches normally — the wire-shaped skip above isn't just refusing
|
||||
every blocked task."""
|
||||
orch = _orch()
|
||||
task: dict[str, Any] = {
|
||||
"id": "t1",
|
||||
"status": "blocked",
|
||||
"blocker_resolver_type": None,
|
||||
"team": "backend",
|
||||
"assigned_to": AGENT_UUIDS["be-pm"],
|
||||
}
|
||||
monkeypatch.setattr(orch, "_fetch_tasks", AsyncMock(return_value=[task]))
|
||||
monkeypatch.setattr(orch, "_is_agent_active", lambda _agent_id: False)
|
||||
monkeypatch.setattr(orch, "_pm_respawn_should_gate", AsyncMock(return_value=False))
|
||||
monkeypatch.setattr(orch, "_build_pm_blocker_prompt", lambda _task: "p")
|
||||
monkeypatch.setattr(orch, "_task_git_context", lambda _task: None)
|
||||
spawn = AsyncMock()
|
||||
monkeypatch.setattr(orch, "spawn_agent", spawn)
|
||||
|
||||
await orch._dispatch_blocker_work(client=MagicMock())
|
||||
|
||||
spawn.assert_awaited_once()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _claimed_task_needs_agent — claimed-but-no-agent detection
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -318,6 +375,43 @@ async def test_dispatch_claimed_without_agent_spawns_at_most_one_per_tick(
|
||||
spawn.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dispatch_claimed_without_agent_has_no_progress_backoff(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Unlike `_dispatch_blocker_work` (every spawn gated through
|
||||
`_pm_respawn_should_gate`), this dispatcher carries no respawn-loop
|
||||
protection of its own — it unconditionally respawns an agentless
|
||||
claimed/in_progress task every tick past the grace window. This is half
|
||||
of why the escalate_up/unblock oscillation defeats the per-(agent, task)
|
||||
breaker: the resolved side of the cycle (the PM restored to in_progress)
|
||||
has no counter here to ever trip, so the loop's other half never runs out
|
||||
of fuel on its own — only a task-scoped breaker that also covers this
|
||||
path (by moving the task to `blocked` entirely, which this dispatcher
|
||||
doesn't fetch) can stop it.
|
||||
"""
|
||||
orch = _orch()
|
||||
task = {
|
||||
"id": "t1",
|
||||
"status": "in_progress",
|
||||
"assigned_to": AGENT_UUIDS["fe-pm"],
|
||||
"updated_at": _STALE,
|
||||
}
|
||||
monkeypatch.setattr(orch, "_fetch_tasks", AsyncMock(return_value=[task]))
|
||||
_stub_git_context(orch, monkeypatch)
|
||||
monkeypatch.setattr(orch, "_get_prompt_for_agent", AsyncMock(return_value="p"))
|
||||
spawn = AsyncMock()
|
||||
monkeypatch.setattr(orch, "spawn_agent", spawn)
|
||||
|
||||
cycles = 10
|
||||
for _ in range(cycles):
|
||||
orch._tick_handled_tasks = set() # a fresh dispatch tick each cycle
|
||||
await orch._dispatch_claimed_without_agent(client=MagicMock())
|
||||
|
||||
# No cycle was ever refused — zero backoff anywhere in this call path.
|
||||
assert spawn.await_count == cycles
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dispatch_claimed_without_agent_releases_unknown_without_spending_budget(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
|
||||
Reference in New Issue
Block a user