fix(orchestrator): task-scoped oscillation breaker for escalate/unblock ping-pong (#685)

* fix(orchestrator): task-scoped oscillation breaker for escalate/unblock ping-pong

An escalation ping-pong oscillates a task between two agents (cell PM
escalate_up -> BLOCKED -> main PM unblock -> restored -> respawn ->
escalate again). The per-(agent, task) respawn gate never trips on it:
the restored side is dispatched by _dispatch_claimed_without_agent,
which consults no respawn counter at all, so one side of the round trip
always has fuel regardless of the other's strikes — and even a tripped
main-PM counter only stalls the task silently at blocked instead of
surfacing the oscillation.

- Strikes are counted task-scoped at the unblock() chokepoint
  (agent-agnostic; legitimate needs_revision rework never calls
  unblock, so it structurally cannot trip this), durable in the
  existing orchestration_markers column — no migration.
- Progress between round-trips (commits / revision_count advancing)
  resets the count: real forward motion is not an oscillation.
- On trip: the task is blocked with a HUMAN resolver (the budget-breach
  posture), both dispatchers stop respawning onto it, further unblock()
  refuses until an admin override clears the marker, and the CEO
  notification names both agents and the cycle count.
- _notification_has_live_work now treats a HITL-blocked related task as
  no live work, closing the same loop for the admin-route escalation
  path.

* fix(orchestrator): wire the oscillation trip to the dispatchers and make recovery reachable

- TaskResponse serializes blocker_resolver_type: the dispatchers' HITL-blocked
  skip and the notification-path live-work check now actually fire over the
  wire instead of only against in-process rows.
- The oscillation marker clears on every human transition out of BLOCKED
  (snapshot or not), and the human unblock route treats a tripped task as
  the requested intervention: clears the marker and proceeds, while the
  agent gateway verb keeps refusing.
- The progress fingerprint includes the terminal-children count, so a
  coordination root whose children advanced between escalations resets
  instead of accruing toward a false trip.

---------

Co-authored-by: Renn F <rennf93@users.noreply.github.com>
This commit is contained in:
Renzo F
2026-07-24 17:20:20 +02:00
committed by GitHub
co-authored by Renn F
parent 23ae0ca217
commit eb0dcb6ecb
11 changed files with 990 additions and 66 deletions
@@ -99,6 +99,63 @@ def test_blocked_task_non_cell_team_unassigned_is_unroutable() -> None:
assert orch._blocker_resolver_slug(task) is None
# ---------------------------------------------------------------------------
# _dispatch_blocker_work — wire-shaped HITL skip. `_fetch_tasks` hands this
# dispatcher plain dicts decoded straight from GET /tasks JSON — this pins
# that shape (blocker_resolver_type as the lowercase enum-value string
# TaskResponse now serializes) rather than an in-process TaskTable/enum.
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_dispatch_blocker_work_skips_wire_shaped_hitl_blocked_task(
monkeypatch: pytest.MonkeyPatch,
) -> None:
orch = _orch()
task: dict[str, Any] = {
"id": "t1",
"status": "blocked",
"blocker_resolver_type": "human",
"team": "backend",
"assigned_to": AGENT_UUIDS["be-dev-1"],
}
monkeypatch.setattr(orch, "_fetch_tasks", AsyncMock(return_value=[task]))
spawn = AsyncMock()
monkeypatch.setattr(orch, "spawn_agent", spawn)
await orch._dispatch_blocker_work(client=MagicMock())
spawn.assert_not_awaited()
@pytest.mark.asyncio
async def test_dispatch_blocker_work_spawns_non_hitl_blocked_task(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Control case: an agent-resolvable block (no HITL marker) still
dispatches normally — the wire-shaped skip above isn't just refusing
every blocked task."""
orch = _orch()
task: dict[str, Any] = {
"id": "t1",
"status": "blocked",
"blocker_resolver_type": None,
"team": "backend",
"assigned_to": AGENT_UUIDS["be-pm"],
}
monkeypatch.setattr(orch, "_fetch_tasks", AsyncMock(return_value=[task]))
monkeypatch.setattr(orch, "_is_agent_active", lambda _agent_id: False)
monkeypatch.setattr(orch, "_pm_respawn_should_gate", AsyncMock(return_value=False))
monkeypatch.setattr(orch, "_build_pm_blocker_prompt", lambda _task: "p")
monkeypatch.setattr(orch, "_task_git_context", lambda _task: None)
spawn = AsyncMock()
monkeypatch.setattr(orch, "spawn_agent", spawn)
await orch._dispatch_blocker_work(client=MagicMock())
spawn.assert_awaited_once()
# ---------------------------------------------------------------------------
# _claimed_task_needs_agent — claimed-but-no-agent detection
# ---------------------------------------------------------------------------
@@ -318,6 +375,43 @@ async def test_dispatch_claimed_without_agent_spawns_at_most_one_per_tick(
spawn.assert_awaited_once()
@pytest.mark.asyncio
async def test_dispatch_claimed_without_agent_has_no_progress_backoff(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Unlike `_dispatch_blocker_work` (every spawn gated through
`_pm_respawn_should_gate`), this dispatcher carries no respawn-loop
protection of its own — it unconditionally respawns an agentless
claimed/in_progress task every tick past the grace window. This is half
of why the escalate_up/unblock oscillation defeats the per-(agent, task)
breaker: the resolved side of the cycle (the PM restored to in_progress)
has no counter here to ever trip, so the loop's other half never runs out
of fuel on its own — only a task-scoped breaker that also covers this
path (by moving the task to `blocked` entirely, which this dispatcher
doesn't fetch) can stop it.
"""
orch = _orch()
task = {
"id": "t1",
"status": "in_progress",
"assigned_to": AGENT_UUIDS["fe-pm"],
"updated_at": _STALE,
}
monkeypatch.setattr(orch, "_fetch_tasks", AsyncMock(return_value=[task]))
_stub_git_context(orch, monkeypatch)
monkeypatch.setattr(orch, "_get_prompt_for_agent", AsyncMock(return_value="p"))
spawn = AsyncMock()
monkeypatch.setattr(orch, "spawn_agent", spawn)
cycles = 10
for _ in range(cycles):
orch._tick_handled_tasks = set() # a fresh dispatch tick each cycle
await orch._dispatch_claimed_without_agent(client=MagicMock())
# No cycle was ever refused — zero backoff anywhere in this call path.
assert spawn.await_count == cycles
@pytest.mark.asyncio
async def test_dispatch_claimed_without_agent_releases_unknown_without_spending_budget(
monkeypatch: pytest.MonkeyPatch,