mirror of
https://github.com/rennf93/roboco.git
synced 2026-08-03 07:23:24 +02:00
fix(orchestrator): task-scoped oscillation breaker for escalate/unblock ping-pong (#685)
* fix(orchestrator): task-scoped oscillation breaker for escalate/unblock ping-pong An escalation ping-pong oscillates a task between two agents (cell PM escalate_up -> BLOCKED -> main PM unblock -> restored -> respawn -> escalate again). The per-(agent, task) respawn gate never trips on it: the restored side is dispatched by _dispatch_claimed_without_agent, which consults no respawn counter at all, so one side of the round trip always has fuel regardless of the other's strikes — and even a tripped main-PM counter only stalls the task silently at blocked instead of surfacing the oscillation. - Strikes are counted task-scoped at the unblock() chokepoint (agent-agnostic; legitimate needs_revision rework never calls unblock, so it structurally cannot trip this), durable in the existing orchestration_markers column — no migration. - Progress between round-trips (commits / revision_count advancing) resets the count: real forward motion is not an oscillation. - On trip: the task is blocked with a HUMAN resolver (the budget-breach posture), both dispatchers stop respawning onto it, further unblock() refuses until an admin override clears the marker, and the CEO notification names both agents and the cycle count. - _notification_has_live_work now treats a HITL-blocked related task as no live work, closing the same loop for the admin-route escalation path. * fix(orchestrator): wire the oscillation trip to the dispatchers and make recovery reachable - TaskResponse serializes blocker_resolver_type: the dispatchers' HITL-blocked skip and the notification-path live-work check now actually fire over the wire instead of only against in-process rows. - The oscillation marker clears on every human transition out of BLOCKED (snapshot or not), and the human unblock route treats a tripped task as the requested intervention: clears the marker and proceeds, while the agent gateway verb keeps refusing. - The progress fingerprint includes the terminal-children count, so a coordination root whose children advanced between escalations resets instead of accruing toward a false trip. --------- Co-authored-by: Renn F <rennf93@users.noreply.github.com>
This commit is contained in:
@@ -32,8 +32,17 @@ from roboco.api.schemas.tasks import (
|
||||
task_to_response,
|
||||
transform_update_data,
|
||||
)
|
||||
from roboco.models.base import Complexity, TaskNature, TaskStatus, TaskType, Team
|
||||
from roboco.db.tables import TaskTable
|
||||
from roboco.models.base import (
|
||||
BlockerResolverType,
|
||||
Complexity,
|
||||
TaskNature,
|
||||
TaskStatus,
|
||||
TaskType,
|
||||
Team,
|
||||
)
|
||||
from roboco.models.product import ProductCellMapping
|
||||
from roboco.runtime.orchestrator import AgentOrchestrator
|
||||
|
||||
_ORDER_DEFAULT = 0
|
||||
|
||||
@@ -311,6 +320,7 @@ def _stub_task(*, with_project: bool = False) -> Any:
|
||||
description="d",
|
||||
acceptance_criteria=["a"],
|
||||
status=TaskStatus.PENDING,
|
||||
blocker_resolver_type=None,
|
||||
priority=1,
|
||||
sequence=0,
|
||||
nature=TaskNature.TECHNICAL,
|
||||
@@ -488,6 +498,90 @@ def test_task_list_to_response_returns_list() -> None:
|
||||
assert len(out) == len(stubs)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# blocker_resolver_type — wire-shaped regression. A hand-rolled stub (like
|
||||
# _stub_task above) can't catch a field TaskResponse silently drops; this
|
||||
# builds a REAL TaskTable row and pushes it through the actual serialization
|
||||
# path the orchestrator's dispatchers see over HTTP.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _hitl_blocked_task_row() -> TaskTable:
|
||||
"""A real TaskTable instance (never added to a session) in the exact
|
||||
shape `unblock`'s oscillation-breaker trip leaves behind: BLOCKED with
|
||||
blocker_resolver_type=HUMAN."""
|
||||
return TaskTable(
|
||||
id=uuid4(),
|
||||
title="t",
|
||||
description="d",
|
||||
acceptance_criteria=["a"],
|
||||
acceptance_criteria_ids=[],
|
||||
parent_ac_refs=[],
|
||||
status=TaskStatus.BLOCKED,
|
||||
blocker_resolver_type=BlockerResolverType.HUMAN,
|
||||
priority=2,
|
||||
sequence=0,
|
||||
nature=TaskNature.TECHNICAL,
|
||||
task_type=TaskType.CODE,
|
||||
project_id=None,
|
||||
product_id=None,
|
||||
docs_complete=False,
|
||||
pr_created=False,
|
||||
board_review_complete=False,
|
||||
team=Team.BACKEND,
|
||||
created_by=uuid4(),
|
||||
assigned_to=None,
|
||||
parent_task_id=None,
|
||||
dependency_ids=[],
|
||||
blocker_ids=[],
|
||||
batch_id=None,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=None,
|
||||
claimed_at=None,
|
||||
claimed_by=None,
|
||||
started_at=None,
|
||||
completed_at=None,
|
||||
target_date=None,
|
||||
estimated_complexity=Complexity.LOW,
|
||||
plan=None,
|
||||
checkpoints=[],
|
||||
progress_updates=[],
|
||||
commits=[],
|
||||
documents=[],
|
||||
dev_notes=None,
|
||||
qa_notes=None,
|
||||
auditor_notes=None,
|
||||
self_verified=False,
|
||||
qa_verified=None,
|
||||
branch_name=None,
|
||||
pr_number=None,
|
||||
pr_url=None,
|
||||
source="manual",
|
||||
confirmed_by_human=True,
|
||||
)
|
||||
|
||||
|
||||
def test_task_to_response_serializes_blocker_resolver_type() -> None:
|
||||
"""A real ORM row's blocker_resolver_type must round-trip — the field was
|
||||
silently missing from TaskResponse, so every dispatcher reading it over
|
||||
the wire saw None regardless of the DB value."""
|
||||
row = _hitl_blocked_task_row()
|
||||
resp = task_to_response(row)
|
||||
assert resp.blocker_resolver_type == BlockerResolverType.HUMAN
|
||||
|
||||
|
||||
def test_wire_shaped_hitl_blocked_task_trips_is_hitl_blocked() -> None:
|
||||
"""End-to-end wire simulation: real TaskTable -> task_to_response ->
|
||||
JSON-mode serialization (what httpx.json() hands the orchestrator) ->
|
||||
AgentOrchestrator._is_hitl_blocked. Before the fix this always read
|
||||
None over the wire and never fired."""
|
||||
row = _hitl_blocked_task_row()
|
||||
resp = task_to_response(row)
|
||||
wire_dict = resp.model_dump(mode="json")
|
||||
assert wire_dict["blocker_resolver_type"] == "human"
|
||||
assert AgentOrchestrator._is_hitl_blocked(wire_dict) is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# enrich_task_with_context — covers the work_session + project lookup branches.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -8,6 +8,7 @@ from roboco.foundation.policy.content import markers as m
|
||||
|
||||
# Named constant — ruff PLR2004 forbids magic-value comparisons.
|
||||
_TWO = 2
|
||||
_THREE = 3
|
||||
|
||||
|
||||
def _task(om: dict | None = None) -> SimpleNamespace:
|
||||
@@ -155,3 +156,47 @@ def test_block_flip_count_bump_and_notify() -> None:
|
||||
assert m.is_block_flip_notified(t) is True
|
||||
# Marking notified must not reset the counter.
|
||||
assert m.get_block_flip_count(t) == _TWO
|
||||
|
||||
|
||||
def test_oscillation_strikes_accrue_on_unchanged_fingerprint() -> None:
|
||||
t = _task()
|
||||
assert m.get_oscillation_strikes(t) == 0
|
||||
assert m.is_oscillation_tripped(t) is False
|
||||
assert m.bump_oscillation_strikes(t, [0, 0]) == 1
|
||||
assert m.bump_oscillation_strikes(t, [0, 0]) == _TWO
|
||||
assert m.bump_oscillation_strikes(t, [0, 0]) == _THREE
|
||||
assert m.get_oscillation_strikes(t) == _THREE
|
||||
assert m.is_oscillation_tripped(t) is False
|
||||
|
||||
|
||||
def test_oscillation_strikes_reset_on_progress() -> None:
|
||||
t = _task()
|
||||
m.bump_oscillation_strikes(t, [0, 0])
|
||||
assert m.bump_oscillation_strikes(t, [0, 0]) == _TWO
|
||||
# A new commit landed between rounds — real progress resets to 1.
|
||||
assert m.bump_oscillation_strikes(t, [1, 0]) == 1
|
||||
# A revision round completing is progress too.
|
||||
assert m.bump_oscillation_strikes(t, [1, 0]) == _TWO
|
||||
assert m.bump_oscillation_strikes(t, [1, 1]) == 1
|
||||
|
||||
|
||||
def test_mark_oscillation_tripped_preserves_strikes_and_fingerprint() -> None:
|
||||
t = _task()
|
||||
m.bump_oscillation_strikes(t, [2, 1])
|
||||
m.bump_oscillation_strikes(t, [2, 1])
|
||||
m.mark_oscillation_tripped(t)
|
||||
assert m.is_oscillation_tripped(t) is True
|
||||
assert m.get_oscillation_strikes(t) == _TWO
|
||||
# tripped survives a subsequent bump (belt-and-suspenders — the guard is
|
||||
# meant to refuse before another bump ever happens).
|
||||
m.bump_oscillation_strikes(t, [2, 1])
|
||||
assert m.is_oscillation_tripped(t) is True
|
||||
|
||||
|
||||
def test_clear_marker_removes_oscillation_state() -> None:
|
||||
t = _task()
|
||||
m.bump_oscillation_strikes(t, [0, 0])
|
||||
m.mark_oscillation_tripped(t)
|
||||
m.clear_marker(t, m.OSCILLATION_STRIKES)
|
||||
assert m.get_oscillation_strikes(t) == 0
|
||||
assert m.is_oscillation_tripped(t) is False
|
||||
|
||||
@@ -0,0 +1,222 @@
|
||||
"""The escalate_up/unblock oscillation breaker.
|
||||
|
||||
Live wedge: a cell PM's escalate_up auto-blocks a task and the Main PM's
|
||||
unblock restores it, repeat, forever — the per-(agent, task) respawn breaker
|
||||
in the orchestrator misses this because the escalator and the resolver each
|
||||
own only half the round trips, so neither individual counter accrues at the
|
||||
cycle's real rate. ``unblock`` now stamps a task-scoped, progress-
|
||||
discriminated strike counter (``markers.oscillation_strikes``) on every
|
||||
restore; past the trip threshold the task is force-blocked for a human
|
||||
instead of restored again, and further ``unblock`` calls on it are refused
|
||||
until an admin override clears it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from roboco.foundation.policy.content import markers
|
||||
from roboco.models.base import BlockerResolverType, TaskStatus
|
||||
from roboco.services.gateway.choreographer import Choreographer, ChoreographerDeps
|
||||
|
||||
# Mirrors _OSCILLATION_TRIP_THRESHOLD in _impl.py — ruff PLR2004 forbids
|
||||
# magic-value comparisons.
|
||||
_TRIP_THRESHOLD = 5
|
||||
|
||||
|
||||
def _make_deps(**overrides: Any) -> ChoreographerDeps:
|
||||
base: dict[str, Any] = {
|
||||
"task": AsyncMock(),
|
||||
"work_session": AsyncMock(),
|
||||
"git": AsyncMock(),
|
||||
"a2a": AsyncMock(),
|
||||
"journal": AsyncMock(),
|
||||
"audit": AsyncMock(),
|
||||
"evidence_repo": AsyncMock(),
|
||||
}
|
||||
base.update(overrides)
|
||||
base["journal"].has_decision_for_task.return_value = True
|
||||
base["journal"].latest_decision_at.return_value = datetime.now(UTC)
|
||||
return ChoreographerDeps(**base)
|
||||
|
||||
|
||||
def _osc_setup() -> tuple[Choreographer, Any, Any, Any, Any]:
|
||||
"""A blocked task whose ``unblock_with_restore`` returns the SAME mock
|
||||
object each call (one real ORM row across requests), with commits/
|
||||
revision_count seeded so the progress fingerprint is well-formed.
|
||||
"""
|
||||
pm_id = uuid4()
|
||||
task_id = uuid4()
|
||||
t = MagicMock(
|
||||
id=task_id,
|
||||
status="blocked",
|
||||
pre_block_state="in_progress",
|
||||
pre_block_assignee=uuid4(),
|
||||
pre_block_metadata={},
|
||||
dependency_ids=[],
|
||||
orchestration_markers=None,
|
||||
commits=[],
|
||||
revision_count=0,
|
||||
blocker_raised_by=uuid4(),
|
||||
)
|
||||
task_svc = AsyncMock()
|
||||
task_svc.get.return_value = t
|
||||
task_svc.unblock_with_restore.return_value = t
|
||||
task_svc.unmet_dependency_ids.return_value = []
|
||||
# A PM coordination root never commits itself — held constant here so
|
||||
# existing scenarios (driven purely by commits/revision_count) are
|
||||
# unaffected; tests targeting the child-count component override it.
|
||||
task_svc.terminal_children_count = AsyncMock(return_value=0)
|
||||
c = Choreographer(_make_deps(task=task_svc))
|
||||
return c, pm_id, task_id, t, task_svc
|
||||
|
||||
|
||||
async def _unblock_once(c: Choreographer, pm_id: Any, task_id: Any, t: Any) -> Any:
|
||||
"""Re-block before each call — a fresh round trip in the cycle."""
|
||||
t.status = "blocked"
|
||||
return await c.unblock(pm_id, task_id, "resolved upstream; restoring")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_progress_trips_after_threshold_cycles() -> None:
|
||||
c, pm_id, task_id, t, task_svc = _osc_setup()
|
||||
cc: Any = c
|
||||
notify = AsyncMock()
|
||||
cc._notify_ceo_oscillation = notify
|
||||
|
||||
envs = [
|
||||
await _unblock_once(c, pm_id, task_id, t) for _ in range(_TRIP_THRESHOLD + 1)
|
||||
]
|
||||
|
||||
for env in envs[:-1]:
|
||||
assert env.error is None, env.as_dict()
|
||||
tripped = envs[-1]
|
||||
assert tripped.error is None, tripped.as_dict()
|
||||
assert "force-blocked" in tripped.next
|
||||
assert markers.is_oscillation_tripped(t) is True
|
||||
assert markers.get_oscillation_strikes(t) == _TRIP_THRESHOLD + 1
|
||||
notify.assert_awaited_once()
|
||||
assert t.blocker_resolver_type == BlockerResolverType.HUMAN
|
||||
task_svc.admin_set_status.assert_awaited_once_with(
|
||||
task_id, TaskStatus.BLOCKED, actor_role="system"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tripped_task_refuses_further_unblock() -> None:
|
||||
c, pm_id, task_id, t, task_svc = _osc_setup()
|
||||
cc: Any = c
|
||||
cc._notify_ceo_oscillation = AsyncMock()
|
||||
markers.mark_oscillation_tripped(t)
|
||||
admin_calls_before = task_svc.admin_set_status.await_count
|
||||
|
||||
env = await _unblock_once(c, pm_id, task_id, t)
|
||||
|
||||
assert env.error == "invalid_state"
|
||||
assert "force-blocked" in env.message
|
||||
# The guard short-circuits before the restore path ever runs again.
|
||||
assert task_svc.admin_set_status.await_count == admin_calls_before
|
||||
task_svc.unblock_with_restore.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_progress_between_rounds_prevents_trip() -> None:
|
||||
"""A commit landing between escalations is real progress, not a loop —
|
||||
strikes reset every round, so the breaker never trips no matter how many
|
||||
rounds occur."""
|
||||
c, pm_id, task_id, t, _task_svc = _osc_setup()
|
||||
cc: Any = c
|
||||
cc._notify_ceo_oscillation = AsyncMock()
|
||||
|
||||
for i in range(_TRIP_THRESHOLD + 3):
|
||||
# A fresh commit lands every round — the commit count strictly grows.
|
||||
t.commits = [{"sha": str(j)} for j in range(i + 1)]
|
||||
env = await _unblock_once(c, pm_id, task_id, t)
|
||||
assert env.error is None, env.as_dict()
|
||||
|
||||
assert markers.is_oscillation_tripped(t) is False
|
||||
assert markers.get_oscillation_strikes(t) == 1
|
||||
cc._notify_ceo_oscillation.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_revision_count_advance_also_counts_as_progress() -> None:
|
||||
c, pm_id, task_id, t, _task_svc = _osc_setup()
|
||||
cc: Any = c
|
||||
cc._notify_ceo_oscillation = AsyncMock()
|
||||
|
||||
for _ in range(_TRIP_THRESHOLD):
|
||||
t.revision_count += 1 # a genuine revision round resolved
|
||||
env = await _unblock_once(c, pm_id, task_id, t)
|
||||
assert env.error is None, env.as_dict()
|
||||
|
||||
assert markers.is_oscillation_tripped(t) is False
|
||||
cc._notify_ceo_oscillation.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_static_terminal_children_still_trips() -> None:
|
||||
"""A coordination root with existing terminal children that DON'T change
|
||||
between rounds still trips — the child count is one more progress
|
||||
signal, not blanket forgiveness for an otherwise stalled cycle."""
|
||||
c, pm_id, task_id, t, task_svc = _osc_setup()
|
||||
cc: Any = c
|
||||
cc._notify_ceo_oscillation = AsyncMock()
|
||||
task_svc.terminal_children_count = AsyncMock(return_value=3)
|
||||
|
||||
envs = [
|
||||
await _unblock_once(c, pm_id, task_id, t) for _ in range(_TRIP_THRESHOLD + 1)
|
||||
]
|
||||
|
||||
tripped = envs[-1]
|
||||
assert tripped.error is None, tripped.as_dict()
|
||||
assert markers.is_oscillation_tripped(t) is True
|
||||
cc._notify_ceo_oscillation.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_terminal_child_completing_between_rounds_resets_strikes() -> None:
|
||||
"""On a PM coordination root the commit/revision_count components are
|
||||
structurally static (PMs never commit) — a child landing COMPLETED
|
||||
between escalations is the only progress signal available, and must
|
||||
still reset strikes like the other two."""
|
||||
c, pm_id, task_id, t, task_svc = _osc_setup()
|
||||
cc: Any = c
|
||||
cc._notify_ceo_oscillation = AsyncMock()
|
||||
|
||||
counts = iter([0, 0, 1, 1, 2]) # a child completes on round 3, then round 5
|
||||
task_svc.terminal_children_count = AsyncMock(side_effect=lambda _tid: next(counts))
|
||||
|
||||
for _ in range(5):
|
||||
env = await _unblock_once(c, pm_id, task_id, t)
|
||||
assert env.error is None, env.as_dict()
|
||||
|
||||
assert markers.is_oscillation_tripped(t) is False
|
||||
# Round 5's fingerprint differs from round 4's (1 -> 2 children), so this
|
||||
# round's strike count reset to 1.
|
||||
assert markers.get_oscillation_strikes(t) == 1
|
||||
cc._notify_ceo_oscillation.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_notify_named_with_strikes_and_both_agents() -> None:
|
||||
c, pm_id, task_id, t, _task_svc = _osc_setup()
|
||||
cc: Any = c
|
||||
notify = AsyncMock()
|
||||
cc._notify_ceo_oscillation = notify
|
||||
escalator_id = t.blocker_raised_by
|
||||
|
||||
for _ in range(_TRIP_THRESHOLD + 1):
|
||||
await _unblock_once(c, pm_id, task_id, t)
|
||||
|
||||
notify.assert_awaited_once_with(
|
||||
task_id,
|
||||
_TRIP_THRESHOLD + 1,
|
||||
t.title,
|
||||
escalator_id=escalator_id,
|
||||
resolver_id=pm_id,
|
||||
)
|
||||
@@ -99,6 +99,63 @@ def test_blocked_task_non_cell_team_unassigned_is_unroutable() -> None:
|
||||
assert orch._blocker_resolver_slug(task) is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _dispatch_blocker_work — wire-shaped HITL skip. `_fetch_tasks` hands this
|
||||
# dispatcher plain dicts decoded straight from GET /tasks JSON — this pins
|
||||
# that shape (blocker_resolver_type as the lowercase enum-value string
|
||||
# TaskResponse now serializes) rather than an in-process TaskTable/enum.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dispatch_blocker_work_skips_wire_shaped_hitl_blocked_task(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
orch = _orch()
|
||||
task: dict[str, Any] = {
|
||||
"id": "t1",
|
||||
"status": "blocked",
|
||||
"blocker_resolver_type": "human",
|
||||
"team": "backend",
|
||||
"assigned_to": AGENT_UUIDS["be-dev-1"],
|
||||
}
|
||||
monkeypatch.setattr(orch, "_fetch_tasks", AsyncMock(return_value=[task]))
|
||||
spawn = AsyncMock()
|
||||
monkeypatch.setattr(orch, "spawn_agent", spawn)
|
||||
|
||||
await orch._dispatch_blocker_work(client=MagicMock())
|
||||
|
||||
spawn.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dispatch_blocker_work_spawns_non_hitl_blocked_task(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Control case: an agent-resolvable block (no HITL marker) still
|
||||
dispatches normally — the wire-shaped skip above isn't just refusing
|
||||
every blocked task."""
|
||||
orch = _orch()
|
||||
task: dict[str, Any] = {
|
||||
"id": "t1",
|
||||
"status": "blocked",
|
||||
"blocker_resolver_type": None,
|
||||
"team": "backend",
|
||||
"assigned_to": AGENT_UUIDS["be-pm"],
|
||||
}
|
||||
monkeypatch.setattr(orch, "_fetch_tasks", AsyncMock(return_value=[task]))
|
||||
monkeypatch.setattr(orch, "_is_agent_active", lambda _agent_id: False)
|
||||
monkeypatch.setattr(orch, "_pm_respawn_should_gate", AsyncMock(return_value=False))
|
||||
monkeypatch.setattr(orch, "_build_pm_blocker_prompt", lambda _task: "p")
|
||||
monkeypatch.setattr(orch, "_task_git_context", lambda _task: None)
|
||||
spawn = AsyncMock()
|
||||
monkeypatch.setattr(orch, "spawn_agent", spawn)
|
||||
|
||||
await orch._dispatch_blocker_work(client=MagicMock())
|
||||
|
||||
spawn.assert_awaited_once()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _claimed_task_needs_agent — claimed-but-no-agent detection
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -318,6 +375,43 @@ async def test_dispatch_claimed_without_agent_spawns_at_most_one_per_tick(
|
||||
spawn.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dispatch_claimed_without_agent_has_no_progress_backoff(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Unlike `_dispatch_blocker_work` (every spawn gated through
|
||||
`_pm_respawn_should_gate`), this dispatcher carries no respawn-loop
|
||||
protection of its own — it unconditionally respawns an agentless
|
||||
claimed/in_progress task every tick past the grace window. This is half
|
||||
of why the escalate_up/unblock oscillation defeats the per-(agent, task)
|
||||
breaker: the resolved side of the cycle (the PM restored to in_progress)
|
||||
has no counter here to ever trip, so the loop's other half never runs out
|
||||
of fuel on its own — only a task-scoped breaker that also covers this
|
||||
path (by moving the task to `blocked` entirely, which this dispatcher
|
||||
doesn't fetch) can stop it.
|
||||
"""
|
||||
orch = _orch()
|
||||
task = {
|
||||
"id": "t1",
|
||||
"status": "in_progress",
|
||||
"assigned_to": AGENT_UUIDS["fe-pm"],
|
||||
"updated_at": _STALE,
|
||||
}
|
||||
monkeypatch.setattr(orch, "_fetch_tasks", AsyncMock(return_value=[task]))
|
||||
_stub_git_context(orch, monkeypatch)
|
||||
monkeypatch.setattr(orch, "_get_prompt_for_agent", AsyncMock(return_value="p"))
|
||||
spawn = AsyncMock()
|
||||
monkeypatch.setattr(orch, "spawn_agent", spawn)
|
||||
|
||||
cycles = 10
|
||||
for _ in range(cycles):
|
||||
orch._tick_handled_tasks = set() # a fresh dispatch tick each cycle
|
||||
await orch._dispatch_claimed_without_agent(client=MagicMock())
|
||||
|
||||
# No cycle was ever refused — zero backoff anywhere in this call path.
|
||||
assert spawn.await_count == cycles
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dispatch_claimed_without_agent_releases_unknown_without_spending_budget(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
|
||||
@@ -530,6 +530,58 @@ async def test_unblock_notifies_once_not_twice_on_repeated_call() -> None:
|
||||
mock_ns.send_unblock_notification.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unblock_clears_tripped_oscillation_marker() -> None:
|
||||
"""The legacy human/panel unblock route never goes through the gateway's
|
||||
_oscillation_unblock_guard — reaching a tripped task here IS the human
|
||||
intervention the breaker demands, so it must clear the marker rather
|
||||
than leave it to refuse the task's next legitimate cycle."""
|
||||
task = _build_task(
|
||||
status=TaskStatus.BLOCKED, branch_name=None, blocker_raised_by=uuid4()
|
||||
)
|
||||
for _ in range(6):
|
||||
markers.bump_oscillation_strikes(task, [0, 0, 0])
|
||||
markers.mark_oscillation_tripped(task)
|
||||
assert markers.is_oscillation_tripped(task) is True
|
||||
|
||||
svc = TaskService(MagicMock(flush=AsyncMock()))
|
||||
_bind(svc, "get", AsyncMock(return_value=task))
|
||||
_bind(svc, "_index_lifecycle_event_background", AsyncMock())
|
||||
mock_ns = MagicMock()
|
||||
mock_ns.send_unblock_notification = AsyncMock()
|
||||
with patch(
|
||||
"roboco.services.notification.NotificationService", return_value=mock_ns
|
||||
):
|
||||
out = await svc.unblock(task.id)
|
||||
|
||||
assert out is task
|
||||
assert markers.is_oscillation_tripped(task) is False
|
||||
assert markers.get_oscillation_strikes(task) == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unblock_leaves_untripped_marker_alone() -> None:
|
||||
"""A normal (non-tripped) unblock must not reset an in-flight, still-live
|
||||
strike count — only a tripped marker is cleared."""
|
||||
task = _build_task(
|
||||
status=TaskStatus.BLOCKED, branch_name=None, blocker_raised_by=uuid4()
|
||||
)
|
||||
markers.bump_oscillation_strikes(task, [0, 0, 0])
|
||||
assert markers.is_oscillation_tripped(task) is False
|
||||
|
||||
svc = TaskService(MagicMock(flush=AsyncMock()))
|
||||
_bind(svc, "get", AsyncMock(return_value=task))
|
||||
_bind(svc, "_index_lifecycle_event_background", AsyncMock())
|
||||
mock_ns = MagicMock()
|
||||
mock_ns.send_unblock_notification = AsyncMock()
|
||||
with patch(
|
||||
"roboco.services.notification.NotificationService", return_value=mock_ns
|
||||
):
|
||||
await svc.unblock(task.id)
|
||||
|
||||
assert markers.get_oscillation_strikes(task) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wire_sibling_collision_dag_notifies_only_for_new_edges() -> None:
|
||||
"""Collision-sequencing notification fires only for freshly-added edges.
|
||||
@@ -2207,6 +2259,101 @@ async def test_admin_set_status_force_no_revision_bump(
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Oscillation breaker — post-trip topology. The trip fires AFTER the restore
|
||||
# already wiped pre_block_assignee/pre_block_state (a fresh re-block via
|
||||
# admin_set_status stamps no new snapshot), so a CEO override out of BLOCKED
|
||||
# must clear the marker even with no snapshot to drive a restore — otherwise
|
||||
# it latently refuses this task's next legitimate gateway unblock forever.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ceo_override_clears_oscillation_marker_with_wiped_snapshot(
|
||||
db_session: AsyncSession,
|
||||
) -> None:
|
||||
agent = AgentTable(
|
||||
id=uuid4(),
|
||||
name="A",
|
||||
slug=f"a-{uuid4().hex[:8]}",
|
||||
role=AgentRole.MAIN_PM,
|
||||
team=Team.BACKEND,
|
||||
status=AgentStatus.ACTIVE,
|
||||
model_config={},
|
||||
system_prompt="pm",
|
||||
capabilities=[],
|
||||
permissions={},
|
||||
metrics={},
|
||||
)
|
||||
db_session.add(agent)
|
||||
await db_session.flush()
|
||||
project = ProjectTable(
|
||||
id=uuid4(),
|
||||
name="P",
|
||||
slug=f"p-{uuid4().hex[:6]}",
|
||||
git_url="https://example.com/r.git",
|
||||
assigned_cell=Team.BACKEND,
|
||||
created_by=agent.id,
|
||||
)
|
||||
db_session.add(project)
|
||||
await db_session.flush()
|
||||
tid = uuid4()
|
||||
task = TaskTable(
|
||||
id=tid,
|
||||
title="t",
|
||||
description="d",
|
||||
acceptance_criteria=["done"],
|
||||
status=TaskStatus.BLOCKED,
|
||||
priority=2,
|
||||
task_type=TaskType.CODE,
|
||||
nature=TaskNature.TECHNICAL,
|
||||
estimated_complexity=Complexity.LOW,
|
||||
team=Team.BACKEND,
|
||||
confirmed_by_human=True,
|
||||
project_id=project.id,
|
||||
created_by=agent.id,
|
||||
assigned_to=agent.id,
|
||||
branch_name="feature/x",
|
||||
# The exact post-trip topology: unblock_with_restore already wiped
|
||||
# the snapshot before the trip fired, and the force-block into
|
||||
# BLOCKED (admin_set_status, from a non-BLOCKED from_status) stamps
|
||||
# no new one of its own.
|
||||
pre_block_state=None,
|
||||
pre_block_assignee=None,
|
||||
blocker_resolver_type=BlockerResolverType.HUMAN,
|
||||
)
|
||||
db_session.add(task)
|
||||
await db_session.flush()
|
||||
# A real trip: strikes accrued past threshold with an unchanging
|
||||
# fingerprint, then force-blocked.
|
||||
for _ in range(6):
|
||||
markers.bump_oscillation_strikes(task, [0, 0, 0])
|
||||
markers.mark_oscillation_tripped(task)
|
||||
assert markers.is_oscillation_tripped(task) is True
|
||||
await db_session.flush()
|
||||
|
||||
svc = get_task_service(db_session)
|
||||
out = await svc.admin_set_status(
|
||||
tid, TaskStatus.IN_PROGRESS, actor_id=cast("UUID", agent.id), actor_role="ceo"
|
||||
)
|
||||
assert out is not None
|
||||
await db_session.flush()
|
||||
|
||||
row = (
|
||||
await db_session.execute(select(TaskTable).where(TaskTable.id == tid))
|
||||
).scalar_one()
|
||||
assert row.status == TaskStatus.IN_PROGRESS
|
||||
assert markers.is_oscillation_tripped(row) is False
|
||||
assert markers.get_oscillation_strikes(row) == 0, (
|
||||
"the marker must be gone entirely, not just its tripped flag, so the "
|
||||
"next cycle's first bump starts a fresh strike count"
|
||||
)
|
||||
# A later legitimate block/unblock cycle counts from fresh, not from the
|
||||
# old strike count.
|
||||
fresh_strikes = markers.bump_oscillation_strikes(row, [1, 0, 0])
|
||||
assert fresh_strikes == 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _extract_completion_learnings — dead-letter on record_learning failure
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user