mirror of
https://github.com/rennf93/roboco.git
synced 2026-08-03 07:23:24 +02:00
fix(orchestrator): stop auditor alert-spawn rotation — ack as auditor on dispatch
The auditor respawned every ~3 min on the same stale rework alerts. Root cause: _dispatch_audit_work's alert path fetched the SYSTEM-wide "not fully acked" view (list_system_notifications), but the auditor is read-only (no ack verb) and auditor_triage never acks — so once an alert existed the CEO was the only party who could clear it, and the CEO hadn't acked. The per-alert cooldown (PR #499) only paced a rotation through the N un-acked alerts; it was a damper, not a fix. Fix: fetch the auditor's OWN pending-ack view (GET /notifications authed as the auditor -> list_for_agent, which filters acked_by for the auditor) and ack the alert as the auditor on dispatch. Each alert is now a one-shot, DB-persistent: the next tick cannot respawn on an alert the auditor already observed — even one the CEO hasn't acked. Authed as the auditor (not the system identity) so the route selects the per-recipient view; HTTP rather than DB-direct so it shares the orchestrator's loop in prod and stays loop-safe in the e2e harness (which runs _dispatch_audit_work in its own asyncio.run loop, away from the app's DB engine). e2e now asserts the alert is in acked_by for the auditor after dispatch — the rotation-stopper itself, not just the spawn.
This commit is contained in:
@@ -26,10 +26,13 @@ def orch() -> AgentOrchestrator:
|
||||
async def _run_dispatch(
|
||||
orch: AgentOrchestrator, *, tasks: list[dict] | None = None
|
||||
) -> MagicMock:
|
||||
"""Run _dispatch_audit_work with notifications empty and return the spawn mock."""
|
||||
"""Run _dispatch_audit_work with no unobserved alerts and return the spawn mock."""
|
||||
client = MagicMock()
|
||||
with (
|
||||
patch.object(orch, "_fetch_notifications", new=AsyncMock(return_value=[])),
|
||||
patch.object(
|
||||
orch, "_next_unobserved_audit_alert", new=AsyncMock(return_value=None)
|
||||
),
|
||||
patch.object(orch, "_ack_alert_as_auditor", new=AsyncMock()),
|
||||
patch.object(orch, "_fetch_tasks", new=AsyncMock(return_value=tasks or [])),
|
||||
patch.object(orch, "_is_agent_active", return_value=False),
|
||||
patch.object(orch, "spawn_agent", new=AsyncMock()) as spawn_mock,
|
||||
@@ -111,7 +114,10 @@ async def test_breaker_skips_when_auditor_active(orch: AgentOrchestrator) -> Non
|
||||
with (
|
||||
patch.object(settings, "audit_interval_seconds", 21600),
|
||||
patch("roboco.runtime.orchestrator.datetime", wraps=datetime) as dt_mock,
|
||||
patch.object(orch, "_fetch_notifications", new=AsyncMock(return_value=[])),
|
||||
patch.object(
|
||||
orch, "_next_unobserved_audit_alert", new=AsyncMock(return_value=None)
|
||||
),
|
||||
patch.object(orch, "_ack_alert_as_auditor", new=AsyncMock()),
|
||||
patch.object(orch, "_fetch_tasks", new=AsyncMock(return_value=[])),
|
||||
patch.object(orch, "_is_agent_active", return_value=True),
|
||||
patch.object(orch, "spawn_agent", new=AsyncMock()) as spawn_mock,
|
||||
@@ -129,8 +135,7 @@ async def test_reactive_alert_stamps_last_spawn_and_blocks_scheduled(
|
||||
"""A reactive alert spawn records _last_audit_spawn_at and returns early."""
|
||||
client = MagicMock()
|
||||
alert = {
|
||||
"id": "a1",
|
||||
"to_agents": ["auditor"],
|
||||
"id": "11111111-1111-1111-1111-111111111111",
|
||||
"subject": "Coverage gap",
|
||||
"body": "Test",
|
||||
}
|
||||
@@ -138,7 +143,10 @@ async def test_reactive_alert_stamps_last_spawn_and_blocks_scheduled(
|
||||
with (
|
||||
patch.object(settings, "audit_interval_seconds", 21600),
|
||||
patch("roboco.runtime.orchestrator.datetime", wraps=datetime) as dt_mock,
|
||||
patch.object(orch, "_fetch_notifications", new=AsyncMock(return_value=[alert])),
|
||||
patch.object(
|
||||
orch, "_next_unobserved_audit_alert", new=AsyncMock(return_value=alert)
|
||||
),
|
||||
patch.object(orch, "_ack_alert_as_auditor", new=AsyncMock()),
|
||||
patch.object(orch, "_fetch_tasks", new=AsyncMock(return_value=[])),
|
||||
patch.object(orch, "_is_agent_active", return_value=False),
|
||||
patch.object(orch, "spawn_agent", new=AsyncMock()) as spawn_mock,
|
||||
@@ -149,6 +157,43 @@ async def test_reactive_alert_stamps_last_spawn_and_blocks_scheduled(
|
||||
assert orch._last_audit_spawn_at == now
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_reactive_alert_acks_so_it_cannot_rotate(
|
||||
orch: AgentOrchestrator,
|
||||
) -> None:
|
||||
"""The alert is acked as the auditor on dispatch, so the next tick —
|
||||
which would otherwise re-fetch the same stale alert and respawn — finds
|
||||
no unobserved alert and falls through to the scheduled path (blocked by
|
||||
the just-stamped _last_audit_spawn_at). This is the rotation-stopper.
|
||||
"""
|
||||
client = MagicMock()
|
||||
alert_id = "22222222-2222-2222-2222-222222222222"
|
||||
alert = {
|
||||
"id": alert_id,
|
||||
"subject": "Rework alert",
|
||||
"body": "Task entered needs_revision",
|
||||
}
|
||||
now = datetime.now(UTC)
|
||||
ack_mock = AsyncMock()
|
||||
fetch_mock = AsyncMock(side_effect=[alert, None])
|
||||
with (
|
||||
patch.object(settings, "audit_interval_seconds", 21600),
|
||||
patch("roboco.runtime.orchestrator.datetime", wraps=datetime) as dt_mock,
|
||||
patch.object(orch, "_next_unobserved_audit_alert", new=fetch_mock),
|
||||
patch.object(orch, "_ack_alert_as_auditor", new=ack_mock),
|
||||
patch.object(orch, "_fetch_tasks", new=AsyncMock(return_value=[])),
|
||||
patch.object(orch, "_is_agent_active", return_value=False),
|
||||
patch.object(orch, "spawn_agent", new=AsyncMock()) as spawn_mock,
|
||||
):
|
||||
dt_mock.now.return_value = now
|
||||
await orch._dispatch_audit_work(client) # tick 1: alert -> spawn + ack
|
||||
await orch._dispatch_audit_work(client) # tick 2: no alert -> scheduled
|
||||
assert spawn_mock.await_count == 1 # not respawned on tick 2
|
||||
assert ack_mock.await_count == 1
|
||||
# acked as the auditor saw it: (client, alert_id)
|
||||
assert ack_mock.await_args.args[1] == alert_id
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_cooldown_zero_disables_scheduled_sweeps(orch: AgentOrchestrator) -> None:
|
||||
"""ROBOCO_AUDIT_INTERVAL_SECONDS=0 disables scheduled sweeps entirely."""
|
||||
|
||||
Reference in New Issue
Block a user