fix(orchestrator): stop auditor alert-spawn rotation — ack as auditor on dispatch

The auditor respawned every ~3 min on the same stale rework alerts.

Root cause: _dispatch_audit_work's alert path fetched the SYSTEM-wide
"not fully acked" view (list_system_notifications), but the auditor is
read-only (no ack verb) and auditor_triage never acks — so once an alert
existed the CEO was the only party who could clear it, and the CEO hadn't
acked. The per-alert cooldown (PR #499) only paced a rotation through the
N un-acked alerts; it was a damper, not a fix.

Fix: fetch the auditor's OWN pending-ack view (GET /notifications authed
as the auditor -> list_for_agent, which filters acked_by for the auditor)
and ack the alert as the auditor on dispatch. Each alert is now a
one-shot, DB-persistent: the next tick cannot respawn on an alert the
auditor already observed — even one the CEO hasn't acked. Authed as the
auditor (not the system identity) so the route selects the per-recipient
view; HTTP rather than DB-direct so it shares the orchestrator's loop in
prod and stays loop-safe in the e2e harness (which runs _dispatch_audit_work
in its own asyncio.run loop, away from the app's DB engine).

e2e now asserts the alert is in acked_by for the auditor after dispatch —
the rotation-stopper itself, not just the spawn.
This commit is contained in:
Renn F
2026-07-14 06:14:27 +02:00
committed by Renzo F
parent f03859c64c
commit 6fe0067f73
3 changed files with 166 additions and 22 deletions
+27
View File
@@ -228,3 +228,30 @@ def test_reactive_alert_producer_spawns_auditor(
prompt = call.kwargs["initial_prompt"]
assert "QUALITY ALERT" in prompt
assert "missing edge-case coverage" in prompt
# Rotation-stopper: dispatch acks the alert as the auditor (the auditor is
# read-only, no ack verb), so the auditor's pending-ack view no longer
# returns it and the next tick cannot respawn on this same alert. Without
# this ack the per-alert cooldown only paced a rotation through every
# stale not-fully-acked alert — the loop being fixed.
async def _acked_by_auditor(session: AsyncSession) -> bool:
from roboco.db.tables import NotificationTable
from sqlalchemy import select
row = (
(
await session.execute(
select(NotificationTable).where(
NotificationTable.related_task_id == task_id,
NotificationTable.type == NotificationType.ALERT,
)
)
)
.scalars()
.one()
)
return auditor_id in row.acked_by
assert stack.run_db(_acked_by_auditor), (
"alert not acked as auditor after dispatch — rotation not stopped"
)
@@ -26,10 +26,13 @@ def orch() -> AgentOrchestrator:
async def _run_dispatch(
orch: AgentOrchestrator, *, tasks: list[dict] | None = None
) -> MagicMock:
"""Run _dispatch_audit_work with notifications empty and return the spawn mock."""
"""Run _dispatch_audit_work with no unobserved alerts and return the spawn mock."""
client = MagicMock()
with (
patch.object(orch, "_fetch_notifications", new=AsyncMock(return_value=[])),
patch.object(
orch, "_next_unobserved_audit_alert", new=AsyncMock(return_value=None)
),
patch.object(orch, "_ack_alert_as_auditor", new=AsyncMock()),
patch.object(orch, "_fetch_tasks", new=AsyncMock(return_value=tasks or [])),
patch.object(orch, "_is_agent_active", return_value=False),
patch.object(orch, "spawn_agent", new=AsyncMock()) as spawn_mock,
@@ -111,7 +114,10 @@ async def test_breaker_skips_when_auditor_active(orch: AgentOrchestrator) -> Non
with (
patch.object(settings, "audit_interval_seconds", 21600),
patch("roboco.runtime.orchestrator.datetime", wraps=datetime) as dt_mock,
patch.object(orch, "_fetch_notifications", new=AsyncMock(return_value=[])),
patch.object(
orch, "_next_unobserved_audit_alert", new=AsyncMock(return_value=None)
),
patch.object(orch, "_ack_alert_as_auditor", new=AsyncMock()),
patch.object(orch, "_fetch_tasks", new=AsyncMock(return_value=[])),
patch.object(orch, "_is_agent_active", return_value=True),
patch.object(orch, "spawn_agent", new=AsyncMock()) as spawn_mock,
@@ -129,8 +135,7 @@ async def test_reactive_alert_stamps_last_spawn_and_blocks_scheduled(
"""A reactive alert spawn records _last_audit_spawn_at and returns early."""
client = MagicMock()
alert = {
"id": "a1",
"to_agents": ["auditor"],
"id": "11111111-1111-1111-1111-111111111111",
"subject": "Coverage gap",
"body": "Test",
}
@@ -138,7 +143,10 @@ async def test_reactive_alert_stamps_last_spawn_and_blocks_scheduled(
with (
patch.object(settings, "audit_interval_seconds", 21600),
patch("roboco.runtime.orchestrator.datetime", wraps=datetime) as dt_mock,
patch.object(orch, "_fetch_notifications", new=AsyncMock(return_value=[alert])),
patch.object(
orch, "_next_unobserved_audit_alert", new=AsyncMock(return_value=alert)
),
patch.object(orch, "_ack_alert_as_auditor", new=AsyncMock()),
patch.object(orch, "_fetch_tasks", new=AsyncMock(return_value=[])),
patch.object(orch, "_is_agent_active", return_value=False),
patch.object(orch, "spawn_agent", new=AsyncMock()) as spawn_mock,
@@ -149,6 +157,43 @@ async def test_reactive_alert_stamps_last_spawn_and_blocks_scheduled(
assert orch._last_audit_spawn_at == now
@pytest.mark.anyio
async def test_reactive_alert_acks_so_it_cannot_rotate(
orch: AgentOrchestrator,
) -> None:
"""The alert is acked as the auditor on dispatch, so the next tick —
which would otherwise re-fetch the same stale alert and respawn — finds
no unobserved alert and falls through to the scheduled path (blocked by
the just-stamped _last_audit_spawn_at). This is the rotation-stopper.
"""
client = MagicMock()
alert_id = "22222222-2222-2222-2222-222222222222"
alert = {
"id": alert_id,
"subject": "Rework alert",
"body": "Task entered needs_revision",
}
now = datetime.now(UTC)
ack_mock = AsyncMock()
fetch_mock = AsyncMock(side_effect=[alert, None])
with (
patch.object(settings, "audit_interval_seconds", 21600),
patch("roboco.runtime.orchestrator.datetime", wraps=datetime) as dt_mock,
patch.object(orch, "_next_unobserved_audit_alert", new=fetch_mock),
patch.object(orch, "_ack_alert_as_auditor", new=ack_mock),
patch.object(orch, "_fetch_tasks", new=AsyncMock(return_value=[])),
patch.object(orch, "_is_agent_active", return_value=False),
patch.object(orch, "spawn_agent", new=AsyncMock()) as spawn_mock,
):
dt_mock.now.return_value = now
await orch._dispatch_audit_work(client) # tick 1: alert -> spawn + ack
await orch._dispatch_audit_work(client) # tick 2: no alert -> scheduled
assert spawn_mock.await_count == 1 # not respawned on tick 2
assert ack_mock.await_count == 1
# acked as the auditor saw it: (client, alert_id)
assert ack_mock.await_args.args[1] == alert_id
@pytest.mark.anyio
async def test_cooldown_zero_disables_scheduled_sweeps(orch: AgentOrchestrator) -> None:
"""ROBOCO_AUDIT_INTERVAL_SECONDS=0 disables scheduled sweeps entirely."""