mirror of
https://github.com/rennf93/roboco.git
synced 2026-08-03 07:23:24 +02:00
Renn F
a47237416e
Smoke run 3 showed agents reaped at the 3-min stale-claim window while they were actively retrying rejected verbs. Two causes: 1. The reaper threshold was hardcoded at 180s via claim_stale_seconds. LLM inference + retry loops routinely take longer than that between verb-successes. Added settings.stale_claim_reap_seconds (default 600s); override via ROBOCO_STALE_CLAIM_REAP_SECONDS env var. claim_stale_seconds (spawn-filter cutoff) is unchanged at 180s. 2. last_heartbeat_at only refreshed on verb SUCCESS. A verb stuck in a rejection loop (e.g. tracing_gap missing journal:decision) showed no heartbeat updates even though the agent was alive. Added a best-effort heartbeat refresh inside _emit_rejection so EVERY verb dispatch — success or rejection — counts as activity. Heartbeat approach: option (b) — touch inside _emit_rejection (single centralized rejection path). Requires no middleware layer, no HTTP body parsing, and no new files. The _touch guard for task_id=None means agent-level rejections (no task context) are a safe no-op. Net effect: agents stop being reaped mid-retry. Genuinely-stuck containers (no verb dispatch at all) still reap normally at 600s. Spec ref: Wave C Task C3.
93 lines
3.0 KiB
Python
93 lines
3.0 KiB
Python
"""Wave C3 (2026-05-12): heartbeat reaper threshold honors config setting.
|
|
|
|
The reaper previously used a hardcoded 180s (= claim_stale_seconds) which
|
|
was also the spawn-filter cutoff. Smoke run 3 showed agents being reaped
|
|
at ~3 minutes while actively retrying rejected verbs; LLM inference alone
|
|
can exceed that window.
|
|
|
|
Two fixes ship together:
|
|
- stale_claim_reap_seconds (default 600) drives the reaper; the
|
|
spawn-filter keeps claim_stale_seconds unchanged.
|
|
- _emit_rejection now touches last_heartbeat_at so every verb attempt
|
|
(success or rejection) counts as agent activity.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from datetime import UTC, datetime, timedelta
|
|
from unittest.mock import AsyncMock
|
|
from uuid import uuid4
|
|
|
|
import pytest
|
|
from roboco.config import settings
|
|
from roboco.runtime.orchestrator import AgentOrchestrator
|
|
|
|
_EXPECTED_DEFAULT_REAP_SECONDS = 600
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_reaper_default_threshold_is_600() -> None:
|
|
"""settings.stale_claim_reap_seconds defaults to 600, not the old 180."""
|
|
assert settings.stale_claim_reap_seconds == _EXPECTED_DEFAULT_REAP_SECONDS
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_orchestrator_init_uses_stale_claim_reap_seconds() -> None:
|
|
"""_claim_heartbeat_ttl is sourced from stale_claim_reap_seconds,
|
|
not claim_stale_seconds."""
|
|
# Build an orchestrator via __new__ to avoid touching the DB, then check
|
|
# the field that _reap_with_service reads.
|
|
orch = AgentOrchestrator.__new__(AgentOrchestrator)
|
|
# Manually run the subset of __init__ that sets _claim_heartbeat_ttl.
|
|
orch._claim_heartbeat_ttl = settings.stale_claim_reap_seconds
|
|
assert orch._claim_heartbeat_ttl == _EXPECTED_DEFAULT_REAP_SECONDS
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_reaper_does_not_reap_under_custom_threshold() -> None:
|
|
"""Task whose heartbeat is 800s old is NOT reaped when threshold=900."""
|
|
now = datetime.now(UTC)
|
|
task_safe = type(
|
|
"T",
|
|
(),
|
|
{
|
|
"id": uuid4(),
|
|
"last_heartbeat_at": now - timedelta(seconds=800),
|
|
},
|
|
)()
|
|
|
|
orch = AgentOrchestrator.__new__(AgentOrchestrator)
|
|
orch._claim_heartbeat_ttl = 900 # custom threshold
|
|
svc = AsyncMock()
|
|
svc.list_in_progress_or_claimed.return_value = [task_safe]
|
|
svc.unclaim_for_reaper = AsyncMock()
|
|
|
|
await orch._reap_with_service(svc)
|
|
|
|
svc.unclaim_for_reaper.assert_not_awaited()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_reaper_does_reap_over_custom_threshold() -> None:
|
|
"""Task whose heartbeat is 1000s old IS reaped when threshold=900."""
|
|
now = datetime.now(UTC)
|
|
stale_id = uuid4()
|
|
task_stale = type(
|
|
"T",
|
|
(),
|
|
{
|
|
"id": stale_id,
|
|
"last_heartbeat_at": now - timedelta(seconds=1000),
|
|
},
|
|
)()
|
|
|
|
orch = AgentOrchestrator.__new__(AgentOrchestrator)
|
|
orch._claim_heartbeat_ttl = 900
|
|
svc = AsyncMock()
|
|
svc.list_in_progress_or_claimed.return_value = [task_stale]
|
|
svc.unclaim_for_reaper = AsyncMock()
|
|
|
|
await orch._reap_with_service(svc)
|
|
|
|
svc.unclaim_for_reaper.assert_awaited_once_with(stale_id)
|