[F035] orchestrator: probe-resume loop actually revives parked agents

_park_provider_unavailable parked the provider + offlined the instance but
never registered a WaitingRecord, so _on_probe_success -> _parked_agents_for
(always filtered on waiting_for=='rate_limit_lifted') returned [] and
resolve_wait revived nobody — recovery fell to the 600s stale-claim reaper
instead of the probe-success path the parking design relied on.

Register + persist a rate_limit_lifted WaitingRecord at park time (mirrors
mark_waiting_long, minus stop_agent — the container is already dead).

Companion reaper guard: _reap_with_service now skips provider-parked
assignees (_assignee_is_provider_parked) so the claim survives until the
probe revives the agent — otherwise the reaper releases the claim to pending
and probe-success respawns on a task the agent no longer owns.
This commit is contained in:
Renn F
2026-06-28 11:03:50 +02:00
parent e3011ec145
commit dd28a1f997
3 changed files with 165 additions and 2 deletions
+52 -1
View File
@@ -18,7 +18,7 @@ from unittest.mock import AsyncMock
from uuid import uuid4
import pytest
from roboco.models.runtime import AgentInstance
from roboco.models.runtime import AgentInstance, WaitingRecord
from roboco.runtime.orchestrator import AgentOrchestrator, AgentState
from roboco.seeds.initial_data import AGENT_UUIDS
@@ -376,6 +376,57 @@ async def test_reap_releases_on_registry_miss_when_container_gone(
svc.unclaim_for_reaper.assert_awaited_once_with(task_id)
@pytest.mark.asyncio
async def test_reap_spares_provider_parked_agent_for_probe_resume(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""F035: a provider-parked agent (dead container, OFFLINE, with a
``rate_limit_lifted`` WaitingRecord) must NOT be reaped by the stale-claim
reaper. The probe-resume loop owns its recovery and respawns it when the
provider recovers; reaping would release the claim to pending, and then
probe-success would respawn the agent on a task it no longer owns.
"""
now = datetime.now(UTC)
task_id = uuid4()
task = type(
"T",
(),
{
"id": task_id,
"last_heartbeat_at": now - timedelta(seconds=1200),
"assigned_to": AGENT_UUIDS["be-dev-1"],
"claimed_by": None,
},
)()
orch = AgentOrchestrator.__new__(AgentOrchestrator)
monkeypatch.setattr(
orch, "_maybe_recover_broken_gateway", AsyncMock(return_value=False)
)
orch._claim_heartbeat_ttl = 300
orch._grok_idle_kill_ttl = 900
orch._instances = {} # parked agent is OFFLINE / not in the registry
orch._waiting_records = {
"be-dev-1": WaitingRecord(
agent_id="be-dev-1",
task_id=str(task_id),
waiting_for="rate_limit_lifted",
waiting_since=now,
context={"provider": "anthropic"},
)
}
monkeypatch.setattr(
orch, "_inspect_container_state", AsyncMock(return_value=(False, 0))
)
svc = AsyncMock()
svc.list_in_progress_or_claimed.return_value = [task]
svc.unclaim_for_reaper = AsyncMock()
await orch._reap_with_service(svc)
svc.unclaim_for_reaper.assert_not_awaited() # spared for the probe loop
@pytest.mark.asyncio
async def test_registry_uninitialised_skips_docker_fallback(
monkeypatch: pytest.MonkeyPatch,