[F035] orchestrator: probe-resume loop actually revives parked agents

_park_provider_unavailable parked the provider + offlined the instance but
never registered a WaitingRecord, so _on_probe_success -> _parked_agents_for
(always filtered on waiting_for=='rate_limit_lifted') returned [] and
resolve_wait revived nobody — recovery fell to the 600s stale-claim reaper
instead of the probe-success path the parking design relied on.

Register + persist a rate_limit_lifted WaitingRecord at park time (mirrors
mark_waiting_long, minus stop_agent — the container is already dead).

Companion reaper guard: _reap_with_service now skips provider-parked
assignees (_assignee_is_provider_parked) so the claim survives until the
probe revives the agent — otherwise the reaper releases the claim to pending
and probe-success respawns on a task the agent no longer owns.
This commit is contained in:
Renn F
2026-06-28 11:03:50 +02:00
parent e3011ec145
commit dd28a1f997
3 changed files with 165 additions and 2 deletions
+50
View File
@@ -6481,6 +6481,29 @@ Start by:
kind=kind, kind=kind,
error=str(exc), error=str(exc),
) )
# F035: register a WaitingRecord so the probe-resume loop can revive
# this agent when the provider recovers. ``_on_probe_success`` reads
# ``_waiting_records`` filtered by ``waiting_for == "rate_limit_lifted"``
# + ``context.provider``; without a record here it resumes nobody and
# recovery falls to the 600s stale-claim reaper instead of the
# probe-success path the parking design relies on. Persisted (mirrors
# ``mark_waiting_long``) so a restart still resolves the wait. We do NOT
# call ``mark_waiting_long`` itself — the container is already dead (it
# exited), so there is nothing to stop, and parking keeps OFFLINE (not
# WAITING_LONG) so the reaper's live-skip / health loop ignore it.
task_id = (
str(instance.current_task_id) if instance.current_task_id else None
)
record = WaitingRecord(
agent_id=agent_id,
task_id=task_id,
waiting_for="rate_limit_lifted",
waiting_since=datetime.now(UTC),
context={"provider": provider, "kind": kind},
)
self._waiting_records[agent_id] = record
with contextlib.suppress(Exception):
await self._persist_waiting_record(record)
logger.warning( logger.warning(
"Provider unavailable; parked (task retried when it recovers)", "Provider unavailable; parked (task retried when it recovers)",
provider=provider, provider=provider,
@@ -7716,6 +7739,26 @@ Start now: evidence(task_id="{task_id}")
instance = instances.get(self._resolve_agent_slug(str(owner))) instance = instances.get(self._resolve_agent_slug(str(owner)))
return instance is not None and instance.state == AgentState.ACTIVE return instance is not None and instance.state == AgentState.ACTIVE
def _assignee_is_provider_parked(self, task: Any) -> bool:
"""True if the task's assignee is parked waiting for a provider to recover.
A provider-parked agent (session-limit / overload / grok-429) is OFFLINE
with a dead container and a ``rate_limit_lifted`` WaitingRecord; the
probe-resume loop owns its recovery. The stale-claim reaper must skip it
so the claim survives until the probe revives the agent reaping would
release the claim to pending and probe-success would then respawn the
agent on a task it no longer owns. Defensive on a missing registry.
"""
owner = getattr(task, "assigned_to", None) or getattr(task, "claimed_by", None)
if not owner:
return False
records = getattr(self, "_waiting_records", None)
if not records:
return False
slug = self._resolve_agent_slug(str(owner))
record = records.get(slug)
return record is not None and record.waiting_for == "rate_limit_lifted"
async def _readopt_running_agents(self) -> int: async def _readopt_running_agents(self) -> int:
"""Re-adopt still-running agent containers into ``_instances`` at startup. """Re-adopt still-running agent containers into ``_instances`` at startup.
@@ -7960,6 +8003,13 @@ Start now: evidence(task_id="{task_id}")
and not await self._maybe_recover_broken_gateway(t) and not await self._maybe_recover_broken_gateway(t)
): ):
continue continue
# F035: a provider-parked agent (session-limit / overload /
# grok-429) is OFFLINE with a dead container and a
# ``rate_limit_lifted`` WaitingRecord. The probe-resume loop
# owns its recovery — do NOT reap the claim, or probe-success
# would later respawn the agent on a task it no longer owns.
if self._assignee_is_provider_parked(t):
continue
task_id = require_uuid(t.id) task_id = require_uuid(t.id)
try: try:
await svc.unclaim_for_reaper(task_id) await svc.unclaim_for_reaper(task_id)
@@ -9,11 +9,12 @@ decision points deterministically (logs + tracker + finalize stubbed).
from __future__ import annotations from __future__ import annotations
from datetime import UTC, datetime
from unittest.mock import AsyncMock from unittest.mock import AsyncMock
import pytest import pytest
from roboco.config import settings from roboco.config import settings
from roboco.models.runtime import AgentInstance from roboco.models.runtime import AgentInstance, WaitingRecord
from roboco.runtime.orchestrator import ( from roboco.runtime.orchestrator import (
_OVERLOAD_RETRY_AFTER_S, _OVERLOAD_RETRY_AFTER_S,
_RATE_LIMIT_RETRY_AFTER_S, _RATE_LIMIT_RETRY_AFTER_S,
@@ -52,6 +53,9 @@ def orch(monkeypatch: pytest.MonkeyPatch) -> AgentOrchestrator:
# transcript fallback empty by default so dev environments with stray # transcript fallback empty by default so dev environments with stray
# transcripts do not make the tests flaky. # transcripts do not make the tests flaky.
monkeypatch.setattr(orch, "_transcript_tail_text", lambda _a, _lines=80: "") monkeypatch.setattr(orch, "_transcript_tail_text", lambda _a, _lines=80: "")
orch._waiting_records = {}
orch._rate_limit_ceo_notified = set()
orch._instances = {}
return orch return orch
@@ -153,6 +157,64 @@ async def test_park_offlines_and_activates_with_kind(
} }
@pytest.mark.asyncio
async def test_park_registers_waiting_record_so_probe_can_resume(
orch: AgentOrchestrator, monkeypatch: pytest.MonkeyPatch
) -> None:
# F035: the probe-resume loop reads _waiting_records filtered by
# waiting_for == "rate_limit_lifted" + context.provider. Without a record
# here, _parked_agents_for(provider) returns [] and _on_probe_success
# resumes nobody — recovery falls to the 600s stale-claim reaper instead of
# the probe-success path the parking design relies on.
orch._waiting_records = {}
inst = _instance()
inst.current_task_id = "task-1"
tracker = _FakeTracker()
monkeypatch.setattr(orch, "_make_tracker", lambda _p: tracker)
monkeypatch.setattr(orch, "_finalize_spawn_session", AsyncMock())
monkeypatch.setattr(orch, "_persist_waiting_record", AsyncMock())
await orch._park_provider_unavailable(
"be-dev-1", inst, provider="anthropic", retry_after=45.0, kind="rate_limited"
)
assert "be-dev-1" in orch._waiting_records
rec = orch._waiting_records["be-dev-1"]
assert rec.waiting_for == "rate_limit_lifted"
assert rec.context.get("provider") == "anthropic"
assert rec.task_id == "task-1"
assert orch._parked_agents_for("anthropic") == ["be-dev-1"]
@pytest.mark.asyncio
async def test_probe_success_respawns_parked_agent(
orch: AgentOrchestrator, monkeypatch: pytest.MonkeyPatch
) -> None:
# F035: once the probe succeeds, the parked agent must be respawned via
# resolve_wait — not left stranded for the 600s reaper.
orch._waiting_records = {
"be-dev-1": WaitingRecord(
agent_id="be-dev-1",
task_id="task-1",
waiting_for="rate_limit_lifted",
waiting_since=datetime.now(UTC),
context={"provider": "anthropic"},
)
}
tracker = _FakeTracker()
tracker.clear = AsyncMock() # type: ignore[method-assign]
monkeypatch.setattr(orch, "_make_tracker", lambda _p: tracker)
monkeypatch.setattr(orch, "_delete_waiting_record", AsyncMock())
monkeypatch.setattr(orch, "_generate_resume_prompt", lambda _r, _res: "resume")
spawn = AsyncMock()
monkeypatch.setattr(orch, "spawn_agent", spawn)
await orch._on_probe_success("anthropic", tracker)
spawn.assert_awaited_once() # parked agent respawned, not stranded
assert "be-dev-1" not in orch._waiting_records
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# _handle_stopped_container — overload short-circuits the crash-retry path # _handle_stopped_container — overload short-circuits the crash-retry path
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
+52 -1
View File
@@ -18,7 +18,7 @@ from unittest.mock import AsyncMock
from uuid import uuid4 from uuid import uuid4
import pytest import pytest
from roboco.models.runtime import AgentInstance from roboco.models.runtime import AgentInstance, WaitingRecord
from roboco.runtime.orchestrator import AgentOrchestrator, AgentState from roboco.runtime.orchestrator import AgentOrchestrator, AgentState
from roboco.seeds.initial_data import AGENT_UUIDS from roboco.seeds.initial_data import AGENT_UUIDS
@@ -376,6 +376,57 @@ async def test_reap_releases_on_registry_miss_when_container_gone(
svc.unclaim_for_reaper.assert_awaited_once_with(task_id) svc.unclaim_for_reaper.assert_awaited_once_with(task_id)
@pytest.mark.asyncio
async def test_reap_spares_provider_parked_agent_for_probe_resume(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""F035: a provider-parked agent (dead container, OFFLINE, with a
``rate_limit_lifted`` WaitingRecord) must NOT be reaped by the stale-claim
reaper. The probe-resume loop owns its recovery and respawns it when the
provider recovers; reaping would release the claim to pending, and then
probe-success would respawn the agent on a task it no longer owns.
"""
now = datetime.now(UTC)
task_id = uuid4()
task = type(
"T",
(),
{
"id": task_id,
"last_heartbeat_at": now - timedelta(seconds=1200),
"assigned_to": AGENT_UUIDS["be-dev-1"],
"claimed_by": None,
},
)()
orch = AgentOrchestrator.__new__(AgentOrchestrator)
monkeypatch.setattr(
orch, "_maybe_recover_broken_gateway", AsyncMock(return_value=False)
)
orch._claim_heartbeat_ttl = 300
orch._grok_idle_kill_ttl = 900
orch._instances = {} # parked agent is OFFLINE / not in the registry
orch._waiting_records = {
"be-dev-1": WaitingRecord(
agent_id="be-dev-1",
task_id=str(task_id),
waiting_for="rate_limit_lifted",
waiting_since=now,
context={"provider": "anthropic"},
)
}
monkeypatch.setattr(
orch, "_inspect_container_state", AsyncMock(return_value=(False, 0))
)
svc = AsyncMock()
svc.list_in_progress_or_claimed.return_value = [task]
svc.unclaim_for_reaper = AsyncMock()
await orch._reap_with_service(svc)
svc.unclaim_for_reaper.assert_not_awaited() # spared for the probe loop
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_registry_uninitialised_skips_docker_fallback( async def test_registry_uninitialised_skips_docker_fallback(
monkeypatch: pytest.MonkeyPatch, monkeypatch: pytest.MonkeyPatch,