[F033] orchestrator: capture container_id at startup re-adoption

_readopt_running_agents registered re-adopted ACTIVE instances with
container_id=None. _check_health skips container_id-is-None instances, so
when a re-adopted container later exited the stopped-container handler
never ran and the task stranded under a phantom ACTIVE instance forever.

Add _resolve_container_id (docker inspect -f '{{.Id}}') and store the real
id on re-adopt. Best-effort: a probe failure degrades to None (still
ACTIVE; the reaper's Docker-liveness fallback covers it).
This commit is contained in:
Renn F
2026-06-28 10:58:30 +02:00
parent 2c3a51df67
commit fa8e567edd
2 changed files with 59 additions and 1 deletions
+41 -1
View File
@@ -5481,6 +5481,30 @@ Start by:
exit_code = None
return is_running, exit_code
@staticmethod
async def _resolve_container_id(container_name: str) -> str | None:
"""Return the Docker container id for ``container_name`` via `docker inspect`.
Used at startup re-adoption (F033) so a re-adopted ACTIVE instance
carries the real container id ``_check_health`` skips
``container_id is None`` instances, so without it a later container exit
is invisible to the health loop and the task strands. Returns ``None``
when the id can't be resolved (caller treats that as best-effort
degraded re-adoption, still covered by the reaper's liveness fallback).
"""
proc = await asyncio.create_subprocess_exec(
"docker",
"inspect",
"-f",
"{{.Id}}",
container_name,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.DEVNULL,
)
stdout, _ = await proc.communicate()
cid = stdout.decode().strip()
return cid or None
@staticmethod
async def _probe_gateway_health(slug: str) -> bool | None:
"""Probe an agent container's gateway out-of-band: healthy / broken / unknown.
@@ -7710,8 +7734,24 @@ Start now: evidence(task_id="{task_id}")
continue
if not is_running:
continue
# F033: capture the real container id. _check_health skips
# ``container_id is None`` instances, so a re-adopted instance
# without the id would be invisible to the health loop — when the
# container later exits the stopped-container handler never runs
# and the task strands under a phantom ACTIVE instance. Best-effort:
# a probe failure degrades to the prior None (still re-adopted as
# ACTIVE; the reaper's Docker-liveness fallback covers it).
container_id: str | None = None
try:
container_id = await self._resolve_container_id(
f"roboco-agent-{slug}"
)
except Exception:
container_id = None
self._instances[slug] = AgentInstance(
agent_id=slug, state=AgentState.ACTIVE
agent_id=slug,
state=AgentState.ACTIVE,
container_id=container_id,
)
readopted += 1
if readopted:
@@ -77,3 +77,21 @@ async def test_readopt_swallows_probe_errors() -> None:
n = await orch._readopt_running_agents()
assert n == 0 # best-effort: a probe failure never raises into startup
@pytest.mark.asyncio
async def test_readopt_records_container_id_so_health_check_can_see_exit() -> None:
# F033: a re-adopted instance registered with container_id=None is skipped by
# _check_health (`if instance.container_id is None: continue`), so when the
# container later exits the stopped-container handler never runs — the task
# is stranded under a phantom ACTIVE instance forever. Re-adopt must capture
# the real container id so the health loop can observe the later exit.
orch = _orch()
orch._inspect_container_state = AsyncMock(return_value=(True, 0)) # type: ignore[method-assign]
orch._resolve_container_id = AsyncMock(return_value="deadbeef1234") # type: ignore[method-assign]
await orch._readopt_running_agents()
inst = orch._instances[next(iter(orch._instances))]
assert inst.container_id == "deadbeef1234"
assert inst.state == AgentState.ACTIVE