mirror of
https://github.com/rennf93/roboco.git
synced 2026-08-03 07:23:24 +02:00
[F033] orchestrator: capture container_id at startup re-adoption
_readopt_running_agents registered re-adopted ACTIVE instances with
container_id=None. _check_health skips container_id-is-None instances, so
when a re-adopted container later exited the stopped-container handler
never ran and the task stranded under a phantom ACTIVE instance forever.
Add _resolve_container_id (docker inspect -f '{{.Id}}') and store the real
id on re-adopt. Best-effort: a probe failure degrades to None (still
ACTIVE; the reaper's Docker-liveness fallback covers it).
This commit is contained in:
@@ -5481,6 +5481,30 @@ Start by:
|
||||
exit_code = None
|
||||
return is_running, exit_code
|
||||
|
||||
@staticmethod
|
||||
async def _resolve_container_id(container_name: str) -> str | None:
|
||||
"""Return the Docker container id for ``container_name`` via `docker inspect`.
|
||||
|
||||
Used at startup re-adoption (F033) so a re-adopted ACTIVE instance
|
||||
carries the real container id — ``_check_health`` skips
|
||||
``container_id is None`` instances, so without it a later container exit
|
||||
is invisible to the health loop and the task strands. Returns ``None``
|
||||
when the id can't be resolved (caller treats that as best-effort
|
||||
degraded re-adoption, still covered by the reaper's liveness fallback).
|
||||
"""
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
"docker",
|
||||
"inspect",
|
||||
"-f",
|
||||
"{{.Id}}",
|
||||
container_name,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.DEVNULL,
|
||||
)
|
||||
stdout, _ = await proc.communicate()
|
||||
cid = stdout.decode().strip()
|
||||
return cid or None
|
||||
|
||||
@staticmethod
|
||||
async def _probe_gateway_health(slug: str) -> bool | None:
|
||||
"""Probe an agent container's gateway out-of-band: healthy / broken / unknown.
|
||||
@@ -7710,8 +7734,24 @@ Start now: evidence(task_id="{task_id}")
|
||||
continue
|
||||
if not is_running:
|
||||
continue
|
||||
# F033: capture the real container id. _check_health skips
|
||||
# ``container_id is None`` instances, so a re-adopted instance
|
||||
# without the id would be invisible to the health loop — when the
|
||||
# container later exits the stopped-container handler never runs
|
||||
# and the task strands under a phantom ACTIVE instance. Best-effort:
|
||||
# a probe failure degrades to the prior None (still re-adopted as
|
||||
# ACTIVE; the reaper's Docker-liveness fallback covers it).
|
||||
container_id: str | None = None
|
||||
try:
|
||||
container_id = await self._resolve_container_id(
|
||||
f"roboco-agent-{slug}"
|
||||
)
|
||||
except Exception:
|
||||
container_id = None
|
||||
self._instances[slug] = AgentInstance(
|
||||
agent_id=slug, state=AgentState.ACTIVE
|
||||
agent_id=slug,
|
||||
state=AgentState.ACTIVE,
|
||||
container_id=container_id,
|
||||
)
|
||||
readopted += 1
|
||||
if readopted:
|
||||
|
||||
@@ -77,3 +77,21 @@ async def test_readopt_swallows_probe_errors() -> None:
|
||||
n = await orch._readopt_running_agents()
|
||||
|
||||
assert n == 0 # best-effort: a probe failure never raises into startup
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_readopt_records_container_id_so_health_check_can_see_exit() -> None:
|
||||
# F033: a re-adopted instance registered with container_id=None is skipped by
|
||||
# _check_health (`if instance.container_id is None: continue`), so when the
|
||||
# container later exits the stopped-container handler never runs — the task
|
||||
# is stranded under a phantom ACTIVE instance forever. Re-adopt must capture
|
||||
# the real container id so the health loop can observe the later exit.
|
||||
orch = _orch()
|
||||
orch._inspect_container_state = AsyncMock(return_value=(True, 0)) # type: ignore[method-assign]
|
||||
orch._resolve_container_id = AsyncMock(return_value="deadbeef1234") # type: ignore[method-assign]
|
||||
|
||||
await orch._readopt_running_agents()
|
||||
|
||||
inst = orch._instances[next(iter(orch._instances))]
|
||||
assert inst.container_id == "deadbeef1234"
|
||||
assert inst.state == AgentState.ACTIVE
|
||||
|
||||
Reference in New Issue
Block a user