mirror of
https://github.com/rennf93/roboco.git
synced 2026-08-03 07:23:24 +02:00
[F033] orchestrator: capture container_id at startup re-adoption
_readopt_running_agents registered re-adopted ACTIVE instances with
container_id=None. _check_health skips container_id-is-None instances, so
when a re-adopted container later exited the stopped-container handler
never ran and the task stranded under a phantom ACTIVE instance forever.
Add _resolve_container_id (docker inspect -f '{{.Id}}') and store the real
id on re-adopt. Best-effort: a probe failure degrades to None (still
ACTIVE; the reaper's Docker-liveness fallback covers it).
This commit is contained in:
@@ -5481,6 +5481,30 @@ Start by:
|
|||||||
exit_code = None
|
exit_code = None
|
||||||
return is_running, exit_code
|
return is_running, exit_code
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
async def _resolve_container_id(container_name: str) -> str | None:
|
||||||
|
"""Return the Docker container id for ``container_name`` via `docker inspect`.
|
||||||
|
|
||||||
|
Used at startup re-adoption (F033) so a re-adopted ACTIVE instance
|
||||||
|
carries the real container id — ``_check_health`` skips
|
||||||
|
``container_id is None`` instances, so without it a later container exit
|
||||||
|
is invisible to the health loop and the task strands. Returns ``None``
|
||||||
|
when the id can't be resolved (caller treats that as best-effort
|
||||||
|
degraded re-adoption, still covered by the reaper's liveness fallback).
|
||||||
|
"""
|
||||||
|
proc = await asyncio.create_subprocess_exec(
|
||||||
|
"docker",
|
||||||
|
"inspect",
|
||||||
|
"-f",
|
||||||
|
"{{.Id}}",
|
||||||
|
container_name,
|
||||||
|
stdout=asyncio.subprocess.PIPE,
|
||||||
|
stderr=asyncio.subprocess.DEVNULL,
|
||||||
|
)
|
||||||
|
stdout, _ = await proc.communicate()
|
||||||
|
cid = stdout.decode().strip()
|
||||||
|
return cid or None
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
async def _probe_gateway_health(slug: str) -> bool | None:
|
async def _probe_gateway_health(slug: str) -> bool | None:
|
||||||
"""Probe an agent container's gateway out-of-band: healthy / broken / unknown.
|
"""Probe an agent container's gateway out-of-band: healthy / broken / unknown.
|
||||||
@@ -7710,8 +7734,24 @@ Start now: evidence(task_id="{task_id}")
|
|||||||
continue
|
continue
|
||||||
if not is_running:
|
if not is_running:
|
||||||
continue
|
continue
|
||||||
|
# F033: capture the real container id. _check_health skips
|
||||||
|
# ``container_id is None`` instances, so a re-adopted instance
|
||||||
|
# without the id would be invisible to the health loop — when the
|
||||||
|
# container later exits the stopped-container handler never runs
|
||||||
|
# and the task strands under a phantom ACTIVE instance. Best-effort:
|
||||||
|
# a probe failure degrades to the prior None (still re-adopted as
|
||||||
|
# ACTIVE; the reaper's Docker-liveness fallback covers it).
|
||||||
|
container_id: str | None = None
|
||||||
|
try:
|
||||||
|
container_id = await self._resolve_container_id(
|
||||||
|
f"roboco-agent-{slug}"
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
container_id = None
|
||||||
self._instances[slug] = AgentInstance(
|
self._instances[slug] = AgentInstance(
|
||||||
agent_id=slug, state=AgentState.ACTIVE
|
agent_id=slug,
|
||||||
|
state=AgentState.ACTIVE,
|
||||||
|
container_id=container_id,
|
||||||
)
|
)
|
||||||
readopted += 1
|
readopted += 1
|
||||||
if readopted:
|
if readopted:
|
||||||
|
|||||||
@@ -77,3 +77,21 @@ async def test_readopt_swallows_probe_errors() -> None:
|
|||||||
n = await orch._readopt_running_agents()
|
n = await orch._readopt_running_agents()
|
||||||
|
|
||||||
assert n == 0 # best-effort: a probe failure never raises into startup
|
assert n == 0 # best-effort: a probe failure never raises into startup
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_readopt_records_container_id_so_health_check_can_see_exit() -> None:
|
||||||
|
# F033: a re-adopted instance registered with container_id=None is skipped by
|
||||||
|
# _check_health (`if instance.container_id is None: continue`), so when the
|
||||||
|
# container later exits the stopped-container handler never runs — the task
|
||||||
|
# is stranded under a phantom ACTIVE instance forever. Re-adopt must capture
|
||||||
|
# the real container id so the health loop can observe the later exit.
|
||||||
|
orch = _orch()
|
||||||
|
orch._inspect_container_state = AsyncMock(return_value=(True, 0)) # type: ignore[method-assign]
|
||||||
|
orch._resolve_container_id = AsyncMock(return_value="deadbeef1234") # type: ignore[method-assign]
|
||||||
|
|
||||||
|
await orch._readopt_running_agents()
|
||||||
|
|
||||||
|
inst = orch._instances[next(iter(orch._instances))]
|
||||||
|
assert inst.container_id == "deadbeef1234"
|
||||||
|
assert inst.state == AgentState.ACTIVE
|
||||||
|
|||||||
Reference in New Issue
Block a user