External/roboco
1
0
Fork 0
mirror of https://github.com/rennf93/roboco.git synced 2026-08-03 07:23:24 +02:00
Code Issues Packages Projects Releases Wiki Activity

[F093] serialize concurrent live-chat spawns under a per-agent lock

Browse Source 
The intake and secretary agent ids are each a single fixed id, so two
concurrent start_intake_session / start_secretary_session calls raced on
the container name (docker run --name roboco-agent-<id>) and the
_instances[<id>] write: both passed the reap-prior check before either
registered, both ran docker run, and the last _instances write won,
orphaning the other container + its relay.

Add _intake_spawn_lock / _secretary_spawn_lock (asyncio.Lock) and wrap the
_spawn_intake_container / _spawn_secretary_container bodies so the second
start waits for the first to fully register before its own reap-prior check
runs. Distinct from self._lock (which stop_agent takes) to avoid a
reentrancy deadlock: the spawn body holds the spawn lock then calls
stop_agent (acquires self._lock) — lock order is always spawn_lock ->
self._lock, never the reverse.
This commit is contained in:
Renn F
2026-06-28 20:19:18 +02:00
parent ba5ac385ed
commit 3c6230d314
3 changed files with 342 additions and 187 deletions
Download Patch File Download Diff File Expand all files Collapse all files
roboco/runtime/orchestrator.py
+219 -187
View File
@@ -826,6 +826,19 @@ class AgentOrchestrator:
# spawn two umbrellas for the same PR (the check is read-then-write
# with no DB-level uniqueness).
self._supersede_lock = asyncio.Lock()
# Serialize concurrent live-chat starts for the single-id interactive
# agents (intake / secretary). Each has a fixed agent id, so two
# concurrent starts race on the container name (``docker run --name
# roboco-agent-<id>``) and the ``_instances[<id>]`` write — orphaning a
# container + relay. The lock makes the second start wait for the first
# to fully register (so the second's reap-prior step sees it) instead of
# both clobbering the registry. Distinct from ``self._lock`` (which
# ``stop_agent`` takes) to avoid a reentrancy deadlock: the spawn body
# holds this lock then calls ``stop_agent`` (acquires ``self._lock``) —
# lock order is always ``_intake_spawn_lock`` -> ``self._lock``, never
# the reverse, so there's no cycle.
self._intake_spawn_lock = asyncio.Lock()
self._secretary_spawn_lock = asyncio.Lock()
# Per-tick set of task_ids already handled by an earlier
# dispatcher. Reset at the start of every _dispatch_all_work.
# Consumed via `self._mark_task_handled` / `_is_task_handled`.
@@ -3466,118 +3479,129 @@ class AgentOrchestrator:
The relay must already be open (``_open_intake_relay``). Heavy + slow
(clone + first-time image build + docker run) — keep it off the request
path via ``start_intake_session``.
Serialized by ``_intake_spawn_lock``: the intake agent id is a single
fixed id, so two concurrent starts would race on the container name and
the ``_instances`` write, orphaning a container + relay. The lock makes a
concurrent start wait for the in-flight one to finish (reap + register)
before it begins its own reap-prior check.
"""
# Single live session: reap any prior intake container before spawning.
if INTAKE_AGENT_ID in self._instances:
await self.stop_agent(INTAKE_AGENT_ID, graceful=False)
async with self._intake_spawn_lock:
# Single live session: reap any prior intake container before spawning.
if INTAKE_AGENT_ID in self._instances:
await self.stop_agent(INTAKE_AGENT_ID, graceful=False)
from roboco.models.base import ModelProvider
from roboco.models.base import ModelProvider
cwd, cloned = await self._clone_intake_scope(
project_slug, product_id, project_ids
)
ambient = await self._resolve_conventions_ambient(
project_slug, product_id=product_id
)
prompt_path = self._generate_composed_prompt(INTAKE_AGENT_ID, ambient=ambient)
route = await self._resolve_agent_route(INTAKE_AGENT_ID)
cli_model = _resolve_agent_cli_model(
route.provider_type.value, route.model_name
)
api_url = (
"http://roboco-orchestrator:8000"
if PROJECT_HOST_PATH
else f"http://127.0.0.1:{settings.port}"
)
# GROK runs the interactive driver on its own grok-CLI prompter image;
# every other provider uses the Claude SDK-driver prompter image.
is_grok = route.provider_type == ModelProvider.GROK
image = GROK_PROMPTER_IMAGE if is_grok else get_agent_image(INTAKE_AGENT_ID)
if is_grok:
await self._ensure_grok_interactive_image(image)
self._ensure_grok_usage_dir(INTAKE_AGENT_ID)
else:
await self._ensure_agent_image(INTAKE_AGENT_ID)
container_name = f"roboco-agent-{INTAKE_AGENT_ID}"
await self._remove_container(container_name)
cmd = self._build_intake_run_cmd(
_IntakeRunSpec(
container_name=container_name,
image=image,
hosts=self._resolve_intake_host_paths(),
session_id=session_id,
cwd=cwd,
cli_model=cli_model,
api_url=api_url,
provider_base_url=route.base_url,
provider_auth_token=route.auth_token,
provider_type=route.provider_type.value,
model=route.model_name,
cwd, cloned = await self._clone_intake_scope(
project_slug, product_id, project_ids
)
)
container_id = await self._run_container_cmd(cmd)
# Shutdown may have begun while this (non-blocking) spawn was in flight
# — the bg coroutine runs concurrently with stop(). If so, remove the
# just-started container and abort WITHOUT registering: stop()'s
# _instances iteration has already run (or is running), so a registration
# now would land a live container nothing tears down (the orphan). The
# stop() drain awaits this coroutine, so the abort surfaces cleanly.
if not self._running:
ambient = await self._resolve_conventions_ambient(
project_slug, product_id=product_id
)
prompt_path = self._generate_composed_prompt(
INTAKE_AGENT_ID, ambient=ambient
)
route = await self._resolve_agent_route(INTAKE_AGENT_ID)
cli_model = _resolve_agent_cli_model(
route.provider_type.value, route.model_name
)
api_url = (
"http://roboco-orchestrator:8000"
if PROJECT_HOST_PATH
else f"http://127.0.0.1:{settings.port}"
)
# GROK runs the interactive driver on its own grok-CLI prompter image;
# every other provider uses the Claude SDK-driver prompter image.
is_grok = route.provider_type == ModelProvider.GROK
image = GROK_PROMPTER_IMAGE if is_grok else get_agent_image(INTAKE_AGENT_ID)
if is_grok:
await self._ensure_grok_interactive_image(image)
self._ensure_grok_usage_dir(INTAKE_AGENT_ID)
else:
await self._ensure_agent_image(INTAKE_AGENT_ID)
container_name = f"roboco-agent-{INTAKE_AGENT_ID}"
await self._remove_container(container_name)
raise _SpawnAbortedDuringShutdown(INTAKE_AGENT_ID)
config = AgentConfig(
agent_id=INTAKE_AGENT_ID,
blueprint_path=prompt_path,
model=route.model_name,
git_context=None,
provider_type=route.provider_type.value,
)
instance = AgentInstance(
agent_id=INTAKE_AGENT_ID,
state=AgentState.ACTIVE,
config=config,
current_task_id=None,
)
instance.container_id = container_id
instance.started_at = datetime.now(UTC)
instance.last_activity = datetime.now(UTC)
self._instances[INTAKE_AGENT_ID] = instance
cmd = self._build_intake_run_cmd(
_IntakeRunSpec(
container_name=container_name,
image=image,
hosts=self._resolve_intake_host_paths(),
session_id=session_id,
cwd=cwd,
cli_model=cli_model,
api_url=api_url,
provider_base_url=route.base_url,
provider_auth_token=route.auth_token,
provider_type=route.provider_type.value,
model=route.model_name,
)
)
container_id = await self._run_container_cmd(cmd)
# Record a usage session (task_id=None) and pin its id on the instance so
# the reap finalizer can look up token usage — without this an interactive
# session finalizes at 0 tokens / $0 (the GROK path reads the captured
# usage.json; the Claude path reads the transcript). Mirrors _launch_spawn.
usage_session_id = await self._record_spawn_session(config, None)
if usage_session_id is not None:
instance.usage_session_id = usage_session_id
# Shutdown may have begun while this (non-blocking) spawn was in flight
# — the bg coroutine runs concurrently with stop(). If so, remove the
# just-started container and abort WITHOUT registering: stop()'s
# _instances iteration has already run (or is running), so a registration
# now would land a live container nothing tears down (the orphan). The
# stop() drain awaits this coroutine, so the abort surfaces cleanly.
if not self._running:
await self._remove_container(container_name)
raise _SpawnAbortedDuringShutdown(INTAKE_AGENT_ID)
# The relay was already opened on the request path (start_intake_session /
# spawn_intake_session) BEFORE the panel connected its SSE stream. Do NOT
# re-open here: a second open would swap in a fresh queue and orphan that
# already-connected stream (the agent's replies would push to the new queue
# while the browser keeps reading the old one). open() is idempotent now as
# a guard, but the redundant call is gone regardless.
logger.info(
"Intake session spawned",
session_id=session_id,
container_id=container_id[:12],
cwd=cwd,
repos=len(cloned),
)
self._fire_audit(
event_type="agent.spawned",
agent_slug=INTAKE_AGENT_ID,
details={"session_id": session_id, "cwd": cwd, "repos": cloned},
)
config = AgentConfig(
agent_id=INTAKE_AGENT_ID,
blueprint_path=prompt_path,
model=route.model_name,
git_context=None,
provider_type=route.provider_type.value,
)
instance = AgentInstance(
agent_id=INTAKE_AGENT_ID,
state=AgentState.ACTIVE,
config=config,
current_task_id=None,
)
instance.container_id = container_id
instance.started_at = datetime.now(UTC)
instance.last_activity = datetime.now(UTC)
self._instances[INTAKE_AGENT_ID] = instance
if initial_message:
self._schedule_intake_first_message(session_id, initial_message)
return instance
# Record a usage session (task_id=None) and pin its id on the instance
# so the reap finalizer can look up token usage — without this an
# interactive session finalizes at 0 tokens / $0 (the GROK path reads the
# captured usage.json; the Claude path reads the transcript). Mirrors
# _launch_spawn.
usage_session_id = await self._record_spawn_session(config, None)
if usage_session_id is not None:
instance.usage_session_id = usage_session_id
# The relay was already opened on the request path
# (start_intake_session / spawn_intake_session) BEFORE the panel
# connected its SSE stream. Do NOT re-open here: a second open would
# swap in a fresh queue and orphan that already-connected stream (the
# agent's replies would push to the new queue while the browser keeps
# reading the old one). open() is idempotent now as a guard, but the
# redundant call is gone regardless.
logger.info(
"Intake session spawned",
session_id=session_id,
container_id=container_id[:12],
cwd=cwd,
repos=len(cloned),
)
self._fire_audit(
event_type="agent.spawned",
agent_slug=INTAKE_AGENT_ID,
details={"session_id": session_id, "cwd": cwd, "repos": cloned},
)
if initial_message:
self._schedule_intake_first_message(session_id, initial_message)
return instance
async def reap_intake_session(self, session_id: str) -> None:
"""End a live chat: close the relay stream and stop the container."""
@@ -3653,100 +3677,108 @@ class AgentOrchestrator:
company state through the API, so its cwd is the baked ``/app`` tree. It
gets an HMAC agent token so its directive tools authenticate as the
Secretary role.
Serialized by ``_secretary_spawn_lock`` for the same reason intake is
serialized by ``_intake_spawn_lock``: a single fixed agent id, so two
concurrent starts would race on the container name and the ``_instances``
write. See ``_spawn_intake_container`` for the deadlock-ordering note.
"""
from roboco.agents_config import issue_agent_token
from roboco.foundation.identity import AGENTS
from roboco.models.base import ModelProvider
async with self._secretary_spawn_lock:
from roboco.agents_config import issue_agent_token
from roboco.foundation.identity import AGENTS
from roboco.models.base import ModelProvider
if SECRETARY_AGENT_ID in self._instances:
await self.stop_agent(SECRETARY_AGENT_ID, graceful=False)
if SECRETARY_AGENT_ID in self._instances:
await self.stop_agent(SECRETARY_AGENT_ID, graceful=False)
prompt_path = self._generate_composed_prompt(SECRETARY_AGENT_ID)
route = await self._resolve_agent_route(SECRETARY_AGENT_ID)
cli_model = _resolve_agent_cli_model(
route.provider_type.value, route.model_name
)
api_url = (
"http://roboco-orchestrator:8000"
if PROJECT_HOST_PATH
else f"http://127.0.0.1:{settings.port}"
)
is_grok = route.provider_type == ModelProvider.GROK
image = GROK_SECRETARY_IMAGE if is_grok else get_agent_image(SECRETARY_AGENT_ID)
if is_grok:
await self._ensure_grok_interactive_image(image)
self._ensure_grok_usage_dir(SECRETARY_AGENT_ID)
else:
await self._ensure_agent_image(SECRETARY_AGENT_ID)
container_name = f"roboco-agent-{SECRETARY_AGENT_ID}"
await self._remove_container(container_name)
agent_uuid = str(AGENTS[SECRETARY_AGENT_ID].uuid)
cmd = self._build_secretary_run_cmd(
_SecretaryRunSpec(
container_name=container_name,
image=image,
hosts=self._resolve_secretary_host_paths(),
session_id=session_id,
cwd="/app",
cli_model=cli_model,
api_url=api_url,
agent_uuid=agent_uuid,
agent_token=issue_agent_token(agent_uuid, "secretary", ""),
provider_base_url=route.base_url,
provider_auth_token=route.auth_token,
provider_type=route.provider_type.value,
model=route.model_name,
prompt_path = self._generate_composed_prompt(SECRETARY_AGENT_ID)
route = await self._resolve_agent_route(SECRETARY_AGENT_ID)
cli_model = _resolve_agent_cli_model(
route.provider_type.value, route.model_name
)
api_url = (
"http://roboco-orchestrator:8000"
if PROJECT_HOST_PATH
else f"http://127.0.0.1:{settings.port}"
)
)
container_id = await self._run_container_cmd(cmd)
# Shutdown may have begun while this (non-blocking) spawn was in flight
# — see the matching guard in _spawn_intake_container. Remove the
# just-started container and abort WITHOUT registering, so it isn't
# orphaned by a stop() that has already iterated _instances.
if not self._running:
is_grok = route.provider_type == ModelProvider.GROK
image = (
GROK_SECRETARY_IMAGE if is_grok else get_agent_image(SECRETARY_AGENT_ID)
)
if is_grok:
await self._ensure_grok_interactive_image(image)
self._ensure_grok_usage_dir(SECRETARY_AGENT_ID)
else:
await self._ensure_agent_image(SECRETARY_AGENT_ID)
container_name = f"roboco-agent-{SECRETARY_AGENT_ID}"
await self._remove_container(container_name)
raise _SpawnAbortedDuringShutdown(SECRETARY_AGENT_ID)
config = AgentConfig(
agent_id=SECRETARY_AGENT_ID,
blueprint_path=prompt_path,
model=route.model_name,
git_context=None,
provider_type=route.provider_type.value,
)
instance = AgentInstance(
agent_id=SECRETARY_AGENT_ID,
state=AgentState.ACTIVE,
config=config,
current_task_id=None,
)
instance.container_id = container_id
instance.started_at = datetime.now(UTC)
instance.last_activity = datetime.now(UTC)
self._instances[SECRETARY_AGENT_ID] = instance
agent_uuid = str(AGENTS[SECRETARY_AGENT_ID].uuid)
cmd = self._build_secretary_run_cmd(
_SecretaryRunSpec(
container_name=container_name,
image=image,
hosts=self._resolve_secretary_host_paths(),
session_id=session_id,
cwd="/app",
cli_model=cli_model,
api_url=api_url,
agent_uuid=agent_uuid,
agent_token=issue_agent_token(agent_uuid, "secretary", ""),
provider_base_url=route.base_url,
provider_auth_token=route.auth_token,
provider_type=route.provider_type.value,
model=route.model_name,
)
)
container_id = await self._run_container_cmd(cmd)
# Pin a usage session id so the reap finalizer can attribute token usage
# (else $0); see the matching note in _spawn_intake_container.
usage_session_id = await self._record_spawn_session(config, None)
if usage_session_id is not None:
instance.usage_session_id = usage_session_id
# Shutdown may have begun while this (non-blocking) spawn was in flight
# — see the matching guard in _spawn_intake_container. Remove the
# just-started container and abort WITHOUT registering, so it isn't
# orphaned by a stop() that has already iterated _instances.
if not self._running:
await self._remove_container(container_name)
raise _SpawnAbortedDuringShutdown(SECRETARY_AGENT_ID)
logger.info(
"Secretary session spawned",
session_id=session_id,
container_id=container_id[:12],
)
self._fire_audit(
event_type="agent.spawned",
agent_slug=SECRETARY_AGENT_ID,
details={"session_id": session_id},
)
if initial_message:
self._schedule_intake_first_message(session_id, initial_message)
return instance
config = AgentConfig(
agent_id=SECRETARY_AGENT_ID,
blueprint_path=prompt_path,
model=route.model_name,
git_context=None,
provider_type=route.provider_type.value,
)
instance = AgentInstance(
agent_id=SECRETARY_AGENT_ID,
state=AgentState.ACTIVE,
config=config,
current_task_id=None,
)
instance.container_id = container_id
instance.started_at = datetime.now(UTC)
instance.last_activity = datetime.now(UTC)
self._instances[SECRETARY_AGENT_ID] = instance
# Pin a usage session id so the reap finalizer can attribute token
# usage (else $0); see the matching note in _spawn_intake_container.
usage_session_id = await self._record_spawn_session(config, None)
if usage_session_id is not None:
instance.usage_session_id = usage_session_id
logger.info(
"Secretary session spawned",
session_id=session_id,
container_id=container_id[:12],
)
self._fire_audit(
event_type="agent.spawned",
agent_slug=SECRETARY_AGENT_ID,
details={"session_id": session_id},
)
if initial_message:
self._schedule_intake_first_message(session_id, initial_message)
return instance
async def reap_secretary_session(self, session_id: str) -> None:
"""End a live Secretary chat: close the relay and stop the container."""
tests/unit/runtime/test_intake_spawn.py
+58
View File
@@ -37,6 +37,9 @@ def _make_minimal_orchestrator() -> AgentOrchestrator:
# (F071); without this the post-docker-run guard would AttributeError on
# the constructor-skipped instance.
orch._running = True
# F093: concurrent intake starts serialize on this lock; the constructor
# (skipped here) initializes it.
orch._intake_spawn_lock = asyncio.Lock()
return orch
@@ -472,6 +475,61 @@ class TestSpawnGuarded:
assert closed == ["sess-B"]
class TestConcurrentSpawnSerialization:
"""Two concurrent intake starts must serialize — the intake agent id is a
single fixed id, so two ``docker run --name roboco-agent-prompter`` calls and
two ``_instances[INTAKE_AGENT_ID]`` writes racing orphan a container + relay.
The spawn body (reap-prior → clone → docker run → register) must run under
a per-agent lock so the second start only begins once the first has fully
registered (or been reaped).
"""
@pytest.mark.asyncio
async def test_concurrent_intake_spawns_do_not_interleave(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
orch = _make_minimal_orchestrator()
_wire_spawn_mocks(monkeypatch, orch, run_calls=[])
# Reap the prior instance on a concurrent start: mock stop_agent so the
# second spawn's reap doesn't need the real self._lock (not set on the
# minimal orchestrator). Records that the prior instance was reaped.
reaped: list[str] = []
async def _stop(aid: str, **_kw: Any) -> None:
reaped.append(aid)
monkeypatch.setattr(orch, "stop_agent", _stop)
# Instrument the first await inside the spawn body (the scope clone) to
# measure how many spawns are inside the body at once. With a serializing
# lock the second spawn is parked on lock.acquire() and can't enter clone
# until the first releases (after fully registering) -> max depth 1.
# Without the lock both spawns reach clone concurrently -> max depth 2.
in_clone = 0
max_depth = 0
async def _clone(*_a: Any, **_kw: Any) -> tuple[str, list[str]]:
nonlocal in_clone, max_depth
in_clone += 1
max_depth = max(max_depth, in_clone)
await asyncio.sleep(0) # yield so the other spawn may enter if not locked
in_clone -= 1
return "/data/workspaces/roboco/board/intake-1", ["/cwd"]
monkeypatch.setattr(orch, "_clone_intake_scope", _clone)
await asyncio.gather(
orch.spawn_intake_session("sess-a", project_slug="roboco"),
orch.spawn_intake_session("sess-b", project_slug="roboco"),
)
assert max_depth == 1 # serialized: never two spawns in the body at once
# The second start reaped the first's registered instance (proves the two
# spawns ran in order, not concurrently clobbering the registry).
assert reaped == [INTAKE_AGENT_ID]
class TestReapIntakeSession:
@pytest.mark.asyncio
async def test_reap_closes_session_and_stops_container(
tests/unit/runtime/test_secretary_spawn_shutdown.py
+65
View File
@@ -16,6 +16,7 @@ and abort WITHOUT registering. The guarded wrapper closes the relay silently
from __future__ import annotations
import asyncio
from pathlib import Path
from types import SimpleNamespace
from typing import Any
@@ -37,6 +38,9 @@ def _make_orchestrator() -> AgentOrchestrator:
orch._instances = {}
orch._bg_tasks = set()
orch._running = True
# F093: concurrent secretary starts serialize on this lock; the constructor
# (skipped here) initializes it.
orch._secretary_spawn_lock = asyncio.Lock()
return orch
@@ -163,3 +167,64 @@ async def test_running_spawn_registers_normally(
# Only the pre-spawn reap remove — the shutdown guard did NOT remove the
# just-started container (the orchestrator stayed running).
assert removed == [f"roboco-agent-{SECRETARY_AGENT_ID}"]
# ---------------------------------------------------------------------------
# F093 — concurrent Secretary starts must serialize. The Secretary agent id is a
# single fixed id, so two concurrent ``spawn_secretary_session`` calls race on
# the container name (``docker run --name roboco-agent-secretary``) and the
# ``_instances[SECRETARY_AGENT_ID]`` write, orphaning a container + relay. The
# spawn body runs under ``_secretary_spawn_lock`` so the second start only begins
# once the first has fully registered (so the second's reap-prior sees it).
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_concurrent_secretary_spawns_do_not_interleave(
monkeypatch: pytest.MonkeyPatch,
) -> None:
orch = _make_orchestrator()
removed: list[str] = []
_wire_secretary_spawn_mocks(monkeypatch, orch, removed, flip_running_on_run=False)
# Reap the prior instance on a concurrent start: mock stop_agent so the
# second spawn's reap doesn't need the real self._lock (not set on the
# minimal orchestrator).
reaped: list[str] = []
async def _stop(aid: str, **_kw: Any) -> None:
reaped.append(aid)
monkeypatch.setattr(orch, "stop_agent", _stop)
# Instrument the first await inside the spawn body (route resolution) to
# measure how many spawns are inside the body at once. With a serializing
# lock the second spawn is parked on lock.acquire() and can't reach the
# route call until the first releases -> max depth 1. Without the lock both
# spawns reach it concurrently -> max depth 2.
in_route = 0
max_depth = 0
async def _route(_aid: str) -> Any:
nonlocal in_route, max_depth
in_route += 1
max_depth = max(max_depth, in_route)
await asyncio.sleep(0) # yield so the other spawn may enter if not locked
in_route -= 1
return SimpleNamespace(
provider_type=SimpleNamespace(value="anthropic"),
model_name="opus",
base_url=None,
auth_token=None,
)
monkeypatch.setattr(orch, "_resolve_agent_route", _route)
await asyncio.gather(
orch.spawn_secretary_session("sess-a", initial_message=None),
orch.spawn_secretary_session("sess-b", initial_message=None),
)
assert max_depth == 1 # serialized: never two spawns in the body at once
# The second start reaped the first's registered instance (ran in order).
assert reaped == [SECRETARY_AGENT_ID]