[F093] serialize concurrent live-chat spawns under a per-agent lock

The intake and secretary agent ids are each a single fixed id, so two
concurrent start_intake_session / start_secretary_session calls raced on
the container name (docker run --name roboco-agent-<id>) and the
_instances[<id>] write: both passed the reap-prior check before either
registered, both ran docker run, and the last _instances write won,
orphaning the other container + its relay.

Add _intake_spawn_lock / _secretary_spawn_lock (asyncio.Lock) and wrap the
_spawn_intake_container / _spawn_secretary_container bodies so the second
start waits for the first to fully register before its own reap-prior check
runs. Distinct from self._lock (which stop_agent takes) to avoid a
reentrancy deadlock: the spawn body holds the spawn lock then calls
stop_agent (acquires self._lock) — lock order is always spawn_lock ->
self._lock, never the reverse.

roboco/runtime/orchestrator.py

@@ -826,6 +826,19 @@ class AgentOrchestrator:
         # spawn two umbrellas for the same PR (the check is read-then-write
         # with no DB-level uniqueness).
         self._supersede_lock = asyncio.Lock()
+        # Serialize concurrent live-chat starts for the single-id interactive
+        # agents (intake / secretary). Each has a fixed agent id, so two
+        # concurrent starts race on the container name (``docker run --name
+        # roboco-agent-<id>``) and the ``_instances[<id>]`` write — orphaning a
+        # container + relay. The lock makes the second start wait for the first
+        # to fully register (so the second's reap-prior step sees it) instead of
+        # both clobbering the registry. Distinct from ``self._lock`` (which
+        # ``stop_agent`` takes) to avoid a reentrancy deadlock: the spawn body
+        # holds this lock then calls ``stop_agent`` (acquires ``self._lock``) —
+        # lock order is always ``_intake_spawn_lock`` -> ``self._lock``, never
+        # the reverse, so there's no cycle.
+        self._intake_spawn_lock = asyncio.Lock()
+        self._secretary_spawn_lock = asyncio.Lock()
         # Per-tick set of task_ids already handled by an earlier
         # dispatcher. Reset at the start of every _dispatch_all_work.
         # Consumed via `self._mark_task_handled` / `_is_task_handled`.
@@ -3466,7 +3479,14 @@ class AgentOrchestrator:
         The relay must already be open (``_open_intake_relay``). Heavy + slow
         (clone + first-time image build + docker run) — keep it off the request
         path via ``start_intake_session``.
+
+        Serialized by ``_intake_spawn_lock``: the intake agent id is a single
+        fixed id, so two concurrent starts would race on the container name and
+        the ``_instances`` write, orphaning a container + relay. The lock makes a
+        concurrent start wait for the in-flight one to finish (reap + register)
+        before it begins its own reap-prior check.
         """
+        async with self._intake_spawn_lock:
             # Single live session: reap any prior intake container before spawning.
             if INTAKE_AGENT_ID in self._instances:
                 await self.stop_agent(INTAKE_AGENT_ID, graceful=False)
@@ -3480,7 +3500,9 @@ class AgentOrchestrator:
             ambient = await self._resolve_conventions_ambient(
                 project_slug, product_id=product_id
             )
-        prompt_path = self._generate_composed_prompt(INTAKE_AGENT_ID, ambient=ambient)
+            prompt_path = self._generate_composed_prompt(
+                INTAKE_AGENT_ID, ambient=ambient
+            )
             route = await self._resolve_agent_route(INTAKE_AGENT_ID)
             cli_model = _resolve_agent_cli_model(
                 route.provider_type.value, route.model_name
@@ -3548,20 +3570,22 @@ class AgentOrchestrator:
             instance.last_activity = datetime.now(UTC)
             self._instances[INTAKE_AGENT_ID] = instance
 
-        # Record a usage session (task_id=None) and pin its id on the instance so
+            # Record a usage session (task_id=None) and pin its id on the instance
-        # the reap finalizer can look up token usage — without this an interactive
+            # so the reap finalizer can look up token usage — without this an
-        # session finalizes at 0 tokens / $0 (the GROK path reads the captured
+            # interactive session finalizes at 0 tokens / $0 (the GROK path reads the
-        # usage.json; the Claude path reads the transcript). Mirrors _launch_spawn.
+            # captured usage.json; the Claude path reads the transcript). Mirrors
+            # _launch_spawn.
             usage_session_id = await self._record_spawn_session(config, None)
             if usage_session_id is not None:
                 instance.usage_session_id = usage_session_id
 
-        # The relay was already opened on the request path (start_intake_session /
+            # The relay was already opened on the request path
-        # spawn_intake_session) BEFORE the panel connected its SSE stream. Do NOT
+            # (start_intake_session / spawn_intake_session) BEFORE the panel
-        # re-open here: a second open would swap in a fresh queue and orphan that
+            # connected its SSE stream. Do NOT re-open here: a second open would
-        # already-connected stream (the agent's replies would push to the new queue
+            # swap in a fresh queue and orphan that already-connected stream (the
-        # while the browser keeps reading the old one). open() is idempotent now as
+            # agent's replies would push to the new queue while the browser keeps
-        # a guard, but the redundant call is gone regardless.
+            # reading the old one). open() is idempotent now as a guard, but the
+            # redundant call is gone regardless.
             logger.info(
                 "Intake session spawned",
                 session_id=session_id,
@@ -3653,7 +3677,13 @@ class AgentOrchestrator:
         company state through the API, so its cwd is the baked ``/app`` tree. It
         gets an HMAC agent token so its directive tools authenticate as the
         Secretary role.
+
+        Serialized by ``_secretary_spawn_lock`` for the same reason intake is
+        serialized by ``_intake_spawn_lock``: a single fixed agent id, so two
+        concurrent starts would race on the container name and the ``_instances``
+        write. See ``_spawn_intake_container`` for the deadlock-ordering note.
         """
+        async with self._secretary_spawn_lock:
             from roboco.agents_config import issue_agent_token
             from roboco.foundation.identity import AGENTS
             from roboco.models.base import ModelProvider
@@ -3673,7 +3703,9 @@ class AgentOrchestrator:
             )
 
             is_grok = route.provider_type == ModelProvider.GROK
-        image = GROK_SECRETARY_IMAGE if is_grok else get_agent_image(SECRETARY_AGENT_ID)
+            image = (
+                GROK_SECRETARY_IMAGE if is_grok else get_agent_image(SECRETARY_AGENT_ID)
+            )
             if is_grok:
                 await self._ensure_grok_interactive_image(image)
                 self._ensure_grok_usage_dir(SECRETARY_AGENT_ID)
@@ -3728,8 +3760,8 @@ class AgentOrchestrator:
             instance.last_activity = datetime.now(UTC)
             self._instances[SECRETARY_AGENT_ID] = instance
 
-        # Pin a usage session id so the reap finalizer can attribute token usage
+            # Pin a usage session id so the reap finalizer can attribute token
-        # (else $0); see the matching note in _spawn_intake_container.
+            # usage (else $0); see the matching note in _spawn_intake_container.
             usage_session_id = await self._record_spawn_session(config, None)
             if usage_session_id is not None:
                 instance.usage_session_id = usage_session_id

tests/unit/runtime/test_intake_spawn.py

@@ -37,6 +37,9 @@ def _make_minimal_orchestrator() -> AgentOrchestrator:
     # (F071); without this the post-docker-run guard would AttributeError on
     # the constructor-skipped instance.
     orch._running = True
+    # F093: concurrent intake starts serialize on this lock; the constructor
+    # (skipped here) initializes it.
+    orch._intake_spawn_lock = asyncio.Lock()
     return orch
 
 
@@ -472,6 +475,61 @@ class TestSpawnGuarded:
         assert closed == ["sess-B"]
 
 
+class TestConcurrentSpawnSerialization:
+    """Two concurrent intake starts must serialize — the intake agent id is a
+    single fixed id, so two ``docker run --name roboco-agent-prompter`` calls and
+    two ``_instances[INTAKE_AGENT_ID]`` writes racing orphan a container + relay.
+    The spawn body (reap-prior → clone → docker run → register) must run under
+    a per-agent lock so the second start only begins once the first has fully
+    registered (or been reaped).
+    """
+
+    @pytest.mark.asyncio
+    async def test_concurrent_intake_spawns_do_not_interleave(
+        self, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        orch = _make_minimal_orchestrator()
+        _wire_spawn_mocks(monkeypatch, orch, run_calls=[])
+
+        # Reap the prior instance on a concurrent start: mock stop_agent so the
+        # second spawn's reap doesn't need the real self._lock (not set on the
+        # minimal orchestrator). Records that the prior instance was reaped.
+        reaped: list[str] = []
+
+        async def _stop(aid: str, **_kw: Any) -> None:
+            reaped.append(aid)
+
+        monkeypatch.setattr(orch, "stop_agent", _stop)
+
+        # Instrument the first await inside the spawn body (the scope clone) to
+        # measure how many spawns are inside the body at once. With a serializing
+        # lock the second spawn is parked on lock.acquire() and can't enter clone
+        # until the first releases (after fully registering) -> max depth 1.
+        # Without the lock both spawns reach clone concurrently -> max depth 2.
+        in_clone = 0
+        max_depth = 0
+
+        async def _clone(*_a: Any, **_kw: Any) -> tuple[str, list[str]]:
+            nonlocal in_clone, max_depth
+            in_clone += 1
+            max_depth = max(max_depth, in_clone)
+            await asyncio.sleep(0)  # yield so the other spawn may enter if not locked
+            in_clone -= 1
+            return "/data/workspaces/roboco/board/intake-1", ["/cwd"]
+
+        monkeypatch.setattr(orch, "_clone_intake_scope", _clone)
+
+        await asyncio.gather(
+            orch.spawn_intake_session("sess-a", project_slug="roboco"),
+            orch.spawn_intake_session("sess-b", project_slug="roboco"),
+        )
+
+        assert max_depth == 1  # serialized: never two spawns in the body at once
+        # The second start reaped the first's registered instance (proves the two
+        # spawns ran in order, not concurrently clobbering the registry).
+        assert reaped == [INTAKE_AGENT_ID]
+
+
 class TestReapIntakeSession:
     @pytest.mark.asyncio
     async def test_reap_closes_session_and_stops_container(

tests/unit/runtime/test_secretary_spawn_shutdown.py

@@ -16,6 +16,7 @@ and abort WITHOUT registering. The guarded wrapper closes the relay silently
 
 from __future__ import annotations
 
+import asyncio
 from pathlib import Path
 from types import SimpleNamespace
 from typing import Any
@@ -37,6 +38,9 @@ def _make_orchestrator() -> AgentOrchestrator:
     orch._instances = {}
     orch._bg_tasks = set()
     orch._running = True
+    # F093: concurrent secretary starts serialize on this lock; the constructor
+    # (skipped here) initializes it.
+    orch._secretary_spawn_lock = asyncio.Lock()
     return orch
 
 
@@ -163,3 +167,64 @@ async def test_running_spawn_registers_normally(
     # Only the pre-spawn reap remove — the shutdown guard did NOT remove the
     # just-started container (the orchestrator stayed running).
     assert removed == [f"roboco-agent-{SECRETARY_AGENT_ID}"]
+
+
+# ---------------------------------------------------------------------------
+# F093 — concurrent Secretary starts must serialize. The Secretary agent id is a
+# single fixed id, so two concurrent ``spawn_secretary_session`` calls race on
+# the container name (``docker run --name roboco-agent-secretary``) and the
+# ``_instances[SECRETARY_AGENT_ID]`` write, orphaning a container + relay. The
+# spawn body runs under ``_secretary_spawn_lock`` so the second start only begins
+# once the first has fully registered (so the second's reap-prior sees it).
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_concurrent_secretary_spawns_do_not_interleave(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    orch = _make_orchestrator()
+    removed: list[str] = []
+    _wire_secretary_spawn_mocks(monkeypatch, orch, removed, flip_running_on_run=False)
+
+    # Reap the prior instance on a concurrent start: mock stop_agent so the
+    # second spawn's reap doesn't need the real self._lock (not set on the
+    # minimal orchestrator).
+    reaped: list[str] = []
+
+    async def _stop(aid: str, **_kw: Any) -> None:
+        reaped.append(aid)
+
+    monkeypatch.setattr(orch, "stop_agent", _stop)
+
+    # Instrument the first await inside the spawn body (route resolution) to
+    # measure how many spawns are inside the body at once. With a serializing
+    # lock the second spawn is parked on lock.acquire() and can't reach the
+    # route call until the first releases -> max depth 1. Without the lock both
+    # spawns reach it concurrently -> max depth 2.
+    in_route = 0
+    max_depth = 0
+
+    async def _route(_aid: str) -> Any:
+        nonlocal in_route, max_depth
+        in_route += 1
+        max_depth = max(max_depth, in_route)
+        await asyncio.sleep(0)  # yield so the other spawn may enter if not locked
+        in_route -= 1
+        return SimpleNamespace(
+            provider_type=SimpleNamespace(value="anthropic"),
+            model_name="opus",
+            base_url=None,
+            auth_token=None,
+        )
+
+    monkeypatch.setattr(orch, "_resolve_agent_route", _route)
+
+    await asyncio.gather(
+        orch.spawn_secretary_session("sess-a", initial_message=None),
+        orch.spawn_secretary_session("sess-b", initial_message=None),
+    )
+
+    assert max_depth == 1  # serialized: never two spawns in the body at once
+    # The second start reaped the first's registered instance (ran in order).
+    assert reaped == [SECRETARY_AGENT_ID]