Files
roboco/roboco/runtime/orchestrator.py
T
Renn F a956083f9f feat(providers): pluggable agent providers + Grok (xAI) backend
Add a roboco/llm/providers/ seam — an AgentProvider lifecycle ABC and a
ProviderRegistry keyed by ModelProvider — so the orchestrator can drive
agent backends other than Claude Code.

The first non-Claude backend is GrokProvider for xAI's grok-build-0.1.
xAI is OpenAI-compatible only (no Anthropic-Messages endpoint), so a Grok
agent runs an OpenAI-protocol runtime pointed at https://api.x.ai/v1
rather than the ANTHROPIC_BASE_URL injection the other providers use. It
reuses the orchestrator's existing mount/auth assembly, so it inherits the
same MCP gateway + tool-manifest wiring as every other agent by
construction, and passes its prompt via env (never an argv positional).

The change is purely additive: only GROK routes through the registry;
Anthropic / Ollama Cloud / self-hosted spawns run the existing
_spawn_container path unchanged.

Includes:
- ModelProvider.GROK (migration 038) + a seeded Grok provider row
  (migration 039) + a grok-build-0.1 catalog entry
- GET/PUT /api/providers/grok-key to store the xAI key (Fernet-encrypted,
  reusing the existing provider-key machinery)
- ClaudeCodeProvider reference adapter over the current spawn
- unit tests for the registry, GrokProvider (gateway wiring, no
  ANTHROPIC_* leak, prompt-injection safety, failure paths) and routing

The dedicated roboco-agent-grok image and the exact OpenAI-protocol CLI
invocation are the remaining piece to finalise with xAI.
2026-06-18 06:14:27 +02:00

8782 lines
354 KiB
Python

"""
Agent Orchestrator
Manages Claude Code containers for all RoboCo agents.
Handles spawning, monitoring, health checks, and graceful shutdown.
The orchestrator is the BRAIN of the system:
- Checks for work BEFORE spawning agents (no wasteful spawns)
- Claims tasks on behalf of agents before spawning
- Agents receive their assignment at spawn time
- Agents scan for more work after completing a task
- Agents only call i_am_idle() when truly no work remains
"""
import asyncio
import contextlib
import json
import os
import shutil
import tempfile
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import TYPE_CHECKING, Any, ClassVar, cast
import httpx
if TYPE_CHECKING:
from collections.abc import Callable, Coroutine
from uuid import UUID
from sqlalchemy.ext.asyncio import AsyncSession
from roboco.llm.providers import AgentProvider, ProviderRegistry
from roboco.services.llm import AgentRoute
from roboco.services.task import TaskService
import structlog
from fastapi import status as http_status
from roboco.agents.factories._base import compose_prompt
from roboco.agents_config import (
ALL_DOCS,
get_agent_role,
get_agent_team,
get_escalation_target,
)
from roboco.config import settings
from roboco.foundation import identity as _foundation
from roboco.foundation.identity import CELL_TEAMS
from roboco.foundation.policy.agent_loop import DEFAULT_BUDGET as _AGENT_LOOP_BUDGET
from roboco.models import AgentRole, Team
from roboco.models.runtime import (
MODEL_MAP,
ROLE_MODEL_MAP,
AgentInstance,
OrchestratorAgentConfig,
OrchestratorAgentState,
SpawnGitContext,
WaitingRecord,
)
from roboco.seeds.initial_data import AGENT_UUIDS
from roboco.services.task import PR_REVIEW_SOURCES, SELF_HEAL_SOURCE
logger = structlog.get_logger()
# Reverse mapping: UUID -> slug
UUID_TO_SLUG = {uuid: slug for slug, uuid in AGENT_UUIDS.items()}
# Re-export for backwards compatibility
AgentState = OrchestratorAgentState
AgentConfig = OrchestratorAgentConfig
# Docker configuration
AGENT_NETWORK = "roboco_default"
AGENT_BASE_IMAGE = "roboco-agent-base"
# Port on which each agent's Claude Code SDK server listens inside its container.
# Referenced by write-hooks (_finalize_spawn_session, _sweep_token_snapshots,
# _sweep_budget_exceeded) to build the SDK health/usage URL.
SDK_PORT: int = 9000
# Rate-limit recovery probe: a free, unmetered liveness call confirms a
# provider has stopped rate-limiting us before parked agents are resumed.
# Listing models / tags costs no tokens; a non-429 response means lifted.
_ANTHROPIC_PROBE_BASE = "https://api.anthropic.com"
_PROBE_TIMEOUT_SECONDS = 10.0
_HTTP_TOO_MANY_REQUESTS = 429
# Consecutive failed recovery probes before the CEO is notified once per episode.
_CEO_NOTIFY_THRESHOLD = 10
# The intake (prompter) agent: a single seeded, board-adjacent interviewer.
# Unlike delivery agents it is never dispatched and runs ONE persistent
# container at a time (single CEO → one live chat). See the INTAKE section
# below and roboco/agent_sdk/intake_main.py.
INTAKE_AGENT_ID = "intake-1"
# The Secretary agent: a single seeded, persistent chief-of-staff container the
# CEO chats with (like intake), but with gated CEO authority. One container at a
# time. Seeded in identity.AGENTS; see roboco/agent_sdk/secretary_main.py.
SECRETARY_AGENT_ID = "secretary-1"
# Role -> Image mapping
# Specialized images extend the base with role-specific tools
AGENT_IMAGES: dict[str, str] = {
# Backend
"be-dev-1": "roboco-agent-dev-be",
"be-dev-2": "roboco-agent-dev-be",
"be-qa": "roboco-agent-qa-be",
"be-pm": "roboco-agent-pm",
"be-doc": "roboco-agent-doc",
# Frontend
"fe-dev-1": "roboco-agent-dev-fe",
"fe-dev-2": "roboco-agent-dev-fe",
"fe-qa": "roboco-agent-qa-fe",
"fe-pm": "roboco-agent-pm",
"fe-doc": "roboco-agent-doc",
# UX/UI
"ux-dev-1": "roboco-agent-ux",
"ux-dev-2": "roboco-agent-ux",
"ux-qa": "roboco-agent-ux", # Uses same as dev for now
"ux-pm": "roboco-agent-pm",
"ux-doc": "roboco-agent-doc",
# Board
"main-pm": "roboco-agent-pm",
"product-owner": "roboco-agent-pm",
"head-marketing": "roboco-agent-pm",
"auditor": "roboco-agent-pm",
# PR Reviewer — read-only reviewer (diff via API, grep, post one
# change-request; never runs code). Its own image for parity with the other
# agents; built FROM the base, no extra toolchain.
"pr-reviewer-1": "roboco-agent-pr-reviewer",
# Intake — persistent Agent-SDK driver, not a one-shot `claude -p`.
INTAKE_AGENT_ID: "roboco-agent-prompter",
# Secretary — persistent Agent-SDK driver with gated CEO authority.
SECRETARY_AGENT_ID: "roboco-agent-secretary",
}
def _qualify_agent_image(bare: str) -> str:
"""Apply the configured registry namespace + tag to a bare agent image.
Default (no ``agent_image_registry``, no ``agent_image_tag``) returns the
bare name unchanged — the local build flow. With a registry set the
orchestrator spawns (and ensures) ``{registry}/roboco-agent-*[:tag]``, the
pre-built images the release workflow publishes, instead of building.
"""
registry = settings.agent_image_registry.rstrip("/")
name = f"{registry}/{bare}" if registry else bare
tag = settings.agent_image_tag
return f"{name}:{tag}" if tag else name
def get_agent_image(agent_id: str) -> str:
"""Get the Docker image for an agent (registry-qualified when configured)."""
return _qualify_agent_image(AGENT_IMAGES.get(agent_id, AGENT_BASE_IMAGE))
# When running in a container, we need host paths for volume mounts.
# These can be overridden via environment variables.
CLAUDE_AUTH_HOST_PATH = os.environ.get(
"ROBOCO_HOST_CLAUDE_DIR",
str(Path.home() / ".claude"),
)
PROJECT_HOST_PATH = os.environ.get("ROBOCO_HOST_PROJECT_DIR", "")
DATA_HOST_PATH = os.environ.get("ROBOCO_HOST_DATA_DIR", "")
# =============================================================================
# ORCHESTRATOR
# =============================================================================
@dataclass(frozen=True)
class _SlaBreach:
"""Per-(role, state) SLA breach payload for _escalate_sla_breach."""
task_id: str
role: str
status: str
age_seconds: int
sla_seconds: int
@dataclass(frozen=True)
class _IntakeRunSpec:
"""Inputs for ``_build_intake_run_cmd``, bundled to keep the signature small."""
container_name: str
image: str
hosts: dict[str, str | None]
session_id: str
cwd: str
cli_model: str
api_url: str
provider_base_url: str | None
provider_auth_token: str | None
@dataclass
class _SecretaryRunSpec:
"""Inputs for ``_build_secretary_run_cmd`` (mirrors ``_IntakeRunSpec``).
Adds the agent uuid + HMAC token: unlike intake, the Secretary's tools call
the backend, so the container needs an authenticated identity.
"""
container_name: str
image: str
hosts: dict[str, str | None]
session_id: str
cwd: str
cli_model: str
api_url: str
agent_uuid: str
agent_token: str
provider_base_url: str | None
provider_auth_token: str | None
def _read_project_slug(task: dict[str, Any]) -> str | None:
"""Extract project slug from a task payload shape-tolerantly."""
slug = task.get("project_slug")
if slug:
return str(slug)
project = task.get("project") or {}
inner = project.get("slug") if isinstance(project, dict) else None
return str(inner) if inner else None
def _is_coordination_task(task: dict[str, Any]) -> bool:
"""True for a board/fan-out task that carries a product but no repo of its own.
Such a task does no git work itself: its cell subtasks each resolve a real
project from the product's cell->project map (see TaskCreate's
project-or-product invariant and migration 018). It therefore has no
project_slug, branch_name, or git token, and must NOT be git-gated at the
spawn-readiness or stuck-detection checks the way a code task is. A task with
neither a project nor a product is genuinely unroutable and stays gated.
"""
return not task.get("project_id") and bool(task.get("product_id"))
# A branch is auto-created only at CLAIM (the claimed->in_progress transition).
# Before that — while a task is still pending/backlog awaiting first dispatch —
# it legitimately has no branch_name, so the readiness / stuck / spawn checks
# must NOT treat a missing branch as a defect. These are the only states where
# a code task is expected to already own a branch.
_BRANCH_EXPECTED_STATES: frozenset[str] = frozenset(
{"claimed", "in_progress", "verifying"}
)
def _branch_is_expected(task: dict[str, Any]) -> bool:
"""True iff this task should already have a branch_name.
A branch only exists at/after claim, and a coordination/fan-out task never
gets one (it does no git of its own). Gating the "missing branch_name"
readiness/stuck condition on this predicate stops the orchestrator from
auto-blocking a never-claimed PENDING code task that simply hasn't reached
the claim transition yet (a pending task sat 13min, auto-blocked
every 30s, never dispatched).
"""
if _is_coordination_task(task):
return False
return str(task.get("status") or "") in _BRANCH_EXPECTED_STATES
def _resolve_agent_cli_model(provider_type: str, model: str) -> str:
"""Translate an agent model name to the string Claude Code expects.
For the Anthropic provider, short names (``opus|sonnet|haiku``) are
translated through ``MODEL_MAP`` as they always were. For non-Anthropic
providers (currently Ollama Cloud) the model identifier is passed verbatim
so raw tags like ``kimi-k2.6:cloud`` reach the Ollama-side integration
intact.
Extracted as a module-level function so both the ``--model`` CLI arg
builder and the ``CLAUDE_CODE_SUBAGENT_MODEL`` env-var injector can call
the same logic without referencing the class by name inside a staticmethod.
"""
if provider_type == "anthropic":
return MODEL_MAP.get(model, model)
return model
def _agent_workspace_path(project_slug: str, team: str, agent_id: str) -> str:
"""Per-agent workspace path inside the container.
Mirrors the bind-mount layout: the host's workspaces dir is mounted at
/data/workspaces (orchestrator.py mount args), so each agent's clone lives
at /data/workspaces/<project>/<team>/<agent>. Used by both
_get_role_permissions (Edit/Write allowlist) and _build_mount_args
(docker ``-w`` flag) so the cwd matches the allowlist scope.
"""
return f"/data/workspaces/{project_slug}/{team}/{agent_id}"
def _cell_workspace_path(project_slug: str, team: str) -> str:
"""Cell-level workspace path (documenter scope).
Same rationale as ``_agent_workspace_path``; documenters work at the cell
branch, not a per-agent dev branch.
"""
return f"/data/workspaces/{project_slug}/{team}"
def _resolve_project_slug_from_git_context(
git_context: "SpawnGitContext | None",
) -> str:
"""Extract project_slug from git_context, falling back to 'default'.
Module-level counterpart to the instance method ``_resolve_project_slug``.
Called by static / classmethod contexts (e.g. ``_build_mount_args``) that
cannot access ``self``. The fallback warning is omitted here because the
instance method already logs it when the full spawn path runs; this helper
is only for the mount-args path where the agent_id/task_id context is not
available.
"""
if git_context and git_context.project_slug:
return git_context.project_slug
return "default"
# =============================================================================
# SPAWN MANIFEST — per-developer tool manifest mounting (Phase 1)
# =============================================================================
# Phase 4: every spawned role gets a gateway manifest. The legacy briefing path
# is gone. A role omitted here gets NO manifest and ROBOCO_GATEWAY_ENABLED=false,
# i.e. none of its flow verbs are pre-registered — so it can never claim its work
# and the dispatcher respawns it on the same task forever. The only roles that
# may be absent are the human-only ones (prompter, secretary) that the
# orchestrator never spawns as delivery agents.
GATEWAY_ENABLED_ROLES: frozenset[str] = frozenset(
{
"developer",
"qa",
"documenter",
"cell_pm",
"main_pm",
"product_owner",
"head_marketing",
"auditor",
"pr_reviewer",
}
)
def _build_manifest_for_agent(agent_id: str, model: str) -> Path | None:
"""Write a SpawnManifest for developer-role agents; return the host path.
Returns ``None`` for roles outside ``GATEWAY_ENABLED_ROLES`` so callers
can skip the manifest mount entirely without extra branching.
Args:
agent_id: Agent slug (e.g. ``be-dev-1``).
model: Resolved model name passed to ``SpawnInputs.agent_model``.
Returns:
Absolute host path to the written JSON file, or ``None``.
"""
from uuid import UUID
from roboco.runtime.spawn_manifest import (
SpawnInputs,
build_for_role,
write_manifest,
)
role = get_agent_role(agent_id) or "developer"
if role not in GATEWAY_ENABLED_ROLES:
return None
team = get_agent_team(agent_id) or "backend"
# UUID for the agent comes from the seeded AGENT_UUIDS map (slug -> UUID
# string). Fall back to uuid4 for unknown agents so the function stays
# callable in tests without seeded data.
raw_uuid = AGENT_UUIDS.get(agent_id)
agent_uuid = UUID(raw_uuid) if raw_uuid else __import__("uuid").uuid4()
workspace_path = Path(settings.workspaces_root) / "roboco" / team / agent_id
manifest = build_for_role(
SpawnInputs(
agent_id=agent_uuid,
role=role,
team=team,
workspace_path=workspace_path,
agent_model=model,
)
)
# Two paths in play:
# - orchestrator-internal: where the file is written inside the
# orchestrator container (settings.manifest_host_dir). The compose
# volume mount makes this dir visible on the host.
# - host-side: what the docker daemon needs for the bind-mount into
# the spawned agent. Computed via DATA_HOST_PATH translation.
write_dir = Path(settings.manifest_host_dir)
write_path = write_dir / f"{agent_id}.json"
write_manifest(manifest, write_path)
if DATA_HOST_PATH:
return Path(f"{DATA_HOST_PATH}/manifests/{agent_id}.json")
return write_path
# =============================================================================
# GATEWAY PRE-SPAWN CHECK (trigger_filter spawn cooldown)
# =============================================================================
async def _count_recent_spawns_for_task(
db_session: Any,
task_id: Any,
cutoff: datetime,
) -> int:
"""Count recent SPAWN decisions for ``task_id`` since ``cutoff``."""
from sqlalchemy import select
from roboco.db.tables import GatewayTriggerTable
result = await db_session.execute(
select(GatewayTriggerTable).where(
GatewayTriggerTable.task_id == task_id,
GatewayTriggerTable.created_at >= cutoff,
GatewayTriggerTable.decision == "spawn",
)
)
return len(result.scalars().all())
async def _count_recent_spawns_for_role(
db_session: Any,
target_role: str,
cutoff: datetime,
) -> int:
"""Count recent SPAWN decisions for ``target_role`` since ``cutoff``."""
from sqlalchemy import select
from roboco.db.tables import GatewayTriggerTable
result = await db_session.execute(
select(GatewayTriggerTable).where(
GatewayTriggerTable.target_role == target_role,
GatewayTriggerTable.created_at >= cutoff,
GatewayTriggerTable.decision == "spawn",
)
)
return len(result.scalars().all())
async def _record_trigger_decision(
db_session: Any,
task_id: Any,
trigger_kind: str,
target_role: str,
decision: Any,
) -> None:
"""Persist a gateway trigger decision row."""
from uuid import uuid4 as _uuid4
from roboco.db.tables import GatewayTriggerTable
row = GatewayTriggerTable(
id=_uuid4(),
trigger_kind=trigger_kind,
task_id=task_id,
target_role=target_role,
decision=decision.outcome.value,
decision_reason=decision.reason,
)
db_session.add(row)
await db_session.flush()
async def gateway_pre_spawn_check(
*,
task_id: str | None,
trigger_kind: str,
target_role: str,
provider: str | None = None,
) -> tuple[str, str]:
"""Consult trigger_filter before spawning a container.
Returns a ``(outcome, reason)`` tuple where ``outcome`` is one of
``"spawn"``, ``"queue"``, or ``"drop"``.
The trigger_filter spawn cooldown runs unconditionally for every spawn.
Args:
provider: Optional provider name (e.g. ``"anthropic"``) for the
agent about to be spawned. When given, the
``RateLimitStateTracker`` is consulted and a QUEUE decision is
returned when that provider is currently rate-limited.
"""
from roboco.db.base import get_session_factory
from roboco.services.gateway.trigger_filter import (
Decision,
SpawnConfig,
SpawnDecision,
TriggerContext,
TriggerKind,
decide_spawn,
)
cutoff = datetime.now(tz=UTC) - timedelta(seconds=settings.spawn_cooldown_seconds)
role_cutoff = datetime.now(tz=UTC) - timedelta(seconds=60)
# When no task_id we cannot query counts; allow (no-task spawns like idle PMs).
if task_id is None:
return SpawnDecision.SPAWN, "no task_id — no-task spawn, skip gate"
try:
from sqlalchemy import select as _select
from roboco.db.tables import TaskTable as _TaskTable
factory = get_session_factory()
async with factory() as db:
recent_for_task = await _count_recent_spawns_for_task(db, task_id, cutoff)
recent_for_role = await _count_recent_spawns_for_role(
db, target_role, role_cutoff
)
# Load the lightweight task proxy needed by is_stale / decide_spawn.
task_result = await db.execute(
_select(_TaskTable).where(_TaskTable.id == task_id)
)
task_row = task_result.scalars().first()
if task_row is None:
return SpawnDecision.SPAWN, "task not found in DB — allow by default"
# Check provider rate-limit status when a provider is known.
# Failure is non-fatal — degrade to False (allow spawn) so Redis
# unavailability never permanently blocks the dispatcher.
provider_rate_limited = False
if provider is not None:
try:
from roboco.services.gateway.rate_limit_tracker import (
RateLimitStateTracker,
)
provider_rate_limited = await RateLimitStateTracker(
provider
).is_rate_limited()
except Exception:
provider_rate_limited = False
trigger = TriggerContext(
kind=TriggerKind(trigger_kind),
skill=None,
recent_spawns_for_task=recent_for_task,
recent_spawns_for_role=recent_for_role,
provider=provider,
provider_rate_limited=provider_rate_limited,
)
config = SpawnConfig(
cooldown_seconds=settings.spawn_cooldown_seconds,
role_rate_per_minute=settings.role_spawn_rate_per_minute,
claim_stale_seconds=settings.claim_stale_seconds,
)
decision: Decision = decide_spawn(
task=task_row, trigger=trigger, config=config
)
await _record_trigger_decision(
db, task_id, trigger_kind, target_role, decision
)
await db.commit()
return decision.outcome.value, decision.reason
except Exception as exc:
# Gateway errors must never block a spawn — degrade gracefully.
logger.warning(
"Gateway pre-spawn check failed; defaulting to spawn",
task_id=task_id,
trigger_kind=trigger_kind,
error=str(exc),
)
return "spawn", f"gateway error (degraded): {exc}"
class AgentReadinessError(Exception):
"""Raised when spawn_agent refuses to spawn because the task isn't ready.
The pre-flight gate auto-blocks the offending task before raising, so the
dispatcher doesn't keep retrying. Callers should log and move on.
"""
class AgentOrchestrator:
"""
Manages Claude Code containers for all agents.
Responsibilities:
- Spawn agents as Docker containers
- Monitor health via docker inspect
- Handle waiting states and respawning
- Provide status API
- Cost-efficient on-demand spawning
"""
def __init__(
self,
mcp_config_dir: Path | None = None,
project_root: Path | None = None,
dispatcher_interval: int = 30,
):
self.mcp_config_dir = mcp_config_dir or Path(".mcp")
self.project_root = project_root or Path.cwd()
self.dispatcher_interval = dispatcher_interval
self._instances: dict[str, AgentInstance] = {}
self._waiting_records: dict[str, WaitingRecord] = {}
self._health_task: asyncio.Task | None = None
self._dispatcher_task: asyncio.Task | None = None
self._sweeper_task: asyncio.Task | None = None
# Last time the transcript-retention prune ran (throttled in the sweep).
self._last_transcript_prune: datetime | None = None
# Rate-limit probe loop: 30-second interval, scans Redis for all
# rate-limited providers and resolves waiting agents on success.
self._rate_limit_probe_task: asyncio.Task | None = None
self._strategy_engine_task: asyncio.Task | None = None
self._external_pr_poll_task: asyncio.Task | None = None
self._self_heal_task: asyncio.Task | None = None
# Provider registry: maps a ModelProvider to a dedicated AgentProvider
# backend. Only providers needing a non-Claude-Code runtime are
# registered (currently GROK, which speaks the OpenAI protocol). Agents
# on unregistered providers (Anthropic / Ollama Cloud / self-hosted) use
# the built-in _spawn_container path unchanged. Built lazily.
self._provider_registry: ProviderRegistry | None = None
# Tracks which providers have already received a CEO notification
# during the current rate-limit episode. Cleared when the probe
# succeeds and the rate limit is lifted (tracker.clear() path).
self._rate_limit_ceo_notified: set[str] = set()
# Strong refs for fire-and-forget audit writes. Without this, the
# event loop only weak-refs the Task and may GC it before it
# commits — audit_log was silently empty because of this.
self._bg_tasks: set[asyncio.Task[None]] = set()
# Wake-up signal for the dispatcher. Set() by API routes immediately
# after status transitions so the dispatcher reacts in milliseconds
# instead of waiting for the next 30-second tick.
self._dispatch_wake: asyncio.Event = asyncio.Event()
self._running = False
self._lock = asyncio.Lock()
# Serializes CEO supersede calls so a double-click can't pass the
# find_supersede_umbrella dedup check twice and cut two branches /
# spawn two umbrellas for the same PR (the check is read-then-write
# with no DB-level uniqueness).
self._supersede_lock = asyncio.Lock()
# Per-tick set of task_ids already handled by an earlier
# dispatcher. Reset at the start of every _dispatch_all_work.
# Consumed via `self._mark_task_handled` / `_is_task_handled`.
self._tick_handled_tasks: set[str] = set()
# Respawn circuit breaker: per (agent_slug, task_id), tracks how
# many times we've spawned without the task status changing. A PM
# that gets re-spawned on the same pending task with no progress
# is in a loop — without this gate the orchestrator re-spawns every
# tick forever (seen in production on 2026-04-22).
self._pm_respawn_tracker: dict[tuple[str, str], dict[str, Any]] = {}
# Board agents (Product Owner / Head of Marketing) get exactly ONE
# review pass per assigned task: they have no verb to claim, plan,
# delegate, or complete, so a respawn cannot advance the task and would
# just loop. Tracks (agent_slug, task_id) already dispatched.
self._board_dispatched: set[tuple[str, str]] = set()
# Cluster C5: a board review is a two-reviewer gate — BOTH the Product
# Owner and the Head of Marketing must review a board/coordination task
# before it is handed to the CEO for Approve & Start. Once both have
# finished (dispatched-and-no-longer-active), the orchestrator emits ONE
# formal CEO notification per task. Tracks task_ids already notified so
# the signal fires exactly once.
self._board_review_ceo_notified: set[str] = set()
# Stale-claim reaper config, sourced from
# stale_claim_reap_seconds (default 600) rather than
# claim_stale_seconds (default 180). The two settings are now
# distinct: claim_stale_seconds drives trigger_filter (spawn
# queueing); stale_claim_reap_seconds drives the reaper.
# Smoke run 3 showed agents reaped at 180s while actively retrying
# rejected verbs — LLM inference routinely exceeds that window.
# Tests bypass `__init__` via `__new__` and set _claim_heartbeat_ttl
# directly; production never uses _task_svc from __init__.
self._claim_heartbeat_ttl: int = settings.stale_claim_reap_seconds
# =========================================================================
# LIFECYCLE
# =========================================================================
async def start(self) -> None:
"""Start the orchestrator."""
self._running = True
# Ensure agent image is built
await self._ensure_agent_image()
# Restore any WaitingRecord rows left by a prior orchestrator run so
# agents that were WAITING_LONG at shutdown can still be resolved.
await self.restore_waiting_records()
# Self-heal: roll back orphan claims left over from a prior crash.
# Tasks that show CLAIMED/IN_PROGRESS but have NO
# branch_name set indicate _finalize_claim flushed the status before
# branch creation failed (before claim-rollback was atomic). Without
# this, the next claim attempt fails non-idempotent on `git checkout -b`.
await self._reconcile_orphan_claims_on_startup()
# Note: Per-agent settings are now generated at spawn time
# via _generate_agent_settings() - no shared settings needed
# Start background tasks
self._health_task = asyncio.create_task(self._health_loop())
self._dispatcher_task = asyncio.create_task(self._dispatcher_loop())
self._sweeper_task = asyncio.create_task(self._sweeper_loop())
self._rate_limit_probe_task = asyncio.create_task(self._rate_limit_probe_loop())
self._strategy_engine_task = asyncio.create_task(self._strategy_engine_loop())
self._external_pr_poll_task = asyncio.create_task(self._external_pr_poll_loop())
self._self_heal_task = asyncio.create_task(self._self_heal_loop())
logger.info(
"Orchestrator started",
dispatcher_interval=self.dispatcher_interval,
internal_api_url=self._api_url,
)
async def stop(self) -> None:
"""Stop the orchestrator and all agents."""
self._running = False
# Cancel background tasks
if self._health_task:
self._health_task.cancel()
with contextlib.suppress(asyncio.CancelledError):
await self._health_task
if self._dispatcher_task:
self._dispatcher_task.cancel()
with contextlib.suppress(asyncio.CancelledError):
await self._dispatcher_task
if self._sweeper_task:
self._sweeper_task.cancel()
with contextlib.suppress(asyncio.CancelledError):
await self._sweeper_task
if self._rate_limit_probe_task:
self._rate_limit_probe_task.cancel()
with contextlib.suppress(asyncio.CancelledError):
await self._rate_limit_probe_task
if self._strategy_engine_task:
self._strategy_engine_task.cancel()
with contextlib.suppress(asyncio.CancelledError):
await self._strategy_engine_task
if self._external_pr_poll_task:
self._external_pr_poll_task.cancel()
with contextlib.suppress(asyncio.CancelledError):
await self._external_pr_poll_task
if self._self_heal_task:
self._self_heal_task.cancel()
with contextlib.suppress(asyncio.CancelledError):
await self._self_heal_task
# Stop all agents
for agent_id in list(self._instances.keys()):
await self.stop_agent(agent_id)
logger.info("Orchestrator stopped")
async def _ensure_agent_image(self, agent_id: str | None = None) -> None:
"""Ensure the agent Docker images are present.
Local mode (no ``agent_image_registry``) builds the base image first,
then the role-specialized image, from ``docker/agent-*.Dockerfile``.
Registry mode pulls the pre-built images instead. Idempotent — skips
anything already present locally.
"""
# Determine build context
if PROJECT_HOST_PATH:
build_context = PROJECT_HOST_PATH
docker_dir = f"{PROJECT_HOST_PATH}/docker"
else:
build_context = str(self.project_root)
docker_dir = str(self.project_root / "docker")
# Always ensure base image exists
await self._ensure_image_present(
AGENT_BASE_IMAGE,
f"{docker_dir}/agent-base.Dockerfile",
build_context,
)
# Ensure the role-specialized image if this agent uses one
if agent_id:
bare = AGENT_IMAGES.get(agent_id, AGENT_BASE_IMAGE)
if bare != AGENT_BASE_IMAGE:
# Map the bare image name to its dockerfile
dockerfile_map = {
"roboco-agent-pm": "agent-pm.Dockerfile",
"roboco-agent-dev-be": "agent-dev-be.Dockerfile",
"roboco-agent-dev-fe": "agent-dev-fe.Dockerfile",
"roboco-agent-qa-be": "agent-qa-be.Dockerfile",
"roboco-agent-qa-fe": "agent-qa-fe.Dockerfile",
"roboco-agent-doc": "agent-doc.Dockerfile",
"roboco-agent-ux": "agent-ux.Dockerfile",
"roboco-agent-prompter": "agent-prompter.Dockerfile",
"roboco-agent-secretary": "agent-secretary.Dockerfile",
"roboco-agent-pr-reviewer": "agent-pr-reviewer.Dockerfile",
}
dockerfile = dockerfile_map.get(bare)
if dockerfile:
await self._ensure_image_present(
bare,
f"{docker_dir}/{dockerfile}",
build_context,
)
async def _ensure_image_present(
self, bare_image: str, dockerfile_path: str, build_context: str
) -> None:
"""Ensure one agent image is present locally.
Pulls it (registry mode) or builds it from its Dockerfile (local mode)
when missing; no-op if already present.
"""
image = _qualify_agent_image(bare_image)
# Check if image exists
proc = await asyncio.create_subprocess_exec(
"docker",
"image",
"inspect",
image,
stdout=asyncio.subprocess.DEVNULL,
stderr=asyncio.subprocess.DEVNULL,
)
await proc.wait()
if proc.returncode == 0:
return
if settings.agent_image_registry:
# Registry mode: pull the pre-built image; never build from source
# (a deployment running pre-built images has no build context).
logger.info("Pulling agent image...", image=image)
proc = await asyncio.create_subprocess_exec(
"docker",
"pull",
image,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
_, stderr = await proc.communicate()
if proc.returncode != 0:
raise RuntimeError(f"Failed to pull image {image}: {stderr.decode()}")
logger.info("Agent image pulled", image=image)
return
logger.info("Building Docker image...", image=image)
proc = await asyncio.create_subprocess_exec(
"docker",
"build",
"-t",
image,
"-f",
dockerfile_path,
build_context,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
_, stderr = await proc.communicate()
if proc.returncode != 0:
raise RuntimeError(f"Failed to build image {image}: {stderr.decode()}")
logger.info("Docker image built successfully", image=image)
# =========================================================================
# PER-AGENT SETTINGS GENERATION
# =========================================================================
def _get_role_permissions(
self, role: str, workspace_path: str, cell_workspace_path: str
) -> dict[str, list[str]]:
"""Get role-specific allow/deny lists for Claude Code tools.
Post-gateway shape: every state-changing operation an agent can
perform routes through ``mcp__roboco-flow__*`` (intent verbs) or
``mcp__roboco-do__*`` (content tools — commit, push, PR, journal,
notify, message), both granted to every role via ``base_allow``.
Role-specific configuration here only governs file IO (Write/Edit
scoping) plus a small handful of legacy native-tool denies that
remain meaningful for weak models. Read-only git lives in
``mcp__roboco-git-readonly__*``.
Args:
role: Agent role (developer, qa, documenter, cell_pm, main_pm, etc.)
workspace_path: Path to agent's own workspace directory
cell_workspace_path: Path to cell's workspace root (for QA/Docs access)
Returns:
Dict with 'allow' and 'deny' lists for Claude Code permissions
"""
# workspace_path: /data/workspaces/{project}/{team}/{agent}
# cell_workspace_path: /data/workspaces/{project}/{team}
configs: dict[str, dict[str, list[str]]] = {
"developer": {
"allow": [
f"Write(/{workspace_path}/**)",
f"Edit(/{workspace_path}/**)",
],
"deny": [],
},
"qa": {
# QA reads code + the open PR via the gateway; never edits.
"allow": [],
"deny": [
"Write(*)",
"Edit(*)",
],
},
"documenter": {
"allow": [
f"Write(/{cell_workspace_path}/**)",
f"Edit(/{cell_workspace_path}/**)",
"Write(//app/docs/**)",
"Edit(//app/docs/**)",
"Write(//app/CHANGELOG.md)",
"Edit(//app/CHANGELOG.md)",
"Write(//app/README.md)",
"Edit(//app/README.md)",
],
"deny": [],
},
"cell_pm": {
# PMs coordinate; they open + merge PRs through the gateway
# but never author code. Edit/Write are denied so weaker
# models can't read the subtask title imperatively and
# start editing source — they have to decompose into a dev
# subtask. Devs are the only role that authors code.
"allow": [],
"deny": [
"Bash(git commit:*)",
"Bash(git push:*)",
"Write(*)",
"Edit(*)",
],
},
"main_pm": {
# Same reasoning as cell_pm — Main PM sits between CEO and
# cell PMs; the work product is coordination + review, not
# commits or edits. Code work routes Main PM → Cell PM →
# Dev only.
"allow": [],
"deny": [
"Bash(git commit:*)",
"Bash(git push:*)",
"Write(*)",
"Edit(*)",
],
},
"product_owner": {
"allow": [
f"Write(/{workspace_path}/**)",
f"Edit(/{workspace_path}/**)",
],
"deny": [],
},
"head_marketing": {
"allow": [
f"Write(/{workspace_path}/**)",
f"Edit(/{workspace_path}/**)",
],
"deny": [],
},
"auditor": {
# Auditor is read-only across the org — observes, never edits.
"allow": [],
"deny": [
"Write(*)",
"Edit(*)",
],
},
"pr_reviewer": {
# PR reviewer reads untrusted external/fork PR diffs and posts a
# change-request via the gateway — it never writes files. Make the
# read-only invariant explicit at the permission layer (it is the
# highest-value prompt-injection target), not just implicit in the
# absence of a writable mount.
"allow": [],
"deny": [
"Write(*)",
"Edit(*)",
],
},
}
if role not in configs:
logger.warning(
"No Claude Code permissions configured for role; "
"agent will be limited to base_allow/base_deny.",
role=role,
)
return configs.get(role, {"allow": [], "deny": []})
def _generate_agent_settings(
self,
agent_id: str,
role: str,
workspace_path: str,
cell_workspace_path: str,
) -> Path:
"""Generate per-agent Claude Code settings file with role-specific permissions.
This replaces the shared settings approach. Each agent gets their own
settings.json with:
- Base MCP tools allowed for all agents
- Role-specific tool permissions
- Explicit deny list blocking native git/file operations
Args:
agent_id: Agent identifier (e.g., "be-dev-1")
role: Agent role (e.g., "developer")
workspace_path: Path to agent's own workspace directory
cell_workspace_path: Path to cell's workspace root (for QA/Docs)
Returns:
Path to the generated settings file
"""
# Base MCP tools for all agents. Post-gateway every role gets the
# full intent-verb + content-tool surface; the orchestrator-side
# API rejects verbs/tools the agent's role isn't authorized for,
# so granting `*` here is safe.
base_allow = [
"mcp__roboco-flow__*",
"mcp__roboco-do__*",
"mcp__roboco-optimal__*",
"mcp__roboco-git-readonly__*",
"Read(*)", # All agents can read any file
]
# Base denials for all agents - block native tools + sensitive reads.
# The Read/Bash denies below are critical: without them an agent can
# read `.git/config` (which, pre-fix, had the PAT embedded in the
# remote URL) or `~/.gitconfig` and exfiltrate project secrets.
# We also block direct curl/wget to github.com — any git-remote op
# must go through the orchestrator's git service, which injects the
# token via bearer header at subprocess time rather than exposing it.
base_deny = [
# Block ALL native git commands - must use roboco_git_* tools
"Bash(git:*)",
# NOTE: Write/Edit are intentionally NOT globally denied here.
# Claude Code evaluates rules deny -> ask -> allow and the first
# match wins, so a deny ALWAYS beats a more-specific allow (the
# glob syntax has no negation). A global Write(*)/Edit(*) here
# therefore unconditionally shadowed the per-role,
# workspace-scoped Write/Edit allows below — every agent (devs
# included) was unable to edit ANY file and fell back to
# destructive bash redirection (clobbering real files). Roles
# that must NOT write (qa, cell_pm, main_pm, auditor) carry
# their own Write(*)/Edit(*) deny in _get_role_permissions.
# Block reads of credential stores, anywhere on the FS
"Read(**/.git/config)",
"Read(**/.gitconfig)",
"Read(/etc/gitconfig)",
"Read(~/.netrc)",
"Read(**/.git-credentials)",
# Block direct GitHub API/wire access — agents must use
# roboco_git_* MCP tools so secrets + traceability stay on the
# orchestrator side.
"Bash(curl:*github.com*)",
"Bash(curl:*api.github.com*)",
"Bash(wget:*github.com*)",
"Bash(wget:*api.github.com*)",
# Same idea for cat-ing credential files in a subshell
"Bash(cat:*.git/config*)",
"Bash(cat:*.gitconfig*)",
"Bash(cat:*.git-credentials*)",
# Block reading env vars that might leak secrets
"Bash(env:*)",
"Bash(printenv:*)",
]
# Get role-specific permissions
role_config = self._get_role_permissions(
role, workspace_path, cell_workspace_path
)
# Combine base + role-specific.
# defaultMode=bypassPermissions lets unlisted operations proceed
# without an interactive prompt (which would hang a non-TTY agent
# container). Explicit deny rules still apply.
settings: dict[str, Any] = {
"permissions": {
"defaultMode": "bypassPermissions",
"allow": base_allow + role_config["allow"],
"deny": base_deny + role_config["deny"],
},
"hooks": {
# Start SDK server on session start (for A2A communication)
"SessionStart": [
{
"hooks": [
{
"type": "command",
"command": "/app/scripts/sdk-startup-hook.sh",
}
]
}
],
# Guard Bash: block shell-level git/curl/wget/env patterns
# that the matcher-based `permissions.deny` can't catch
# (e.g. `cd X && git fetch`). Redirects agents to the MCP
# equivalents instead of bloating prompts with rules.
"PreToolUse": [
{
"matcher": "Bash",
"hooks": [
{
"type": "command",
"command": "/app/scripts/bash-guard-hook.sh",
}
],
},
],
"PostToolUse": [
# Check for incoming A2A messages after each tool use
{
"matcher": "*",
"hooks": [
{
"type": "command",
"command": "/app/scripts/a2a-check-hook.sh",
}
],
},
# Per-session budget counter + loop detector. Shared SDK
# state lets this hook emit [Budget]/[Loop]/[Halt]
# reminders that the orchestrator's kill-switch corroborates.
{
"matcher": "*",
"hooks": [
{
"type": "command",
"command": "/app/scripts/post-tool-budget-hook.sh",
}
],
},
# Sync token usage from the transcript so /usage/status
# (and the cost dashboard) reflect real spend. Idempotent
# absolute set — running it per tool keeps mid-run
# snapshots and reaped-agent sessions accurate.
{
"matcher": "*",
"hooks": [
{
"type": "command",
"command": "/app/scripts/usage-report-hook.sh",
}
],
},
],
# Stop guard: refuse silent exits unless a terminal tool was
# just called (idle/substitute/escalate/pause/...). Second
# attempt auto-substitutes via SDK so the task doesn't rot.
"Stop": [
{
"hooks": [
{
"type": "command",
"command": "/app/scripts/stop-hook.sh",
},
# Final token-usage sync at turn end — guarantees
# the session total is captured before the agent
# idles and the orchestrator finalizes the row.
{
"type": "command",
"command": "/app/scripts/usage-report-hook.sh",
},
]
}
],
# Prompt-injection guard — rejects turns that look like
# another agent's content trying to override our rules.
"UserPromptSubmit": [
{
"hooks": [
{
"type": "command",
"command": "/app/scripts/user-prompt-hook.sh",
}
]
}
],
# Snapshot budget / terminal state before compact so the
# next session resumes with continuity.
"PreCompact": [
{
"hooks": [
{
"type": "command",
"command": "/app/scripts/pre-compact-hook.sh",
}
]
}
],
# Post-mortem: write a reflect-journal entry summarising the
# session (tools called, halt/loop triggered, last tool).
"SessionEnd": [
{
"hooks": [
{
"type": "command",
"command": "/app/scripts/session-end-hook.sh",
}
]
}
],
},
}
# Write to per-agent settings file
# When running in container: write to /app/agent-settings (mounted to host)
# When running on host: use temp directory
if DATA_HOST_PATH:
settings_dir = Path("/app/agent-settings")
else:
settings_dir = Path(tempfile.gettempdir()) / "roboco-agent-settings"
settings_dir.mkdir(parents=True, exist_ok=True)
settings_path = settings_dir / f"{agent_id}-settings.json"
# Handle case where Docker auto-created a directory instead of a file
if settings_path.is_dir():
shutil.rmtree(settings_path)
settings_path.write_text(json.dumps(settings, indent=2))
logger.debug(
"Generated per-agent settings",
agent_id=agent_id,
role=role,
settings_path=str(settings_path),
allow_count=len(settings["permissions"]["allow"]),
deny_count=len(settings["permissions"]["deny"]),
)
return settings_path
# =========================================================================
# AGENT SPAWNING
# =========================================================================
def _task_git_context(self, task: dict[str, Any]) -> SpawnGitContext | None:
"""Build SpawnGitContext from a task dict for workspace mounting.
Without this, spawned agents fall back to project_slug="default"
and get a Write/Edit permission lock to /data/workspaces/default/...
which does not exist, so the agent's file tools fail.
"""
project_slug = task.get("project_slug")
if not project_slug:
return None
return SpawnGitContext(
project_slug=project_slug,
branch_name=task.get("branch_name"),
)
def _fire_audit(
self,
*,
event_type: str,
agent_slug: str,
task_id: str | None = None,
details: dict[str, Any] | None = None,
severity: str = "info",
) -> None:
"""Emit an agent-lifecycle audit event without blocking the caller.
Strong-refs the Task so it isn't garbage-collected before it
commits to `audit_log`. Silently skips if there's no running loop
(e.g. sync unit tests).
"""
import contextlib as _ctx
from roboco.services.audit import get_audit_service
with _ctx.suppress(RuntimeError):
bg = asyncio.get_running_loop().create_task(
get_audit_service().log_agent_event(
event_type=event_type,
agent_slug=agent_slug,
task_id=task_id,
details=details or {},
severity=severity,
)
)
self._bg_tasks.add(bg)
bg.add_done_callback(self._bg_tasks.discard)
async def _git_context_default_project(self) -> SpawnGitContext | None:
"""Return git context for the 'default' project when no task is known.
Used by no-task spawns (idle PM, scanner-only agents). Picks the
first active project in the DB — the common case is a single-project
deployment, where this resolves to the correct slug; for multi-
project deployments the caller should pass task_id to disambiguate.
"""
from sqlalchemy import select
from roboco.db.base import get_db_context
from roboco.db.tables import ProjectTable
try:
async with get_db_context() as db:
result = await db.execute(
select(ProjectTable.slug, ProjectTable.default_branch)
.where(ProjectTable.is_active.is_(True))
.order_by(ProjectTable.created_at.asc())
.limit(1)
)
row = result.first()
if row is None:
return None
slug, default_branch = row
if not slug:
return None
return SpawnGitContext(
project_slug=slug,
branch_name=default_branch,
)
except Exception as e:
logger.warning(
"Could not derive default project git context",
error=str(e),
)
return None
async def _git_context_from_task_id(self, task_id: str) -> SpawnGitContext | None:
"""Load a task by ID and derive git context for spawning.
Used by `spawn_agent` when called without an explicit git_context
(e.g. the /agents/{slug}/spawn API endpoint). Without this, agents
spawned via that endpoint get project_slug="default" and their
workspace mount points at a path that doesn't exist.
"""
from sqlalchemy import select
from roboco.db.base import get_db_context
from roboco.db.tables import ProjectTable, TaskTable
try:
async with get_db_context() as db:
result = await db.execute(
select(TaskTable.branch_name, ProjectTable.slug)
.select_from(TaskTable)
.join(ProjectTable, TaskTable.project_id == ProjectTable.id)
.where(TaskTable.id == task_id)
)
row = result.first()
if row is None:
return None
branch_name, project_slug = row
if not project_slug:
return None
return SpawnGitContext(
project_slug=project_slug,
branch_name=branch_name,
)
except Exception as e:
logger.warning(
"Could not derive git context from task_id",
task_id=task_id,
error=str(e),
)
return None
async def _safe_spawn(
self,
*,
agent_id: str,
task_id: str | None = None,
initial_prompt: str | None = None,
git_context: SpawnGitContext | None = None,
context_label: str = "dispatcher",
) -> AgentInstance | None:
"""Spawn an agent, absorbing errors so one bad spawn doesn't abort the
rest of the dispatcher's loop.
Each dispatcher iterates many tasks; if `spawn_agent` raised, the
remaining tasks were skipped until the next tick. This wrapper logs
and returns None on failure so siblings still get dispatched.
The gateway pre-spawn check runs first; a QUEUE or DROP outcome skips
the container launch.
"""
# Gateway pre-spawn cooldown gate.
target_role = get_agent_role(agent_id) or "unknown"
# Map context_label to one of the TriggerKind string values; unknown
# labels fall back to "scan" which is the least-specific kind.
trigger_kind_map = {
"a2a": "a2a",
"escalation": "escalation",
"notification": "notification",
}
trigger_kind = trigger_kind_map.get(context_label, "scan")
outcome, reason = await gateway_pre_spawn_check(
task_id=task_id,
trigger_kind=trigger_kind,
target_role=target_role,
provider=self.get_provider_for_agent(agent_id),
)
if outcome != "spawn":
logger.info(
"Gateway pre-spawn check suppressed spawn",
agent_id=agent_id,
task_id=task_id,
outcome=outcome,
reason=reason,
)
return None
try:
return await self.spawn_agent(
agent_id=agent_id,
task_id=task_id,
initial_prompt=initial_prompt,
git_context=git_context,
)
except Exception as e:
logger.error(
"Spawn failed during dispatch; continuing with next task",
context=context_label,
agent_id=agent_id,
task_id=task_id,
error=str(e),
)
return None
async def _resolve_spawn_git_context(
self,
git_context: SpawnGitContext | None,
task_id: str | None,
) -> SpawnGitContext | None:
"""Auto-derive git context if the caller didn't supply one."""
if git_context is not None and git_context.project_slug:
return git_context
derived: SpawnGitContext | None = None
if task_id:
derived = await self._git_context_from_task_id(task_id)
if derived is None:
derived = await self._git_context_default_project()
return derived if derived is not None else git_context
def _existing_running_instance(self, agent_id: str) -> AgentInstance | None:
"""Return the running instance for agent_id, or None if it can be respawned."""
existing = self._instances.get(agent_id)
if existing is None:
return None
if existing.state in (AgentState.OFFLINE, AgentState.WAITING_LONG):
return None
logger.warning(
"Agent already running",
agent_id=agent_id,
state=existing.state,
)
return existing
def _resolve_project_slug(
self,
git_context: SpawnGitContext | None,
agent_id: str,
task_id: str | None,
) -> str:
"""Pull project_slug from context, or fall back to 'default' with a warning."""
project_slug = (
git_context.project_slug
if git_context and git_context.project_slug
else None
)
if not project_slug:
logger.warning(
"Spawning agent without project_slug; workspace fallback used. "
"Agent file tools will be locked to a nonexistent path.",
agent_id=agent_id,
task_id=task_id,
)
project_slug = "default"
return project_slug
async def _prepare_agent_spawn(
self,
agent_id: str,
task_id: str | None,
model: str | None,
git_context: SpawnGitContext | None,
) -> tuple[AgentConfig, AgentInstance, Path | None]:
"""Build AgentConfig + AgentInstance and surface per-agent settings path."""
blueprint_path = self._generate_composed_prompt(agent_id)
canonical_role = get_agent_role(agent_id)
team = get_agent_team(agent_id) or "backend"
# Resolve the provider route for this agent. Caller-supplied `model`
# wins (dispatcher overrides, tests). Otherwise the routing service
# resolves (agent_slug | role | global) assignments, falling back
# internally to `ROLE_MODEL_MAP` when no rows exist — so a fresh
# deployment with an empty `model_assignments` table behaves exactly
# as before.
route = await self._resolve_agent_route(agent_id)
if not model:
model = route.model_name
project_slug = self._resolve_project_slug(git_context, agent_id, task_id)
workspace_path = _agent_workspace_path(project_slug, team, agent_id)
cell_workspace_path = _cell_workspace_path(project_slug, team)
agent_settings_path = self._generate_agent_settings(
agent_id, canonical_role, workspace_path, cell_workspace_path
)
briefing_path = await self._write_agent_briefing(
agent_id, task_id, workspace_path
)
await self._ensure_agent_image(agent_id)
mcp_config_path = await self._generate_mcp_config(agent_id, git_context)
from uuid import uuid4
config = AgentConfig(
agent_id=agent_id,
blueprint_path=blueprint_path,
model=model,
mcp_config_path=mcp_config_path,
claude_session_id=str(uuid4()),
git_context=git_context,
briefing_path=briefing_path,
provider_type=route.provider_type.value,
provider_base_url=route.base_url,
provider_auth_token=route.auth_token,
)
instance = AgentInstance(
agent_id=agent_id,
state=AgentState.STARTING,
config=config,
current_task_id=task_id,
)
self._instances[agent_id] = instance
return config, instance, agent_settings_path
async def _launch_spawn(
self,
task_id: str | None,
config: AgentConfig,
instance: AgentInstance,
initial_prompt: str | None,
agent_settings_path: Path | None,
) -> AgentInstance:
"""Launch the container and emit spawn audit events.
`agent_id` was dropped as a redundant parameter — `config.agent_id`
is the same value and was always the caller's source.
"""
agent_slug = config.agent_id
try:
container_id = await self._spawn_container(
config, initial_prompt, agent_settings_path
)
instance.container_id = container_id
instance.state = AgentState.ACTIVE
instance.started_at = datetime.now(UTC)
instance.last_activity = datetime.now(UTC)
logger.info(
"Agent spawned",
agent_id=agent_slug,
container_id=container_id[:12],
model=config.model,
task_id=task_id,
)
self._fire_audit(
event_type="agent.spawned",
agent_slug=agent_slug,
task_id=task_id,
details={
"container_id": container_id[:12],
"model": config.model,
},
)
# Record a token-usage session row in the DB and bind its UUID to
# the instance so _finalize_spawn_session can look it up directly.
usage_session_id = await self._record_spawn_session(config, task_id)
if usage_session_id is not None:
instance.usage_session_id = usage_session_id
return instance
except Exception as e:
instance.state = AgentState.OFFLINE
instance.error_count += 1
logger.error(
"Failed to spawn agent",
agent_id=agent_slug,
error=str(e),
)
self._fire_audit(
event_type="agent.spawn_failed",
agent_slug=agent_slug,
task_id=task_id,
details={"error": str(e)},
severity="error",
)
raise
async def spawn_agent(
self,
agent_id: str,
initial_prompt: str | None = None,
task_id: str | None = None,
model: str | None = None,
git_context: SpawnGitContext | None = None,
) -> AgentInstance:
"""
Spawn a Claude Code container for an agent.
Args:
agent_id: Agent identifier (e.g., "be-dev-1")
initial_prompt: Optional initial prompt
task_id: Optional task ID being worked on
model: Override model selection
git_context: Optional git context (project_slug, branch_name)
Returns:
AgentInstance handle
Raises:
AgentReadinessError: task is not spawn-ready (missing criteria,
missing git token, no branch plan, role mismatch). The task
is auto-blocked before we raise so the dispatcher doesn't
keep retrying.
"""
# Pre-flight: refuse to spawn if the task isn't ready. Auto-block
# on refusal so the dispatcher doesn't keep spinning a container
# that will immediately fail (wasted image pull + startup tokens).
readiness_reason = await self._readiness_gate(agent_id, task_id)
if readiness_reason:
raise AgentReadinessError(
f"spawn refused for {agent_id} (task={task_id}): {readiness_reason}"
)
# Auto-derive git_context when the caller didn't supply one. Two
# paths:
# (a) task_id present → look up the task's project;
# (b) no task_id → fall back to the sole active project (or
# the first one if there are multiple).
# Without (b), no-task spawns (e.g. idle PM bootstrapping) hit the
# "workspace fallback used" path and get mounted at
# /data/workspaces/default/... which doesn't exist.
git_context = await self._resolve_spawn_git_context(git_context, task_id)
async with self._lock:
existing = self._existing_running_instance(agent_id)
if existing is not None:
return existing
config, instance, agent_settings_path = await self._prepare_agent_spawn(
agent_id, task_id, model, git_context
)
# Record the task as handled so later dispatchers in the same
# tick don't act on it again. Safe even if _launch_spawn fails
# — the next tick starts fresh.
self._mark_task_handled(task_id)
return await self._launch_spawn(
task_id,
config,
instance,
initial_prompt,
agent_settings_path,
)
def _resolve_host_paths(
self, config: AgentConfig, agent_settings_path: Path | None
) -> dict[str, str | None]:
"""Compute host mount paths for both containerized and host runtime."""
mcp_name = config.mcp_config_path.name if config.mcp_config_path else ""
if PROJECT_HOST_PATH:
return {
"docs": f"{PROJECT_HOST_PATH}/docs",
"workspaces": f"{DATA_HOST_PATH}/workspaces",
"claude": CLAUDE_AUTH_HOST_PATH,
"mcp_config": f"{DATA_HOST_PATH}/mcp-configs/{mcp_name}",
"prompt": (
f"{DATA_HOST_PATH}/prompts-generated/{config.agent_id}-prompt.md"
),
"settings": (
f"{DATA_HOST_PATH}/agent-settings/{config.agent_id}-settings.json"
if agent_settings_path
else None
),
"briefing": (
f"{DATA_HOST_PATH}/briefings/{config.agent_id}.md"
if config.briefing_path
else None
),
}
return {
"docs": str((self.project_root / "docs").absolute()),
"workspaces": str(Path(settings.workspaces_root)),
"claude": CLAUDE_AUTH_HOST_PATH,
"mcp_config": str(config.mcp_config_path),
"prompt": str(
Path(tempfile.gettempdir())
/ "roboco-prompts"
/ f"{config.agent_id}-prompt.md"
),
"settings": str(agent_settings_path) if agent_settings_path else None,
"briefing": (str(config.briefing_path) if config.briefing_path else None),
}
@staticmethod
def _build_mount_args(
container_name: str, config: AgentConfig, hosts: dict[str, str | None]
) -> list[str]:
"""Compose `docker run -v/-e` mount + env args for the agent."""
cmd: list[str] = [
"docker",
"run",
"-d",
"--name",
container_name,
"--network",
AGENT_NETWORK,
# Mount Claude auth directory (for API keys, etc.)
"-v",
f"{hosts['claude']}:/home/agent/.claude",
]
AgentOrchestrator._append_claude_json_mount(cmd, hosts)
AgentOrchestrator._append_optional_host_mounts(cmd, hosts)
role = get_agent_role(config.agent_id) or "developer"
cmd.extend(AgentOrchestrator._core_volume_and_env_args(config, hosts, role))
AgentOrchestrator._append_provider_env(cmd, config)
subagent_model = _resolve_agent_cli_model(config.provider_type, config.model)
cmd.extend(["-e", f"CLAUDE_CODE_SUBAGENT_MODEL={subagent_model}"])
AgentOrchestrator._append_manifest_args(cmd, config, subagent_model)
AgentOrchestrator._append_workspace_cwd(cmd, config)
return cmd
@staticmethod
def _append_claude_json_mount(cmd: list[str], hosts: dict[str, str | None]) -> None:
"""Mount host's ~/.claude.json sibling FILE if present."""
claude_dir = hosts["claude"]
if not claude_dir:
return
claude_json_host = f"{claude_dir.rstrip('/')}.json"
if Path(claude_json_host).exists():
cmd.extend(["-v", f"{claude_json_host}:/home/agent/.claude.json"])
@staticmethod
def _append_optional_host_mounts(
cmd: list[str], hosts: dict[str, str | None]
) -> None:
"""Mount agent settings.json and briefing.md when their hosts exist."""
settings_host = hosts.get("settings")
if settings_host:
cmd.extend(["-v", f"{settings_host}:/home/agent/.claude/settings.json:ro"])
briefing_host = hosts.get("briefing")
if briefing_host:
cmd.extend(["-v", f"{briefing_host}:/app/briefing.md:ro"])
@staticmethod
def _core_volume_and_env_args(
config: AgentConfig, hosts: dict[str, str | None], role: str
) -> list[str]:
"""The always-on -v/-e block (prompt, docs, workspaces, env)."""
docs_ro = "" if config.agent_id in ALL_DOCS else ":ro"
return [
"-v",
f"{hosts['prompt']}:/app/system-prompt.md:ro",
"-v",
f"{hosts['docs']}:/app/docs{docs_ro}",
"-v",
f"{hosts['workspaces']}:/data/workspaces",
"-v",
f"{hosts['mcp_config']}:/app/mcp-config.json:ro",
"-e",
f"ROBOCO_AGENT_ID={config.agent_id}",
"-e",
f"ROBOCO_AGENT_ROLE={role}",
"-e",
"ROBOCO_API_URL=http://roboco-orchestrator:8000",
"-e",
"ROBOCO_SDK_PORT=9000",
"-e",
"ROBOCO_SDK_URL=http://localhost:9000",
"-e",
f"ROBOCO_AGENT_TOOL_CALL_WARN={settings.agent_tool_call_warn}",
"-e",
f"ROBOCO_AGENT_TOOL_CALL_HALT={settings.agent_tool_call_halt}",
"-e",
f"ROBOCO_AGENT_LOOP_THRESHOLD={settings.agent_loop_threshold}",
"-e",
f"ROBOCO_AGENT_LOOP_WINDOW={settings.agent_loop_window}",
"-e",
f"ROBOCO_AGENT_STOP_ATTEMPT_ALLOWANCE={settings.agent_stop_attempt_allowance}",
]
@staticmethod
def _append_provider_env(cmd: list[str], config: AgentConfig) -> None:
"""Inject ANTHROPIC_* env only on non-Anthropic providers."""
# Provider routing: only inject ANTHROPIC_* env vars when the
# resolved provider is non-Anthropic (i.e. Ollama Cloud). For the
# Anthropic default path both fields are None and Claude Code
# inside the container continues to use its mounted ~/.claude
# credentials — preserving legacy behaviour byte-for-byte.
if config.provider_base_url:
cmd.extend(["-e", f"ANTHROPIC_BASE_URL={config.provider_base_url}"])
if config.provider_auth_token:
cmd.extend(["-e", f"ANTHROPIC_AUTH_TOKEN={config.provider_auth_token}"])
@staticmethod
def _append_manifest_args(
cmd: list[str], config: AgentConfig, subagent_model: str
) -> None:
"""Write the spawn manifest and flip the gateway flag."""
# Spawn manifest + gateway flag — developer role only in Phase 1.
# _build_manifest_for_agent writes the JSON file to the host and
# returns the path; other roles get None and the gateway flag stays off.
manifest_host_path = _build_manifest_for_agent(config.agent_id, subagent_model)
if manifest_host_path:
cmd.extend(
[
"-v",
f"{manifest_host_path}:/app/tool-manifest.json:ro",
"-e",
"ROBOCO_GATEWAY_ENABLED=true",
"-e",
"ROBOCO_TOOL_MANIFEST_PATH=/app/tool-manifest.json",
]
)
else:
cmd.extend(["-e", "ROBOCO_GATEWAY_ENABLED=false"])
_ROLES_WITH_AGENT_WORKSPACE: ClassVar[frozenset[str]] = frozenset(
{"developer", "product_owner", "head_marketing"}
)
_ROLES_WITH_CELL_WORKSPACE: ClassVar[frozenset[str]] = frozenset({"documenter"})
@staticmethod
def _append_workspace_cwd(cmd: list[str], config: AgentConfig) -> None:
"""Set the container -w to the agent or cell workspace by role."""
# Pre-gateway parity: set the container's cwd
# to the agent's task workspace so Edit/Write resolve to paths that
# match _get_role_permissions allowlist, and `git add` operates inside
# the workspace clone. Without this, container WORKDIR (/app from the
# Dockerfile) shadows the workspace and every file op fails.
#
# Mirror the workspace-path selection in _get_role_permissions exactly:
# - developer / product_owner / head_marketing: per-agent workspace
# - documenter: cell workspace
# - qa / cell_pm / main_pm / auditor: no write workspace → omit -w
role = get_agent_role(config.agent_id) or "developer"
team = get_agent_team(config.agent_id) or ""
project = _resolve_project_slug_from_git_context(config.git_context)
if role in AgentOrchestrator._ROLES_WITH_AGENT_WORKSPACE:
cmd.extend(["-w", _agent_workspace_path(project, team, config.agent_id)])
elif role in AgentOrchestrator._ROLES_WITH_CELL_WORKSPACE:
cmd.extend(["-w", _cell_workspace_path(project, team)])
@staticmethod
def _append_agent_auth_env(cmd: list[str], config: AgentConfig) -> None:
"""Append agent HMAC token env var to the docker run cmd."""
# Agent HMAC auth token — bound to (agent_id, role, team). The
# API middleware refuses requests whose headers don't match the
# token, which stops one agent on the Docker network from
# spoofing another agent's role. Token is stable per agent as
# long as the secret doesn't rotate, so it's fine to compute at
# spawn time and inject once.
from roboco.agents_config import (
get_agent_role as _get_role,
)
from roboco.agents_config import (
get_agent_team as _get_team,
)
from roboco.agents_config import (
issue_agent_token,
)
_role = _get_role(config.agent_id)
_team = _get_team(config.agent_id) or ""
_token = issue_agent_token(config.agent_id, _role, _team)
cmd.extend(["-e", f"ROBOCO_AGENT_TOKEN={_token}"])
@staticmethod
def _append_git_context_env(cmd: list[str], config: AgentConfig) -> None:
"""Append git-context env vars to the docker run cmd."""
if not config.git_context:
return
if config.git_context.project_slug:
cmd.extend(["-e", f"ROBOCO_PROJECT_SLUG={config.git_context.project_slug}"])
if config.git_context.branch_name:
cmd.extend(["-e", f"ROBOCO_BRANCH={config.git_context.branch_name}"])
@staticmethod
def _default_spawn_prompt() -> str:
"""Fallback prompt when the caller provided none."""
return (
"You may have been spawned without a specific task assignment. "
"Follow your standard workflow:\n\n"
"1. Call `give_me_work()` to find work for your role\n"
"2. Begin the assigned task (its details arrive in the "
"response): UNDERSTAND -> PLAN -> EXECUTE -> VERIFY -> HANDOFF\n"
"3. If no tasks available, call `i_am_idle()` "
"to shutdown gracefully\n\n"
"Start now by scanning for work."
)
@classmethod
def _append_image_and_claude_args(
cls, cmd: list[str], config: AgentConfig, initial_prompt: str | None
) -> None:
"""Append the image + Claude Code CLI args to the docker run cmd.
`--tools` explicitly enumerates the built-in tools loaded at session
start. Without it, Claude CLI's default behavior leaves Edit/Write
in the deferred pool, so an agent that doesn't reliably call
ToolSearch (e.g. weaker non-Anthropic models routed via
Ollama-cloud) ends up unable to modify any file. The set below is
the minimum every agent role needs:
- Read/Write/Edit : file IO inside the workspace
- Bash : shell commands (gated by bash-guard hook)
- Grep/Glob : code navigation
- TodoWrite : per-session planning
Permissions still gate *which* paths Edit/Write can touch (see
`_get_role_permissions`), so this is purely about loading vs
denying.
"""
claude_args = [
get_agent_image(config.agent_id),
"--model",
cls._resolve_cli_model(config),
"--system-prompt-file",
"/app/system-prompt.md",
"--mcp-config",
"/app/mcp-config.json",
"--strict-mcp-config",
"--tools",
"Read,Write,Edit,Bash,Grep,Glob,TodoWrite",
"--output-format",
"stream-json",
"--verbose",
]
# Pin the Claude session id so the agent's transcript is locatable by id
# at finalize, regardless of which project/cwd dir Claude Code writes it
# to (review/coordinate roles run at /app, not a per-agent workspace).
if config.claude_session_id:
claude_args += ["--session-id", config.claude_session_id]
claude_args += ["-p", initial_prompt or cls._default_spawn_prompt()]
cmd.extend(claude_args)
@staticmethod
def _resolve_cli_model(config: AgentConfig) -> str:
"""Return the string to pass to `claude --model`."""
return _resolve_agent_cli_model(config.provider_type, config.model)
def _ensure_provider_registry(self) -> "ProviderRegistry":
"""Build (once) the registry of dedicated provider backends.
Only providers that need a runtime other than the built-in Claude Code
container are registered. Today that is GROK (xAI, OpenAI protocol).
"""
if self._provider_registry is None:
from roboco.llm.providers import GrokProvider, ProviderRegistry
from roboco.models.base import ModelProvider
registry = ProviderRegistry()
registry.register(ModelProvider.GROK, GrokProvider(self))
self._provider_registry = registry
return self._provider_registry
def _provider_for(self, provider_type: str) -> "AgentProvider | None":
"""Resolve a dedicated provider for a route's ``provider_type`` string.
Returns ``None`` for providers that use the built-in Claude Code spawn
(Anthropic / Ollama Cloud / self-hosted) or any unrecognised value — the
caller then runs the existing container path unchanged.
"""
from roboco.models.base import ModelProvider
try:
model_provider = ModelProvider(provider_type)
except ValueError:
return None
return self._ensure_provider_registry().get_or_none(model_provider)
async def _spawn_container(
self,
config: AgentConfig,
initial_prompt: str | None = None,
agent_settings_path: Path | None = None,
) -> str:
"""Spawn a Docker container for the agent.
Args:
config: Agent configuration
initial_prompt: Optional initial prompt for the agent
agent_settings_path: Path to per-agent Claude settings file
"""
# A dedicated provider backend (e.g. GROK / OpenAI protocol) handles its
# own spawn. Anthropic / Ollama Cloud / self-hosted have no dedicated
# provider registered and fall through to the Claude Code body below,
# byte-for-byte unchanged.
provider = self._provider_for(config.provider_type)
if provider is not None:
result = await provider.spawn(config, initial_prompt, agent_settings_path)
return result.instance_id
container_name = f"roboco-agent-{config.agent_id}"
await self._remove_container(container_name)
if not config.mcp_config_path:
raise RuntimeError("MCP config path not set")
hosts = self._resolve_host_paths(config, agent_settings_path)
cmd = self._build_mount_args(container_name, config, hosts)
self._append_agent_auth_env(cmd, config)
self._append_git_context_env(cmd, config)
self._append_image_and_claude_args(cmd, config, initial_prompt)
proc = await asyncio.create_subprocess_exec(
*cmd,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
stdout, stderr = await proc.communicate()
if proc.returncode != 0:
raise RuntimeError(f"Failed to start container: {stderr.decode()}")
return stdout.decode().strip()
async def _remove_container(self, container_name: str) -> None:
"""Remove a container if it exists, dumping its logs to disk first.
Docker deletes the container's json-file log when we `docker rm`, so
before removal we copy the current log to /data/logs/agents/{slug}/
with a timestamp. That gives us persistent history across respawns
without needing an entrypoint wrapper inside the agent image.
"""
# Check the container actually exists before trying to dump logs;
# _remove_container is routinely called pre-spawn to clear stale
# containers, and on first spawn there's nothing to dump.
inspect = await asyncio.create_subprocess_exec(
"docker",
"inspect",
"--format={{.Id}}",
container_name,
stdout=asyncio.subprocess.DEVNULL,
stderr=asyncio.subprocess.DEVNULL,
)
exists = (await inspect.wait()) == 0
if exists:
slug = container_name.removeprefix("roboco-agent-")
log_dir = Path("/data/logs/agents") / slug
try:
log_dir.mkdir(parents=True, exist_ok=True)
timestamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
log_path = log_dir / f"{timestamp}.log"
with log_path.open("wb") as out:
dump_proc = await asyncio.create_subprocess_exec(
"docker",
"logs",
container_name,
stdout=out,
stderr=out,
)
await dump_proc.wait()
if log_path.stat().st_size == 0:
log_path.unlink(missing_ok=True)
except Exception as e:
logger.warning(
"Could not dump container logs before removal",
container=container_name,
error=str(e),
)
proc = await asyncio.create_subprocess_exec(
"docker",
"rm",
"-f",
container_name,
stdout=asyncio.subprocess.DEVNULL,
stderr=asyncio.subprocess.DEVNULL,
)
await proc.wait()
async def _generate_mcp_config(
self,
agent_id: str,
git_context: SpawnGitContext | None = None,
) -> Path:
"""Generate MCP config for an agent.
Post-gateway: every state-changing tool routes through one of two
servers, and read-only views go through two more:
- roboco-flow intent verbs (lifecycle transitions)
- roboco-do content tools (commit, push, PR, journal,
notify, message)
- roboco-git-readonly status, log, diff, branch list
- roboco-optimal knowledge base, RAG, semantic search
- roboco-docs documentation file management (panel docs)
The agent's role is asserted by the orchestrator API on every
verb/tool call, so all roles get the same MCP surface from this
registration; verbs the agent's role can't run return a
not-authorized error rather than 404. Git context is forwarded
only as a fallback for tools that resolve project/branch from env.
"""
# MCP servers run inside agent containers, need to connect via Docker network
if PROJECT_HOST_PATH:
api_url = "http://roboco-orchestrator:8000"
else:
api_url = f"http://127.0.0.1:{settings.port}"
agent_role = get_agent_role(agent_id) or ""
# Gateway v1 endpoints declare X-Agent-ID as Annotated[UUID, Header(...)],
# so the MCP server has to forward the agent's UUID — not the slug — or
# every gateway call 422s on header parse. Resolve via AGENT_UUIDS map;
# if the slug isn't in the map (custom agents), fall back to the slug
# and let the API surface the unknown-agent error.
agent_uuid = AGENT_UUIDS.get(agent_id, agent_id)
mcp_env: dict[str, str] = {
"ROBOCO_API_URL": api_url,
"ROBOCO_ORCHESTRATOR_URL": api_url,
"ROBOCO_AGENT_ID": agent_uuid,
"ROBOCO_AGENT_ROLE": agent_role,
# Every MCP server is launched as `uv run python -m
# roboco.mcp.<server>` by Claude Code, with cwd = the agent's
# WORKSPACE (not /app). Without this, `uv run` resolves a
# cwd-relative `.venv` (≠ the baked /app/.venv), ignores the
# image's VIRTUAL_ENV with a warning, and RE-SYNCS the full
# dependency set (torch/lancedb/pyarrow/scipy, ~350MB) into a
# fresh venv on every spawn — masked by a warm uv wheel cache,
# but on a cold cache (first spawn after an image rebuild) the
# download takes minutes and the MCP servers never come up
# before the agent burns its budget. Pinning the project env
# to the pre-baked venv is necessary but NOT sufficient: `uv run`
# still resolves the project from the workspace cwd and re-syncs
# when the clone's uv.lock drifts from the image — leaving the MCP
# servers stuck at status="pending" so the agent gets zero gateway
# verbs. Each server is therefore launched with `uv run --no-sync`
# (below) to use /app/.venv as-is and start instantly.
"UV_PROJECT_ENVIRONMENT": "/app/.venv",
}
# Add git context if available
if git_context:
if git_context.project_slug:
mcp_env["ROBOCO_PROJECT_SLUG"] = git_context.project_slug
if git_context.branch_name:
mcp_env["ROBOCO_BRANCH"] = git_context.branch_name
mcp_servers: dict[str, dict[str, Any]] = {
# Intent verbs — every role-scoped lifecycle transition.
"roboco-flow": {
"command": "uv",
"args": ["run", "--no-sync", "python", "-m", "roboco.mcp.flow_server"],
"env": mcp_env,
},
# Content tools — commit, push, PR, journal, notify, message.
"roboco-do": {
"command": "uv",
"args": ["run", "--no-sync", "python", "-m", "roboco.mcp.do_server"],
"env": mcp_env,
},
# Read-only git views — status, log, diff, branches.
"roboco-git-readonly": {
"command": "uv",
"args": ["run", "--no-sync", "python", "-m", "roboco.mcp.git_readonly"],
"env": mcp_env,
},
# Knowledge base — RAG / semantic search / ask_mentor.
"roboco-optimal": {
"command": "uv",
"args": [
"run",
"--no-sync",
"python",
"-m",
"roboco.mcp.optimal_server",
agent_id,
],
"env": mcp_env,
},
}
# Docs server — documentation file management. Registered only for
# roles that touch panel docs; handlers still enforce per-role
# access so the surface is fail-closed.
docs_roles = (
"documenter",
"cell_pm",
"main_pm",
"product_owner",
"head_marketing",
)
if agent_role in docs_roles:
mcp_servers["roboco-docs"] = {
"command": "uv",
"args": [
"run",
"--no-sync",
"python",
"-m",
"roboco.mcp.docs_server",
agent_id,
],
"env": mcp_env,
}
# Web research — external search/fetch for Board + PM roles. The
# provider key stays server-side (the route holds it); the agent only
# ever talks to the backend, so the container needs no external egress.
research_roles = (
"cell_pm",
"main_pm",
"product_owner",
"head_marketing",
)
if settings.research_enabled and agent_role in research_roles:
mcp_servers["roboco-search"] = {
"command": "uv",
"args": [
"run",
"--no-sync",
"python",
"-m",
"roboco.mcp.search_server",
agent_id,
],
"env": mcp_env,
}
config: dict[str, Any] = {"mcpServers": mcp_servers}
# Write to shared config directory (mounted in both orchestrator and agents)
# When running in container: /app/mcp-configs -> host's ./data/mcp-configs
# When running on host: use temp directory
if DATA_HOST_PATH:
# Running in container - use shared mounted directory
config_dir = Path("/app/mcp-configs")
config_dir.mkdir(parents=True, exist_ok=True)
else:
# Running on host - use temp directory
config_dir = Path(tempfile.gettempdir())
config_path = config_dir / f"roboco-mcp-{agent_id}.json"
config_path.write_text(json.dumps(config, indent=2))
return config_path
def _generate_composed_prompt(self, agent_id: str) -> Path:
"""Generate composed system prompt for an agent.
Uses the layered prompt composition system:
base.md + roles/{role}.md + teams/{team}.md + identities/{agent}.md
Returns:
Path to the generated prompt file
"""
# Get role and team from canonical config
role_str = get_agent_role(agent_id)
team_str = get_agent_team(agent_id)
# Convert to enums
role_enum = AgentRole(role_str) if role_str else None
team_enum = Team(team_str) if team_str else None
if not role_enum:
raise ValueError(f"Unknown role for agent: {agent_id}")
# Compose the prompt from layers
prompt_content = compose_prompt(role_enum, team_enum, agent_id)
# Determine output directory
if PROJECT_HOST_PATH:
# Running in container - use shared directory that maps to host
config_dir = Path("/app/prompts-generated")
config_dir.mkdir(parents=True, exist_ok=True)
else:
# Running directly on host
config_dir = Path(tempfile.gettempdir()) / "roboco-prompts"
config_dir.mkdir(parents=True, exist_ok=True)
# Write to file
prompt_path = config_dir / f"{agent_id}-prompt.md"
prompt_path.write_text(prompt_content)
logger.debug(
"Generated composed prompt",
agent_id=agent_id,
role=role_str,
team=team_str,
path=str(prompt_path),
size=len(prompt_content),
)
return prompt_path
async def _readiness_gate(self, agent_id: str, task_id: str | None) -> str | None:
"""Return a reason string if the spawn must be refused, else None.
Checks run only when a task is being spawned for. No-task spawns
(idle PM bootstrap, etc.) are always ready. On any refusal that
represents a persistent problem we auto-block the task so the
dispatcher stops retrying — the PM sees the block notification.
"""
if not task_id:
return None
try:
async with httpx.AsyncClient(timeout=5.0) as client:
task_or_reason = await self._readiness_fetch_task(client, task_id)
if isinstance(task_or_reason, str):
return task_or_reason
task = task_or_reason
# Universal dependency gate: refuse to spawn an agent of ANY role
# onto a task whose cross-task dependencies are not yet terminal.
# This check previously lived only on the dev dispatch path, so
# cell-PM, Main-PM and board agents were spawned onto
# dependency-blocked tasks and flailed unblock / escalate / notify
# against an unfinished upstream. Auto-block so the task leaves the
# pending pool (no per-tick spawn-refusal that would starve
# siblings); `_unblock_dependents` revives it the moment the
# upstream reaches a terminal state.
if dep_reason := await self._check_dependencies_terminal(client, task):
return await self._readiness_block(client, task_id, dep_reason)
persistent = self._readiness_check_task(agent_id, task)
# Skip the git-token gate for coordination tasks — they have no
# project of their own, so there's no token to require.
if persistent is None and not _is_coordination_task(task):
project_slug = _read_project_slug(task)
persistent = await self._readiness_check_git_token(project_slug)
if persistent is not None:
return await self._readiness_block(client, task_id, persistent)
except httpx.HTTPError as e:
# Transient — retry on next dispatch without auto-blocking.
return f"readiness check HTTP error: {e}"
return None
async def _readiness_fetch_task(
self, client: httpx.AsyncClient, task_id: str
) -> dict[str, Any] | str:
"""Fetch the task or return a reason string.
404 → "task not found" (caller should auto-block).
Other non-200s → transient; caller returns the reason verbatim
without auto-blocking so the dispatcher can retry next tick.
"""
resp = await client.get(f"{self._api_url}/tasks/{task_id}")
if resp.status_code == http_status.HTTP_404_NOT_FOUND:
await self._readiness_block(client, task_id, "task not found")
return "task not found"
if resp.status_code != http_status.HTTP_200_OK:
return f"task-fetch returned {resp.status_code}"
task = resp.json()
return task if isinstance(task, dict) else "task payload not an object"
@staticmethod
@staticmethod
def _readiness_check_acceptance_criteria(task: dict[str, Any]) -> str | None:
"""Return blocker reason for missing acceptance criteria, else None."""
criteria = task.get("acceptance_criteria") or []
if isinstance(criteria, str):
criteria = [criteria] if criteria.strip() else []
if not criteria:
return "missing acceptance_criteria"
return None
@staticmethod
def _readiness_check_role_for_status(
agent_id: str, role: str, status: str, *, is_coordination: bool = False
) -> str | None:
"""Verify agent role matches the role expected for the task status.
Handoff states are role-specific. Dev-owned states (in_progress,
verifying, needs_revision, paused, blocked) are restricted to
developer/documenter to defang the bug where QA got
respawned on a `needs_revision` task via the crash-restart path
and immediately hit ``role 'qa' may not claim from status
'needs_revision'`` at the gateway. A coordination root (no code; product
fan-out owned by a PM) is the exception: it has no dev, so a CEO-rejected
one returns to its PM — the dev-owned states also accept the PM roles for
it (a pure widening; nothing currently allowed is blocked).
"""
role_mismatch: dict[str, str | set[str]] = {
"awaiting_qa": "qa",
"awaiting_documentation": "documenter",
"awaiting_pm_review": {"cell_pm", "main_pm"},
"awaiting_ceo_approval": "ceo",
# Dev-owned states — only developer/documenter may claim or
# resume work here. PMs / QA spawning on these is a misroute.
"needs_revision": {"developer", "documenter"},
"verifying": {"developer", "documenter"},
}
required = role_mismatch.get(status)
if required is None:
return None
if is_coordination and status in ("needs_revision", "verifying"):
required = set(required) | {"cell_pm", "main_pm"}
ok = role in required if isinstance(required, set) else role == required
if ok:
return None
return (
f"state={status} requires role in {required!r} "
f"but agent {agent_id} is {role!r}"
)
def _readiness_check_task(self, agent_id: str, task: dict[str, Any]) -> str | None:
"""Return a persistent blocker reason on the task itself, else None."""
status = task.get("status", "")
role = get_agent_role(agent_id) or ""
if reason := self._readiness_check_acceptance_criteria(task):
return reason
# A coordination task (product, no repo of its own) does no git: skip the
# project-slug and branch-name gates that only apply to code tasks.
if not _is_coordination_task(task):
if not _read_project_slug(task):
return "task has no project"
# Branch is auto-created at claim, so only states at/after claim are
# expected to own one. _branch_is_expected centralizes this gate so
# the readiness and stuck-detection paths agree.
if _branch_is_expected(task) and not task.get("branch_name"):
return f"state={status} but branch_name is unset"
return self._readiness_check_role_for_status(
agent_id, role, status, is_coordination=_is_coordination_task(task)
)
@staticmethod
async def _readiness_check_git_token(project_slug: str | None) -> str | None:
"""Ensure the project has a decryptable git token, else blocker reason."""
if not project_slug:
return "task has no project"
from roboco.db.base import get_session_factory
from roboco.services.project import get_project_service
session_factory = get_session_factory()
async with session_factory() as db:
project_svc = get_project_service(db)
try:
token = await project_svc.get_decrypted_token_by_slug(project_slug)
except Exception as e:
return f"project '{project_slug}' git-token decrypt failed: {e}"
if not token:
return f"project '{project_slug}' has no git token configured"
return None
async def _readiness_block(
self, client: httpx.AsyncClient, task_id: str, reason: str
) -> str:
"""Auto-block the task and return the human-readable reason."""
await self._auto_block_task(client, task_id, f"readiness: {reason}")
return reason
async def _resolve_agent_route(self, agent_id: str) -> "AgentRoute":
"""Resolve (provider, model) for `agent_id` via `ModelRoutingService`.
Errors are contained: any DB/session failure degrades to a legacy
Anthropic-default AgentRoute so spawn never stalls on routing.
"""
from roboco.db.base import get_session_factory
from roboco.models.base import ModelProvider
from roboco.models.runtime import MODEL_MAP
from roboco.services.llm import (
AgentRoute,
get_model_routing_service,
)
try:
factory = get_session_factory()
async with factory() as db:
router = get_model_routing_service(db)
return await router.resolve_for_agent(agent_id)
except Exception as e: # pragma: no cover
role = get_agent_role(agent_id) or ""
short = ROLE_MODEL_MAP.get(role, "sonnet")
logger.warning(
"Model routing resolve failed; using legacy Anthropic path",
agent_id=agent_id,
error=str(e),
)
return AgentRoute(
provider_id=None,
provider_type=ModelProvider.ANTHROPIC,
base_url=None,
auth_token=None,
model_name=MODEL_MAP.get(short, short),
)
_TOOL_LOAD_CACHE: ClassVar[dict[str, str]] = {}
# Per-role built-in tools, enumerated in the briefing so the agent
# knows exactly what it has. These are pre-loaded at spawn via the
# Claude Code `--tools` flag and gated only by the per-role
# permission rules — NOT by ToolSearch (MCP-only; never gates
# built-ins). Mirrors the system-prompt layer's _ROLE_BUILTIN_TOOLS
# in roboco/agents/factories/_base.py — kept in sync because the
# briefing and the system prompt are independent code paths.
_COMMON_BUILTIN_TOOLS: ClassVar[tuple[str, ...]] = (
"Read",
"Bash",
"Grep",
"Glob",
"TodoWrite",
)
_ROLE_BUILTIN_TOOLS: ClassVar[dict[str, tuple[str, ...]]] = {
"developer": (*_COMMON_BUILTIN_TOOLS, "Edit", "Write"),
"documenter": (*_COMMON_BUILTIN_TOOLS, "Edit", "Write"),
"qa": _COMMON_BUILTIN_TOOLS,
"main_pm": _COMMON_BUILTIN_TOOLS,
"cell_pm": _COMMON_BUILTIN_TOOLS,
"product_owner": _COMMON_BUILTIN_TOOLS,
"head_marketing": _COMMON_BUILTIN_TOOLS,
"auditor": _COMMON_BUILTIN_TOOLS,
"pr_reviewer": _COMMON_BUILTIN_TOOLS,
}
def _build_tool_load_block(self, role: str) -> str:
"""Briefing block affirming the role's built-in tools are ready.
Built-in tools are pre-loaded at spawn via the Claude Code
`--tools` flag and gated only by the per-role permission rules.
ToolSearch is MCP-only and never gates built-ins — an earlier
revision instructed agents to "run ToolSearch to activate
Edit/Write", which was false (ToolSearch is not even callable
here), so weak models chased a nonexistent tool and fell back to
destructive shell file-writes. This states the tools are live and
steers away from that failure. Cached per role.
"""
if role in self._TOOL_LOAD_CACHE:
return self._TOOL_LOAD_CACHE[role]
tools = self._ROLE_BUILTIN_TOOLS.get(role)
if not tools:
block = ""
else:
tool_list = ", ".join(tools)
edit_line = (
"Make file changes with Edit/Write — never rewrite a "
"whole file via shell redirection (>, heredoc, tee); "
"that destroys content and is unnecessary.\n"
if "Edit" in tools
else "You read and review; you do not author files.\n"
)
block = (
"## Your tools are ready\n"
"\n"
f"Loaded and available now: {tool_list}. Use them "
"directly. Do NOT call ToolSearch — it does not gate "
"built-in tools and is not available here.\n"
f"{edit_line}"
"\n"
)
self._TOOL_LOAD_CACHE[role] = block
return block
@staticmethod
def _format_task_briefing_block(task_id: str, task: dict[str, Any]) -> str:
"""Build the ``## Current task`` markdown block from a fetched task."""
criteria_list = task.get("acceptance_criteria") or []
if isinstance(criteria_list, str):
criteria_list = [criteria_list]
criteria = (
"\n".join(f"- {c}" for c in criteria_list)
if criteria_list
else "- (none listed — ask PM before proceeding)"
)
branch = task.get("branch_name") or "(to be created)"
project_slug = task.get("project_slug") or "(unset — ask PM)"
return (
"\n## Current task\n"
f"- **ID:** `{task.get('id', task_id)}`\n"
f"- **Title:** {task.get('title', '(untitled)')}\n"
f"- **Status:** {task.get('status', 'unknown')}\n"
f"- **Type:** {task.get('task_type', 'unknown')}\n"
f"- **Project slug:** `{project_slug}` "
"(pass this as `project_slug=` on every git/task tool)\n"
f"- **Branch:** `{branch}`\n"
"\n### Acceptance criteria\n"
f"{criteria}\n"
)
async def _fetch_task_for_briefing(
self, agent_id: str, task_id: str
) -> dict[str, Any] | None:
"""Best-effort GET /tasks/{id}; returns task dict or None on failure."""
try:
async with httpx.AsyncClient(timeout=5.0) as client:
resp = await client.get(f"{self._api_url}/tasks/{task_id}")
if resp.status_code == http_status.HTTP_200_OK:
payload: dict[str, Any] = resp.json()
return payload
except Exception as e:
logger.debug(
"Briefing task-fetch failed — falling back to role-only",
agent_id=agent_id,
task_id=task_id,
error=str(e),
)
return None
async def _write_agent_briefing(
self,
agent_id: str,
task_id: str | None,
workspace_path: str,
) -> Path | None:
"""Write a compact task briefing to be read by SessionStart hook.
The briefing saves the agent from burning its first 2-3 tool calls on
`give_me_work` (whose Envelope already carries the task details). If
`task_id` is known we fetch
the task and include title, status, branch, and acceptance criteria.
On fetch failure we still emit the role-level part (role, escalation
target, terminal tools, workspace path) — strictly better than nothing.
"""
role = get_agent_role(agent_id) or "agent"
team = get_agent_team(agent_id) or "-"
escalate_to = get_escalation_target(agent_id) or "main-pm"
tool_load_block = self._build_tool_load_block(role)
task_block = ""
if task_id:
task = await self._fetch_task_for_briefing(agent_id, task_id)
if task is not None:
task_block = self._format_task_briefing_block(task_id, task)
content = (
f"# Session briefing — {agent_id}\n"
"\n"
f"{tool_load_block}"
"## You are\n"
f"- **Agent:** `{agent_id}`\n"
f"- **Role:** {role}\n"
f"- **Team:** {team}\n"
f"- **Escalate to:** `{escalate_to}`\n"
f"- **Workspace:** `{workspace_path}`\n"
f"{task_block}"
"\n## Terminal tools (how to exit cleanly)\n"
"- `i_am_idle()` — no work remaining (every role)\n"
"- `i_am_blocked(task_id, reason, ...)` — stuck (developer)\n"
"- `unclaim(task_id)` — release a claim back to the pool\n"
"- Role handoffs:\n"
" - developer → `i_am_done(task_id, notes)` (submit for QA)\n"
" - qa → `pass(task_id, notes)` / `fail(task_id, issues)`\n"
" - documenter → `i_documented(task_id, notes, files)`\n"
" - cell_pm → `complete(task_id, notes)` / `submit_up(...)`"
" / `escalate_up(...)`\n"
" - main_pm → `complete(...)` / `escalate_to_ceo(...)`\n"
"\n"
"A Stop without a terminal tool will be rejected; a second Stop\n"
"auto-substitutes the task so it can be picked up elsewhere.\n"
"\n"
"## Budget\n"
f"Soft-warn at {settings.agent_tool_call_warn} tool calls, "
f"hard cap at {settings.agent_tool_call_halt}. Loops — same "
f"tool+args {settings.agent_loop_threshold}x within "
f"{settings.agent_loop_window} calls — are flagged; stop and "
"escalate instead of retrying.\n"
)
if PROJECT_HOST_PATH:
briefings_dir = Path("/app/briefings")
else:
briefings_dir = Path(tempfile.gettempdir()) / "roboco-briefings"
briefings_dir.mkdir(parents=True, exist_ok=True)
path = briefings_dir / f"{agent_id}.md"
path.write_text(content)
logger.debug(
"Wrote agent briefing",
agent_id=agent_id,
path=str(path),
has_task=bool(task_block),
)
return path
# Slug -> team string for ROUTING purposes. Derived from
# foundation.AGENTS so adding/renaming an agent edits exactly one
# file (foundation/identity.py). The dispatcher relies on this for
# task assignment routing categories.
_AGENT_TEAM_MAP: ClassVar[dict[str, str]] = {
slug: row.team.value for slug, row in _foundation.AGENTS.items()
}
def _get_agent_team(self, agent_id: str) -> str | None:
"""Get team from agent_id. Returns None for unknown slugs."""
try:
return _foundation.team_for_slug(agent_id).value
except KeyError:
return None
def _resolve_agent_slug(self, agent_id_or_uuid: str) -> str:
"""Resolve agent UUID to slug. Returns input if already a slug."""
# Check if it's a known UUID and convert to slug
if agent_id_or_uuid in UUID_TO_SLUG:
return UUID_TO_SLUG[agent_id_or_uuid]
# Already a slug or unknown UUID
return agent_id_or_uuid
def _mark_task_handled(self, task_id: str | None) -> None:
"""Record that `task_id` was acted on earlier in this dispatch tick."""
if task_id:
self._tick_handled_tasks.add(task_id)
def _is_task_handled_this_tick(self, task_id: str | None) -> bool:
"""True if a prior dispatcher already handled this task this tick."""
return bool(task_id and task_id in self._tick_handled_tasks)
def _is_parallel_phase_claim(
self, task: dict[str, Any], dev_uuid: str | None
) -> bool:
"""True if a `claimed` task is actually in the doc/PR parallel phase.
The `original_developer:` quick_context marker is set pre-QA by
`open_pr`, so it alone cannot distinguish a QA-claimed
awaiting_qa task (wrong) from a doc-claimed awaiting_documentation
task (right). Require the claimant to be a documenter.
"""
if not dev_uuid:
return False
claimed_by = task.get("claimed_by")
if not claimed_by:
return False
claimed_slug = self._resolve_agent_slug(claimed_by)
return bool(claimed_slug) and "doc" in claimed_slug
async def _respawn_dev_for_pr_half(
self, task: dict[str, Any], dev_uuid: str | None
) -> None:
"""Respawn the original developer if they still owe the PR half.
`pr_number` is set by the PR-create handler as soon as GitHub
confirms the PR, even if the status-gated `pr_created` flag never
flips — without that second check we'd respawn the dev forever
after they've already created the PR (the handler refuses to set
pr_created=True when the doc's claim moved status out of
awaiting_documentation).
"""
if not dev_uuid or task.get("pr_created") or task.get("pr_number"):
return
dev_slug = self._resolve_agent_slug(dev_uuid)
if not dev_slug or self._is_agent_active(dev_slug):
return
await self.spawn_agent(
agent_id=dev_slug,
task_id=task["id"],
initial_prompt=self._build_dev_prompt(task),
git_context=self._task_git_context(task),
)
# =========================================================================
# INTAKE (PROMPTER) LIVE SESSION
#
# The intake agent is not task-driven and is never dispatched. It is a
# persistent Claude-Agent-SDK driver the CEO chats with live (the container
# entrypoint is roboco.agent_sdk.intake_main). One fixed container —
# `intake-1`, the seeded board-adjacent interviewer — serves one live
# session at a time (single CEO; one-session-per-CEO).
#
# This spawn is a DELIBERATELY separate path from spawn_agent: no task, no
# readiness gate, no `claude -p` CLI args (the image ENTRYPOINT is the
# driver), no settings.json/hook mount (the driver owns the receiver on
# port 9000, not the inbox sidecar), and no MCP/gateway surface (the live
# agent reads code with Read/Grep/Glob and talks only to the human).
# =========================================================================
async def start_intake_session(
self,
session_id: str,
*,
project_slug: str | None = None,
product_id: str | None = None,
initial_message: str | None = None,
) -> None:
"""Non-blocking start: open the relay now, spawn the container in the bg.
The panel's ``POST /live/start`` returns immediately rather than blocking
on the workspace clone + first-time image build + ``docker run`` (which
can exceed the HTTP timeout — the cause of the "Request timed out" the
panel showed). The panel opens the SSE stream right away; the agent's
first reply arrives once the container is up. A spawn failure is pushed
onto the relay as an ``error`` event and closes the session, so the panel
shows it instead of hanging. Exactly one of ``project_slug`` /
``product_id`` must be given.
"""
if bool(project_slug) == bool(product_id):
raise ValueError(
"intake scope requires exactly one of project_slug / product_id"
)
self._open_intake_relay(session_id)
self._schedule_bg(
self._spawn_intake_container_guarded(
session_id,
project_slug=project_slug,
product_id=product_id,
initial_message=initial_message,
)
)
async def spawn_intake_session(
self,
session_id: str,
*,
project_slug: str | None = None,
product_id: str | None = None,
initial_message: str | None = None,
) -> AgentInstance:
"""Spawn the intake container for one live chat, **synchronously**.
Opens the relay then clones + launches the container, awaiting the whole
thing. Prefer ``start_intake_session`` on the request path; this blocking
variant is for direct/internal callers and tests. Exactly one of
``project_slug`` / ``product_id`` must be given.
"""
if bool(project_slug) == bool(product_id):
raise ValueError(
"intake scope requires exactly one of project_slug / product_id"
)
self._open_intake_relay(session_id)
return await self._spawn_intake_container(
session_id,
project_slug=project_slug,
product_id=product_id,
initial_message=initial_message,
)
@staticmethod
def _open_intake_relay(session_id: str) -> None:
"""Register the live relay session so the SSE stream connects immediately."""
from roboco.services.prompter_live import get_live_registry
get_live_registry().open(session_id, INTAKE_AGENT_ID)
async def _spawn_intake_container_guarded(
self,
session_id: str,
*,
project_slug: str | None,
product_id: str | None,
initial_message: str | None,
) -> None:
"""Background container spawn; surface failures on the relay, not silently."""
from roboco.services.prompter_live import get_live_registry
try:
await self._spawn_intake_container(
session_id,
project_slug=project_slug,
product_id=product_id,
initial_message=initial_message,
)
except Exception as exc:
logger.error(
"Intake container spawn failed", session_id=session_id, error=str(exc)
)
registry = get_live_registry()
registry.push(
session_id,
{"kind": "error", "text": f"Couldn't start the intake agent: {exc}"},
)
registry.close(session_id)
async def _spawn_intake_container(
self,
session_id: str,
*,
project_slug: str | None,
product_id: str | None,
initial_message: str | None,
) -> AgentInstance:
"""Clone the scope, launch the SDK-driver container, track the instance.
The relay must already be open (``_open_intake_relay``). Heavy + slow
(clone + first-time image build + docker run) — keep it off the request
path via ``start_intake_session``.
"""
# Single live session: reap any prior intake container before spawning.
if INTAKE_AGENT_ID in self._instances:
await self.stop_agent(INTAKE_AGENT_ID, graceful=False)
cwd, cloned = await self._clone_intake_scope(project_slug, product_id)
prompt_path = self._generate_composed_prompt(INTAKE_AGENT_ID)
route = await self._resolve_agent_route(INTAKE_AGENT_ID)
cli_model = _resolve_agent_cli_model(
route.provider_type.value, route.model_name
)
api_url = (
"http://roboco-orchestrator:8000"
if PROJECT_HOST_PATH
else f"http://127.0.0.1:{settings.port}"
)
await self._ensure_agent_image(INTAKE_AGENT_ID)
container_name = f"roboco-agent-{INTAKE_AGENT_ID}"
await self._remove_container(container_name)
cmd = self._build_intake_run_cmd(
_IntakeRunSpec(
container_name=container_name,
image=get_agent_image(INTAKE_AGENT_ID),
hosts=self._resolve_intake_host_paths(),
session_id=session_id,
cwd=cwd,
cli_model=cli_model,
api_url=api_url,
provider_base_url=route.base_url,
provider_auth_token=route.auth_token,
)
)
container_id = await self._run_container_cmd(cmd)
config = AgentConfig(
agent_id=INTAKE_AGENT_ID,
blueprint_path=prompt_path,
model=route.model_name,
git_context=None,
)
instance = AgentInstance(
agent_id=INTAKE_AGENT_ID,
state=AgentState.ACTIVE,
config=config,
current_task_id=None,
)
instance.container_id = container_id
instance.started_at = datetime.now(UTC)
instance.last_activity = datetime.now(UTC)
self._instances[INTAKE_AGENT_ID] = instance
# The relay was already opened on the request path (start_intake_session /
# spawn_intake_session) BEFORE the panel connected its SSE stream. Do NOT
# re-open here: a second open would swap in a fresh queue and orphan that
# already-connected stream (the agent's replies would push to the new queue
# while the browser keeps reading the old one). open() is idempotent now as
# a guard, but the redundant call is gone regardless.
logger.info(
"Intake session spawned",
session_id=session_id,
container_id=container_id[:12],
cwd=cwd,
repos=len(cloned),
)
self._fire_audit(
event_type="agent.spawned",
agent_slug=INTAKE_AGENT_ID,
details={"session_id": session_id, "cwd": cwd, "repos": cloned},
)
if initial_message:
self._schedule_intake_first_message(session_id, initial_message)
return instance
async def reap_intake_session(self, session_id: str) -> None:
"""End a live chat: close the relay stream and stop the container."""
from roboco.services.prompter_live import get_live_registry
get_live_registry().close(session_id)
await self.stop_agent(INTAKE_AGENT_ID, graceful=True)
logger.info("Intake session reaped", session_id=session_id)
# ------------------------------------------------------------------ #
# Secretary live session (mirrors intake; no scope clone; auth token)
# ------------------------------------------------------------------ #
async def start_secretary_session(
self, session_id: str, *, initial_message: str | None = None
) -> None:
"""Non-blocking start: open the relay now, spawn the container in the bg."""
from roboco.services.prompter_live import get_live_registry
get_live_registry().open(session_id, SECRETARY_AGENT_ID)
self._schedule_bg(
self._spawn_secretary_container_guarded(
session_id, initial_message=initial_message
)
)
async def spawn_secretary_session(
self, session_id: str, *, initial_message: str | None = None
) -> AgentInstance:
"""Spawn the Secretary container synchronously (internal callers/tests)."""
from roboco.services.prompter_live import get_live_registry
get_live_registry().open(session_id, SECRETARY_AGENT_ID)
return await self._spawn_secretary_container(
session_id, initial_message=initial_message
)
async def _spawn_secretary_container_guarded(
self, session_id: str, *, initial_message: str | None
) -> None:
"""Background spawn; surface failures on the relay, not silently."""
from roboco.services.prompter_live import get_live_registry
try:
await self._spawn_secretary_container(
session_id, initial_message=initial_message
)
except Exception as exc:
logger.error(
"Secretary container spawn failed",
session_id=session_id,
error=str(exc),
)
registry = get_live_registry()
registry.push(
session_id,
{"kind": "error", "text": f"Couldn't start the Secretary: {exc}"},
)
registry.close(session_id)
async def _spawn_secretary_container(
self, session_id: str, *, initial_message: str | None
) -> AgentInstance:
"""Launch the Secretary SDK-driver container and track the instance.
Unlike intake there is no workspace scope to clone — the Secretary reads
company state through the API, so its cwd is the baked ``/app`` tree. It
gets an HMAC agent token so its directive tools authenticate as the
Secretary role.
"""
from roboco.agents_config import issue_agent_token
from roboco.foundation.identity import AGENTS
if SECRETARY_AGENT_ID in self._instances:
await self.stop_agent(SECRETARY_AGENT_ID, graceful=False)
prompt_path = self._generate_composed_prompt(SECRETARY_AGENT_ID)
route = await self._resolve_agent_route(SECRETARY_AGENT_ID)
cli_model = _resolve_agent_cli_model(
route.provider_type.value, route.model_name
)
api_url = (
"http://roboco-orchestrator:8000"
if PROJECT_HOST_PATH
else f"http://127.0.0.1:{settings.port}"
)
await self._ensure_agent_image(SECRETARY_AGENT_ID)
container_name = f"roboco-agent-{SECRETARY_AGENT_ID}"
await self._remove_container(container_name)
agent_uuid = str(AGENTS[SECRETARY_AGENT_ID].uuid)
cmd = self._build_secretary_run_cmd(
_SecretaryRunSpec(
container_name=container_name,
image=get_agent_image(SECRETARY_AGENT_ID),
hosts=self._resolve_secretary_host_paths(),
session_id=session_id,
cwd="/app",
cli_model=cli_model,
api_url=api_url,
agent_uuid=agent_uuid,
agent_token=issue_agent_token(agent_uuid, "secretary", ""),
provider_base_url=route.base_url,
provider_auth_token=route.auth_token,
)
)
container_id = await self._run_container_cmd(cmd)
config = AgentConfig(
agent_id=SECRETARY_AGENT_ID,
blueprint_path=prompt_path,
model=route.model_name,
git_context=None,
)
instance = AgentInstance(
agent_id=SECRETARY_AGENT_ID,
state=AgentState.ACTIVE,
config=config,
current_task_id=None,
)
instance.container_id = container_id
instance.started_at = datetime.now(UTC)
instance.last_activity = datetime.now(UTC)
self._instances[SECRETARY_AGENT_ID] = instance
logger.info(
"Secretary session spawned",
session_id=session_id,
container_id=container_id[:12],
)
self._fire_audit(
event_type="agent.spawned",
agent_slug=SECRETARY_AGENT_ID,
details={"session_id": session_id},
)
if initial_message:
self._schedule_intake_first_message(session_id, initial_message)
return instance
async def reap_secretary_session(self, session_id: str) -> None:
"""End a live Secretary chat: close the relay and stop the container."""
from roboco.services.prompter_live import get_live_registry
get_live_registry().close(session_id)
await self.stop_agent(SECRETARY_AGENT_ID, graceful=True)
logger.info("Secretary session reaped", session_id=session_id)
def _resolve_secretary_host_paths(self) -> dict[str, str | None]:
"""Host paths for the Secretary container's mounts (claude + prompt).
No workspaces mount: the Secretary reads company state via the API and
runs from the baked ``/app`` tree.
"""
if PROJECT_HOST_PATH:
return {
"claude": CLAUDE_AUTH_HOST_PATH,
"prompt": (
f"{DATA_HOST_PATH}/prompts-generated/{SECRETARY_AGENT_ID}-prompt.md"
),
}
return {
"claude": CLAUDE_AUTH_HOST_PATH,
"prompt": str(
Path(tempfile.gettempdir())
/ "roboco-prompts"
/ f"{SECRETARY_AGENT_ID}-prompt.md"
),
}
@staticmethod
def _build_secretary_run_cmd(spec: _SecretaryRunSpec) -> list[str]:
"""Compose the `docker run` argv for the persistent Secretary container."""
cmd: list[str] = [
"docker",
"run",
"-d",
"--name",
spec.container_name,
"--network",
AGENT_NETWORK,
"-v",
f"{spec.hosts['claude']}:/home/agent/.claude",
]
AgentOrchestrator._append_claude_json_mount(cmd, spec.hosts)
cmd.extend(
[
"-v",
f"{spec.hosts['prompt']}:/app/system-prompt.md:ro",
"-e",
f"ROBOCO_AGENT_ID={spec.agent_uuid}",
"-e",
"ROBOCO_AGENT_ROLE=secretary",
"-e",
f"ROBOCO_AGENT_TOKEN={spec.agent_token}",
"-e",
f"ROBOCO_API_URL={spec.api_url}",
"-e",
f"ROBOCO_SECRETARY_SESSION_ID={spec.session_id}",
"-e",
f"ROBOCO_WORKSPACE={spec.cwd}",
"-e",
f"CLAUDE_CODE_SUBAGENT_MODEL={spec.cli_model}",
]
)
if spec.provider_base_url:
cmd.extend(["-e", f"ANTHROPIC_BASE_URL={spec.provider_base_url}"])
if spec.provider_auth_token:
cmd.extend(["-e", f"ANTHROPIC_AUTH_TOKEN={spec.provider_auth_token}"])
cmd.append(spec.image)
return cmd
async def _clone_intake_scope(
self, project_slug: str | None, product_id: str | None
) -> tuple[str, list[str]]:
"""Clone the chat scope's repo(s); return (container cwd, all paths).
``project`` → one repo; ``product`` → each distinct cell project (the
Main-PM-style distinct-repo set, kept in its deterministic team order so
the primary is stable). The agent's cwd is the primary project's intake
workspace; for a product the sibling repos sit alongside it under
``/data/workspaces`` and are readable via Grep/Glob/Read.
"""
from roboco.db.base import get_session_factory
from roboco.services.workspace import WorkspaceService
team = get_agent_team(INTAKE_AGENT_ID) or "board"
factory = get_session_factory()
async with factory() as db:
slugs = await self._intake_scope_slugs(db, project_slug, product_id)
ws = WorkspaceService(db)
for slug in slugs:
await ws.ensure_workspace(slug, INTAKE_AGENT_ID)
# Container-side paths (the workspaces tree is mounted at
# /data/workspaces inside the container, regardless of the host root).
paths = [_agent_workspace_path(slug, team, INTAKE_AGENT_ID) for slug in slugs]
return paths[0], paths
@staticmethod
async def _intake_scope_slugs(
db: Any, project_slug: str | None, product_id: str | None
) -> list[str]:
"""Resolve the chat scope to the project slug(s) to clone."""
if project_slug:
return [project_slug]
if not product_id:
raise ValueError("intake scope requires project_slug or product_id")
from uuid import UUID
from roboco.services.product import ProductService
from roboco.services.project import get_project_service
project_ids = await ProductService(db).distinct_project_ids(UUID(product_id))
project_svc = get_project_service(db)
slugs: list[str] = []
for pid in project_ids:
project = await project_svc.get(pid)
if project and project.slug:
slugs.append(project.slug)
if not slugs:
raise ValueError(f"product {product_id} resolves to no projects")
return slugs
def _resolve_intake_host_paths(self) -> dict[str, str | None]:
"""Host paths for the intake container's three mounts (claude/prompt/ws).
Mirrors ``_resolve_host_paths`` but only for what the driver needs —
there is no settings.json, MCP config, or briefing for the intake agent.
"""
if PROJECT_HOST_PATH:
return {
"claude": CLAUDE_AUTH_HOST_PATH,
"prompt": (
f"{DATA_HOST_PATH}/prompts-generated/{INTAKE_AGENT_ID}-prompt.md"
),
"workspaces": f"{DATA_HOST_PATH}/workspaces",
}
return {
"claude": CLAUDE_AUTH_HOST_PATH,
"prompt": str(
Path(tempfile.gettempdir())
/ "roboco-prompts"
/ f"{INTAKE_AGENT_ID}-prompt.md"
),
"workspaces": str(Path(settings.workspaces_root)),
}
@staticmethod
def _build_intake_run_cmd(spec: _IntakeRunSpec) -> list[str]:
"""Compose the `docker run` argv for the persistent intake container.
No claude CLI args (the image ENTRYPOINT is the SDK driver), no
settings.json/hook mount (the driver owns port 9000), no MCP config.
The driver reads ``/app/system-prompt.md`` and the env below.
"""
cmd: list[str] = [
"docker",
"run",
"-d",
"--name",
spec.container_name,
"--network",
AGENT_NETWORK,
"-v",
f"{spec.hosts['claude']}:/home/agent/.claude",
]
AgentOrchestrator._append_claude_json_mount(cmd, spec.hosts)
cmd.extend(
[
"-v",
f"{spec.hosts['prompt']}:/app/system-prompt.md:ro",
"-v",
f"{spec.hosts['workspaces']}:/data/workspaces",
"-e",
f"ROBOCO_AGENT_ID={INTAKE_AGENT_ID}",
"-e",
f"ROBOCO_AGENT_ROLE={get_agent_role(INTAKE_AGENT_ID) or 'prompter'}",
"-e",
f"ROBOCO_API_URL={spec.api_url}",
"-e",
f"ROBOCO_PROMPTER_SESSION_ID={spec.session_id}",
"-e",
f"ROBOCO_WORKSPACE={spec.cwd}",
"-e",
f"CLAUDE_CODE_SUBAGENT_MODEL={spec.cli_model}",
]
)
# Non-Anthropic providers need explicit endpoint/token; the Anthropic
# default uses the mounted ~/.claude login (same as every agent).
if spec.provider_base_url:
cmd.extend(["-e", f"ANTHROPIC_BASE_URL={spec.provider_base_url}"])
if spec.provider_auth_token:
cmd.extend(["-e", f"ANTHROPIC_AUTH_TOKEN={spec.provider_auth_token}"])
cmd.append(spec.image)
return cmd
async def _run_container_cmd(self, cmd: list[str]) -> str:
"""Run a detached `docker run` and return the container id."""
proc = await asyncio.create_subprocess_exec(
*cmd,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
stdout, stderr = await proc.communicate()
if proc.returncode != 0:
raise RuntimeError(f"Failed to start intake container: {stderr.decode()}")
return stdout.decode().strip()
def _schedule_bg(self, coro: "Coroutine[Any, Any, None]") -> None:
"""Fire-and-forget a coroutine, strong-reffed so it isn't GC'd mid-flight.
Silently no-ops when there's no running loop (sync unit tests); the coro
is closed to avoid a "never awaited" warning.
"""
import contextlib as _ctx
try:
loop = asyncio.get_running_loop()
except RuntimeError:
with _ctx.suppress(Exception):
coro.close()
return
bg = loop.create_task(coro)
self._bg_tasks.add(bg)
bg.add_done_callback(self._bg_tasks.discard)
def _schedule_intake_first_message(self, session_id: str, text: str) -> None:
"""Fire-and-forget the opening message once the container is reachable."""
self._schedule_bg(self._deliver_when_ready(session_id, text))
async def _deliver_when_ready(
self,
session_id: str,
text: str,
*,
attempts: int = 30,
delay: float = 1.0,
) -> None:
"""Retry-deliver the first message until the container receiver is up."""
from roboco.services.prompter_live import get_live_registry
registry = get_live_registry()
for _ in range(attempts):
if await registry.deliver(session_id, text):
return
await asyncio.sleep(delay)
logger.warning(
"Intake first message never delivered (receiver never came up)",
session_id=session_id,
)
# =========================================================================
# AGENT STOPPING
# =========================================================================
async def stop_agent(
self,
agent_id: str,
graceful: bool = True,
exit_reason: str = "stopped",
) -> None:
"""Stop an agent container.
Finalization (the HTTP call to the agent SDK's /usage/status endpoint)
is performed BEFORE acquiring self._lock so that the network I/O does
not block other operations that need the lock.
"""
# Finalize the spawn-session row before the container is removed so we
# can still query the SDK's /usage/status endpoint. This must happen
# outside self._lock — the HTTP round-trip would otherwise hold the
# lock for the full network timeout.
instance = self._instances.get(agent_id)
if instance is None:
return
if instance.container_id:
await self._finalize_spawn_session(agent_id, exit_reason=exit_reason)
async with self._lock:
if agent_id not in self._instances:
return
instance = self._instances[agent_id]
if instance.container_id:
instance.state = AgentState.STOPPING
container_name = f"roboco-agent-{agent_id}"
if graceful:
# Graceful stop with timeout
proc = await asyncio.create_subprocess_exec(
"docker",
"stop",
"-t",
"10",
container_name,
stdout=asyncio.subprocess.DEVNULL,
stderr=asyncio.subprocess.DEVNULL,
)
await proc.wait()
else:
# Force kill
proc = await asyncio.create_subprocess_exec(
"docker",
"kill",
container_name,
stdout=asyncio.subprocess.DEVNULL,
stderr=asyncio.subprocess.DEVNULL,
)
await proc.wait()
# Remove container
await self._remove_container(container_name)
instance.state = AgentState.OFFLINE
instance.container_id = None
logger.info("Agent stopped", agent_id=agent_id)
# =========================================================================
# WAITING STATE MANAGEMENT
# =========================================================================
async def mark_waiting_long(
self,
agent_id: str,
waiting_for: str,
task_id: str | None = None,
context: dict[str, Any] | None = None,
) -> None:
"""
Mark an agent as WAITING_LONG and terminate.
The agent will be respawned when the wait condition is resolved.
The record is mirrored to `waiting_records` in Postgres so a later
orchestrator restart can still resolve the wait.
"""
record = WaitingRecord(
agent_id=agent_id,
task_id=task_id,
waiting_for=waiting_for,
waiting_since=datetime.now(UTC),
context=context or {},
)
self._waiting_records[agent_id] = record
await self._persist_waiting_record(record)
# Stop the agent
await self.stop_agent(agent_id)
# Update state
if agent_id in self._instances:
self._instances[agent_id].state = AgentState.WAITING_LONG
self._instances[agent_id].waiting_for = waiting_for
self._instances[agent_id].waiting_context = context or {}
logger.info(
"Agent marked as waiting_long",
agent_id=agent_id,
waiting_for=waiting_for,
task_id=task_id,
)
async def _persist_waiting_record(self, record: WaitingRecord) -> None:
"""Upsert a WaitingRecord into the waiting_records table."""
try:
from uuid import UUID as _UUID
from sqlalchemy import delete
from roboco.db.base import get_session_factory
from roboco.db.tables import WaitingRecordTable
session_factory = get_session_factory()
async with session_factory() as db:
# One record per agent; delete prior then insert.
await db.execute(
delete(WaitingRecordTable).where(
WaitingRecordTable.agent_id == record.agent_id
)
)
row = WaitingRecordTable(
agent_id=record.agent_id,
task_id=(_UUID(record.task_id) if record.task_id else None),
waiting_for=record.waiting_for,
waiting_since=record.waiting_since,
context=record.context,
)
db.add(row)
await db.commit()
except Exception as e:
logger.error(
"Failed to persist waiting record",
agent_id=record.agent_id,
error=str(e),
)
async def _delete_waiting_record(self, agent_id: str) -> None:
"""Delete a persisted waiting record when its wait resolves."""
try:
from sqlalchemy import delete
from roboco.db.base import get_session_factory
from roboco.db.tables import WaitingRecordTable
session_factory = get_session_factory()
async with session_factory() as db:
await db.execute(
delete(WaitingRecordTable).where(
WaitingRecordTable.agent_id == agent_id
)
)
await db.commit()
except Exception as e:
logger.error(
"Failed to delete waiting record",
agent_id=agent_id,
error=str(e),
)
# =========================================================================
# PROVIDER QUERY HELPERS (used by the choreographer rate-limit path)
# =========================================================================
def get_provider_for_agent(self, agent_slug: str) -> str | None:
"""Return the ``provider_type`` for a currently-tracked agent, or None.
Reads the in-memory ``_instances`` dict so this is synchronous and
O(1). Returns None when the agent is not tracked or has no config.
Args:
agent_slug: The agent slug (e.g. ``"be-dev-1"``).
"""
instance = self._instances.get(agent_slug)
if instance is None or instance.config is None:
return None
return instance.config.provider_type
def get_active_agent_slugs_for_provider(self, provider: str) -> list[str]:
"""Return slugs of all active agents currently using ``provider``.
"Active" means the instance's state is ACTIVE or STARTING (i.e.
the container is running or spinning up — not IDLE, WAITING_LONG,
STOPPING, or OFFLINE).
Args:
provider: Provider type string, e.g. ``"anthropic"`` or
``"ollama_cloud"``.
"""
active_states = {AgentState.ACTIVE, AgentState.STARTING}
return [
slug
for slug, inst in self._instances.items()
if inst.state in active_states
and inst.config is not None
and inst.config.provider_type == provider
]
# =========================================================================
# TOKEN USAGE INSTRUMENTATION
# =========================================================================
async def _record_spawn_session(
self,
config: "OrchestratorAgentConfig",
task_id: str | None,
) -> "UUID | None":
"""Insert a row into agent_spawn_sessions after a successful spawn.
Returns the UUID of the created row so the caller can store it on
the AgentInstance for later direct-by-id lookup in
_finalize_spawn_session. Returns None when the insert fails; a
missing session row must never block the spawn path.
"""
try:
from uuid import uuid4 as _uuid4
from roboco.db.base import get_session_factory
from roboco.db.tables import AgentSpawnSessionTable
agent_slug = config.agent_id
team = get_agent_team(agent_slug) or "backend"
role = get_agent_role(agent_slug) or "developer"
session_id = _uuid4()
session_factory = get_session_factory()
async with session_factory() as db:
row = AgentSpawnSessionTable(
id=session_id,
agent_slug=agent_slug,
team=team,
role=role,
model=config.model or "unknown",
task_id=task_id,
started_at=datetime.now(UTC),
)
db.add(row)
await db.commit()
logger.debug(
"Spawn session recorded",
agent_slug=agent_slug,
session_id=str(session_id),
task_id=task_id,
)
return session_id
except Exception as exc:
logger.warning(
"Failed to record spawn session",
agent_slug=config.agent_id,
error=str(exc),
)
return None
def _claude_session_id_for(self, agent_id: str) -> str | None:
"""The orchestrator-assigned Claude session id for a running agent."""
instance = self._instances.get(agent_id)
return (
instance.config.claude_session_id if instance and instance.config else None
)
@staticmethod
def _usage_from_transcript(
agent_id: str, claude_session_id: str | None = None
) -> tuple[int, int, int, int]:
"""Sum token usage from the agent's Claude Code transcript.
The host ``~/.claude`` is mounted into the orchestrator, so transcripts
are readable here under ``projects/<cwd-dir>/<session-id>.jsonl``. When
the orchestrator-assigned ``claude_session_id`` is known we locate the
exact transcript by id across ANY project dir — review/coordinate roles
run at cwd ``/app`` so theirs lands in ``projects/-app``, not in a
per-agent ``projects/*-{slug}`` dir. Without an id we fall back to the
newest transcript in the agent's own workspace dir. Durable fallback for
the live SDK ``/usage/status`` fetch, which misses for short-lived or
torn-down agents. Returns zeros when no transcript is found.
"""
from roboco.agent_sdk.transcript_usage import sum_transcript_usage
projects = Path.home() / ".claude" / "projects"
try:
if claude_session_id:
by_id = list(projects.glob(f"*/{claude_session_id}.jsonl"))
if by_id:
return sum_transcript_usage(by_id[0])
jsonl = [
f
for d in projects.glob(f"*-{agent_id}")
if d.is_dir()
for f in d.glob("*.jsonl")
]
if not jsonl:
return (0, 0, 0, 0)
newest = max(jsonl, key=lambda f: f.stat().st_mtime)
return sum_transcript_usage(newest)
except OSError:
return (0, 0, 0, 0)
async def _resolve_final_token_usage(
self, agent_id: str
) -> tuple[int, int, int, int]:
"""Resolve final token counts for a stopping agent.
Tries the live SDK ``/usage/status`` first; if that misses — the SDK's
in-memory counts race container teardown for short-lived agents — it
falls back to the agent's Claude Code transcript, which is durable and
mounted into this container. Returns
``(input, output, cache_read, cache_write)``.
"""
tokens = (0, 0, 0, 0)
sdk_url = f"http://roboco-agent-{agent_id}:{SDK_PORT}/usage/status"
try:
async with httpx.AsyncClient(timeout=3.0) as client:
resp = await client.get(sdk_url)
if resp.status_code == http_status.HTTP_200_OK:
data = resp.json()
tokens = (
data.get("tokens_input", 0),
data.get("tokens_output", 0),
data.get("tokens_cache_read", 0),
data.get("tokens_cache_write", 0),
)
except Exception as sdk_exc:
logger.debug(
"Could not fetch final token counts from SDK",
agent_id=agent_id,
error=str(sdk_exc),
)
if not tokens[0] and not tokens[1]:
tin, tout, cr, cw = self._usage_from_transcript(
agent_id, self._claude_session_id_for(agent_id)
)
if tin or tout:
tokens = (tin, tout, cr, cw)
return tokens
async def _finalize_spawn_session(
self,
agent_id: str,
exit_reason: str = "stopped",
) -> None:
"""Close the open agent_spawn_sessions row for this agent.
Resolves final token counts (live SDK, with a durable transcript
fallback), calculates cost via the pricing module, then updates the DB
row with ended_at, token totals, exit_reason, and estimated_cost_usd.
Errors are caught and logged — finalization must never block stop_agent.
"""
try:
from roboco.billing.pricing import calculate_cost
from roboco.db.base import get_session_factory
from roboco.db.tables import AgentSpawnSessionTable
# Resolve final token counts (live SDK, with transcript fallback).
(
tokens_input,
tokens_output,
tokens_cache_read,
tokens_cache_write,
) = await self._resolve_final_token_usage(agent_id)
# Look up the model and usage_session_id from the running instance config.
model = "unknown"
instance = self._instances.get(agent_id)
if instance and instance.config:
model = instance.config.model or "unknown"
usage_session_id = instance.usage_session_id if instance else None
cost = calculate_cost(
model=model,
tokens_input=tokens_input,
tokens_output=tokens_output,
tokens_cache_read=tokens_cache_read,
tokens_cache_write=tokens_cache_write,
)
session_factory = get_session_factory()
async with session_factory() as db:
from sqlalchemy import select, update
# Prefer a direct lookup by the session UUID captured at spawn
# time; fall back to the (agent_slug, ended_at IS NULL) query
# for instances that pre-date the usage_session_id field.
if usage_session_id is not None:
result = await db.execute(
select(AgentSpawnSessionTable).where(
AgentSpawnSessionTable.id == usage_session_id
)
)
else:
result = await db.execute(
select(AgentSpawnSessionTable)
.where(
AgentSpawnSessionTable.agent_slug == agent_id,
AgentSpawnSessionTable.ended_at.is_(None),
)
.order_by(AgentSpawnSessionTable.started_at.desc())
.limit(1)
)
session_row = result.scalar_one_or_none()
if session_row is not None:
await db.execute(
update(AgentSpawnSessionTable)
.where(AgentSpawnSessionTable.id == session_row.id)
.values(
ended_at=datetime.now(UTC),
tokens_input=tokens_input,
tokens_output=tokens_output,
tokens_cache_read=tokens_cache_read,
tokens_cache_write=tokens_cache_write,
exit_reason=exit_reason,
estimated_cost_usd=cost,
)
)
await db.commit()
logger.debug(
"Spawn session finalized",
agent_id=agent_id,
session_id=str(session_row.id),
tokens_input=tokens_input,
tokens_output=tokens_output,
estimated_cost_usd=cost,
)
except Exception as exc:
logger.warning(
"Failed to finalize spawn session",
agent_id=agent_id,
error=str(exc),
)
@staticmethod
async def _fetch_agent_tokens(
client: httpx.AsyncClient, agent_id: str
) -> tuple[int, int, int, int] | None:
"""Fetch cumulative token counts from an agent's SDK usage endpoint.
Returns ``(input, output, cache_read, cache_write)`` or ``None`` when the
agent returns a non-200 status or has not accrued any tokens yet.
"""
sdk_url = f"http://roboco-agent-{agent_id}:{SDK_PORT}/usage/status"
resp = await client.get(sdk_url)
if resp.status_code != http_status.HTTP_200_OK:
return None
data = resp.json()
tokens = (
data.get("tokens_input", 0),
data.get("tokens_output", 0),
data.get("tokens_cache_read", 0),
data.get("tokens_cache_write", 0),
)
if sum(tokens) == 0:
return None
return tokens
async def _resolve_active_tokens(
self, client: httpx.AsyncClient, agent_id: str
) -> tuple[int, int, int, int] | None:
"""Resolve live token counts for an active agent.
Tries the agent SDK's ``/usage/status`` first; on a zero/miss falls
back to the durable transcript (the SDK can report zero mid-run, the
same race the finalize path handles). Returns ``None`` when neither
source has any usage yet.
"""
tokens = await self._fetch_agent_tokens(client, agent_id)
if tokens is not None:
return tokens
transcript = self._usage_from_transcript(
agent_id, self._claude_session_id_for(agent_id)
)
return transcript if any(transcript) else None
@staticmethod
async def _persist_token_snapshot(
session_factory: Any,
agent_id: str,
instance: AgentInstance,
tokens: tuple[int, int, int, int],
) -> bool:
"""Insert a token_usage_snapshots row and refresh the open session totals.
Returns True when a snapshot was written; False when the agent has no
open spawn-session row to attach it to.
"""
from uuid import uuid4
from sqlalchemy import select, update
from roboco.db.tables import AgentSpawnSessionTable, TokenUsageSnapshotTable
tokens_input, tokens_output, tokens_cache_read, tokens_cache_write = tokens
async with session_factory() as db:
# Prefer a direct lookup by the session UUID captured at spawn time;
# fall back to the agent_slug heuristic for instances that pre-date
# the usage_session_id field.
if instance.usage_session_id is not None:
result = await db.execute(
select(AgentSpawnSessionTable).where(
AgentSpawnSessionTable.id == instance.usage_session_id
)
)
else:
result = await db.execute(
select(AgentSpawnSessionTable)
.where(
AgentSpawnSessionTable.agent_slug == agent_id,
AgentSpawnSessionTable.ended_at.is_(None),
)
.order_by(AgentSpawnSessionTable.started_at.desc())
.limit(1)
)
session_row = result.scalar_one_or_none()
if session_row is None:
return False
db.add(
TokenUsageSnapshotTable(
id=uuid4(),
agent_spawn_session_id=session_row.id,
snapshotted_at=datetime.now(UTC),
tokens_input=tokens_input,
tokens_output=tokens_output,
tokens_cache_read=tokens_cache_read,
tokens_cache_write=tokens_cache_write,
)
)
await db.execute(
update(AgentSpawnSessionTable)
.where(AgentSpawnSessionTable.id == session_row.id)
.values(
tokens_input=tokens_input,
tokens_output=tokens_output,
tokens_cache_read=tokens_cache_read,
tokens_cache_write=tokens_cache_write,
)
)
await db.commit()
return True
async def _sweep_token_snapshots(self) -> None:
"""Write a token_usage_snapshots row for each active agent with non-zero tokens.
Called from _run_sweep() every ~60 s. Also updates the cumulative
token counts on the open agent_spawn_sessions row so the DB reflects
current progress without waiting for session close.
Errors per-agent are caught so one bad agent doesn't abort the whole sweep.
Also publishes a USAGE_SNAPSHOT aggregate event after the loop so the
/ws/system dashboard updates live for active agents.
"""
if not self._instances:
return
try:
from roboco.db.base import get_session_factory
except ImportError:
return
session_factory = get_session_factory()
# Accumulators for the post-loop USAGE_SNAPSHOT event.
_usage_by_agent: list[dict[str, Any]] = []
_usage_total_input = 0
_usage_total_output = 0
_usage_total_cost = 0.0
async with httpx.AsyncClient(timeout=3.0) as client:
for agent_id, instance in list(self._instances.items()):
if instance.state not in (
AgentState.ACTIVE,
AgentState.WAITING_SHORT,
):
continue
try:
tokens = await self._resolve_active_tokens(client, agent_id)
if tokens is None:
continue
persisted = await self._persist_token_snapshot(
session_factory, agent_id, instance, tokens
)
if not persisted:
continue
tokens_input, tokens_output = tokens[0], tokens[1]
model = instance.config.model if instance.config else "unknown"
# Accumulate per-agent data for the aggregate snapshot.
with contextlib.suppress(Exception):
from roboco.billing.pricing import calculate_cost
agent_cost = calculate_cost(
model=model,
tokens_input=tokens_input,
tokens_output=tokens_output,
)
_usage_by_agent.append(
{
"agent_id": agent_id,
"input_tokens": tokens_input,
"output_tokens": tokens_output,
"model": model,
"cost_estimate": agent_cost,
}
)
_usage_total_input += tokens_input
_usage_total_output += tokens_output
_usage_total_cost += agent_cost
except Exception as agent_exc:
logger.debug(
"Token snapshot failed for agent",
agent_id=agent_id,
error=str(agent_exc),
)
# Publish a USAGE_SNAPSHOT aggregate if any active agents had token data.
if _usage_by_agent:
with contextlib.suppress(Exception):
from roboco.events import get_event_bus
from roboco.services.usage_events import (
UsageSnapshot,
publish_usage_snapshot,
)
await publish_usage_snapshot(
get_event_bus(),
UsageSnapshot(
period="live",
totals={
"input_tokens": _usage_total_input,
"output_tokens": _usage_total_output,
},
cost_estimate=_usage_total_cost,
by_agent=_usage_by_agent,
),
)
async def _sweep_daily_rollup(self) -> None:
"""Upsert daily_usage_rollups from closed agent_spawn_sessions.
Groups ended sessions by (date, agent_slug, team, model) and sums
their token counts + cost. Uses a Python-side upsert to stay
compatible with asyncpg / SQLAlchemy without raw INSERT ... ON CONFLICT
dialect-specific SQL.
Errors are caught so a bad rollup doesn't abort the sweeper.
"""
try:
from roboco.db.base import get_session_factory
from roboco.db.tables import AgentSpawnSessionTable
except ImportError:
return
try:
from uuid import uuid4 as _uuid4
from sqlalchemy import func, select
session_factory = get_session_factory()
async with session_factory() as db:
# Aggregate closed sessions by (date, agent_slug, team, model).
# Limit to the last 7 days to avoid re-aggregating all-time
# history on every sweep — older days are already stable.
rollup_window_start = datetime.now(UTC) - timedelta(days=7)
result = await db.execute(
select(
func.date(AgentSpawnSessionTable.started_at).label("date"),
AgentSpawnSessionTable.agent_slug,
AgentSpawnSessionTable.team,
AgentSpawnSessionTable.model,
func.sum(AgentSpawnSessionTable.tokens_input).label(
"tokens_input"
),
func.sum(AgentSpawnSessionTable.tokens_output).label(
"tokens_output"
),
func.sum(AgentSpawnSessionTable.tokens_cache_read).label(
"tokens_cache_read"
),
func.sum(AgentSpawnSessionTable.tokens_cache_write).label(
"tokens_cache_write"
),
func.sum(AgentSpawnSessionTable.estimated_cost_usd).label(
"total_cost_usd"
),
func.count(AgentSpawnSessionTable.id).label("session_count"),
)
.where(
AgentSpawnSessionTable.ended_at.isnot(None),
AgentSpawnSessionTable.started_at >= rollup_window_start,
)
.group_by(
func.date(AgentSpawnSessionTable.started_at),
AgentSpawnSessionTable.agent_slug,
AgentSpawnSessionTable.team,
AgentSpawnSessionTable.model,
)
)
rows = result.fetchall()
for row in rows:
await self._upsert_rollup_row(db, row, _uuid4)
await db.commit()
logger.debug("Daily usage rollup complete", rows_processed=len(rows))
except Exception as exc:
logger.warning("Daily usage rollup failed", error=str(exc))
async def _upsert_rollup_row(self, db: Any, row: Any, uuid4: Any) -> None:
"""Insert or update a single daily_usage_rollups row from an aggregate.
Looks up the existing rollup for (date, agent_slug, team, model) and
either updates its summed columns or inserts a fresh row.
"""
from sqlalchemy import select, update
from roboco.db.tables import DailyUsageRollupTable
key = {
"date": row.date,
"agent_slug": row.agent_slug,
"team": row.team,
"model": row.model,
}
values = {
"tokens_input": int(row.tokens_input or 0),
"tokens_output": int(row.tokens_output or 0),
"tokens_cache_read": int(row.tokens_cache_read or 0),
"tokens_cache_write": int(row.tokens_cache_write or 0),
"total_cost_usd": float(row.total_cost_usd or 0.0),
"session_count": int(row.session_count or 0),
}
existing_result = await db.execute(
select(DailyUsageRollupTable).where(
DailyUsageRollupTable.date == key["date"],
DailyUsageRollupTable.agent_slug == key["agent_slug"],
DailyUsageRollupTable.team == key["team"],
DailyUsageRollupTable.model == key["model"],
)
)
existing = existing_result.scalar_one_or_none()
if existing is not None:
await db.execute(
update(DailyUsageRollupTable)
.where(DailyUsageRollupTable.id == existing.id)
.values(**values)
)
else:
db.add(DailyUsageRollupTable(id=uuid4(), **key, **values))
async def restore_waiting_records(self) -> int:
"""Load persisted waiting records into memory on orchestrator start.
Call this from `start()` so agents marked WAITING_LONG before the
previous orchestrator exited can still be resolved.
"""
try:
from sqlalchemy import select
from roboco.db.base import get_session_factory
from roboco.db.tables import WaitingRecordTable
session_factory = get_session_factory()
async with session_factory() as db:
rows = await db.execute(select(WaitingRecordTable))
count = 0
for row in rows.scalars().all():
self._waiting_records[row.agent_id] = WaitingRecord(
agent_id=row.agent_id,
task_id=str(row.task_id) if row.task_id else None,
waiting_for=row.waiting_for,
waiting_since=row.waiting_since,
context=dict(row.context or {}),
)
count += 1
if count:
logger.info(
"Restored waiting records from database",
count=count,
)
return count
except Exception as e:
logger.error("Failed to restore waiting records", error=str(e))
return 0
async def resolve_wait(
self,
agent_id: str,
resolution: dict[str, Any],
) -> AgentInstance | None:
"""
Resolve a wait condition and respawn the agent.
Args:
agent_id: The waiting agent
resolution: Details about the resolution
Returns:
Respawned AgentInstance or None
"""
if agent_id not in self._waiting_records:
return None
record = self._waiting_records[agent_id]
del self._waiting_records[agent_id]
await self._delete_waiting_record(agent_id)
# Generate resume prompt
resume_prompt = self._generate_resume_prompt(record, resolution)
# Preserve the original git_context from the prior instance so the
# respawned agent keeps the same workspace mount path.
prior = self._instances.get(agent_id)
prior_git_context = prior.config.git_context if prior and prior.config else None
# Respawn
return await self.spawn_agent(
agent_id=agent_id,
initial_prompt=resume_prompt,
task_id=record.task_id,
git_context=prior_git_context,
)
def _generate_resume_prompt(
self,
record: WaitingRecord,
resolution: dict[str, Any],
) -> str:
"""Generate a resume prompt for a respawning agent."""
if record.waiting_for == "blocker_resolution":
return f"""
You were working on TASK-{record.task_id} and got blocked.
The blocker has been resolved: {resolution.get("details", "Resolved")}
Resume by:
1. Reading your checkpoint from .tasks/active/TASK-{record.task_id}/
2. Call unblock("{record.task_id}")
3. Continue from where you left off
"""
elif record.waiting_for == "qa_result":
if resolution.get("passed"):
return f"""
TASK-{record.task_id} has passed QA review.
The task is now awaiting documentation.
You may return to scanning for new work with give_me_work().
"""
else:
return f"""
TASK-{record.task_id} needs revision based on QA feedback.
QA notes: {resolution.get("notes", "See task for details")}
Resume by:
1. Reading the QA feedback
2. Updating your TODOs to address each issue
3. Making the fixes
4. Re-submitting for QA
"""
elif record.waiting_for == "answer":
return f"""
You asked a question about TASK-{record.task_id}:
Your question: {record.context.get("question", "Unknown")}
Answer received: {resolution.get("answer", "Unknown")}
Resume by incorporating this information and continuing from where you stopped.
"""
elif record.waiting_for == "assignment":
return f"""
You have been assigned a new task: TASK-{resolution.get("task_id")}
Start by:
1. Review the task details provided in your briefing / context_briefing
2. Follow the standard workflow: UNDERSTAND → PLAN → EXECUTE → VERIFY → NOTES
"""
else:
return f"Resuming. Wait condition '{record.waiting_for}' resolved."
# =========================================================================
# HEALTH MONITORING
# =========================================================================
async def _health_loop(self) -> None:
"""Background health check loop."""
while self._running:
try:
await asyncio.sleep(30) # Check every 30 seconds
await self._check_health()
except asyncio.CancelledError:
break
except Exception as e:
logger.error("Health check error", error=str(e))
async def _sweeper_loop(self) -> None:
"""Background sweeper for session timeouts and stale notifications.
Addresses two silent-failure surfaces:
- SessionTable.timeout_seconds / max_time_window were never enforced;
sessions stayed ACTIVE forever.
- NotificationTable.expires_at existed but no job ever acted on it.
Runs on its own interval so a slow sweep can't delay agent dispatch.
"""
sweep_interval = 60 # seconds
while self._running:
try:
await asyncio.sleep(sweep_interval)
await self._run_sweep()
except asyncio.CancelledError:
break
except Exception as e:
logger.error("Sweeper loop error", error=str(e))
async def _run_sweep(self) -> None:
"""Run one pass of session + notification sweepers."""
from roboco.db.base import get_session_factory
from roboco.services.messaging import get_messaging_service
from roboco.services.notification_delivery import (
get_notification_delivery_service,
)
session_factory = get_session_factory()
async with session_factory() as db:
msg_svc = get_messaging_service(db)
try:
closed = await msg_svc.sweep_timed_out_sessions()
if closed:
await db.commit()
except Exception as e:
await db.rollback()
logger.warning("Session sweep failed", error=str(e))
deliv_svc = get_notification_delivery_service(db)
try:
expired = await deliv_svc.sweep_expired_notifications()
if expired:
await db.commit()
except Exception as e:
await db.rollback()
logger.warning("Notification sweep failed", error=str(e))
# Budget kill-switch — runs every sweep. Any agent whose SDK reports
# halt=true has breached its per-session tool-call cap; terminate the
# container so the next dispatcher tick doesn't waste tokens on the
# same session.
await self._sweep_budget_exceeded()
# Token-usage instrumentation: snapshot active agents and roll up
# closed sessions into the daily aggregation table.
await self._sweep_token_snapshots()
await self._sweep_daily_rollup()
# Prune old agent transcripts (throttled internally to ~hourly) so the
# operator's bind-mounted ~/.claude doesn't grow without bound.
await self._sweep_transcript_retention()
# Close-on-land for landed supersedes — runs here (always-on sweeper)
# rather than the default-off external-PR poll loop, so a supersede that
# lands after external_pr_enabled is toggled off is still reconciled.
await self._sweep_superseded_prs()
async def _sweep_superseded_prs(self) -> None:
"""Retire the contributor PR for any supersede umbrella that landed.
Dormant in a standard deployment: when no ``external_pr_supersede``
umbrellas exist the lookup returns nothing and no GitHub call is made,
so this is safe to run unconditionally on every sweep.
"""
from roboco.db.base import get_session_factory
from roboco.services.git import GitService
from roboco.services.task import get_task_service
system_id = _foundation.AGENTS["system"].uuid
session_factory = get_session_factory()
async with session_factory() as db:
try:
git = GitService(db)
task_service = get_task_service(db)
closed = await self._close_superseded_prs(git, task_service, system_id)
if closed:
await db.commit()
except Exception as e:
await db.rollback()
logger.warning("Supersede close-on-land sweep failed", error=str(e))
async def _sweep_transcript_retention(self) -> None:
"""Prune agent transcripts older than the retention window.
Throttled to ``settings.transcript_prune_interval_seconds``. Reads the
window from the ``system_settings`` store (panel-editable), falling back
to ``settings.transcript_retention_days``. Only agent-owned project dirs
(``-app`` + per-workspace dirs) are touched — never the operator's own
Claude sessions. Best-effort: any failure is logged, never raised.
"""
if not settings.transcript_prune_enabled:
return
now = datetime.now(UTC)
last = self._last_transcript_prune
if (
last is not None
and (now - last).total_seconds()
< settings.transcript_prune_interval_seconds
):
return
self._last_transcript_prune = now
retention_days = settings.transcript_retention_days
with contextlib.suppress(Exception):
from roboco.db.base import get_session_factory
from roboco.services.settings import get_settings_service
session_factory = get_session_factory()
async with session_factory() as db:
retention_days = await get_settings_service(db).get_int(
"transcript_retention_days", settings.transcript_retention_days
)
from roboco.runtime.transcript_retention import select_prunable_transcripts
projects_root = Path.home() / ".claude" / "projects"
cutoff = (now - timedelta(days=retention_days)).timestamp()
prunable = select_prunable_transcripts(
projects_root, settings.workspaces_root, cutoff
)
pruned = 0
for transcript in prunable:
try:
transcript.unlink()
pruned += 1
except OSError as exc:
logger.debug(
"Transcript prune failed", path=str(transcript), error=str(exc)
)
if pruned:
logger.info(
"Pruned old agent transcripts",
count=pruned,
retention_days=retention_days,
)
@staticmethod
async def _fetch_budget_status(
client: httpx.AsyncClient, url: str, agent_id: str
) -> dict[str, Any] | None:
"""Read an agent's SDK budget status; None if unreachable/not-JSON.
The SDK being unreachable is benign (container not yet started, already
gone, or a transient blip) and the health loop covers genuine failures,
so the failure is swallowed — but logged at debug so it is observable
rather than silent (the bare try/except/continue it replaced was not).
"""
try:
resp = await client.get(url)
except httpx.HTTPError as exc:
logger.debug(
"Budget status unreachable; skipping agent this sweep",
agent_id=agent_id,
error=str(exc),
)
return None
if resp.status_code != http_status.HTTP_200_OK:
return None
try:
data = resp.json()
except ValueError as exc:
logger.debug(
"Budget status not JSON; skipping agent this sweep",
agent_id=agent_id,
error=str(exc),
)
return None
return data if isinstance(data, dict) else None
async def _sweep_budget_exceeded(self) -> None:
"""Stop agents whose per-session SDK budget reports halt=true.
Each agent's SDK server is reachable at
`http://roboco-agent-{agent_id}:9000/budget/status` on the shared
agent network. A budget-exceeded agent gets a forced stop with a
`budget_exceeded` reason; the task is already being auto-substituted
by the post-tool hook on the agent side.
"""
if not self._instances:
return
async with httpx.AsyncClient(timeout=3.0) as client:
for agent_id, instance in list(self._instances.items()):
if instance.state not in (
AgentState.ACTIVE,
AgentState.WAITING_SHORT,
):
continue
url = f"http://roboco-agent-{agent_id}:{SDK_PORT}/budget/status"
data = await self._fetch_budget_status(client, url, agent_id)
if data is None or not data.get("halt"):
continue
logger.warning(
"Agent budget exceeded; terminating container",
agent_id=agent_id,
total_calls=data.get("total"),
halt_threshold=data.get("halt_threshold"),
)
try:
await self.stop_agent(agent_id, graceful=True)
except Exception as e:
logger.warning(
"Failed to stop budget-exceeded agent",
agent_id=agent_id,
error=str(e),
)
@staticmethod
async def _inspect_container_state(
container_name: str,
) -> tuple[bool, int | None]:
"""Return (is_running, exit_code) from `docker inspect`.
exit_code is None when the output is missing or unparseable; the
caller treats None as a crash for safety.
"""
proc = await asyncio.create_subprocess_exec(
"docker",
"inspect",
"-f",
"{{.State.Running}} {{.State.ExitCode}}",
container_name,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.DEVNULL,
)
stdout, _ = await proc.communicate()
parts = stdout.decode().strip().split()
is_running = bool(parts) and parts[0] == "true"
try:
exit_code = int(parts[1]) if len(parts) > 1 and parts[1] else None
except ValueError:
exit_code = None
return is_running, exit_code
async def _handle_stopped_container(
self, agent_id: str, instance: Any, exit_code: int | None
) -> None:
"""Update state + auto-restart only when the exit was non-zero.
Graceful exits (exit 0 — agent called i_am_idle)
were treated as crashes by the old logic. The health check bumped
error_count and respawned the agent with the prior task_id even if
the task had since moved into a state the role can't claim from
(e.g. QA → needs_revision). Now: clean exits reset error_count and
do nothing; non-zero exits keep the existing crash-retry behaviour.
"""
cid = instance.container_id[:12] if instance.container_id else None
graceful = exit_code == 0
if graceful:
logger.info(
"Agent container exited gracefully",
agent_id=agent_id,
container_id=cid,
exit_code=exit_code,
)
else:
logger.warning(
"Agent container stopped unexpectedly",
agent_id=agent_id,
container_id=cid,
exit_code=exit_code,
)
# The agent self-exited (a graceful i_am_idle shutdown, or a crash), so
# stop_agent() — which normally finalizes — was never called. Finalize
# here to capture token usage from the transcript; otherwise the
# spawn-session row is left open with zero tokens.
await self._finalize_spawn_session(
agent_id, exit_reason="completed" if graceful else "crashed"
)
instance.state = AgentState.OFFLINE
instance.container_id = None
if graceful:
instance.error_count = 0
return
instance.error_count += 1
max_retries = 3
if instance.error_count < max_retries:
logger.info("Auto-restarting crashed agent", agent_id=agent_id)
await self.spawn_agent(
agent_id=agent_id,
task_id=instance.current_task_id,
git_context=(instance.config.git_context if instance.config else None),
)
elif instance.error_count == max_retries:
# Exactly at the threshold — escalate once to humans so a
# stranded agent doesn't die silently. Subsequent crashes
# stay quiet to avoid notification spam.
logger.error(
"Agent exceeded max restart attempts; escalating",
agent_id=agent_id,
error_count=instance.error_count,
task_id=instance.current_task_id,
)
await self._notify_agent_stranded(
agent_id=agent_id,
error_count=instance.error_count,
task_id=instance.current_task_id,
)
async def _check_health(self) -> None:
"""Check health of all running agents."""
for agent_id, instance in list(self._instances.items()):
if instance.state not in (AgentState.ACTIVE, AgentState.WAITING_SHORT):
continue
if instance.container_id is None:
continue
is_running, exit_code = await self._inspect_container_state(
f"roboco-agent-{agent_id}"
)
if not is_running:
await self._handle_stopped_container(agent_id, instance, exit_code)
async def _notify_agent_stranded(
self,
agent_id: str,
error_count: int,
task_id: str | None,
) -> None:
"""Create a notification for humans when an agent can't be restarted.
Posts a high-priority notification addressed to the auditor and CEO.
Fire-and-forget: the agent is already dead; don't let our own failure
stop the health loop.
"""
try:
from sqlalchemy import select
from roboco.db.base import get_session_factory
from roboco.db.tables import AgentTable, NotificationTable
from roboco.models.base import (
AgentRole,
NotificationPriority,
NotificationType,
)
from roboco.services.notification_delivery import (
get_notification_delivery_service,
)
from roboco.utils.converters import require_uuid
session_factory = get_session_factory()
async with session_factory() as db:
orch_agent = await db.execute(
select(AgentTable).where(AgentTable.role == AgentRole.AUDITOR)
)
auditor = orch_agent.scalar_one_or_none()
ceo_result = await db.execute(
select(AgentTable).where(AgentTable.role == AgentRole.CEO)
)
ceo = ceo_result.scalar_one_or_none()
recipients = [a.id for a in (auditor, ceo) if a is not None]
if not recipients:
logger.warning(
"No auditor/ceo found for stranded-agent notification",
agent_id=agent_id,
)
return
# recipients is non-empty (guarded above) and already holds the
# non-None ids in auditor-then-ceo order — its first entry is the
# same value as `auditor.id if auditor else ceo.id`, without the
# union-narrowing mypy can't prove.
from_agent = recipients[0]
notification = NotificationTable(
type=NotificationType.ALERT,
priority=NotificationPriority.HIGH,
from_agent=from_agent,
to_agents=recipients,
subject=f"Agent stranded: {agent_id}",
body=(
f"Agent '{agent_id}' exceeded max restart attempts "
f"({error_count}) and will not auto-recover. "
f"Task: {task_id or 'none'}. Manual intervention needed."
),
requires_ack=True,
)
db.add(notification)
await db.flush()
delivery = get_notification_delivery_service(db)
await delivery.deliver(require_uuid(notification.id))
await db.commit()
except Exception as e:
logger.error(
"Failed to send stranded-agent notification",
agent_id=agent_id,
error=str(e),
)
# =========================================================================
# RATE-LIMIT PROBE LOOP
# =========================================================================
async def _strategy_engine_loop(self) -> None:
"""Engine 2: periodically surface goal drift / idle / stranded work.
Dormant by default — returns immediately unless ``strategy_engine_enabled``
is set, so it adds zero behaviour to a standard deployment. Notify-only;
it never spends or builds.
"""
if not settings.strategy_engine_enabled:
return
from roboco.db import get_db_context
from roboco.services.strategy_engine import get_strategy_engine
interval = settings.strategy_engine_interval_seconds
while self._running:
try:
await asyncio.sleep(interval)
async with get_db_context() as db:
await get_strategy_engine(db).run_cycle()
except asyncio.CancelledError:
break
except Exception:
logger.exception("strategy engine cycle failed")
async def _external_pr_poll_loop(self) -> None:
"""Engine 3: discover inbound PRs and open review tasks.
Dormant by default — returns immediately unless ``external_pr_enabled``
OR ``internal_pr_enabled``, so a standard deployment makes no inbound
GitHub call. This only lists open PRs and records a review task per
newly-seen reviewable one (external/fork PRs, and — when internal review
is on — org-repo PRs not tied to an active task); it never fetches or
runs contributor code (that waits on a human confirmation downstream).
New review tasks wake the dispatcher.
"""
if not (settings.external_pr_enabled or settings.internal_pr_enabled):
return
from roboco.db import get_db_context
interval = settings.external_pr_poll_interval_seconds
while self._running:
try:
await asyncio.sleep(interval)
async with get_db_context() as db:
ingested = await self._poll_external_prs_once(db)
if ingested:
self._dispatch_wake.set()
except asyncio.CancelledError:
break
except Exception:
logger.exception("external-PR poll cycle failed")
async def _self_heal_loop(self) -> None:
"""Engine 4: watch RoboCo's OWN CI, surface regressions, open fix tasks.
Dormant by default — returns immediately unless ``self_heal_enabled``, so
a standard deployment makes no CI call and adds zero behaviour. It only
NOTIFIES the CEO and (behind ``self_heal_originate_enabled``) opens a
PENDING fix task into RoboCo's own lifecycle; it never starts, merges, or
deploys. The per-cycle session commits any opened task here.
"""
if not settings.self_heal_enabled:
return
from roboco.db import get_db_context
from roboco.services.self_heal_engine import get_self_heal_engine
# Operability: self-heal is armed but has no target → it will silently
# no-op every cycle. Say so once at startup so a misconfiguration (unset
# or wrong ROBOCO_SELF_HEAL_PROJECT_SLUG) isn't mistaken for "all green".
if not settings.self_heal_project_slug.strip():
logger.warning(
"self-heal enabled but self_heal_project_slug is unset — the loop "
"will not detect anything until the target project is configured"
)
interval = settings.self_heal_interval_seconds
while self._running:
try:
await asyncio.sleep(interval)
async with get_db_context() as db:
await get_self_heal_engine(db).run_cycle()
await db.commit()
except asyncio.CancelledError:
break
except Exception:
logger.exception("self-heal cycle failed")
@staticmethod
def _repo_key(git_url: str) -> str:
"""Normalized repo identity (case/.git/trailing-slash insensitive)."""
return git_url.lower().rstrip("/").removesuffix(".git")
@classmethod
def _projects_one_per_repo(cls, projects: list[Any]) -> list[Any]:
"""One canonical project per distinct repo.
Many projects can point at the SAME repo — a monorepo product's
backend/frontend/ux cells each have their own Project mapping to one
git_url. Polling per-project would then ingest one review task per cell
for a single external PR (the per-(project,pr) dedup can't see across
projects). Collapse to one canonical project per repo (deterministic by
slug so the pick is stable across polls); genuinely separate repos
(multi-repo) each keep their own. Projects without a git_url are skipped.
"""
seen: set[str] = set()
canonical: list[Any] = []
for project in sorted(projects, key=lambda p: str(p.slug)):
git_url = getattr(project, "git_url", None)
if not git_url:
continue
key = cls._repo_key(git_url)
if key in seen:
continue
seen.add(key)
canonical.append(project)
return canonical
async def _poll_external_prs_once(self, db: "AsyncSession") -> int:
"""One discovery pass across active repos; returns tasks ingested.
Repo-aware: collapses active projects to one canonical project per
distinct repo (so a monorepo product yields ONE review per PR, not one
per cell-project), lists each repo's open PRs, and ingests a de-duped
review task for each reviewable one — external/fork PRs, and (when
internal review is on) org-repo PRs not tied to an active task. Commits
once at the end.
"""
from roboco.services.git import GitService
from roboco.services.project import get_project_service
from roboco.services.task import get_task_service
git = GitService(db)
task_service = get_task_service(db)
projects = await get_project_service(db).list_all(active_only=True)
system_id = _foundation.AGENTS["system"].uuid
allowlist = {a.lower() for a in settings.external_pr_author_allowlist}
ingested = 0
for project in self._projects_one_per_repo(projects):
for pr in await git.list_open_prs(project.slug):
if await self._ingest_pr_if_reviewable(
task_service, project, pr, system_id, allowlist
):
ingested += 1
await db.commit()
return ingested
async def _ingest_pr_if_reviewable(
self,
task_service: "TaskService",
project: Any,
pr: dict[str, Any],
system_id: "UUID",
allowlist: set[str],
) -> bool:
"""Ingest a review task for one open PR if it qualifies; True if ingested.
External/fork PRs (when external review is on and the author is allowed)
are ingested as ``external_pr``. Org-repo PRs whose head branch no active
task owns (when internal review is on) are ingested as ``internal_pr`` —
the org's own in-flight integration PRs are skipped, since a live task
owns their branch and they already pass QA + PM review.
"""
if pr.get("number") is None:
return False
if self._is_external_pr(pr):
if not settings.external_pr_enabled or not self._pr_author_allowed(
pr, allowlist
):
return False
source = "external_pr"
else:
if not settings.internal_pr_enabled:
return False
if await task_service.active_task_owns_branch(
str(pr.get("head_ref") or "")
):
return False
source = "internal_pr"
created = await task_service.ingest_external_pr(
project_id=cast("UUID", project.id),
pr=pr,
created_by=system_id,
team=Team.BOARD,
source=source,
)
return created is not None
async def _close_superseded_prs(
self, git: Any, task_service: Any, system_id: "UUID"
) -> int:
"""Close + link the contributor PR for each landed supersede umbrella.
Idempotent: each umbrella is marked ``closed=1`` after its contributor PR
is closed, so it is processed once. ``delete_branch=False`` — the
contributor's branch lives on their fork; we never touch it. Caller
commits.
"""
closed = 0
for umbrella in await task_service.supersede_umbrellas_pending_close():
pr_number = self._parse_supersede_pr(umbrella.quick_context or "")
if pr_number is None:
continue
try:
await git.close_pull_request(
pr_number,
comment=(
"Superseded by the roboco team's own PR — the work was "
"finished and hardened to our standards. Thanks for the "
"contribution!"
),
delete_branch=False,
actor_agent_id=system_id,
# PR numbers are per-repo — scope the close to THIS
# umbrella's project so a same-numbered PR in another
# project's repo is never resolved (and closed) by mistake.
project_id=cast("UUID", umbrella.project_id),
)
except Exception:
# A permanent close failure (deleted PR, revoked PAT) would
# otherwise re-fire + re-log every tick forever; keep it a single
# warning rather than a per-tick stack trace.
logger.warning("close-on-land failed", pr_number=pr_number)
continue
await task_service.mark_supersede_pr_closed(cast("UUID", umbrella.id))
closed += 1
return closed
@staticmethod
def _parse_supersede_pr(quick_context: str) -> int | None:
"""Extract the contributor PR number from a supersede umbrella marker.
Anchored to the marker line so a CEO note containing ``pr=`` on a later
line of the multi-writer ``quick_context`` can't be misread as the PR.
"""
for raw in quick_context.splitlines():
line = raw.strip()
if not line.startswith("external_pr_supersede"):
continue
for part in line.split():
if part.startswith("pr="):
try:
return int(part[3:])
except ValueError:
return None
return None
return None
@staticmethod
def _pr_author_allowed(pr: dict[str, Any], allowlist: set[str]) -> bool:
"""With a non-empty allowlist, only those GitHub authors are reviewed.
An empty allowlist (the default) reviews every external PR — the review
is read-only, so it is safe; the ``confirmed_by_human`` gate still
protects any later supersede that would run the contributor's code.
"""
if not allowlist:
return True
return (pr.get("user_login") or "").lower() in allowlist
@staticmethod
def _is_external_pr(pr: dict[str, Any]) -> bool:
"""A PR the org did not author: a fork head or a non-member author."""
if pr.get("is_fork"):
return True
trusted = {"OWNER", "MEMBER", "COLLABORATOR"}
assoc = (pr.get("author_association") or "").upper()
return assoc not in trusted
async def supersede_external_pr(self, review_task_id: "UUID") -> dict[str, Any]:
"""CEO-authorized takeover of a reviewed external PR.
Confirms the review task (this CEO action is the human confirmation that
authorizes running the contributor's code), cuts a roboco-owned branch
off the contributor's fork head (refs/pull/{n}/head — the only point
untrusted code enters a roboco branch), and creates the supersede
umbrella for Main PM to delegate to a cell. Returns a status dict.
"""
from roboco.db import get_db_context
from roboco.models.base import TaskStatus
from roboco.services.git import GitService
from roboco.services.project import get_project_service
from roboco.services.task import get_task_service
# Serialize concurrent CEO calls (double-click) — the dedup check and
# the umbrella/branch creation are not atomic across DB sessions.
async with self._supersede_lock, get_db_context() as db:
task_service = get_task_service(db)
review = await task_service.get(review_task_id)
if review is None or getattr(review, "source", "") not in PR_REVIEW_SOURCES:
return {"ok": False, "error": "not a PR-review task"}
if not review.project_id or not review.pr_number:
return {
"ok": False,
"error": "review task missing project or pr_number",
}
# Review-first: only supersede a PR the org has actually reviewed.
if review.status != TaskStatus.COMPLETED:
return {
"ok": False,
"error": "review not complete — review the PR first",
}
project = await get_project_service(db).get(cast("UUID", review.project_id))
if project is None:
return {"ok": False, "error": "project not found"}
pr_number = int(review.pr_number)
project_id = cast("UUID", review.project_id)
# Idempotent: a repeat call returns the existing umbrella — no second
# branch cut, no duplicate cell takeover.
existing = await task_service.find_supersede_umbrella(project_id, pr_number)
if existing is not None:
return {
"ok": True,
"supersede_task_id": str(existing.id),
"branch": existing.branch_name,
"already_superseded": True,
}
system_id = _foundation.AGENTS["system"].uuid
branch_name = f"feature/main_pm/supersede-pr-{pr_number}"
# The CEO authorized fetching + finishing the contributor's code.
review.confirmed_by_human = True
# Create the umbrella BEFORE the push: a create failure then can't
# orphan a pushed branch. Only a commit failure after the push could
# (rare) — the branch is logged so an orphan stays discoverable.
umbrella = await task_service.create_supersede_umbrella(
review_task_id=review_task_id,
branch_name=branch_name,
created_by=system_id,
)
umbrella_id = str(umbrella.id) if umbrella is not None else None
git = GitService(db)
workspace = await git.get_workspace(project.slug, agent_id=system_id)
logger.warning(
"supersede: cutting roboco branch off untrusted fork PR head",
branch=branch_name,
pr_number=pr_number,
project=project.slug,
)
await git.create_branch_from_pr_head(
workspace, project.slug, pr_number, branch_name
)
await db.commit()
self._dispatch_wake.set()
return {"ok": True, "supersede_task_id": umbrella_id, "branch": branch_name}
async def _rate_limit_probe_loop(self) -> None:
"""Background loop: probe rate-limited providers every ~30 seconds.
Runs independently of the 60-second session/notification sweeper so
rate limits can be cleared on their own cadence without blocking
other sweep work.
"""
probe_interval = 30 # seconds
while self._running:
try:
await asyncio.sleep(probe_interval)
await self._sweep_rate_limit_probes()
except asyncio.CancelledError:
break
except Exception as e:
logger.error("Rate-limit probe loop error", error=str(e))
async def _sweep_rate_limit_probes(self) -> None:
"""One probe pass: check every rate-limited provider.
For each provider whose estimated_lift_at has passed:
- Call ``_do_probe(provider)`` to test connectivity.
- **Success**: clear the tracker, resolve all parked agents, publish
``RATE_LIMIT_LIFTED``.
- **Failure**: increment probe_failures; if the count reaches 10 and
we haven't already sent a CEO notification for this episode, send
one now.
"""
from roboco.services.gateway.rate_limit_tracker import RateLimitStateTracker
try:
providers = await RateLimitStateTracker.list_rate_limited_providers()
except Exception as e:
logger.warning("Failed to list rate-limited providers", error=str(e))
return
for provider, state in providers:
try:
await self._probe_one_provider(provider, state)
except Exception as e:
logger.error(
"Unhandled error probing provider",
provider=provider,
error=str(e),
)
def _make_tracker(self, provider: str) -> Any:
"""Return a RateLimitStateTracker for *provider*.
Extracted as its own method so unit tests can monkeypatch it to
return an async mock without needing to intercept lazy imports.
"""
from roboco.services.gateway.rate_limit_tracker import RateLimitStateTracker
return RateLimitStateTracker(provider)
@staticmethod
def _too_early_to_probe(state: dict[str, Any]) -> bool:
"""True while the estimated lift time (activated_at + retry_after) is future.
Missing or malformed timestamps fall through to allow the probe.
"""
activated_at_raw = state.get("activated_at")
retry_after = state.get("retry_after")
if not activated_at_raw or retry_after is None:
return False
try:
activated_at = datetime.fromisoformat(activated_at_raw)
except (ValueError, TypeError):
return False
return datetime.now(UTC) < activated_at + timedelta(seconds=retry_after)
def _parked_agents_for(self, provider: str) -> list[str]:
"""Agent slugs parked waiting for *provider*'s rate limit to lift."""
return [
agent_id
for agent_id, record in list(self._waiting_records.items())
if record.waiting_for == "rate_limit_lifted"
and record.context.get("provider") == provider
]
async def _on_probe_success(self, provider: str, tracker: Any) -> None:
"""Clear the limit, resume parked agents, publish RATE_LIMIT_LIFTED."""
logger.info("Rate-limit probe succeeded; clearing provider", provider=provider)
await tracker.clear()
# New episodes should get a fresh CEO notification.
self._rate_limit_ceo_notified.discard(provider)
resumed = self._parked_agents_for(provider)
for agent_id in resumed:
with contextlib.suppress(Exception):
await self.resolve_wait(
agent_id,
{
"reason": "rate_limit_lifted",
"provider": provider,
"lifted_at": datetime.now(UTC).isoformat(),
},
)
with contextlib.suppress(Exception):
from roboco.events import get_event_bus
from roboco.models.events import Event, EventType
await get_event_bus().publish(
Event(
type=EventType.RATE_LIMIT_LIFTED,
data={
"provider": provider,
"resumedAgents": resumed,
"timestamp": datetime.now(UTC).isoformat(),
},
)
)
logger.info(
"RATE_LIMIT_LIFTED published",
provider=provider,
resumed_agents=len(resumed),
)
async def _on_probe_failure(
self, provider: str, tracker: Any, activated_at_raw: str | None
) -> None:
"""Count a failed probe; notify the CEO once at the failure threshold."""
failure_count = await tracker.increment_probe_failures()
logger.debug(
"Rate-limit probe failed", provider=provider, probe_failures=failure_count
)
if (
failure_count >= _CEO_NOTIFY_THRESHOLD
and provider not in self._rate_limit_ceo_notified
):
self._rate_limit_ceo_notified.add(provider)
await self._notify_rate_limit_ceo(
provider=provider,
activated_at_str=activated_at_raw or "unknown",
paused_agent_count=len(self._parked_agents_for(provider)),
)
async def _probe_one_provider(self, provider: str, state: dict[str, Any]) -> None:
"""Probe a single rate-limited provider and handle the outcome."""
if self._too_early_to_probe(state):
return # Wait until after the estimated lift time.
tracker = self._make_tracker(provider)
if await self._do_probe(provider):
await self._on_probe_success(provider, tracker)
else:
await self._on_probe_failure(provider, tracker, state.get("activated_at"))
@staticmethod
def _probe_target(provider: str) -> tuple[str | None, dict[str, str]]:
"""Resolve the (url, headers) for a free liveness probe of ``provider``.
Returns ``(None, {})`` when the provider can't be probed — an unknown
provider, or Anthropic with no API key configured. The caller then
falls back to time-expiry optimism rather than parking forever.
"""
p = provider.lower()
if p == "anthropic":
key = settings.anthropic_api_key
if not key:
return None, {}
return (
f"{_ANTHROPIC_PROBE_BASE}/v1/models",
{"x-api-key": key, "anthropic-version": "2023-06-01"},
)
if p.startswith("ollama"):
return f"{settings.ollama_base_url.rstrip('/')}/api/tags", {}
return None, {}
async def _do_probe(self, provider: str) -> bool:
"""Return True if ``provider`` is accepting requests again (not 429).
Makes a free, unmetered liveness call — Anthropic ``GET /v1/models``
or Ollama ``GET /api/tags`` — and treats any non-429 response as the
rate limit having lifted. A 429 keeps the provider parked; a network
error stays parked too (retry next sweep). When the provider can't be
probed (no key / unknown), fall back to time-expiry optimism: the
caller only reaches this after ``estimated_lift_at`` has passed.
Injectable boundary — tests monkeypatch this to force outcomes.
"""
url, headers = self._probe_target(provider)
if url is None:
return True # cannot probe — trust the elapsed retry_after window
try:
async with httpx.AsyncClient(timeout=_PROBE_TIMEOUT_SECONDS) as client:
resp = await client.get(url, headers=headers)
except httpx.HTTPError as exc:
logger.debug(
"Rate-limit probe request failed", provider=provider, error=str(exc)
)
return False # unreachable — stay parked, retry on the next sweep
return resp.status_code != _HTTP_TOO_MANY_REQUESTS
async def _notify_rate_limit_ceo(
self,
provider: str,
activated_at_str: str,
paused_agent_count: int,
) -> None:
"""Send a high-priority notification to the CEO about a persistent rate limit.
Fires once per rate-limit episode. Follows the same pattern as
``_notify_stranded_agent`` — direct DB insert + delivery.deliver().
"""
try:
from sqlalchemy import select as _select
from roboco.db.base import get_session_factory
from roboco.db.tables import AgentTable, NotificationTable
from roboco.models.base import (
AgentRole,
NotificationPriority,
NotificationType,
)
from roboco.services.notification_delivery import (
get_notification_delivery_service,
)
from roboco.utils.converters import require_uuid
# Compute human-friendly duration
duration_desc = "unknown duration"
try:
activated_at = datetime.fromisoformat(activated_at_str)
elapsed = datetime.now(UTC) - activated_at
total_minutes = int(elapsed.total_seconds() / 60)
if total_minutes < 60: # noqa: PLR2004
duration_desc = f"{total_minutes} minute(s)"
else:
duration_desc = f"{total_minutes // 60}h {total_minutes % 60}m"
except (ValueError, TypeError):
pass
session_factory = get_session_factory()
async with session_factory() as db:
ceo_result = await db.execute(
_select(AgentTable).where(AgentTable.role == AgentRole.CEO)
)
ceo = ceo_result.scalar_one_or_none()
if ceo is None:
logger.warning(
"CEO agent not found; skipping rate-limit CEO notification",
provider=provider,
)
return
notification = NotificationTable(
type=NotificationType.ALERT,
priority=NotificationPriority.HIGH,
from_agent=ceo.id,
to_agents=[ceo.id],
subject=f"Rate limit persisting: {provider}",
body=(
f"Provider '{provider}' has been rate-limited for "
f"{duration_desc}. "
f"{paused_agent_count} agent(s) are currently paused. "
f"10 consecutive probe attempts have failed. "
f"Manual intervention may be required."
),
requires_ack=True,
)
db.add(notification)
await db.flush()
delivery = get_notification_delivery_service(db)
await delivery.deliver(require_uuid(notification.id))
await db.commit()
logger.info(
"Rate-limit CEO notification sent",
provider=provider,
paused_agents=paused_agent_count,
)
except Exception as e:
logger.error(
"Failed to send rate-limit CEO notification",
provider=provider,
error=str(e),
)
# =========================================================================
# STATUS API
# =========================================================================
def get_state(self, agent_id: str) -> AgentState:
"""Get current state of an agent."""
if agent_id not in self._instances:
return AgentState.OFFLINE
return self._instances[agent_id].state
def get_instance(self, agent_id: str) -> AgentInstance | None:
"""Get instance for an agent."""
return self._instances.get(agent_id)
def get_waiting_agents(self) -> dict[str, WaitingRecord]:
"""Get all waiting agents."""
return dict(self._waiting_records)
def get_status_summary(self) -> dict[str, Any]:
"""Get summary of all agent states."""
by_state: dict[str, int] = {}
agents: list[dict[str, Any]] = []
for state in AgentState:
count = sum(1 for i in self._instances.values() if i.state == state)
if count > 0:
by_state[state.value] = count
for agent_id, instance in self._instances.items():
cid = instance.container_id[:12] if instance.container_id else None
agents.append(
{
"agent_id": agent_id,
"state": instance.state.value,
"container_id": cid,
"task_id": instance.current_task_id,
"error_count": instance.error_count,
"started_at": instance.started_at.isoformat()
if instance.started_at
else None,
}
)
return {
"total": len(self._instances),
"by_state": by_state,
"waiting_count": len(self._waiting_records),
"agents": agents,
}
# =========================================================================
# SMART DISPATCHER - API HELPERS
# =========================================================================
@property
def _api_url(self) -> str:
"""Get the internal API URL for task/notification queries."""
return settings.internal_api_url
def _is_agent_active(self, agent_id: str) -> bool:
"""Check if an agent is currently running."""
if agent_id not in self._instances:
return False
return self._instances[agent_id].state == AgentState.ACTIVE
async def _check_parent_branch_ready(
self, client: httpx.AsyncClient, task_id: str, parent_id: str
) -> str | None:
"""Verify the parent task has a branch; auto-block + return msg if not.
Race window: the PM's `i_will_plan` claims the parent (transitions
status -> in_progress, sets assigned_to) and then `_finalize_claim`
creates the branch via `_ensure_branch_for_task`. Both actions land
in the same DB transaction but a child dev's spawn dispatch can fire
microseconds before that transaction commits and see branch_name=None.
Without retry we'd auto-block the child unnecessarily.
When the parent is clearly mid-claim (in_progress + assigned_to set)
re-fetch up to 3 times with a 250ms delay before giving up. Total
worst-case wait is 750ms — well inside the dispatcher's tick budget
and only paid when the race actually triggers. Real misses (parent
still pending or unassigned) auto-block immediately as before.
"""
parent_resp = await client.get(f"{self._api_url}/tasks/{parent_id}")
if not parent_resp.is_success:
return None
parent = parent_resp.json()
if parent.get("branch_name"):
return None
# A coordination/fan-out parent (product, no repo of its own) never gets
# a branch: the child resolves its own real project and cuts from that
# project's default branch, not from the parent. Blocking the child on a
# branch the parent will never have wedges the cell↔Main-PM loop.
if _is_coordination_task(parent):
return None
if parent.get("status") == "in_progress" and parent.get("assigned_to"):
for _ in range(3):
await asyncio.sleep(0.25)
parent_resp = await client.get(f"{self._api_url}/tasks/{parent_id}")
if not parent_resp.is_success:
continue
parent = parent_resp.json()
if parent.get("branch_name"):
return None
await self._auto_block_task(
client,
task_id,
"Parent task must be claimed first to create its branch",
)
return f"Task {task_id} waiting for parent branch"
async def _check_dev_needs_subtasks(
self, client: httpx.AsyncClient, task: dict[str, Any]
) -> str | None:
"""Block non-trivial root tasks routed to a dev without subtasks."""
complexity = task.get("estimated_complexity", "low")
parent_task_id = task.get("parent_task_id")
if complexity not in ("medium", "high") or parent_task_id:
return None
task_id = task.get("id")
try:
resp = await client.get(f"{self._api_url}/tasks/{task_id}/subtasks")
subtasks = resp.json() if resp.is_success else []
except Exception:
subtasks = []
if subtasks:
return None
await self._auto_block_task(
client,
str(task_id),
f"Task complexity is {complexity} but no subtasks. "
"Cell PM must break down work first.",
)
return (
f"Task {task_id} is {complexity} complexity "
"without subtasks - Cell PM must break it down"
)
async def _validate_task_for_spawn(
self,
client: httpx.AsyncClient,
task: dict,
agent_slug: str,
) -> str | None:
"""
Validate task is ready for agent spawn.
Returns None if valid, or error message if task cannot proceed.
This prevents spawning agents on tasks that are missing prerequisites.
"""
from roboco.agents_config import get_agent_role
if shape_err := await self._check_spawn_task_shape(client, task):
return shape_err
if dep_err := await self._check_dependencies_terminal(client, task):
return dep_err
# _check_spawn_task_shape guarantees a non-empty id past this point.
task_id = str(task.get("id"))
parent_id = task.get("parent_task_id")
if parent_id:
err = await self._check_parent_branch_ready(client, task_id, parent_id)
if err:
return err
logger.info("Task ready for hierarchical branch creation", task_id=task_id)
if get_agent_role(agent_slug) == "developer":
err = await self._check_dev_needs_subtasks(client, task)
if err:
return err
return None # All validations passed
async def _check_spawn_task_shape(
self, client: httpx.AsyncClient, task: dict[str, Any]
) -> str | None:
"""Reject a task that is structurally unroutable (id/description/repo)."""
task_id = task.get("id")
if not task_id:
return "Task missing ID"
min_description_len = 10
description = (task.get("description") or "").strip()
if len(description) < min_description_len:
return (
f"Task {task_id} has inadequate description ({len(description)} chars)"
)
# A coordination task carries a product instead of a repo; only a task
# with neither is genuinely unroutable.
if not task.get("project_id") and not _is_coordination_task(task):
await self._auto_block_task(
client, task_id, "Task needs a project_id or product_id"
)
return f"Task {task_id} needs a project or product"
return None
async def _check_dependencies_terminal(
self, client: httpx.AsyncClient, task: dict[str, Any]
) -> str | None:
"""Hold a pre-assigned task whose dependencies are not yet terminal.
A dev subtask is always pre-assigned, so it never passes through the
unassigned claim pool's dependency filter. Without this gate the
dispatcher would spawn the dev container while a cross-cell dependency
(e.g. the UX/UI design the frontend dev waits on) is still open. Return
a skip reason while ANY dependency is non-terminal; allow the spawn
once every dependency reaches completed/cancelled.
"""
dependency_ids = task.get("dependency_ids") or []
if not dependency_ids:
return None
terminal = ("completed", "cancelled")
for dep_id in dependency_ids:
dep_resp = await client.get(f"{self._api_url}/tasks/{dep_id}")
# A dependency we cannot read is treated as unmet — fail closed
# rather than spawn ahead of work whose state is unknown.
if not dep_resp.is_success or dep_resp.json().get("status") not in terminal:
return (
f"Task {task.get('id')} waiting on non-terminal dependency {dep_id}"
)
return None
async def _auto_block_task(
self, client: httpx.AsyncClient, task_id: str, reason: str
) -> None:
"""Auto-block a task that cannot proceed due to missing prerequisites."""
try:
await client.patch(
f"{self._api_url}/tasks/{task_id}",
json={
"status": "blocked",
"dev_notes": f"[AUTO-BLOCKED] {reason}",
},
)
logger.info(
"Auto-blocked task with missing prerequisites",
task_id=task_id,
reason=reason,
)
except Exception as e:
logger.error(
"Failed to auto-block task",
task_id=task_id,
error=str(e),
)
async def _auto_resume_paused_parent(
self, client: httpx.AsyncClient, task_id: str
) -> None:
"""Resume a paused parent right before its PM is respawned for closure.
A PM auto-pauses its owned parent on i_am_idle (by design,
so the closure dispatcher knows to respawn it). Pre-gateway the
parent was resumed at respawn so the PM landed actionable; the
gateway refactor dropped that, so the respawned PM had to issue
``resume()`` itself — which weak models reliably fail,
wedging the whole chain. Restore the auto-resume:
paused -> in_progress before spawn so the PM can directly
submit_up / complete / escalate. Best-effort; a resume failure
must not block the spawn (the PM can still resume manually).
"""
try:
await client.patch(
f"{self._api_url}/tasks/{task_id}",
json={"status": "in_progress"},
)
logger.info(
"Auto-resumed paused parent for PM closure respawn",
task_id=task_id,
)
except Exception as e:
logger.error(
"Failed to auto-resume paused parent",
task_id=task_id,
error=str(e),
)
async def _auto_recover_blocked_parent(
self, client: httpx.AsyncClient, task_id: str
) -> None:
"""Recover a blocked parent right before its PM is respawned for closure.
Symmetric to ``_auto_resume_paused_parent``. The
closure dispatcher only reaches this point once every descendant
is terminal, so a parent still ``blocked`` here is an errant /
stale block (e.g. a child's i_am_blocked propagated, or a PM
blocked it and never unblocked) — the real dependency is already
done. That resume path handled only ``paused`` parents, so a ``blocked``
one wedged the whole chain forever: the respawned PM cannot
submit_up / complete a blocked parent and must first ``unblock``
it (needs journal:decision), which weak models never reliably do
(a dogfood run wedged exactly here). ``blocked -> in_progress``
is lifecycle-valid — it is precisely what ``unblock(restore=True)``
performs. Best-effort; a failure must not block the spawn (the PM
can still ``unblock`` manually).
"""
try:
await client.patch(
f"{self._api_url}/tasks/{task_id}",
json={"status": "in_progress"},
)
logger.info(
"Auto-recovered blocked parent for PM closure respawn",
task_id=task_id,
)
except Exception as e:
logger.error(
"Failed to auto-recover blocked parent",
task_id=task_id,
error=str(e),
)
def _select_agent_for_cell(self, cell: str, role: str) -> str | None:
"""
Select the best available agent for a cell and role.
Prefers agents that are not currently active.
For developers, uses round-robin among candidates.
"""
prefix_map = {"backend": "be", "frontend": "fe", "ux_ui": "ux"}
prefix = prefix_map.get(cell)
if not prefix:
return None
# Build candidate list based on role
if role == "dev":
candidates = [f"{prefix}-dev-1", f"{prefix}-dev-2"]
elif role == "qa":
candidates = [f"{prefix}-qa"]
elif role == "doc":
candidates = [f"{prefix}-doc"]
elif role == "pm":
candidates = [f"{prefix}-pm"]
else:
return None
# Prefer non-active agents
for agent_id in candidates:
if not self._is_agent_active(agent_id):
return agent_id
# All active - return first (task will queue for them via scan)
return candidates[0]
async def _claim_task_for_agent(
self,
client: httpx.AsyncClient,
task_id: str,
agent_id: str,
) -> bool:
"""Claim a task on behalf of an agent before spawning."""
try:
resp = await client.post(
f"{self._api_url}/tasks/{task_id}/claim",
json={"agent_id": agent_id},
)
if resp.status_code == http_status.HTTP_200_OK:
logger.info(
"Task claimed for agent",
task_id=task_id,
agent_id=agent_id,
)
return True
logger.warning(
"Failed to claim task",
task_id=task_id,
agent_id=agent_id,
status=resp.status_code,
)
except Exception as e:
logger.error("Claim task error", task_id=task_id, error=str(e))
return False
async def _fetch_tasks(
self,
client: httpx.AsyncClient,
status: str | list[str],
team: str | None = None,
) -> list[dict[str, Any]]:
"""Fetch tasks by status and optional team filter."""
# If multiple statuses, make separate requests and combine results
statuses = status if isinstance(status, list) else [status]
all_tasks: list[dict[str, Any]] = []
for single_status in statuses:
params: dict[str, Any] = {"status": single_status}
if team:
params["team"] = team
try:
resp = await client.get(f"{self._api_url}/tasks", params=params)
if resp.status_code == http_status.HTTP_200_OK:
tasks: list[dict[str, Any]] = resp.json()
all_tasks.extend(tasks)
except Exception as e:
logger.error(
"Fetch tasks error", status=single_status, team=team, error=str(e)
)
return all_tasks
async def _fetch_notifications(
self,
client: httpx.AsyncClient,
notification_type: str,
unacknowledged: bool = True,
) -> list[dict[str, Any]]:
"""Fetch notifications by type."""
params: dict[str, Any] = {
"type_filter": notification_type,
"pending_ack_only": str(unacknowledged).lower(),
}
try:
resp = await client.get(
f"{self._api_url}/notifications",
params=params,
)
if resp.status_code == http_status.HTTP_200_OK:
data = resp.json()
items: list[dict[str, Any]] = data.get("items", [])
return items
except Exception as e:
logger.error(
"Fetch notifications error",
notification_type=notification_type,
error=str(e),
)
return []
# =========================================================================
# SMART ROUTING - TASK CLASSIFICATION
# =========================================================================
# Keywords that indicate strategic/board-level tasks
_BOARD_KEYWORDS = frozenset(
{
"roadmap",
"architecture",
"security",
"budget",
"hiring",
"strategy",
"vision",
"milestone",
"release",
"launch",
}
)
# Keywords that indicate PM coordination is needed
_PM_KEYWORDS = frozenset(
{
"coordinate",
"integration",
"cross-team",
"sync",
"planning",
"milestone",
"dependencies",
"review",
}
)
# Keywords that indicate cross-cell work (requires Main PM)
_CROSS_CELL_KEYWORDS = frozenset(
{
"all teams",
"all cells",
"every team",
"every cell",
"all departments",
"cross-cell",
"company-wide",
"organization-wide",
"backend and frontend",
"frontend and backend",
"all three",
}
)
def _has_board_keywords(self, text: str) -> bool:
"""Check if text contains board-level keywords."""
return any(kw in text for kw in self._BOARD_KEYWORDS)
def _has_pm_keywords(self, text: str) -> bool:
"""Check if text contains PM coordination keywords."""
return any(kw in text for kw in self._PM_KEYWORDS)
def _has_cross_cell_keywords(self, text: str) -> bool:
"""Check if text indicates work spanning multiple cells."""
return any(kw in text for kw in self._CROSS_CELL_KEYWORDS)
# Direct team-to-routing mappings (explicit assignments bypass keyword analysis)
_TEAM_ROUTING_MAP: ClassVar[dict[str, str]] = {
"main_pm": "main_pm",
"board": "board",
"marketing": "marketing",
}
@staticmethod
def _route_by_task_type(task_type: str, team: str | None) -> str | None:
"""Route based on task_type field alone; returns None if no match."""
cell_teams = tuple(
sorted(t.value for t in CELL_TEAMS)
) # ("backend", "frontend", "ux_ui")
if task_type in ("planning", "research", "administrative"):
return "cell_pm" if team in cell_teams else "main_pm"
if task_type == "design" and team not in ("backend", "frontend"):
return "cell_pm"
return None
def _classify_cell_code_task(self, text: str, complexity: str) -> str:
"""Route a cell-owned code task WITHIN its cell (dev or cell_pm).
Implementation work that belongs to a CELL never escalates to the
board or main_pm by keyword — a dev task whose description says
"Create & Launch" or "auth/security" is still a dev task. Letting the
board/main_pm keyword heuristics fire on it is how a cell code task
ended up "reviewed" by the board and a PM ended up owning (and
deadlocking) a dev code task.
"""
if self._has_pm_keywords(text) or complexity == "high":
return "cell_pm"
return "dev"
def _classify_strategic_code_task(
self, text: str, team: str | None, complexity: str
) -> str:
"""Route a team-less / "all" top-level code task by strategic heuristics."""
if self._has_board_keywords(text):
return "board"
if (
self._has_cross_cell_keywords(text)
or complexity == "high"
or not team
or team == "all"
):
return "main_pm"
if self._has_pm_keywords(text) or complexity == "medium":
return "cell_pm"
return "dev"
def _classify_code_task(self, task: dict[str, Any]) -> str:
"""Classify a generic `code` task via keyword/complexity heuristics."""
team = task.get("team")
title = (task.get("title") or "").lower()
description = (task.get("description") or "").lower()
text = f"{title} {description}"
complexity = task.get("estimated_complexity", "medium").lower()
cell_teams = frozenset(t.value for t in CELL_TEAMS)
if team in cell_teams:
return self._classify_cell_code_task(text, complexity)
return self._classify_strategic_code_task(text, team, complexity)
def _classify_task_routing(self, task: dict[str, Any]) -> str:
"""
Classify a task for routing based on task_type, team, complexity, and keywords.
Returns one of: "board", "main_pm", "cell_pm", "dev", "marketing"
"""
team = task.get("team")
task_type = task.get("task_type", "code")
# Task type takes precedence for non-code work
by_type = self._route_by_task_type(task_type, team)
if by_type:
return by_type
if team in self._TEAM_ROUTING_MAP:
return self._TEAM_ROUTING_MAP[team]
return self._classify_code_task(task)
# Team to PM mapping for routing
_TEAM_PM_MAP: ClassVar[dict[str, str]] = {
"backend": "be-pm",
"frontend": "fe-pm",
"ux_ui": "ux-pm",
}
def _get_routing_target(self, routing: str, task: dict[str, Any]) -> str | None:
"""
Resolve a routing decision to a specific agent slug.
Args:
routing: One of "board", "main_pm", "cell_pm", "dev", "marketing"
task: The task being routed
Returns:
Agent slug (e.g., "main-pm", "be-pm", "be-dev-1") or None
"""
team = task.get("team")
# Static routing targets
static_targets = {
"board": "product-owner",
"main_pm": "main-pm",
"marketing": "head-marketing",
}
if routing in static_targets:
return static_targets[routing]
# Cell PM routing - requires team lookup
if routing == "cell_pm":
return self._TEAM_PM_MAP.get(team, "main-pm") if team else "main-pm"
# Dev routing - select a cell agent.
if routing == "dev":
agent = self._select_agent_for_cell(team, "dev") if team else None
if agent:
return agent
# No cell agent — team is missing or a non-cell team (fullstack /
# system). Fall back to main-pm to triage rather than leaving the
# task ownerless-and-dormant: the dispatcher never re-spawns an
# unrouted pending task, so a None here strands it. Mirrors the
# cell_pm / escalation `... or "main-pm"` default.
logger.warning(
"dev routing found no cell agent; falling back to main-pm",
task_id=task.get("id"),
team=team,
)
return "main-pm"
# Unrecognized routing classification — never strand the task; main-pm
# triages it instead of it going dormant.
logger.warning(
"unrecognized routing classification; falling back to main-pm",
routing=routing,
task_id=task.get("id"),
)
return "main-pm"
def _build_main_pm_triage_prompt(self, task: dict[str, Any]) -> str:
"""Build prompt for MAIN PM to triage and distribute to Cell PMs."""
task_id = task.get("id", "unknown")
title = task.get("title", "Untitled")
complexity = task.get("complexity", "medium")
description = task.get("description", "")
return f"""You are the MAIN PM at RoboCo. This task is assigned to YOU.
TASK: {task_id}
TITLE: {title}
COMPLEXITY: {complexity}
DESCRIPTION: {description[:500]}
YOUR JOB: Break this down and delegate to Cell PMs. You do NOT implement
code. You do NOT assign directly to developers — Cell PMs manage their
teams. For purely-PM work (validation, announcements, cross-cell sync) you
may keep the task and work it via your gateway verbs.
== DELEGATION TARGETS ==
- Backend work → be-pm (who delegates to be-dev-1 / be-dev-2)
- Frontend work → fe-pm (who delegates to fe-dev-1 / fe-dev-2)
- UX/UI work → ux-pm (who delegates to ux-dev-1 / ux-dev-2)
NEVER assign to a dev slug from this seat — only Cell PM slugs.
== TOOLS ==
Gateway verbs (already loaded):
- evidence(task_id="{task_id}") — inspect the task
- triage_all() — see what's pending across cells
- note(text, scope='decision', task_id="{task_id}")
REQUIRED before i_will_plan / complete / escalate
- i_will_plan(task_id="{task_id}", plan="<your detailed plan as a string>")
claim + record plan + start your own root task
- delegate(parent_task_id="{task_id}", title=..., description=...,
assigned_to=<one of "be-pm" / "fe-pm" / "ux-pm">,
team=<one of "backend" / "frontend" / "ux_ui">,
task_type=<one of "code" / "documentation" / "research" /
"planning" / "design" / "administrative">,
acceptance_criteria=[...],
estimated_complexity=<one of "low" / "medium" / "high">)
creates a subtask under your root and assigns it to a Cell PM.
Use the EXACT enum strings above — invented values like
"development" or "small" are rejected by the gateway. Repeat
once per cell that needs work.
- unblock(task_id, restore=True)
- complete(task_id="{task_id}", notes=...) for root awaiting_pm_review
- escalate_to_ceo(task_id="{task_id}", reason=...) for root tasks
- say(channel, text), dm(recipient, text)
- i_am_idle() — when delegated and waiting
== WORKFLOW ==
1. evidence(task_id="{task_id}")
2. note(scope='decision', task_id="{task_id}",
text="<plan summary: cells X/Y get subtasks A/B>")
3. i_will_plan(task_id="{task_id}",
plan="<detailed plan: scope, cell breakdown, sequencing, risks>")
4. delegate(parent_task_id="{task_id}", title="Backend slice of <root>",
description="What be-pm should coordinate.",
assigned_to="be-pm", team="backend", task_type="code",
acceptance_criteria=["c1", "c2"], estimated_complexity="medium")
— repeat per cell that needs work. ONE subtask per cell; the Cell PM
breaks it down further.
5. say("#main-pm-board", "Delegated <root> to be-pm/fe-pm — see subtasks")
6. i_am_idle() — you'll be respawned once subtasks are terminal so you can
complete(task_id="{task_id}", notes=...) or escalate_to_ceo on the root.
== RULES ==
- Never `commit`, never write code, never run `git`. PMs coordinate.
- Never assign a code subtask directly to a developer slug — always to a Cell PM.
- delegate / complete / escalate will fail unless you've logged a journal
decision for this task — read the `remediate` field on errors.
Start now: evidence(task_id="{task_id}")
"""
def _build_pm_triage_prompt(self, task: dict[str, Any]) -> str:
"""Build prompt for CELL PM to triage and delegate a task."""
task_id = task.get("id", "unknown")
title = task.get("title", "Untitled")
complexity = task.get("complexity", "medium")
team = task.get("team", "unknown")
# Build team-specific info
channel = f"{team}-cell" if team != "ux_ui" else "uxui-cell"
dev_map = {
"backend": ("be-dev-1", "be-dev-2"),
"frontend": ("fe-dev-1", "fe-dev-2"),
"ux_ui": ("ux-dev-1", "ux-dev-2"),
}
devs = dev_map.get(team, ("be-dev-1",))
primary_dev = devs[0]
dev_options = " or ".join(devs)
return f"""You are the PM for {team} team. This task is assigned to YOU.
TASK: {task_id}
TITLE: {title}
COMPLEXITY: {complexity}
TEAM: {team}
YOUR JOB: Break this down into concrete subtasks and delegate each to a
developer in your cell. You do NOT code. You do NOT run git. You coordinate.
Available developers in your cell: {dev_options}
== TOOLS ==
Gateway verbs (already loaded):
- evidence(task_id="{task_id}") — read PR + commits + diff
- triage() — see what your cell needs next
- note(text, scope='decision', task_id="{task_id}")
REQUIRED before i_will_plan / unblock / complete / escalate
- i_will_plan(task_id="{task_id}", plan="<detailed plan as a string>")
claim + record plan + start your own cell-PM task
- delegate(parent_task_id="{task_id}", title=..., description=...,
assigned_to=<dev slug in your cell, e.g. "be-dev-1">,
team="{team}",
task_type=<one of "code" / "documentation" / "research" /
"planning" / "design" / "administrative">,
acceptance_criteria=[...],
estimated_complexity=<one of "low" / "medium" / "high">)
creates a subtask under your cell-PM task and assigns it to a developer.
Use the EXACT enum strings above — invented values like
"development" or "small" are rejected by the gateway.
Repeat 2 to 5 times for focused subtasks.
- unblock(task_id, restore=True)
when a dev signals i_am_blocked
- complete(task_id, notes)
review a SUBTASK in awaiting_pm_review (auto-merges its leaf PR)
- submit_up(task_id="{task_id}", notes=...)
when YOUR OWN cell-PM task's subtasks are all terminal: opens cell-level
PR up to Main PM's branch and transitions to awaiting_pm_review.
- escalate_up(task_id, reason) — to Main PM
- say("{channel}", text), dm(recipient, text)
- i_am_idle() — when delegated and waiting
== WORKFLOW ==
1. evidence(task_id="{task_id}")
2. note(scope='decision', task_id="{task_id}",
text="<approach>; subtasks: A→{primary_dev}, B→...")
3. i_will_plan(task_id="{task_id}",
plan="<detailed plan: scope, subtask breakdown, sequencing, risks>")
4. delegate(parent_task_id="{task_id}", title="Add login endpoint",
description="Implement POST /login that issues a session token.",
assigned_to="{primary_dev}", team="{team}", task_type="code",
acceptance_criteria=["c1", "c2"], estimated_complexity="medium")
— repeat 2 to 5 times for focused subtasks under your cell-PM task.
5. say("{channel}", "Broke down <task>: subtasks created and assigned")
6. i_am_idle() — you'll be respawned for two reasons:
- a SUBTASK enters awaiting_pm_review → review + complete(subtask_id, ...)
- all subtasks terminal → submit_up(task_id="{task_id}", notes=...) on YOUR task
== RULES ==
- Never `commit`, never write code, never run `git`. PMs coordinate.
- Subtasks MUST go to a developer slug in YOUR cell, not another cell's PM.
- delegate / complete / submit_up / escalate will fail unless you've logged
a journal decision for the relevant task — read the `remediate` field.
Start now: evidence(task_id="{task_id}")
"""
# =========================================================================
# SMART DISPATCHER - MAIN LOOP
# =========================================================================
def trigger_dispatch(self) -> None:
"""Wake the dispatcher up immediately for a single pass.
Called by API routes right after a task status transition so the
orchestrator reacts in milliseconds — e.g. a PM creates a subtask
and the assignee spawns within a second instead of after the next
30-second poll. Safe to call multiple times; the Event coalesces.
"""
self._dispatch_wake.set()
async def _dispatcher_loop(self) -> None:
"""
Main dispatcher loop - periodically checks for work and spawns agents.
This is the BRAIN of the orchestrator. It:
1. Queries for tasks needing work (pending, awaiting_qa, etc.)
2. Queries for events needing attention (blockers, escalations)
3. Spawns appropriate agents with task assignments
Hybrid timing: a poll (dispatcher_interval) guarantees progress even
without external signals, while `_dispatch_wake` lets API routes
kick the loop for immediate reactions after status transitions.
"""
while self._running:
try:
# Wait either for an explicit wake signal or the poll timeout,
# whichever comes first. asyncio.wait_for re-raises
# TimeoutError when the poll window expires, which we treat
# as "run dispatch anyway".
import contextlib
with contextlib.suppress(TimeoutError):
await asyncio.wait_for(
self._dispatch_wake.wait(),
timeout=self.dispatcher_interval,
)
self._dispatch_wake.clear()
await self._dispatch_all_work()
except asyncio.CancelledError:
break
except Exception as e:
logger.error("Dispatcher loop error", error=str(e))
async def _reconcile_orphan_claims_on_startup(self) -> None:
"""Roll back tasks left in CLAIMED/IN_PROGRESS without a branch.
A task in CLAIMED/IN_PROGRESS with ``branch_name IS NULL`` is an
orphan: ``_finalize_claim`` flushed the status before branch creation
failed (or before claim rollback became atomic). The next claim then
fails non-idempotent on ``git checkout -b`` because the on-disk
branch may exist while the DB state is stale.
Opens its own session via the factory; the logic itself lives in
``_reconcile_with_service`` so tests can drive it against an
injected session without the factory dance. Best-effort: if
reconciliation fails, log and continue — startup must not be
blocked by a single bad row.
"""
from roboco.db.base import get_session_factory
from roboco.services.task import TaskService
factory = get_session_factory()
try:
async with factory() as db:
svc = TaskService(db)
await self._reconcile_with_service(svc)
await db.commit()
except Exception as exc:
logger.error("startup reconcile failed; continuing", error=str(exc))
async def _reconcile_with_service(self, svc: "TaskService") -> None:
"""Inner reconcile loop, parameterised by the TaskService to use.
Same shape as ``_reap_with_service`` — extracted so tests can
bypass ``get_session_factory`` and drive the logic directly.
"""
from roboco.utils.converters import require_uuid
candidates = await svc.list_in_progress_or_claimed()
orphans = [t for t in candidates if not t.branch_name]
if not orphans:
logger.info("startup reconcile: no orphan claims")
return
for t in orphans:
task_id = require_uuid(t.id)
try:
await svc.unclaim_for_reaper(task_id)
logger.warning(
"startup reconcile: orphan claim rolled back",
task_id=str(task_id),
had_status=str(t.status),
)
except Exception as exc:
logger.error(
"startup reconcile: rollback failed",
task_id=str(t.id),
error=str(exc),
)
async def _reap_stale_claims(self) -> None:
"""Release claimed/in_progress tasks whose holder hasn't heart-beat in TTL.
Closes the "dead container squats task forever" failure mode that
the schema hinted at (``last_heartbeat_at`` since migration 006) but
no code enforced. The runtime decision (cutoff, iteration) lives
here in the orchestrator; the actual UPDATE statements live in
``TaskService.unclaim_for_reaper``.
Opens a fresh per-tick session — short-lived because the reaper
runs on every dispatch cycle and the work is cheap (one SELECT
plus N UPDATEs for the typically-empty stale set). Tests that
need to inject a mock service do so by building an instance via
``__new__`` (bypassing this method) and calling
``_reap_with_service`` directly.
"""
from roboco.db.base import get_session_factory
from roboco.services.task import TaskService
factory = get_session_factory()
async with factory() as db:
svc = TaskService(db)
await self._reap_with_service(svc)
await db.commit()
def _assignee_has_active_instance(self, task: Any) -> bool:
"""True if the task's assignee currently holds a live (ACTIVE) container.
The heartbeat only approximates liveness. A developer deep in an
edit/test cycle can go longer than the heartbeat TTL between gateway
calls, so a heartbeat-only reaper releases claims out from under agents
that are alive and working — churning the task (and risking a double
spawn against the still-running container). The agent-instance registry
is the ground truth; defer to it when present. Defensive on missing
fields so a heartbeat-only caller (and the reaper's own unit tests)
behave exactly as before.
"""
owner = getattr(task, "assigned_to", None) or getattr(task, "claimed_by", None)
if not owner:
return False
instances = getattr(self, "_instances", None)
if not instances:
return False
instance = instances.get(self._resolve_agent_slug(str(owner)))
return instance is not None and instance.state == AgentState.ACTIVE
async def _reap_with_service(self, svc: "TaskService") -> None:
"""Inner reap loop, parameterized by the TaskService to use.
Wraps each ``unclaim_for_reaper`` in try/except so a single bad row
doesn't abort the dispatch tick — the reaper must keep ticking even
if one task's release somehow fails. A claim whose assignee still has
a live container is skipped: the heartbeat is a stale proxy there, and
reaping a working agent only churns the task.
"""
from roboco.utils.converters import require_uuid
cutoff = datetime.now(UTC) - timedelta(seconds=self._claim_heartbeat_ttl)
candidates = await svc.list_in_progress_or_claimed()
for t in candidates:
ts = t.last_heartbeat_at
if ts is None or ts < cutoff:
if self._assignee_has_active_instance(t):
continue
task_id = require_uuid(t.id)
try:
await svc.unclaim_for_reaper(task_id)
logger.warning(
"stale claim reaped",
task_id=str(task_id),
last_heartbeat=ts.isoformat() if ts else None,
)
except Exception as exc:
logger.error(
"stale-claim reap failed; continuing",
task_id=str(task_id),
error=str(exc),
)
async def _dispatch_all_work(self) -> None:
"""Run all dispatchers to check for and assign work.
Each dispatcher is isolated: if one raises (e.g., a transient API
error), the rest still run in this tick instead of waiting for the
next one.
`_tick_handled_tasks` gives downstream dispatchers a way to
skip tasks that an earlier dispatcher already acted on this
tick. Order-dependent bugs (like the Fix-B scenario where
`_dispatch_qa_work` claimed for QA and the next dispatcher
re-spawned the dev on the same claimed row) are defanged by
early dispatchers marking the task handled.
The stale-claim reaper runs first, before any dispatcher tries to
spawn an agent for a task whose previous holder is dead. Without
this ordering, the spawn pass could race against a stale claim and
skip work the reaper would have freed in the same tick.
"""
self._tick_handled_tasks = set()
# Free any tasks whose claim went stale before the spawn pass runs.
# Wrapped because a reaper failure must not block dispatch — the
# next tick will retry.
try:
await self._reap_stale_claims()
except Exception as e:
logger.error("Stale-claim reaper failed; continuing tick", error=str(e))
# Orchestrator uses SYSTEM role for internal API calls
# Using a well-known UUID for the orchestrator identity
headers = {
"X-Agent-ID": "00000000-0000-0000-0000-000000000000",
"X-Agent-Role": "system",
}
dispatchers: list[tuple[str, Any]] = []
async with httpx.AsyncClient(timeout=30.0, headers=headers) as client:
dispatchers = [
("pm_work", self._dispatch_pm_work(client)),
("pm_closure_work", self._dispatch_pm_closure_work(client)),
(
"revision_coordination",
self._dispatch_revision_coordination_roots(client),
),
("dev_work", self._dispatch_dev_work(client)),
("qa_work", self._dispatch_qa_work(client)),
("pr_review_work", self._dispatch_pr_review_work(client)),
("doc_work", self._dispatch_doc_work(client)),
("pm_review_work", self._dispatch_pm_review_work(client)),
("marketing_work", self._dispatch_marketing_work(client)),
("blocker_work", self._dispatch_blocker_work(client)),
(
"claimed_without_agent",
self._dispatch_claimed_without_agent(client),
),
("escalation_work", self._dispatch_escalation_work(client)),
("approval_work", self._dispatch_approval_work(client)),
("a2a_work", self._dispatch_a2a_work(client)),
("audit_work", self._dispatch_audit_work(client)),
("detect_stuck_tasks", self._detect_stuck_tasks(client)),
]
for name, coro in dispatchers:
try:
await coro
except Exception as e:
logger.error(
"Dispatcher raised; continuing with next dispatcher",
dispatcher=name,
error=str(e),
)
# =========================================================================
# SMART DISPATCHER - TASK-BASED DISPATCHERS
# =========================================================================
_PM_AGENTS: ClassVar[frozenset[str]] = frozenset(
{
"main-pm",
"be-pm",
"fe-pm",
"ux-pm",
}
)
# Board reviewers. They advise — review + record requirements + escalate —
# but do not build or delegate. Dispatched once per assigned board task.
_BOARD_AGENTS: ClassVar[frozenset[str]] = frozenset(
{
"product-owner",
"head-marketing",
}
)
# Use foundation's default; keep the local name for back-compat.
_PM_RESPAWN_MAX_UNPRODUCTIVE = _AGENT_LOOP_BUDGET.pm_respawn_max_unproductive
_PM_RESPAWN_MAX_TRACING_RESETS = _AGENT_LOOP_BUDGET.pm_respawn_max_tracing_resets
async def _pm_respawn_should_gate(
self, agent_slug: str, task: dict[str, Any]
) -> bool:
"""Return True when the respawn should be skipped (loop detected).
Tracks (agent_slug, task_id) -> count of consecutive spawns where
the task's status did not advance. When the task status changes,
the counter resets. Once the count hits the threshold, the spawn
is skipped and a warning logged; operators must intervene.
Tracing-gap reset
-----------------
With the gateway claim-time gates installed, a rule-following PM
will hit ``PARENT_NOT_CLAIMED`` (a ``tracing_gap`` envelope) and
the prompt will tell it to call the prerequisite verb first.
Each retry leaves the task status unchanged but the agent IS
making progress through the verb chain. Counting that as a
strike kills rule-followers.
Solution: before incrementing on a same-status spawn, check
``audit_log`` for a ``gateway.rejected`` row tagged
``reason == "tracing_gap"`` from this (agent, task) since the
last check. If found, reset the counter — the agent followed
the rules, not stuck.
Audit lookup is best-effort: any failure falls through to the
legacy strike behavior so audit problems don't break the gate.
"""
task_id = task.get("id")
if not task_id:
return False
key = (agent_slug, task_id)
current_status = task.get("status")
record = self._pm_respawn_tracker.get(key)
now = datetime.now(UTC)
if record is None or record.get("last_status") != current_status:
self._pm_respawn_tracker[key] = {
"count": 1,
"last_status": current_status,
"last_check": now,
}
return False
# Same status as last spawn — could be a stuck loop OR a
# rule-following retry. A tracing_gap normally means the agent is
# advancing through a verb chain, so reset the strike counter — but
# only up to a bound. A task whose EVERY respawn trips the same gap is
# wedged, not progressing (e.g. the unblock journal-decision gate a
# cold-respawned PM can never satisfy), so cap the resets and let
# strikes accrue once the budget is exhausted. Without this cap the
# gate never fires for a tracing_gap loop and respawns run forever.
if await self._pm_made_rule_following_retry(agent_slug, task_id, record):
resets = record.get("tracing_resets", 0)
if resets < self._PM_RESPAWN_MAX_TRACING_RESETS:
record["tracing_resets"] = resets + 1
record["count"] = 1
record["last_check"] = now
record["notified"] = False
return False
logger.warning(
"PM respawn tracing_gap reset budget exhausted — "
"treating recurring gap as a stuck loop",
agent_id=agent_slug,
task_id=task_id,
task_status=current_status,
tracing_resets=resets,
)
record["count"] += 1
record["last_check"] = now
if record["count"] > self._PM_RESPAWN_MAX_UNPRODUCTIVE:
logger.warning(
"PM respawn loop detected — skipping spawn",
agent_id=agent_slug,
task_id=task_id,
task_status=current_status,
spawn_attempts=record["count"],
threshold=self._PM_RESPAWN_MAX_UNPRODUCTIVE,
hint=(
"Agent repeatedly spawned without advancing task state. "
"Investigate prompt/schema drift or escalate manually."
),
)
# A skipped spawn pauses the loop but can't advance the task; alert
# an overseer once so a wedged agent isn't silently stranded.
if not record.get("notified"):
record["notified"] = True
await self._notify_stuck_agent(agent_slug, task_id, current_status)
return True
return False
async def _notify_stuck_agent(
self, agent_slug: str, task_id: str, task_status: str | None
) -> None:
"""One-shot alert to the CEO that an agent is wedged in a respawn loop.
Best-effort: a notification failure must not wedge dispatch, so any
error is logged and swallowed.
"""
from roboco.services.notification import NotificationService
try:
await NotificationService().send_stuck_agent_notification(
task_id=task_id,
agent_slug=agent_slug,
task_status=task_status or "unknown",
to_agent="ceo",
)
except Exception as exc:
logger.warning(
"Failed to send stuck-agent notification",
agent_id=agent_slug,
task_id=task_id,
error=str(exc),
)
async def _pm_made_rule_following_retry(
self,
agent_slug: str,
task_id: str,
record: dict[str, Any],
) -> bool:
"""Did the agent emit a ``tracing_gap`` envelope since the last check?
Returns ``False`` for unknown slugs (defensive — the audit query
needs an agent UUID, and we'd rather fall through to the legacy
strike behavior than crash). Returns ``False`` if the audit
lookup raises — observability must never block the gate.
"""
agent_uuid_str = AGENT_UUIDS.get(agent_slug)
if not agent_uuid_str:
return False
from uuid import UUID
try:
agent_uuid = UUID(agent_uuid_str)
task_uuid = UUID(task_id)
except (ValueError, TypeError):
return False
since = record.get("last_check") or datetime.now(UTC)
from roboco.services.audit import get_audit_service
audit = get_audit_service()
try:
return await audit.has_recent_tracing_gap(
agent_id=agent_uuid,
task_id=task_uuid,
since=since,
)
except Exception as exc:
logger.debug(
"audit.has_recent_tracing_gap failed; falling back to strike count",
agent_slug=agent_slug,
task_id=task_id,
error=str(exc),
)
return False
async def _handle_pm_assigned_task(
self, task: dict[str, Any], assigned_to: str
) -> None:
"""Spawn an already-assigned PM agent if it isn't running."""
agent_slug = self._resolve_agent_slug(assigned_to)
if agent_slug not in self._PM_AGENTS or self._is_agent_active(agent_slug):
return
if await self._pm_respawn_should_gate(agent_slug, task):
return
logger.info(
"Spawning assigned PM agent",
task_id=task.get("id"),
agent_id=agent_slug,
)
pm_prompt = (
self._build_main_pm_triage_prompt(task)
if agent_slug == "main-pm"
else self._build_pm_triage_prompt(task)
)
await self.spawn_agent(
agent_id=agent_slug,
task_id=task["id"],
initial_prompt=pm_prompt,
git_context=self._task_git_context(task),
)
async def _handle_board_assigned_task(
self, task: dict[str, Any], assigned_to: str
) -> None:
"""Review an assigned board task with the FULL board (PO + HoM), ONCE each.
A board/coordination task — especially one with a UI / user-facing
dimension — must be reviewed by BOTH the Product Owner AND the Head of
Marketing before it is handed to the CEO. The task is assigned to one
board agent, but the review is a two-reviewer gate, so this dispatches
both regardless of which one ``assigned_to`` names.
Board roles advise: they can triage, record notes, and discuss, but have
NO verb to claim, plan, delegate, or complete. A respawn cannot advance
the task — it would just loop — so dispatch is one-shot per (agent, task).
The board reviews and records requirements; the CEO then approves and
hands the task to Main PM for delegation to the cells.
Once BOTH reviewers have finished (each dispatched and no longer active),
the board-review handoff fires: the task is flagged board-reviewed and a
single formal CEO notification is emitted so Approve & Start is an
actionable signal rather than buried chatter.
"""
# `assigned_to` only gates that this IS a board task; the review itself
# always involves the whole board, not just the named assignee.
if self._resolve_agent_slug(assigned_to) not in self._BOARD_AGENTS:
return
task_id = str(task.get("id"))
for board_slug in sorted(self._BOARD_AGENTS):
await self._dispatch_board_reviewer(board_slug, task_id, task)
await self._maybe_handoff_board_review_to_ceo(task_id)
async def _dispatch_board_reviewer(
self, board_slug: str, task_id: str, task: dict[str, Any]
) -> None:
"""One-shot spawn of a single board reviewer for a board task.
Skips when the reviewer is already running or has already been
dispatched for this task (board roles have no progression verb, so a
respawn would loop). Records the (agent, task) pair so the
review-completion detector can tell which reviewers have run.
"""
if self._is_agent_active(board_slug):
return
key = (board_slug, task_id)
if key in self._board_dispatched:
return
self._board_dispatched.add(key)
logger.info(
"Spawning board agent for review",
task_id=task_id,
agent_id=board_slug,
)
await self.spawn_agent(
agent_id=board_slug,
task_id=task["id"],
initial_prompt=self._build_board_prompt(task),
git_context=self._task_git_context(task),
)
def _board_review_complete(self, task_id: str) -> bool:
"""True once EVERY board reviewer has reviewed and gone idle.
A reviewer has finished when it was dispatched for this task
(``_board_dispatched``) and is no longer running (``_is_agent_active``).
Both PO and HoM must satisfy this before the task is handoff-ready.
"""
return all(
(board_slug, task_id) in self._board_dispatched
and not self._is_agent_active(board_slug)
for board_slug in self._BOARD_AGENTS
)
async def _maybe_handoff_board_review_to_ceo(self, task_id: str) -> None:
"""Unlock the CEO's Approve & Start gate when the board review is done.
Two one-shot effects fire once BOTH board reviewers have finished:
1. Persist ``board_review_complete`` on the task. The task stays
pending (its pending state is what hands it to Main PM on approval),
so this flag is the only thing that makes the CEO's Approve & Start
button appear — it never shows on a board task the board hasn't
finished reviewing.
2. Emit an ack-required APPROVAL notification to the CEO. Board agents
only post channel dialogue + journal notes during review, which
left the CEO with no actionable signal; this is that signal.
Fires at most once per task; a failure clears the guard so a later tick
retries, and never blocks the dispatch loop.
"""
if task_id in self._board_review_ceo_notified:
return
if not self._board_review_complete(task_id):
return
self._board_review_ceo_notified.add(task_id)
from uuid import UUID
from roboco.db.base import get_db_context
from roboco.services.notification import NotificationService
from roboco.services.task import TaskService
try:
async with get_db_context() as db:
await TaskService(db).mark_board_review_complete(UUID(task_id))
await db.commit()
await NotificationService().send_board_review_complete_notification(
task_id=task_id,
)
except Exception as exc:
# Don't wedge dispatch on a failure; allow a retry by clearing the
# one-shot guard so a later tick can re-run the handoff.
self._board_review_ceo_notified.discard(task_id)
logger.warning(
"Failed to hand board-review completion to CEO",
task_id=task_id,
error=str(exc),
)
return
logger.info(
"Board review complete — CEO Approve & Start unlocked",
task_id=task_id,
)
# Keep-alive re-draft: if an intake chat is parked awaiting this review,
# inject the board's feedback so the still-resident prompter re-drafts
# in-context. Best-effort; the cold "Re-draft" path covers the rest.
await self._inject_board_brief_into_parked_intake(task_id)
async def _inject_board_brief_into_parked_intake(self, task_id: str) -> None:
"""Inject the board's review into a parked intake session, if one exists.
No-op when no session is parked for this task (it was reaped, the
container died, or the draft never used the board route) — the CEO then
re-drafts via the cold ``/re-interview`` path instead. Never raises.
"""
from roboco.services.prompter_live import get_live_registry
session = get_live_registry().find_by_task(task_id)
if session is None:
return
from uuid import UUID
from roboco.db.base import get_db_context
from roboco.services.journal import get_journal_service
from roboco.services.prompter import compose_redraft_message
from roboco.services.task import get_task_service
try:
async with get_db_context() as db:
task = await get_task_service(db).get(UUID(task_id))
if task is None:
return
entries = await get_journal_service(db).board_review_brief(
UUID(task_id)
)
message = compose_redraft_message(task, entries)
delivered = await get_live_registry().deliver(session.session_id, message)
logger.info(
"Injected board feedback into parked intake",
task_id=task_id,
delivered=delivered,
)
except Exception as exc:
logger.warning(
"Failed to inject board feedback into parked intake",
task_id=task_id,
error=str(exc),
)
def _pm_spawn_prompt(
self, routing: str, agent_id: str, task: dict[str, Any]
) -> str:
"""Pick the correct prompt for a classified spawn."""
if routing == "dev":
return self._build_dev_prompt(task)
if routing == "main_pm" or agent_id == "main-pm":
return self._build_main_pm_triage_prompt(task)
return self._build_pm_triage_prompt(task)
async def _route_unassigned_pm_task(
self, client: httpx.AsyncClient, task: dict[str, Any]
) -> None:
"""Classify and route an unassigned pending task to its target agent."""
routing = self._classify_task_routing(task)
agent_id = self._get_routing_target(routing, task)
if not agent_id:
logger.warning(
"No routing target found",
task_id=task.get("id"),
routing=routing,
)
return
# Board work is a two-reviewer gate (PO + Head of Marketing), not a
# single-assignee claim. Routing only ever names one board agent
# (product-owner), so claiming + spawning that one here would leave the
# Head of Marketing out (finding #4). Delegate to the board handler,
# which dispatches BOTH reviewers one-shot and leaves the task pending
# for the CEO's Approve & Start. ``agent_id`` is the routed board slug.
if routing == "board":
await self._handle_board_assigned_task(task, agent_id)
return
# Don't auto-claim back to the creator. A PM that just created this
# task is about to assign it (e.g. be-pm creating a code subtask to
# hand to be-dev-1 one tool-call later). Racing in and claiming for
# the PM hijacks the delegation — the PM ends up owning a code task
# it never intended to work on itself. Skip this tick and let the
# next dispatch pick it up once assigned_to is set, OR re-evaluate
# when we have a clearer signal the creator won't route it.
created_by = task.get("created_by")
if created_by:
creator_slug = self._resolve_agent_slug(str(created_by))
if creator_slug == agent_id:
logger.info(
"Skipping auto-claim: routing target is the creator",
task_id=task.get("id"),
creator=creator_slug,
routing=routing,
)
return
logger.info(
"Routing task",
task_id=task.get("id"),
routing=routing,
agent_id=agent_id,
)
if self._is_agent_active(agent_id):
await self._claim_task_for_agent(client, task["id"], agent_id)
return
if await self._claim_task_for_agent(client, task["id"], agent_id):
prompt = self._pm_spawn_prompt(routing, agent_id, task)
await self.spawn_agent(
agent_id=agent_id,
task_id=task["id"],
initial_prompt=prompt,
git_context=self._task_git_context(task),
)
async def _dispatch_pm_work(self, client: httpx.AsyncClient) -> None:
"""
Dispatch PM triage work - routes new tasks to appropriate level.
This is the FIRST dispatcher called - it classifies unassigned tasks
and routes them to Board, Main PM, Cell PM, or directly to devs.
Also handles already-assigned pending tasks for PM agents.
Monitors: pending tasks (both assigned and unassigned)
Spawns: product-owner, main-pm, be-pm, fe-pm, ux-pm (or devs for simple)
"""
tasks = await self._fetch_tasks(client, "pending")
for task in tasks:
if self._is_task_handled_this_tick(task.get("id")):
continue
# External-PR review tasks are owned by _dispatch_pr_review_work; the
# PM hierarchy never routes or spawns them.
if task.get("source") in PR_REVIEW_SOURCES:
continue
# A self-heal fix task is opened by the loop but must stay INERT until
# the CEO Approve-&-Starts it (which flips confirmed_by_human). Until
# then the PM hierarchy must not route, assign, or spawn it — even
# though it's already team=main_pm. It still appears in the panel so
# the CEO can see and approve it. This guard sits before the
# assigned-vs-unassigned split, so it holds whether or not the task
# carries an assignee.
if task.get("source") == SELF_HEAL_SOURCE and not task.get(
"confirmed_by_human"
):
continue
assigned_to = task.get("assigned_to")
if assigned_to:
if self._resolve_agent_slug(assigned_to) in self._BOARD_AGENTS:
await self._handle_board_assigned_task(task, assigned_to)
else:
await self._handle_pm_assigned_task(task, assigned_to)
continue
await self._route_unassigned_pm_task(client, task)
async def _dispatch_revision_coordination_roots(
self, client: httpx.AsyncClient
) -> None:
"""Re-spawn the owning PM for a CEO-rejected coordination root.
A coordination root (team=main_pm, product-linked, no repo) the CEO sends
back lands in ``needs_revision``. The dev dispatcher skips it (not a cell
team) and the closure path only handles paused parents, so without this
it would sit in needs_revision forever — the deadlock. Respawn its PM so
it re-coordinates the revision. Cell/code needs_revision tasks are left
to the dev dispatcher; this handles only coordination roots.
"""
tasks = await self._fetch_tasks(client, "needs_revision")
for task in tasks:
if self._is_task_handled_this_tick(task.get("id")):
continue
if not _is_coordination_task(task):
continue
owner = task.get("assigned_to") or task.get("claimed_by")
agent_slug = self._resolve_agent_slug(owner) if owner else None
if not agent_slug or self._is_agent_active(agent_slug):
continue
if get_agent_role(agent_slug) not in ("cell_pm", "main_pm"):
continue
await self.spawn_agent(
agent_id=agent_slug,
task_id=task["id"],
initial_prompt=self._get_prompt_for_agent(agent_slug, task),
git_context=self._task_git_context(task),
)
@staticmethod
def _all_descendants_terminal(descendants: list[dict[str, Any]]) -> bool:
"""Every descendant in a closure-complete state?"""
return all(st.get("status") in ("completed", "cancelled") for st in descendants)
@staticmethod
def _already_promoted_for_closure(task: dict[str, Any]) -> bool:
"""Skip closure respawn when PR+status show task has moved up."""
return bool(
task.get("pr_number")
and task.get("status")
in ("awaiting_pm_review", "awaiting_ceo_approval", "completed")
)
@staticmethod
def _coerce_heartbeat(value: Any) -> datetime | None:
"""Normalize ``last_heartbeat_at`` to an aware UTC datetime.
The dispatcher reads tasks via the HTTP API, which serializes
datetimes as ISO-8601 strings; direct service callers (and tests)
may pass ``datetime`` objects. Anything else is treated as
absent so a malformed value can't accidentally arm the gate.
"""
if value is None:
return None
if isinstance(value, datetime):
return value if value.tzinfo else value.replace(tzinfo=UTC)
if isinstance(value, str):
try:
parsed = datetime.fromisoformat(value)
except ValueError:
return None
return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC)
return None
def _is_recently_paused(self, task: dict[str, Any]) -> bool:
"""A paused task whose heartbeat is fresher than the stale cutoff.
Closes the ``i_am_idle`` vs closure-respawn race:
``i_am_idle`` auto-pauses in-flight tasks and then sets the agent
IDLE. If the dispatcher ticks between those two writes it sees a
paused parent and would spawn the closure PM against a session
that is mid-shutdown. A fresh ``last_heartbeat_at`` (newer than
``settings.claim_stale_seconds``) is the signal that the agent
was alive moments ago and a respawn now would race the existing
session. Genuinely-stale paused tasks (or tasks with no heartbeat
recorded) fall through and follow the regular closure path.
"""
if task.get("status") != "paused":
return False
last_hb = self._coerce_heartbeat(task.get("last_heartbeat_at"))
if last_hb is None:
return False
cutoff = datetime.now(UTC) - timedelta(seconds=self._claim_heartbeat_ttl)
return last_hb > cutoff
def _closure_pm_for_team(self, team: str | None) -> str:
"""Pick the PM that owns closure for a given team."""
if team in ("backend", "frontend", "ux_ui"):
return self._TEAM_PM_MAP.get(team, "be-pm")
return "main-pm"
async def _maybe_spawn_pm_closure(
self, client: httpx.AsyncClient, task: dict[str, Any]
) -> None:
"""If this parent task is ready for closure, spawn its PM."""
task_id = task.get("id")
if not task_id:
return
if self._is_recently_paused(task):
logger.debug(
"Skipping closure spawn for recently-paused parent",
task_id=task_id,
last_heartbeat_at=task.get("last_heartbeat_at"),
)
return
descendants = await self._fetch_all_descendants(client, task_id)
if not descendants:
return
if not self._all_descendants_terminal(descendants):
return
if self._already_promoted_for_closure(task):
return
pm_id = self._closure_pm_for_team(task.get("team"))
if self._is_agent_active(pm_id):
return
logger.info(
"Parent task ready for closure",
task_id=task_id,
descendants_count=len(descendants),
pm_id=pm_id,
)
# The parent auto-paused when its PM idled (by design). Resume
# it before respawn so the PM lands actionable (in_progress) and can
# directly submit_up / complete / escalate — pre-gateway behaviour the
# gateway refactor dropped, which wedged a dogfood run (the model
# never issued resume() itself).
# A parent that is `blocked` at closure (all descendants
# terminal) is an errant/stale block — recover it symmetrically so
# the chain can't wedge forever waiting for a PM to manually unblock.
parent_status = task.get("status")
if parent_status == "paused":
await self._auto_resume_paused_parent(client, task_id)
elif parent_status == "blocked":
await self._auto_recover_blocked_parent(client, task_id)
prompt = self._build_pm_closure_prompt(task, descendants)
await self.spawn_agent(
agent_id=pm_id,
task_id=task_id,
initial_prompt=prompt,
git_context=self._task_git_context(task),
)
async def _dispatch_pm_closure_work(self, client: httpx.AsyncClient) -> None:
"""
Dispatch PM closure work - check parent tasks ready to close.
When all subtasks of a parent task are completed, spawn the PM
to review and close the parent task.
Monitors: tasks with completed subtasks but parent still open
Spawns: be-pm, fe-pm, ux-pm, main-pm (based on parent team)
"""
# Find parent tasks that might have children ready for closure
# Include "paused" - PM pauses while waiting, respawned when subtasks done
# Include "awaiting_pm_review" - parent awaiting review when children done
parent_statuses = ["claimed", "in_progress", "paused", "awaiting_pm_review"]
for status in parent_statuses:
tasks = await self._fetch_tasks(client, status)
for task in tasks:
await self._maybe_spawn_pm_closure(client, task)
async def _fetch_subtasks(
self, client: httpx.AsyncClient, parent_id: str
) -> list[dict[str, Any]]:
"""Fetch direct subtasks for a parent task."""
try:
resp = await client.get(
f"{self._api_url}/tasks",
params={"parent_task_id": parent_id},
)
if resp.status_code == http_status.HTTP_200_OK:
data = resp.json()
tasks = data.get("tasks", data) if isinstance(data, dict) else data
return list(tasks) if tasks else []
except Exception as e:
logger.warning(
"Failed to fetch subtasks", parent_id=parent_id, error=str(e)
)
return []
async def _fetch_all_descendants(
self, client: httpx.AsyncClient, task_id: str
) -> list[dict[str, Any]]:
"""Fetch ALL descendants (children, grandchildren, etc.) recursively.
Uses the /tasks/{id}/descendants endpoint which does BFS traversal.
"""
try:
resp = await client.get(f"{self._api_url}/tasks/{task_id}/descendants")
if resp.status_code == http_status.HTTP_200_OK:
data = resp.json()
# Endpoint returns list directly
return list(data) if data else []
except Exception as e:
logger.warning("Failed to fetch descendants", task_id=task_id, error=str(e))
return []
def _build_pm_closure_prompt(
self, task: dict[str, Any], subtasks: list[dict[str, Any]]
) -> str:
"""Prompt for PM closing their own parent task (subtasks terminal)."""
task_id = task.get("id", "unknown")
title = task.get("title", "Untitled")
team = task.get("team", "unknown")
subtask_summary = "\n".join(
f" - {st.get('title', 'Untitled')} ({st.get('status', 'unknown')})"
for st in subtasks
)
is_root = not task.get("parent_task_id")
project_slug = task.get("project_slug", "")
if is_root:
target_line = (
"submit_up promotes to awaiting_ceo_approval; the CEO reviews "
"and merges to master. You do NOT merge to master yourself."
)
submit_step = (
f'4. submit_up(task_id="{task_id}",\n'
' notes="<aggregate summary: what shipped across the '
'cells, evidence, risk callouts>")\n'
" — promotes to awaiting_ceo_approval. "
"CEO is the final approver."
)
else:
target_line = (
"submit_up opens your cell-level PR into the parent task's "
"branch and transitions you to awaiting_pm_review for the "
"parent PM."
)
submit_step = (
f'4. submit_up(task_id="{task_id}",\n'
' notes="<cell summary: what your cell shipped, '
'evidence>")\n'
" — opens cell-level PR up to the parent's branch and "
"transitions to awaiting_pm_review."
)
return f"""You are closing YOUR OWN parent task. All subtasks are
terminal — promote the merged work one level up the hierarchy.
TASK: {task_id}
TITLE: {title}
TEAM: {team}
PROJECT: {project_slug}
ROOT TASK: {"yes" if is_root else "no"}
SUBTASK SUMMARY:
{subtask_summary}
PROMOTION TARGET: {target_line}
== PM CLOSURE WORKFLOW ==
1. evidence(task_id="{task_id}")
— review aggregate state, every acceptance criterion, and each
subtask's terminal status. Returns the inline diff for your branch
(all merged subtask work).
2. If any subtask is still in awaiting_pm_review, review + close it FIRST:
- APPROVE leaf: complete(task_id="<subtask_id>",
notes="<merge rationale>")
(auto-merges the leaf PR into your cell branch).
- NEEDS REWORK: leave a clear note(scope='decision',
task_id="<subtask_id>", text="...") and rely on the dispatcher to
respawn the dev for revision.
3. note(scope='decision', task_id="{task_id}",
text="Closure: {title} — <rationale, AC coverage, risks>")
— REQUIRED before submit_up().
{submit_step}
5. i_am_idle()
Never `commit`, never write code, never run `git`. PMs coordinate.
"""
def _get_prompt_for_agent(self, agent_slug: str, task: dict[str, Any]) -> str:
"""Get the prompt appropriate to the agent's ACTUAL role.
A respawn must hand each role the prompt it can act on — a PM or board
agent handed the developer prompt is told to write code and call verbs
it does not own. Reuses the same per-role prompt builders the role
dispatchers use so a respawn matches a fresh dispatch:
developer → dev prompt
qa → QA prompt
documenter → doc prompt
cell_pm → cell-PM triage prompt
main_pm → main-PM triage prompt
product_owner → board-review prompt
head_marketing → marketing prompt for a marketing task, else board
auditor → audit prompt
Unknown roles fall back to the dev prompt (safe default for an
executable task).
"""
role = get_agent_role(agent_slug)
# head_marketing is the one role whose prompt depends on the task, so it
# is resolved before the static role→builder table.
if role == "head_marketing":
if task.get("team") == "marketing":
return self._build_marketing_prompt(task)
return self._build_board_prompt(task)
builders: dict[str, Callable[[dict[str, Any]], str]] = {
"developer": self._build_dev_prompt,
"qa": self._build_qa_prompt,
"documenter": self._build_doc_prompt,
"cell_pm": self._build_pm_triage_prompt,
"main_pm": self._build_main_pm_triage_prompt,
"product_owner": self._build_board_prompt,
"auditor": lambda _task: self._build_audit_prompt(),
"pr_reviewer": self._build_pr_review_prompt,
}
builder = builders.get(role, self._build_dev_prompt)
return builder(task)
async def _dispatch_dev_work(self, client: httpx.AsyncClient) -> None:
"""
Dispatch assigned work to the assigned agent.
NOTE: This handles PRE-ASSIGNED tasks (assigned by PM),
needs_revision tasks, and in_progress tasks where agent is not active
(e.g., after unblock). New unassigned pending tasks are handled by
_dispatch_pm_work() which routes them through the PM hierarchy.
Monitors: assigned pending tasks, needs_revision tasks, orphaned in_progress
Spawns: Any assigned agent (dev, doc, qa) with appropriate prompt
"""
# Get tasks needing attention. Includes:
# - `claimed` — PM-delegated claims where the assignee was never spawned
# - `blocked` — but only when another agent can resolve (see below)
# `pending`, `needs_revision`, `in_progress` are the classic cases.
tasks = await self._fetch_tasks(
client,
["pending", "claimed", "needs_revision", "in_progress", "blocked"],
)
for task in tasks:
if self._is_task_handled_this_tick(task.get("id")):
continue
# External-PR review tasks belong to the pr_reviewer, never a dev —
# _dispatch_pr_review_work owns them.
if task.get("source") in PR_REVIEW_SOURCES:
continue
await self._dev_dispatch_one(client, task)
@staticmethod
def _resolve_dev_owner_uuid(task: dict[str, Any]) -> str | None:
"""Pick the right owner UUID for dev dispatch based on status.
Always falls back to ``claimed_by`` when ``assigned_to`` is missing, so
a task left half-reaped (assigned_to nulled but still claimed) still
dispatches to its rightful owner instead of going dormant — the
orchestrator knows who to call even when one ownership field was cleared.
"""
status = task.get("status")
if status in ("claimed", "blocked"):
return task.get("claimed_by") or task.get("assigned_to")
return task.get("assigned_to") or task.get("claimed_by")
async def _respawn_dev_if_inactive(
self, task: dict[str, Any], agent_slug: str
) -> None:
"""Respawn a dev agent on an existing task when it isn't running."""
if self._is_agent_active(agent_slug):
return
await self.spawn_agent(
agent_id=agent_slug,
task_id=task["id"],
initial_prompt=self._build_dev_prompt(task),
git_context=self._task_git_context(task),
)
async def _spawn_pending_dev(
self,
client: httpx.AsyncClient,
task: dict[str, Any],
agent_slug: str,
) -> None:
"""Validate and spawn a dev agent for a pending, pre-assigned task."""
if self._is_agent_active(agent_slug):
return
# Per-dev queue order: hold a dev's higher-sequence code leaf while it
# still has an earlier non-terminal code sibling under the same parent,
# so the dev works its queue one task at a time, in order. Loop-free —
# just not dispatched this tick.
if await self._blocked_by_earlier_lane_sibling(task):
return
validation_issue = await self._validate_task_for_spawn(client, task, agent_slug)
if validation_issue:
logger.warning(
"Skipping spawn due to validation failure",
task_id=task["id"],
agent=agent_slug,
reason=validation_issue,
)
return
await self.spawn_agent(
agent_id=agent_slug,
task_id=task["id"],
initial_prompt=self._get_prompt_for_agent(agent_slug, task),
git_context=self._task_git_context(task),
)
@staticmethod
def _is_hitl_blocked(task: dict[str, Any]) -> bool:
"""HITL-blocked tasks wait for human resolution; skip respawn."""
return (
task.get("status") == "blocked"
and task.get("blocker_resolver_type") == "human"
)
async def _handle_dev_existing_owner(
self, task: dict[str, Any], status: str, agent_slug: str
) -> None:
"""Respawn existing dev for needs_revision / in_progress / claimed."""
# A `blocked` task is waiting for its blocker to clear (PM / dependency);
# the owner has no legal move from `blocked`, so respawning it does
# nothing but churn. It is revived only when unblocked back to
# in_progress, or released to the pool (unclaim) for re-delegation.
if status == "blocked":
return
if status in (
"in_progress",
"claimed",
) and not self._is_agent_active(agent_slug):
logger.info(
"Respawning agent for orphaned task",
task_id=task["id"],
agent=agent_slug,
status=status,
)
await self._respawn_dev_if_inactive(task, agent_slug)
async def _dev_dispatch_one(
self, client: httpx.AsyncClient, task: dict[str, Any]
) -> None:
"""Dispatch a single task from `_dispatch_dev_work`'s fetch set."""
team = task.get("team")
if team not in ["backend", "frontend", "ux_ui"]:
return
if self._is_hitl_blocked(task):
logger.debug(
"Skipping HITL-blocked task; waiting for human",
task_id=task["id"],
)
return
status = task.get("status")
owner_uuid = self._resolve_dev_owner_uuid(task)
agent_slug = self._resolve_agent_slug(owner_uuid) if owner_uuid else None
# Role/task_type mismatch guard. The dispatcher
# previously trusted whatever ``assigned_to`` named, so a
# documentation task accidentally assigned to a developer agent
# would silently spawn the dev. Reject the dispatch if the
# assignee's role doesn't match the task type — the PM that
# mis-assigned needs to fix it before any agent runs.
# Tasks owned by PM/board/QA roles aren't this dispatcher's lane;
# `_dispatch_pm_work` and the QA-pool path own them. Silently skip
# so the warning only fires on actual dev/doc misassignments.
if agent_slug:
assignee_role = get_agent_role(agent_slug)
if assignee_role not in ("developer", "documenter", "unknown"):
return
if not self._dev_dispatch_role_matches(task, agent_slug):
logger.warning(
"dev dispatch: role/task_type mismatch — skipping spawn",
task_id=task.get("id"),
task_type=task.get("task_type"),
assignee_slug=agent_slug,
assignee_role=assignee_role,
)
return
if agent_slug and status in (
"needs_revision",
"in_progress",
"claimed",
"blocked",
):
await self._handle_dev_existing_owner(task, status, agent_slug)
return
# Pending tasks pre-assigned by PM.
if agent_slug:
await self._spawn_pending_dev(client, task, agent_slug)
@staticmethod
def _dev_dispatch_role_matches(task: dict[str, Any], agent_slug: str) -> bool:
"""Return True if the assignee role matches the task's task_type.
Dev dispatcher only spawns developer-role agents. A doc/qa task
assigned to a dev (or vice versa) should be flagged, not silently
spawned. Returns True when the type is unknown or the assignee role
is unknown — the validation runs as a guard, not a strict gate, so
an unknown classification doesn't block work that would otherwise
proceed.
"""
role = get_agent_role(agent_slug)
if role == "unknown":
return True
task_type = task.get("task_type")
if task_type == "documentation":
return role == "documenter"
# `code` / `research` / `planning` / `administrative` / `design` all
# route through dev or PM; only the doc-task case is unambiguous.
return role == "developer"
async def _spawn_assigned_qa(self, task: dict[str, Any], assigned_to: str) -> bool:
"""If task.assigned_to is a QA slug, spawn/skip-if-running; else False.
Returns True when the dispatch decision for this task was
handled at the assignee level (spawned or already running).
Returns False when the assigned_to is NOT a QA agent — caller
then falls through to the unassigned-select path.
"""
assigned_slug = self._resolve_agent_slug(assigned_to)
if not assigned_slug or "qa" not in assigned_slug:
logger.warning(
"awaiting_qa task assigned to non-QA slug; reassigning via QA pool",
task_id=task["id"],
assigned_slug=assigned_slug,
)
return False
if self._is_agent_active(assigned_slug):
return True
await self.spawn_agent(
agent_id=assigned_slug,
task_id=task["id"],
initial_prompt=self._build_qa_prompt(task),
git_context=self._task_git_context(task),
)
return True
async def _dispatch_qa_work(self, client: httpx.AsyncClient) -> None:
"""
Dispatch QA work to QA agents.
Monitors: awaiting_qa tasks
Spawns: be-qa, fe-qa, ux-qa
"""
tasks = await self._fetch_tasks(client, "awaiting_qa")
for task in tasks:
if self._is_task_handled_this_tick(task.get("id")):
continue
team = task.get("team")
if team not in ["backend", "frontend", "ux_ui"]:
continue
assigned_to = task.get("assigned_to")
if assigned_to and await self._spawn_assigned_qa(task, assigned_to):
continue
# Unassigned task - select QA agent for this team
agent_id = self._select_agent_for_cell(team, "qa")
if not agent_id:
continue
if self._is_agent_active(agent_id):
# QA already running, they'll pick up on scan
continue
# Claim the task for QA agent BEFORE spawning
if not await self._claim_task_for_agent(client, task["id"], agent_id):
logger.warning(
"Failed to claim awaiting_qa task for QA",
task_id=task["id"],
agent_id=agent_id,
)
continue
# Spawn QA agent with task assignment
await self.spawn_agent(
agent_id=agent_id,
task_id=task["id"],
initial_prompt=self._build_qa_prompt(task),
git_context=self._task_git_context(task),
)
# Only spawn one QA at a time per cell
break
async def _dispatch_pr_review_work(self, client: httpx.AsyncClient) -> None:
"""Dispatch inbound external-PR review tasks to the PR reviewer.
Monitors: pending tasks with ``source='external_pr'``.
Spawns: the single global reviewer ``pr-reviewer-1`` (one review at a
time). No pre-claim — the task stays PENDING until the reviewer claims
it itself via ``claim_pr_review``; the prompt carries the task id. The
``is_agent_active`` guard prevents a double-spawn across ticks.
"""
reviewer = "pr-reviewer-1"
if self._is_agent_active(reviewer):
return
tasks = await self._fetch_tasks(client, "pending")
for task in tasks:
if task.get("source") not in PR_REVIEW_SOURCES:
continue
if self._is_task_handled_this_tick(task.get("id")):
continue
if task.get("assigned_to"):
continue
await self.spawn_agent(
agent_id=reviewer,
task_id=task["id"],
initial_prompt=self._build_pr_review_prompt(task),
git_context=self._task_git_context(task),
)
break
async def _dispatch_doc_work(self, client: httpx.AsyncClient) -> None:
"""
Dispatch documentation + developer work during the parallel
awaiting_documentation phase.
`awaiting_documentation` requires BOTH docs_complete=True AND
pr_created=True to advance to awaiting_pm_review. Doc writes the
docs; original developer pushes and creates the PR. Whoever
finishes last triggers the state transition. Previously this
dispatcher only spawned the documenter — if the documenter
finished first, the task would sit indefinitely with pr_created=
False and nothing would spawn the dev to finish the other half.
Monitors: awaiting_documentation tasks
Spawns:
- documenter (be-doc, fe-doc, ux-doc) if docs_complete=False
- original_developer if pr_created=False (tracked in
quick_context as "original_developer:<uuid>")
"""
from roboco.services.task import extract_original_developer
# Fetch both `awaiting_documentation` and `claimed` because the
# doc's claim transitions status from awaiting_documentation →
# claimed. Without including `claimed` we'd miss tasks where doc
# already grabbed it but pr_created is still false (dev hasn't
# pushed/created PR yet). The `original_developer:` marker in
# quick_context identifies tasks that are actually in the parallel
# phase vs unrelated claimed tasks.
tasks = await self._fetch_tasks(client, ["awaiting_documentation", "claimed"])
for task in tasks:
if self._is_task_handled_this_tick(task.get("id")):
continue
await self._doc_dispatch_one(client, task, extract_original_developer)
async def _auto_assign_doc(
self, client: httpx.AsyncClient, task: dict[str, Any], team: str
) -> None:
"""
Auto-select and spawn a documenter for an unassigned awaiting_documentation task
"""
agent_id = self._select_agent_for_cell(team, "doc")
if not agent_id or self._is_agent_active(agent_id):
return
if not await self._claim_task_for_agent(client, task["id"], agent_id):
logger.warning(
"Failed to claim awaiting_documentation task for doc",
task_id=task["id"],
agent_id=agent_id,
)
return
await self.spawn_agent(
agent_id=agent_id,
task_id=task["id"],
initial_prompt=self._build_doc_prompt(task),
git_context=self._task_git_context(task),
)
async def _doc_dispatch_one(
self,
client: httpx.AsyncClient,
task: dict[str, Any],
extract_original_developer: Any,
) -> None:
"""Process a single task for `_dispatch_doc_work`."""
team = task.get("team")
if team not in ["backend", "frontend", "ux_ui"]:
return
quick_context = task.get("quick_context") or ""
dev_uuid = extract_original_developer(quick_context)
status = task.get("status")
# Only consider `claimed` tasks actually in the doc/PR parallel
# phase. See `_is_parallel_phase_claim` docstring for the why.
if status == "claimed" and not self._is_parallel_phase_claim(task, dev_uuid):
return
# Developer half: push + create PR
await self._respawn_dev_for_pr_half(task, dev_uuid)
# Documenter half: write docs
if task.get("docs_complete"):
return
if await self._respawn_doc_if_assigned(task):
return
# Auto-assign a documenter only when still in awaiting_documentation.
if status != "awaiting_documentation":
return
await self._auto_assign_doc(client, task, team)
async def _respawn_doc_if_assigned(self, task: dict[str, Any]) -> bool:
"""If task is assigned to an inactive documenter, respawn them.
Returns True when the task is already assigned (whether or not a
respawn happened) so the caller can stop processing. Returns
False when the task is unassigned so the caller can auto-select
a documenter for it.
"""
assigned_to = task.get("assigned_to")
if not assigned_to:
return False
assigned_slug = self._resolve_agent_slug(assigned_to)
if self._is_agent_active(assigned_slug):
return True
if assigned_slug and "doc" in assigned_slug:
await self.spawn_agent(
agent_id=assigned_slug,
task_id=task["id"],
initial_prompt=self._build_doc_prompt(task),
git_context=self._task_git_context(task),
)
return True
async def _blocked_by_earlier_sibling(self, task: dict[str, Any]) -> bool:
"""True if a lower-sequence, same-team sibling is not yet terminal.
Sequence-ordered merge: leaf siblings share one cell branch, so merging
a later sibling before an earlier one diverges the branch and wedges the
loser. Hold a higher-sequence sibling's review/merge dispatch until the
earlier ones land (or are cancelled). Loop-free: the task simply isn't
dispatched this tick — no reject, no respawn churn.
Only same-team siblings block (they target the same branch). Terminal
siblings (completed/cancelled) never block, so a cancelled sibling can't
deadlock the rest. Best-effort: any lookup failure falls through to
dispatch — the ordering check must never wedge the dispatcher.
"""
parent_id = task.get("parent_task_id")
seq = task.get("sequence")
team = task.get("team")
if not parent_id or seq is None:
return False
from uuid import UUID
from roboco.db.base import get_session_factory
from roboco.models.base import TaskStatus
from roboco.services.task import get_task_service
terminal = {TaskStatus.COMPLETED, TaskStatus.CANCELLED}
try:
session_factory = get_session_factory()
async with session_factory() as db:
task_svc = get_task_service(db)
siblings = await task_svc.get_subtasks(UUID(str(parent_id)))
except Exception as exc:
logger.debug(
"sibling-order check failed; dispatching anyway",
task_id=task.get("id"),
error=str(exc),
)
return False
for sib in siblings:
sib_seq = getattr(sib, "sequence", 0) or 0
sib_team = getattr(sib, "team", None)
sib_status = getattr(sib, "status", None)
sib_team_val = getattr(sib_team, "value", sib_team)
if (
str(sib_team_val) == str(team)
and sib_seq < seq
and sib_status not in terminal
):
return True
return False
async def _blocked_by_earlier_lane_sibling(self, task: dict[str, Any]) -> bool:
"""True if the SAME dev has an earlier non-terminal code sibling.
Per-dev sequenced queues (Spec 3): a PM delegates a full queue of code
subtasks to each cell dev up front. This BUILD/dispatch barrier holds a
dev's higher-sequence code leaf until its own lower-sequence code
siblings under the same parent are terminal, so the dev works its queue
one live task at a time, in order — while the other dev's lane runs
concurrently (true two-dev parallelism).
Distinct from :meth:`_blocked_by_earlier_sibling` (the MERGE barrier,
keyed on team): this is keyed on the assignee and only gates ``code``.
Loop-free (skip this tick — no reject, no respawn churn) and best-effort
(any lookup failure falls through to dispatch so the check never wedges).
"""
if str(task.get("task_type") or "") != "code":
return False
parent_id = task.get("parent_task_id")
seq = task.get("sequence")
owner = task.get("assigned_to") or task.get("claimed_by")
if not parent_id or seq is None or not owner:
return False
from uuid import UUID
from roboco.db.base import get_session_factory
from roboco.models.base import TaskStatus
from roboco.services.task import get_task_service
terminal = {TaskStatus.COMPLETED, TaskStatus.CANCELLED}
try:
session_factory = get_session_factory()
async with session_factory() as db:
task_svc = get_task_service(db)
siblings = await task_svc.get_subtasks(UUID(str(parent_id)))
except Exception as exc:
logger.debug(
"lane-order check failed; dispatching anyway",
task_id=task.get("id"),
error=str(exc),
)
return False
task_id = str(task.get("id"))
return any(
self._is_earlier_live_lane_sibling(
sib, task_id=task_id, owner=str(owner), seq=seq, terminal=terminal
)
for sib in siblings
)
@staticmethod
def _is_earlier_live_lane_sibling(
sib: Any, *, task_id: str, owner: str, seq: int, terminal: set[Any]
) -> bool:
"""True if ``sib`` is a lower-sequence non-terminal code task for ``owner``."""
if str(sib.id) == task_id:
return False
sib_type = getattr(sib, "task_type", None)
sib_type_val = getattr(sib_type, "value", sib_type)
return (
str(getattr(sib, "assigned_to", None)) == owner
and str(sib_type_val) == "code"
and (getattr(sib, "sequence", 0) or 0) < seq
and getattr(sib, "status", None) not in terminal
)
async def _dispatch_pm_review_work(self, client: httpx.AsyncClient) -> None:
"""
Dispatch PM review work to cell PMs or Main PM.
Monitors: awaiting_pm_review tasks
Spawns: be-pm, fe-pm, ux-pm, main-pm
"""
tasks = await self._fetch_tasks(client, "awaiting_pm_review")
for task in tasks:
team = task.get("team")
assigned_to = task.get("assigned_to")
# Sequence-ordered merge: don't review/merge a leaf until its
# earlier same-team siblings have landed, so they merge into the
# shared cell branch in order instead of racing and wedging.
if await self._blocked_by_earlier_sibling(task):
continue
# If already assigned, check if that agent is running
if assigned_to:
assigned_slug = self._resolve_agent_slug(assigned_to)
if self._is_agent_active(assigned_slug):
continue
# Loop guard: a review task that keeps re-surfacing without
# advancing (e.g. an unmergeable PR that re-blocks every cycle)
# must stop respawning the reviewer, else it burns tokens
# forever. The gate notifies the CEO once it trips.
if await self._pm_respawn_should_gate(assigned_slug, task):
continue
# Agent not running - spawn them to continue
await self.spawn_agent(
agent_id=assigned_slug,
task_id=task["id"],
initial_prompt=self._build_pm_review_prompt(task),
git_context=self._task_git_context(task),
)
continue
# Unassigned task - select PM based on team
# Cell tasks go to Cell PM, cross-cell/main_pm tasks go to Main PM
if team in ["backend", "frontend", "ux_ui"]:
pm_id = self._TEAM_PM_MAP.get(team, "be-pm")
else:
# main_pm, board, or no team → Main PM handles it
pm_id = "main-pm"
if self._is_agent_active(pm_id):
continue
# Claim the task for PM BEFORE spawning
if not await self._claim_task_for_agent(client, task["id"], pm_id):
logger.warning(
"Failed to claim awaiting_pm_review task for PM",
task_id=task["id"],
agent_id=pm_id,
)
continue
await self.spawn_agent(
agent_id=pm_id,
task_id=task["id"],
initial_prompt=self._build_pm_review_prompt(task),
git_context=self._task_git_context(task),
)
break
async def _dispatch_marketing_work(self, client: httpx.AsyncClient) -> None:
"""
Dispatch marketing work to head-marketing.
Monitors: pending tasks with team=marketing
Spawns: head-marketing
"""
tasks = await self._fetch_tasks(client, "pending", team="marketing")
for task in tasks:
# Skip already claimed/assigned tasks
if task.get("assigned_to"):
continue
if self._is_agent_active("head-marketing"):
# Already running, they'll pick up on scan
continue
await self.spawn_agent(
agent_id="head-marketing",
task_id=task["id"],
initial_prompt=self._build_marketing_prompt(task),
git_context=self._task_git_context(task),
)
break
# =========================================================================
# SMART DISPATCHER - EVENT-BASED DISPATCHERS
# =========================================================================
def _blocker_resolver_slug(self, task: dict[str, Any]) -> str | None:
"""Pick the agent that should be dispatched to unblock ``task``.
The unblock content gate (note/unblock) is assignee-only: the
dispatched agent must be the task's CURRENT ``assigned_to``, or its
required pre-unblock decision note returns not_authorized and the
orchestrator respawns it forever (a livelock — a task escalated to
Main PM kept respawning the ex-assignee cell PM, which could not author
the note). So whenever the blocked task carries an assignee that is a
PM or board role, dispatch THAT assignee. Only a task with no PM/board
assignee (e.g. still held by the dev who raised i_am_blocked) falls back
to the cell PM for its team.
"""
assignee_uuid = task.get("assigned_to") or task.get("claimed_by")
if assignee_uuid:
assignee_slug = self._resolve_agent_slug(str(assignee_uuid))
if assignee_slug in self._PM_AGENTS or assignee_slug in self._BOARD_AGENTS:
return assignee_slug
team = task.get("team")
if team not in ("backend", "frontend", "ux_ui"):
return None
return self._select_agent_for_cell(team, "pm")
async def _dispatch_blocker_work(self, client: httpx.AsyncClient) -> None:
"""
Dispatch blocker resolution to the task's current unblock authority.
Monitors: blocked tasks
Spawns: the task's current PM/board assignee, else the cell PM
"""
tasks = await self._fetch_tasks(client, "blocked")
for task in tasks:
# HITL-blocked tasks wait for a human; never spawn an agent on them.
if self._is_hitl_blocked(task):
continue
agent_id = self._blocker_resolver_slug(task)
if not agent_id:
continue
if self._is_agent_active(agent_id):
continue
# Loop guard: a blocked task whose unblock can never succeed (e.g.
# a cold-respawned PM that can't satisfy the unblock decision gate,
# or an unresolvable merge conflict) must stop respawning the
# resolver. The gate notifies the CEO once it trips so the wedged
# task surfaces instead of silently burning tokens.
if await self._pm_respawn_should_gate(agent_id, task):
continue
await self.spawn_agent(
agent_id=agent_id,
task_id=task["id"],
initial_prompt=self._build_pm_blocker_prompt(task),
git_context=self._task_git_context(task),
)
break
def _claimed_task_needs_agent(self, task: dict[str, Any]) -> str | None:
"""Return the assignee slug to (re)spawn for an agentless claimed task.
A task left CLAIMED/IN_PROGRESS with an assignee but no running
container (e.g. a reassignment that didn't spawn) is invisibly stuck —
only PENDING tasks get fresh dispatch, and the heartbeat reaper can't
see it because the claim seeded a fresh heartbeat. Returns the assignee
slug when the task has sat past the grace window with no active agent;
``None`` when it is healthy, too fresh, or HITL-blocked.
"""
if self._is_hitl_blocked(task):
return None
owner_uuid = task.get("assigned_to") or task.get("claimed_by")
if not owner_uuid:
return None
agent_slug = self._resolve_agent_slug(str(owner_uuid))
# The assignee is running, and on THIS task — healthy.
instance = self._instances.get(agent_slug)
if instance is not None and instance.state == AgentState.ACTIVE:
return None
# Grace window: a just-claimed task whose spawn is still in flight must
# not be churned. _time_in_state under-counts (any update bumps it),
# which biases toward "agent is working" — exactly the safe direction.
age = self._time_in_state(task)
grace = settings.claimed_no_agent_grace_seconds
if age is None or age.total_seconds() < grace:
return None
return agent_slug
async def _dispatch_claimed_without_agent(self, client: httpx.AsyncClient) -> None:
"""(Re)spawn or release claimed/in_progress tasks that have no agent.
Net for the invisible-stuck case the other dispatchers miss: a task
held CLAIMED/IN_PROGRESS by an assignee with no running container. If
the assignee is a known spawnable agent, respawn it on the task; if not
(unknown slug — e.g. a stale UUID), release the claim to PENDING so the
normal routing reclaims it with a role match.
Throttle: spawns at most ONE container per tick (``break`` after the
first respawn), matching every sibling dispatcher. A restart leaves
many agentless claims at once; without the cap this single tick would
burst-spawn a container for every one of them. The release-to-pending
path spawns nothing, so it does not consume the per-tick spawn budget
and keeps draining stale claims.
"""
tasks = await self._fetch_tasks(client, ["claimed", "in_progress"])
for task in tasks:
task_id = task.get("id")
if self._is_task_handled_this_tick(task_id):
continue
agent_slug = self._claimed_task_needs_agent(task)
if agent_slug is None:
continue
if get_agent_role(agent_slug) in (None, "unknown"):
# Unknown assignee — no agent to spawn; release for re-dispatch.
await self._release_claim_to_pending(str(task_id))
continue
logger.warning(
"Claimed/in_progress task has no running agent; respawning assignee",
task_id=task_id,
agent=agent_slug,
status=task.get("status"),
)
await self.spawn_agent(
agent_id=agent_slug,
task_id=str(task_id),
initial_prompt=self._get_prompt_for_agent(agent_slug, task),
git_context=self._task_git_context(task),
)
break
async def _release_claim_to_pending(self, task_id: str) -> None:
"""Release a stuck claim back to PENDING via the lifecycle-safe path.
Reuses ``TaskService.unclaim_for_reaper`` (claimed/in_progress ->
pending, clears assignee + work session) so the state machine records
the transition rather than a raw status PATCH. Opens its own short-lived
session, mirroring ``_reap_stale_claims``.
"""
from roboco.db.base import get_session_factory
from roboco.services.task import TaskService
from roboco.utils.converters import require_uuid
try:
factory = get_session_factory()
async with factory() as db:
svc = TaskService(db)
await svc.unclaim_for_reaper(require_uuid(task_id))
await db.commit()
logger.warning(
"Released agentless claim to pending for re-dispatch",
task_id=task_id,
)
except Exception as exc:
logger.error(
"Failed to release agentless claim; will retry next tick",
task_id=task_id,
error=str(exc),
)
async def _dispatch_escalation_work(self, client: httpx.AsyncClient) -> None:
"""
Dispatch escalations to appropriate managers.
Monitors: escalation notifications (unacknowledged)
Spawns: be-pm, fe-pm, ux-pm, main-pm, product-owner, head-marketing
"""
notifications = await self._fetch_notifications(client, "blocker_escalation")
for notif in notifications:
targets = notif.get("to_agents", [])
for agent_id in targets:
# Resolve UUID to slug - to_agents contains UUIDs from database
agent_slug = self._resolve_agent_slug(str(agent_id))
valid_targets = [
"be-pm",
"fe-pm",
"ux-pm",
"main-pm",
"product-owner",
"head-marketing",
]
if agent_slug not in valid_targets:
continue
if self._is_agent_active(agent_slug):
continue
await self.spawn_agent(
agent_id=agent_slug,
initial_prompt=self._build_escalation_prompt(notif),
)
break
async def _dispatch_approval_work(self, client: httpx.AsyncClient) -> None:
"""
Dispatch approval requests to approvers.
Monitors: approval notifications (unacknowledged)
Spawns: product-owner, head-marketing, main-pm
"""
notifications = await self._fetch_notifications(client, "approval")
for notif in notifications:
targets = notif.get("to_agents", [])
for agent_id in targets:
# Resolve UUID to slug - to_agents contains UUIDs from database
agent_slug = self._resolve_agent_slug(str(agent_id))
if agent_slug not in ["product-owner", "head-marketing", "main-pm"]:
continue
if self._is_agent_active(agent_slug):
continue
await self.spawn_agent(
agent_id=agent_slug,
initial_prompt=self._build_approval_prompt(notif),
)
break
async def _dispatch_audit_work(self, client: httpx.AsyncClient) -> None:
"""
Dispatch audit work to the auditor.
Monitors: quality alert notifications
Spawns: auditor
Note: Periodic scheduled audits can be added here in the future.
"""
alerts = await self._fetch_notifications(client, "alert")
for alert in alerts:
targets = alert.get("to_agents", [])
# Resolve UUIDs to slugs and check if auditor is a target
target_slugs = [self._resolve_agent_slug(str(t)) for t in targets]
if "auditor" in target_slugs and not self._is_agent_active("auditor"):
await self.spawn_agent(
agent_id="auditor",
initial_prompt=self._build_audit_prompt(alert),
)
return
# TODO: Add scheduled periodic audits
# Check last audit time, spawn if overdue
async def _detect_stuck_tasks(self, client: httpx.AsyncClient) -> None:
"""
Detect and auto-block tasks that are stuck.
This is a proactive enforcement mechanism that finds tasks which
have been pending without progress and have prerequisite issues.
Runs every dispatcher cycle but only takes action on truly stuck tasks.
CEO-approved timeout: 10 minutes
"""
STUCK_THRESHOLD_MINUTES = 10 # CEO-approved threshold
tasks = await self._fetch_tasks(client, "pending")
for task in tasks:
age = self._get_task_age(task)
if age is None or age < timedelta(minutes=STUCK_THRESHOLD_MINUTES):
continue
issues = self._check_stuck_conditions(task)
issues.extend(await self._check_dev_subtask_issue(client, task))
if issues:
task_id = task.get("id")
if not task_id:
continue
age_mins = int(age.total_seconds() // 60)
reason = f"Task stuck for {age_mins} minutes: " + ", ".join(issues)
await self._auto_block_task(client, task_id, reason)
logger.warning(
"Auto-blocked stuck task",
task_id=task_id,
age_minutes=age_mins,
issues=issues,
)
# Per-(role, state) SLA check. Independent from the pending-task
# sweep above — different states, different action (escalate vs
# auto-block).
await self._detect_sla_exceeded(client)
async def _check_sla_for_task(
self,
client: httpx.AsyncClient,
task: dict[str, Any],
status: str,
) -> None:
"""Check one task's SLA; escalate if exceeded. No-ops on missing data."""
from roboco.enforcement.task_lifecycle import sla_seconds_for
assigned = task.get("assigned_to")
if not assigned:
return
assigned_slug = self._resolve_agent_slug(assigned)
role = get_agent_role(assigned_slug or "")
sla = sla_seconds_for(role, status)
if sla is None:
return
age = self._time_in_state(task)
if age is None or age.total_seconds() < sla:
return
task_id = task.get("id")
if not task_id:
return
await self._escalate_sla_breach(
client,
_SlaBreach(
task_id=str(task_id),
role=role or "",
status=status,
age_seconds=int(age.total_seconds()),
sla_seconds=sla,
),
)
async def _detect_sla_exceeded(self, client: httpx.AsyncClient) -> None:
"""Auto-escalate tasks that exceeded their per-role SLA.
Uses ROLE_STATE_SLA_KEYS in enforcement/task_lifecycle.py. Dev tasks
stuck in `in_progress`/`verifying`, QA tasks in `claimed`, doc tasks
in `claimed`, and cell-PM tasks in `claimed` all get a soft bump so
work doesn't silently rot.
"""
from roboco.enforcement.task_lifecycle import ROLE_STATE_SLA_KEYS
# Fetch each (role, state) combo we care about. One API call per
# unique status so we don't fan out pointlessly.
statuses = sorted({state for _, state in ROLE_STATE_SLA_KEYS})
for status in statuses:
try:
tasks = await self._fetch_tasks(client, status)
except Exception as e:
logger.debug(
"SLA sweep fetch failed; skipping status",
status=status,
error=str(e),
)
continue
for task in tasks:
await self._check_sla_for_task(client, task, status)
def _time_in_state(self, task: dict[str, Any]) -> timedelta | None:
"""Approximate time in current state via task.updated_at.
Not perfect — any field update bumps `updated_at`, not just status
changes — but it's the coarse signal we have, and it under-counts
(biased toward "agent is working") rather than over-counts, which
matches the soft-SLA intent.
"""
updated_at = task.get("updated_at") or task.get("created_at")
if not updated_at:
return None
try:
if updated_at.endswith("Z"):
updated_at = updated_at[:-1] + "+00:00"
parsed = datetime.fromisoformat(updated_at)
if parsed.tzinfo is None:
parsed = parsed.replace(tzinfo=UTC)
return datetime.now(UTC) - parsed
except (ValueError, TypeError):
return None
async def _escalate_sla_breach(
self, client: httpx.AsyncClient, breach: _SlaBreach
) -> None:
"""Record SLA breach in dev_notes and nudge state forward.
We don't force a state transition here — the MCP lifecycle rules are
still authoritative. We log, annotate the task, and notify the
assignee's escalation target. The agent's next spawn picks up the
updated notes and usually self-escalates.
"""
age_mins = breach.age_seconds // 60
sla_mins = breach.sla_seconds // 60
note = (
f"[SLA] role={breach.role} status={breach.status} "
f"time_in_state={age_mins}m sla={sla_mins}m. "
"Escalating — agent should call escalate_up() "
"or unclaim()."
)
try:
await client.patch(
f"{self._api_url}/tasks/{breach.task_id}",
json={"dev_notes": note},
)
logger.warning(
"SLA breach noted on task",
task_id=breach.task_id,
role=breach.role,
status=breach.status,
age_minutes=age_mins,
sla_minutes=sla_mins,
)
except Exception as e:
logger.debug(
"SLA breach annotation failed",
task_id=breach.task_id,
error=str(e),
)
def _get_task_age(self, task: dict[str, Any]) -> timedelta | None:
"""Parse task created_at and return age, or None if unparseable."""
created_at_str = task.get("created_at")
if not created_at_str:
return None
try:
if created_at_str.endswith("Z"):
created_at_str = created_at_str[:-1] + "+00:00"
created_at = datetime.fromisoformat(created_at_str)
if created_at.tzinfo is None:
created_at = created_at.replace(tzinfo=UTC)
return datetime.now(UTC) - created_at
except (ValueError, TypeError):
return None
_MIN_DESCRIPTION_LEN = 10
def _check_stuck_conditions(self, task: dict[str, Any]) -> list[str]:
"""Check for common stuck conditions (git, description)."""
issues: list[str] = []
# A branch only exists once a task is claimed; a coordination task does
# no git at all. A pending, never-claimed code task therefore has no
# branch by design — flagging that here auto-blocked tasks before their
# first dispatch. Only flag a missing branch when the task is in a
# state where it should already own one.
if not task.get("branch_name") and _branch_is_expected(task):
issues.append("Task missing branch_name")
description = (task.get("description") or "").strip()
if len(description) < self._MIN_DESCRIPTION_LEN:
issues.append("Empty or inadequate description")
return issues
async def _check_dev_subtask_issue(
self, client: httpx.AsyncClient, task: dict[str, Any]
) -> list[str]:
"""Check if complex dev task is missing subtasks."""
from roboco.agents_config import get_agent_role
assigned_to = task.get("assigned_to")
if not assigned_to:
return []
agent_slug = self._resolve_agent_slug(assigned_to)
if not agent_slug or get_agent_role(agent_slug) != "developer":
return []
complexity = task.get("estimated_complexity", "low")
is_low_complexity = complexity not in ("medium", "high")
if is_low_complexity or task.get("parent_task_id"):
return []
try:
resp = await client.get(f"{self._api_url}/tasks/{task.get('id')}/subtasks")
subtasks = resp.json() if resp.is_success else []
except Exception:
subtasks = []
if not subtasks:
return [f"{complexity} complexity task without subtasks"]
return []
async def _dispatch_a2a_work(self, client: httpx.AsyncClient) -> None:
"""
Dispatch A2A (Agent-to-Agent) requests to target agents.
Monitors: a2a_request notifications (unacknowledged)
Spawns: Any agent that is the target of an A2A request
This is a fallback mechanism - primary A2A routing happens via events.
If the event-based spawn fails, these notifications will be picked up here.
"""
notifications = await self._fetch_notifications(client, "a2a_request")
for notif in notifications:
targets = notif.get("to_agents", [])
for agent_id in targets:
# Resolve UUID to slug - to_agents contains UUIDs from database
agent_slug = self._resolve_agent_slug(str(agent_id))
if self._is_agent_active(agent_slug):
# Agent is online - SDK handles A2A delivery directly
# No action needed here, SDK server receives messages
continue
# Agent is offline - spawn them with A2A context
await self.spawn_agent(
agent_id=agent_slug,
initial_prompt=self._build_a2a_prompt(notif),
)
break
# =========================================================================
# SMART DISPATCHER - PROMPT BUILDERS
# =========================================================================
def _get_workflow_state(
self,
status: str,
has_plan: bool,
) -> str:
"""Determine developer workflow state from task attributes.
Args:
status: Task status (claimed, in_progress, needs_revision, etc.)
has_plan: Whether task has a plan submitted
Returns:
Workflow state string (NEEDS_PLAN, READY_TO_START, EXECUTING, etc.)
"""
# Direct status mappings
status_map = {
"in_progress": "EXECUTING",
"needs_revision": "REVISION_REQUIRED",
"verifying": "VERIFYING",
}
if status in status_map:
return status_map[status]
# Handle claimed status with sub-states
if status == "claimed":
if not has_plan:
return "NEEDS_PLAN"
return "READY_TO_START"
return status.upper()
def _get_workflow_instructions(self, state: str, task_id: str) -> str:
"""Get workflow instructions for the given state.
Args:
state: Workflow state (NEEDS_PLAN, READY_TO_START, etc.)
task_id: Task ID for tool call examples
Returns:
Markdown-formatted instructions for the current state
"""
instructions = {
"NEEDS_PLAN": f"""## NEXT STEP: Claim + Plan + Start
Call i_will_work_on(task_id="{task_id}",
plan="<approach, ordered steps, risks, open questions>").
This single verb claims the task, records your plan, and transitions
to in_progress.
""",
"READY_TO_START": f"""## NEXT STEP: Start Work
Call i_will_work_on(task_id="{task_id}", plan="<your plan as a string>")
to begin.
""",
"EXECUTING": """## IN PROGRESS
Continue development. Required gates before i_am_done() will succeed
(enforced server-side — `remediate` tells you what's missing):
1. commit("<type(scope): subject, >=20 chars>")
— makes the git commit, auto-prefixes task ID, records progress.
Repeat per meaningful chunk.
2. note(scope='decision'|'learning'|'reflect', task_id="...", text=...)
as you make trade-offs.
When acceptance criteria are met, call
open_pr(task_id="...") to push your branch and open the PR,
then i_am_done(task_id="...", notes="<self-verification summary>")
to submit for QA review.
If you hit something you can't unblock yourself:
i_am_blocked(task_id="...",
reason="<blocked_external|low_context|...>").
""",
"REVISION_REQUIRED": f"""## REVISION REQUESTED
QA or PM requested changes:
1. evidence(task_id="{task_id}") — read qa_notes / pm_notes / inline diff
2. i_will_work_on(task_id="{task_id}",
plan="<revised plan addressing each issue>")
3. commit() the fixes, then
i_am_done(task_id="{task_id}", notes="<what was fixed>")
""",
"VERIFYING": f"""## SELF-VERIFICATION
Run the project's quality checks against acceptance criteria:
1. Run tests, lint, type checks in your workspace.
2. evidence(task_id="{task_id}") — sanity-check inline diff + commits.
3. If everything passes:
i_am_done(task_id="{task_id}", notes="<verification summary>")
— chains submit_verification + push + create_pr + submit_qa.
4. If issues found: commit() the fixes and retry.
""",
}
return instructions.get(
state, f'Call evidence(task_id="{task_id}") to check status.'
)
def _build_dev_prompt(self, task: dict[str, Any]) -> str:
"""Build state-aware initial prompt for a developer."""
task_id = task.get("id", "unknown")
title = task.get("title", "Untitled")
status = task.get("status", "unknown")
# Determine workflow state based on task attributes
has_plan = bool(task.get("plan"))
workflow_state = self._get_workflow_state(status, has_plan)
instructions = self._get_workflow_instructions(workflow_state, task_id)
return f"""You have been assigned a development task.
TASK ID: {task_id}
TITLE: {title}
STATUS: {status}
WORKFLOW STATE: {workflow_state}
{instructions}
Start by calling evidence(task_id="{task_id}") for full details and acceptance criteria.
When out of work: i_am_idle().
"""
def _build_qa_prompt(self, task: dict[str, Any]) -> str:
"""Build initial prompt for a QA agent."""
task_id = task.get("id", "unknown")
title = task.get("title", "Untitled")
assigned_to = task.get("assigned_to", "unknown")
team = task.get("team", "unknown")
return f"""A task is ready for QA review.
TASK ID: {task_id}
TITLE: {title}
DEVELOPER: {assigned_to}
TEAM: {team}
== QA WORKFLOW ==
1. claim_review(task_id="{task_id}")
— assigns the QA seat; returns inline diff + PR + commits as evidence.
The PR is already open (dev opened it before submitting QA);
review on GitHub if you need more context.
2. Review the implementation against EVERY acceptance criterion.
Run/read tests; sanity-check the diff for regressions, security,
and scope creep.
3. Decide:
- PASS: pass(task_id="{task_id}",
notes="<>=80 chars: what you verified, which AC, evidence>")
— transitions awaiting_qa → awaiting_documentation.
- FAIL: fail(task_id="{task_id}",
issues=["concrete issue 1", "concrete issue 2", ...])
— transitions to needs_revision; each issue must be specific and
actionable.
4. note(scope='reflect'|'learning', task_id="{task_id}", text=...)
for anything worth flagging.
5. give_me_work() to pick up the next QA item,
or i_am_idle() if the queue is empty.
"""
def _build_pr_review_prompt(self, task: dict[str, Any]) -> str:
"""Build the initial prompt for the PR reviewer on an external PR."""
task_id = task.get("id", "unknown")
title = task.get("title", "Untitled")
pr_number = task.get("pr_number", "?")
pr_url = task.get("pr_url", "")
return f"""An external contributor opened a pull request. Review it.
TASK ID: {task_id}
TITLE: {title}
EXTERNAL PR: #{pr_number} {pr_url}
== TRUST BOUNDARY ==
This PR is from OUTSIDE the org — the code is untrusted. The review is
READ-ONLY: you read the diff, you do NOT fetch, check out, build, or run the
contributor's code. Do not push to their fork. You never merge.
== REVIEW WORKFLOW ==
1. claim_pr_review(task_id="{task_id}")
— starts the review; returns the contributor's unified diff inline.
2. Review the diff adversarially: correctness, security (injection, secret
leaks, supply-chain/dependency risk), scope, and the codebase's standards.
Reason about it from the diff alone — do not run it.
3. note(scope="learning", task_id="{task_id}", text="<what the review surfaced>")
— required before you can post.
4. post_pr_review(task_id="{task_id}",
body="<one complete change-request: per-finding file + line + expected
vs actual; be specific and actionable>",
event="REQUEST_CHANGES")
— posts ONE complete review to the PR and finishes the task. Use
event="APPROVE" only if the PR is genuinely ready as-is.
5. i_am_idle() when done.
"""
def _build_doc_prompt(self, task: dict[str, Any]) -> str:
"""Build initial prompt for a documenter."""
task_id = task.get("id", "unknown")
title = task.get("title", "Untitled")
team = task.get("team", "unknown")
return f"""A task is ready for documentation. The dev's PR is already open
— you're documenting alongside the QA-passed branch.
TASK ID: {task_id}
TITLE: {title}
TEAM: {team}
== DOC WORKFLOW ==
1. claim_doc_task(task_id="{task_id}")
— assigns the doc seat and opens your workspace on the task's branch.
2. evidence(task_id="{task_id}") — read dev handoff notes, qa_notes,
and the inline diff so the docs reflect what actually shipped.
3. Write/update docs in your workspace: README sections, API references,
code comments, migration notes, or new docs files as the change requires.
4. commit("docs(scope): <subject, >=20 chars>") per logical doc chunk
— auto-prefixes the task ID and stages tracked changes.
5. i_documented(task_id="{task_id}",
notes="<>=20 chars: what you documented and where>",
files=["docs/foo.md", "README.md", ...])
— transitions awaiting_documentation → awaiting_pm_review.
6. give_me_work() for the next doc item,
or i_am_idle() if the queue is empty.
"""
def _build_pm_review_prompt(self, task: dict[str, Any]) -> str:
"""Prompt for PM reviewing a SUBTASK in awaiting_pm_review."""
task_id = task.get("id", "unknown")
title = task.get("title", "Untitled")
team = task.get("team", "unknown")
return f"""A SUBTASK in your cell is awaiting your PM review.
It has passed QA and documentation; the leaf PR is open and ready to merge.
TASK ID: {task_id}
TITLE: {title}
TEAM: {team}
== PM REVIEW WORKFLOW (leaf subtask) ==
1. evidence(task_id="{task_id}")
— review PR, commits, inline diff, dev_notes, qa_notes, doc files.
2. Spot-check that:
- every acceptance criterion is satisfied,
- QA's pass notes line up with the actual diff,
- docs reflect what shipped.
3. note(scope='decision', task_id="{task_id}",
text="<approve rationale or rejection reason>")
— REQUIRED before complete().
4. Decide:
- APPROVE: complete(task_id="{task_id}", notes="<merge rationale>")
— auto-merges the leaf PR and finalizes the subtask.
- NEEDS REWORK: leave a clear note(scope='decision', text="...") and
rely on the dispatcher to respawn the dev for revision.
Use escalate_up only if the issue is truly outside your cell.
5. give_me_work() / triage() for the next item, or i_am_idle().
Never `commit`, never write code, never run `git`. PMs coordinate.
"""
def _build_board_prompt(self, task: dict[str, Any]) -> str:
"""Prompt for a board agent (Product Owner / Head of Marketing) to
review and SHAPE a strategic task. Board roles advise — they do not
build, code, or delegate."""
task_id = task.get("id", "unknown")
title = task.get("title", "Untitled")
description = task.get("description", "No description")
return f"""\
You are on the Board. This strategic task is under board review.
TASK: {task_id}
TITLE: {title}
DESCRIPTION: {description}
THE BOARD REVIEWS AS A PAIR: the Product Owner AND the Head of Marketing both
review every board task before it reaches the CEO. The Product Owner owns
product requirements + acceptance scope; the Head of Marketing owns the UX /
user-facing / positioning dimension. The CEO only gets the handoff after BOTH
of you have recorded a review.
YOUR ROLE: review and shape this work. You do NOT build, code, claim, or
delegate — those verbs are not yours. Your deliverable is a recorded review.
== WHAT TO DO ==
1. triage()
— see your board-level work and context.
2. note(text="<the product requirements and acceptance criteria you expect, the
scope, the must-haves, and what 'done' looks like — Head of Marketing:
the UX, user-facing impact, and how the feature is positioned>",
scope='decision', task_id="{task_id}")
— this recorded review is how the CEO and Main PM act on your input.
3. say(...) in your board channel to flag UX, positioning, or risk concerns and
to coordinate with your fellow board reviewer.
4. i_am_idle()
— when your review is recorded. Once both board reviewers are done, the
CEO is notified the task is ready for Approve & Start, then routes it to
Main PM for delegation to the cells; you do NOT hand it off yourself.
Do NOT attempt to claim, plan, complete, or delegate — the gateway will reject
those, and a substantive recorded note IS your job here.
"""
def _build_marketing_prompt(self, task: dict[str, Any]) -> str:
"""Build initial prompt for head-marketing with a marketing task."""
task_id = task.get("id", "unknown")
title = task.get("title", "Untitled")
description = task.get("description", "No description")
return f"""You have been assigned a marketing task.
TASK ID: {task_id}
TITLE: {title}
DESCRIPTION: {description}
Begin work:
1. Review the task details above (full acceptance criteria arrive in your
briefing / the give_me_work response)
2. Execute the marketing task (content, campaigns, research, etc.)
3. Coordinate with Product Owner or Main PM if needed
4. Call i_am_done() when done
5. Call give_me_work() to check for more marketing work
6. If no more work, call i_am_idle() to shutdown gracefully
"""
def _build_pm_blocker_prompt(self, task: dict[str, Any]) -> str:
"""Build initial prompt for a Cell PM handling a blocker."""
task_id = task.get("id", "unknown")
title = task.get("title", "Untitled")
assigned_to = task.get("assigned_to", "unknown")
blocker = task.get("blocker", {})
reason = blocker.get("reason", "Unknown")
what_needed = blocker.get("what_needed", "Unknown")
return f"""A task in your cell is BLOCKED and needs your attention.
TASK ID: {task_id}
TITLE: {title}
ASSIGNED TO: {assigned_to}
BLOCKER REASON: {reason}
WHAT'S NEEDED: {what_needed}
Your job:
1. Understand the blocker by reviewing task details
2. Communicate with the blocked developer if needed
3. Resolve the blocker (coordinate resources, make decisions, escalate if needed)
4. Once resolved, call unblock("{task_id}") to release the task back to the developer
5. Call triage() to check for other blocked tasks in your cell
6. If no more blockers, call i_am_idle() to shutdown gracefully
"""
def _build_escalation_prompt(self, notification: dict[str, Any]) -> str:
"""Build initial prompt for handling an escalation."""
notif_id = notification.get("id", "unknown")
from_agent = notification.get("from_agent", "unknown")
subject = notification.get("subject", "No subject")
priority = notification.get("priority", "normal")
body = notification.get("body", "No details provided")
return f"""You have received an ESCALATION that requires your attention.
FROM: {from_agent}
SUBJECT: {subject}
PRIORITY: {priority}
DETAILS:
{body}
Your job:
1. Acknowledge the notification with notify_ack("{notif_id}")
2. Assess the escalation and determine action needed
3. Communicate decisions via appropriate channels
4. If this requires further escalation, use escalate_up()
5. When resolved, call triage() for other work
6. If no more work, call i_am_idle() to shutdown gracefully
"""
def _build_approval_prompt(self, notification: dict[str, Any]) -> str:
"""Build initial prompt for handling an approval request."""
notif_id = notification.get("id", "unknown")
from_agent = notification.get("from_agent", "unknown")
subject = notification.get("subject", "No subject")
related_task_id = notification.get("related_task_id", "None")
body = notification.get("body", "No details provided")
return f"""You have received an APPROVAL REQUEST.
FROM: {from_agent}
SUBJECT: {subject}
RELATED TASK: {related_task_id}
REQUEST:
{body}
Your job:
1. Review the approval request carefully
2. If related to a task, use the task context provided in your briefing
3. Make your decision and communicate it
4. Acknowledge with notify_ack("{notif_id}")
5. Call triage() for other work
6. If no more work, call i_am_idle() to shutdown gracefully
"""
def _build_audit_prompt(self, alert: dict[str, Any] | None = None) -> str:
"""Build initial prompt for the auditor."""
if alert:
subject = alert.get("subject", "Quality issue detected")
body = alert.get("body", "Review system quality metrics")
return f"""QUALITY ALERT triggered your attention.
ALERT: {subject}
DETAILS: {body}
Your job:
1. Investigate the quality issue
2. Review relevant channels and task history (you have read access to all)
3. Compile your findings
4. Report to CEO via appropriate channel
5. Call i_am_idle() when complete
"""
return """Periodic AUDIT requested.
Your job:
1. Review recent activity across all cells
2. Check quality metrics (QA pass/fail rates, blocker frequency, etc.)
3. Identify any concerns or patterns
4. Compile audit report for CEO
5. Call i_am_idle() when complete
"""
def _build_a2a_prompt(self, notification: dict[str, Any]) -> str:
"""Build initial prompt for handling an A2A (Agent-to-Agent) request.
Reads `priority` directly off the notification row (set by
NotificationService.send_a2a_notification). Pre-Phase-3 this
consumed a non-existent `metadata.urgent` and always rendered
urgency_note=False; the column-level priority is now the source
of truth.
"""
notif_id = notification.get("id", "unknown")
from_agent = notification.get("from_agent", "unknown")
body = notification.get("body", "No message provided")
related_task_id = notification.get("related_task_id")
metadata = notification.get("metadata", {})
skill = metadata.get("skill", "general")
priority_raw = notification.get("priority", "normal")
# URGENT gets the bold attention-grabber; HIGH gets a quieter
# "higher priority" hint; NORMAL gets no prefix.
if priority_raw == "urgent":
urgency_note = "**URGENT** - This request has priority.\n\n"
elif priority_raw == "high":
urgency_note = "**HIGH PRIORITY** - Please handle promptly.\n\n"
else:
urgency_note = ""
task_note = f"RELATED TASK: {related_task_id}\n" if related_task_id else ""
return f"""You have received an A2A (Agent-to-Agent) REQUEST.
{urgency_note}FROM: {from_agent}
SKILL: {skill}
{task_note}
REQUEST:
{body}
Your job:
1. Acknowledge the notification with notify_ack("{notif_id}")
2. Process the request using your {skill} capabilities
3. Respond to {from_agent} using dm("{from_agent}", ...)
4. If you need task context, it is provided in your briefing for the related task
5. When done, call give_me_work() for other work
6. If no more work, call i_am_idle() to shutdown gracefully
"""