mirror of
https://github.com/rennf93/roboco.git
synced 2026-08-03 07:23:24 +02:00
[sweep] strip Fxxx audit-ID tokens + trim bloated comments/docstrings + add behavior-change docs
Post-audit sweep over the 135 audit-fix commits since19a474d3: 1. Stripped every # Fxxx: audit-ID token from comments AND every Fxxx token from docstring openings across 211 blocks / ~626 lines. The CEO flagged these twice: audit-issue IDs in code confuse future devs/agents. The descriptive text is preserved; only the Fxxx token is removed (and bloated narrative blocks trimmed to 1-3 lines keeping the one non-obvious invariant). 2. Trimmed bloated comments/docstrings to the concise standard (1-3 lines). 3. Added missing behavior-change docs for the audit-fix batch: prompts/roles (documenter, pr_reviewer, qa), user-facing docs (api auth, websockets, agent-gateway, megatask, merge-model, task-lifecycle, grok, resilience, conventions, panel, security, troubleshooting), and the RAG corpus (cell-pm, main-pm, pr-reviewer, qa roles; conventions; messaging-tools; escalation; megatask; task-claiming workflows). Comment/docstring/prose ONLY — zero code-line edits (verified: the diff contains no def/class/return/if/for/await/assignment/call lines). Gates green: ruff format + ruff check clean, mypy clean on roboco/. The only pytest failures are the pre-existing sync_branch tracing-decision gap (B1,250be5c2) — not sweep-caused and tracked separately.
This commit is contained in:
@@ -196,16 +196,9 @@ def test_in_progress_task_with_no_agent_returns_assignee() -> None:
|
||||
|
||||
|
||||
def test_claimed_task_with_unknown_assignee_returns_slug_for_release() -> None:
|
||||
# F032: a claimed/in_progress task whose assignee is a stale/unknown UUID
|
||||
# (no seeded agent) must reach the release-to-pending path. The human-only
|
||||
# guard (role_for_slug_or_none) returns None for an unknown slug, and
|
||||
# ``None in (CEO, PROMPTER, SECRETARY)`` is False — so it does NOT
|
||||
# short-circuit, the slug falls through the grace window, and the resolver
|
||||
# returns the unknown slug. _dispatch_claimed_without_agent then sees
|
||||
# get_agent_role(slug) == "unknown" and releases the claim to pending for
|
||||
# a role-matched reclaim. Before F031's role_for_slug_or_none fix the guard
|
||||
# raised KeyError on the unknown slug, crashing the whole tick before the
|
||||
# release path could run.
|
||||
# A claimed/in_progress task whose assignee is a stale/unknown UUID (no
|
||||
# seeded agent) must reach the release-to-pending path: the human-only guard
|
||||
# returns None for unknown slugs, so the slug falls through and is released.
|
||||
orch = _orch()
|
||||
unknown_uuid = str(uuid4())
|
||||
task: dict[str, Any] = {
|
||||
|
||||
@@ -58,11 +58,11 @@ async def test_load_watch_set_filters_enabled_one_per_repo() -> None:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_load_watch_set_keeps_distinct_workflows_per_repo() -> None:
|
||||
"""F115: a monorepo's several cell-projects each carrying their OWN
|
||||
``ci_watch_workflow`` must ALL be watched — collapsing to the canonical
|
||||
cell's workflow would miss a red on the other cells' workflows (under-count).
|
||||
Same repo, DIFFERENT workflows → one entry per (repo, workflow). The engine's
|
||||
per-git_url fix-task dedup still prevents duplicate fix tasks for the repo."""
|
||||
"""A monorepo's several cell-projects each carrying their OWN
|
||||
``ci_watch_workflow`` must ALL be watched — collapsing to the canonical cell's
|
||||
workflow would miss a red on the other cells' workflows (under-count). Same
|
||||
repo, DIFFERENT workflows → one entry per (repo, workflow); per-git_url dedup
|
||||
still prevents duplicate fix tasks for the repo."""
|
||||
orch = _orch()
|
||||
be = MagicMock(
|
||||
slug="be",
|
||||
|
||||
@@ -46,12 +46,12 @@ async def test_load_set_filters_command_one_per_repo() -> None:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_load_set_keeps_distinct_commands_per_repo() -> None:
|
||||
"""F115: a monorepo's several cell-projects each carrying their OWN
|
||||
``dep_update_command`` (different ecosystems → different lockfiles) must
|
||||
ALL be probed — collapsing to the canonical cell's command would miss the
|
||||
other cells' lockfile drift (under-count). Same repo, DIFFERENT commands →
|
||||
one entry per (repo, command). The engine's per-git_url open-task dedup
|
||||
still prevents duplicate update tasks for the repo."""
|
||||
"""A monorepo's several cell-projects each carrying their OWN
|
||||
``dep_update_command`` (different ecosystems → different lockfiles) must ALL
|
||||
be probed — collapsing to the canonical cell's command would miss the other
|
||||
cells' lockfile drift (under-count). Same repo, DIFFERENT commands → one
|
||||
entry per (repo, command); per-git_url open-task dedup still prevents
|
||||
duplicate update tasks for the repo."""
|
||||
orch = _orch()
|
||||
be = MagicMock(
|
||||
slug="be", git_url="https://x/a.git", dep_update_command="uv lock --upgrade"
|
||||
|
||||
@@ -60,11 +60,9 @@ async def test_cost_over_cap_kills_and_evicts(monkeypatch: pytest.MonkeyPatch) -
|
||||
async def test_cost_over_cap_finalizes_spawn_session_before_evict(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
# F040: a cost-cap-killed grok container must finalize its spawn session so
|
||||
# the captured usage/cost is recorded in the DB/dashboard — otherwise the
|
||||
# session row stays open (ended_at IS NULL) and the burn is invisible.
|
||||
# Finalization must run BEFORE the instance is popped: _finalize_spawn_session
|
||||
# reads self._instances[agent_id] for the model + usage_session_id.
|
||||
# Cost-cap-killed grok container must finalize its spawn session BEFORE the
|
||||
# instance is popped: _finalize_spawn_session reads _instances[agent_id] for
|
||||
# the model + usage_session_id; otherwise the burn stays invisible.
|
||||
orch, _remove_mock = _orch(monkeypatch, cap=5.0, cost=7.5)
|
||||
finalize = AsyncMock()
|
||||
monkeypatch.setattr(orch, "_finalize_spawn_session", finalize)
|
||||
|
||||
@@ -116,7 +116,7 @@ async def test_park_grok_rate_limited_activates_and_offlines(
|
||||
# needs the dict + persist stub to exercise that without AttributeError.
|
||||
orch._waiting_records = {}
|
||||
orch._rate_limit_ceo_notified = set()
|
||||
# F097 backoff state — the constructor (skipped here) initializes these.
|
||||
# Backoff state — the constructor (skipped here) initializes these.
|
||||
orch._grok_last_park_at = None
|
||||
orch._grok_repark_count = 0
|
||||
inst = _grok_instance()
|
||||
@@ -159,7 +159,7 @@ async def test_handle_stopped_container_parks_on_grok_429(
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# F041: exit 78 (auth missing/expired) parks instead of crash-retrying
|
||||
# Exit 78 (auth missing/expired) parks instead of crash-retrying
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@@ -178,11 +178,9 @@ def test_is_grok_auth_exit() -> None:
|
||||
async def test_handle_stopped_container_parks_on_grok_auth_exit(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
# F041: a grok container whose entrypoint ran `grok_auth --check` and found
|
||||
# the token missing/expired exits 78 (EX_CONFIG). Crash-retrying 3x burns
|
||||
# tokens for zero progress (the agent can't start without a valid token);
|
||||
# park it like the 429 exit-75 path so the probe-resume loop revives the
|
||||
# task once grok_auth.refresh_if_stale mints a fresh token.
|
||||
# A grok container whose entrypoint ran `grok_auth --check` and found the
|
||||
# token missing/expired exits 78 (EX_CONFIG); park it (like the 429 exit-75
|
||||
# path) so the probe-resume loop revives the task once a fresh token is minted.
|
||||
orch = AgentOrchestrator.__new__(AgentOrchestrator)
|
||||
inst = _grok_instance()
|
||||
park = AsyncMock()
|
||||
@@ -224,9 +222,9 @@ async def test_park_grok_auth_unavailable_activates_with_auth_missing_kind(
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# F097 — grok has no real probe, so an optimistic clear respawns into a still-
|
||||
# active xAI 429 every ~90s. Back off the re-park retry_after within one rate-
|
||||
# limit episode so the churn dampens instead of spinning flat at 60s.
|
||||
# Grok has no real probe, so an optimistic clear respawns into a still-active
|
||||
# xAI 429 every ~90s; back off the re-park retry_after within one rate-limit
|
||||
# episode so the churn dampens instead of spinning flat at 60s.
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
|
||||
@@ -37,8 +37,8 @@ def _make_minimal_orchestrator() -> AgentOrchestrator:
|
||||
# (F071); without this the post-docker-run guard would AttributeError on
|
||||
# the constructor-skipped instance.
|
||||
orch._running = True
|
||||
# F093: concurrent intake starts serialize on this lock; the constructor
|
||||
# (skipped here) initializes it.
|
||||
# Concurrent intake starts serialize on this lock; the constructor (skipped
|
||||
# here) initializes it.
|
||||
orch._intake_spawn_lock = asyncio.Lock()
|
||||
return orch
|
||||
|
||||
@@ -572,11 +572,9 @@ class TestDeliverWhenReady:
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# F071 — non-blocking intake spawn must not orphan a container if shutdown
|
||||
# arrives between ``docker run`` and the _instances registration. The guarded
|
||||
# wrapper runs concurrently with stop(); without a post-docker-run shutdown
|
||||
# check, the just-started container is never recorded in _instances (which
|
||||
# stop() already iterated) so nothing tears it down — a leaked container.
|
||||
# Non-blocking intake spawn must not orphan a container if shutdown arrives
|
||||
# between ``docker run`` and _instances registration: without a post-docker-run
|
||||
# shutdown check the just-started container is never recorded so leaks.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
|
||||
@@ -82,7 +82,7 @@ def test_intake_grok_mounts_subscription_auth_when_present(
|
||||
cmd = AgentOrchestrator._build_intake_run_cmd(
|
||||
_intake_spec("grok", base_url="https://api.x.ai/v1", token="xai-key")
|
||||
)
|
||||
# F005: directory mount (ro), not the single-file inode-pinning mount.
|
||||
# directory mount (ro), not the single-file inode-pinning mount.
|
||||
assert f"{grok_dir}:/home/agent/.grok-auth-ro:ro" in cmd
|
||||
|
||||
|
||||
|
||||
@@ -1,22 +1,10 @@
|
||||
"""F070 — fire-and-forget ``_bg_tasks`` (respawn_tracker upserts, audit-log
|
||||
writes, intake first-message delivery) were never cancelled or drained on
|
||||
shutdown. ``Orchestrator.stop()`` cancelled only the named loop tasks and the
|
||||
agents, then returned, abandoning any in-flight ``_schedule_bg`` work.
|
||||
"""Drain ``_bg_tasks`` on shutdown so fire-and-forget writes (respawn_tracker
|
||||
upserts, audit-log writes, intake first-message delivery) are not abandoned.
|
||||
|
||||
The data-loss tail: an in-flight ``_persist_respawn_record`` upsert dropped at
|
||||
shutdown means the last few gate-mutation strikes never reach the DB. The
|
||||
in-memory counter dies with the process; ``restore_respawn_tracker()`` on the
|
||||
next start repopulates a stale lower count and the dispatcher re-burns the
|
||||
full strike threshold (4 spawns) against a still-wedged task — the exact
|
||||
re-burn the durable tracker exists to stop. Audit-log writes (load-bearing for
|
||||
the cycle-time / rework metrics) are similarly dropped.
|
||||
|
||||
The fix DRAINs ``_bg_tasks`` with a bounded timeout on shutdown — short DB
|
||||
writes finish before the process exits (data preserved), while a stuck task
|
||||
can't hang shutdown (it is cancelled once the drain deadline passes). Cancels
|
||||
outright would lose the data (the opposite of the goal), so the drain tries to
|
||||
let work complete first. The ``stop_agent`` loop is also wrapped so one agent's
|
||||
stop error can't skip the drain (which would still drop the data).
|
||||
Invariant: ``Orchestrator.stop()`` drains ``_bg_tasks`` with a bounded timeout —
|
||||
short DB writes finish before the process exits (data preserved), a stuck task
|
||||
is cancelled once the deadline passes (can't hang shutdown). The ``stop_agent``
|
||||
loop is wrapped so one agent's stop error can't skip the drain.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -152,10 +140,9 @@ async def test_stop_failing_agent_does_not_skip_drain() -> None:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stop_is_idempotent_double_call_is_noop() -> None:
|
||||
"""F117: stop() is idempotent. The lifespan shutdown path now stops the
|
||||
orchestrator before closing the DB, and bootstrap's finally block re-calls
|
||||
stop() as a safety net. The second call must be a clean no-op — not a
|
||||
re-drain, not a re-stop of already-stopped agents — guarded by ``_stopped``."""
|
||||
"""stop() is idempotent: the lifespan path and bootstrap's finally block both
|
||||
call it, so the second call must be a clean no-op — not a re-drain or re-stop
|
||||
of already-stopped agents — guarded by ``_stopped``."""
|
||||
orch = _make_orchestrator()
|
||||
real_drain = orch._drain_bg_tasks
|
||||
drain_calls = 0
|
||||
|
||||
@@ -568,10 +568,9 @@ def _stop_agent_patches(orch: AgentOrchestrator) -> Any:
|
||||
|
||||
|
||||
async def test_stop_agent_releases_claim_when_release_claim_true() -> None:
|
||||
"""F120: stop_agent(release_claim=True) hands the agent's claimed task back
|
||||
to the pool immediately. A SIGTERM/budget-kill mid-verb otherwise leaves
|
||||
the task CLAIMED/IN_PROGRESS with no running agent for up to
|
||||
stale_claim_reap_seconds (the reaper's heartbeat TTL)."""
|
||||
"""stop_agent(release_claim=True) releases the agent's claimed task to the
|
||||
pool immediately, so a mid-verb SIGTERM/budget-kill doesn't strand the task
|
||||
CLAIMED/IN_PROGRESS until the reaper's heartbeat TTL expires."""
|
||||
orch = _make_orchestrator()
|
||||
instance = _make_instance(_AGENT_ID)
|
||||
instance.current_task_id = str(uuid4())
|
||||
@@ -631,10 +630,9 @@ async def test_stop_agent_does_not_release_claim_by_default() -> None:
|
||||
|
||||
|
||||
async def test_stop_agent_skips_release_for_provider_parked_agent() -> None:
|
||||
"""F120: a provider-parked agent (rate_limit_lifted WaitingRecord) must NOT
|
||||
have its claim released even when release_claim=True. The probe-resume loop
|
||||
owns its recovery and the claim must survive so probe-success revives the
|
||||
SAME agent on the SAME task — reaping would let another agent claim it."""
|
||||
"""A provider-parked agent (rate_limit_lifted WaitingRecord) must NOT have
|
||||
its claim released even when release_claim=True — the probe-resume loop
|
||||
revives the SAME agent on the SAME task, so reaping would lose the claim."""
|
||||
orch = _make_orchestrator()
|
||||
instance = _make_instance(_AGENT_ID)
|
||||
instance.current_task_id = str(uuid4())
|
||||
|
||||
@@ -108,11 +108,9 @@ async def test_clean_output_is_not_overload(
|
||||
async def test_detects_overload_marker_in_transcript(
|
||||
orch: AgentOrchestrator, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
# F036: the SDK server writes model-API errors to /tmp/sdk-server.log, not
|
||||
# stdout, so the overload marker (529/500/503) may appear only in the durable
|
||||
# Claude transcript — exactly the rationale already applied to the
|
||||
# session-limit detector. Without reading the transcript here an overload
|
||||
# is missed and the agent crash-respawns straight back into it.
|
||||
# The overload marker may appear only in the durable Claude transcript, not
|
||||
# stdout; without reading it an overload is missed and the agent
|
||||
# crash-respawns straight back into it.
|
||||
monkeypatch.setattr(settings, "overload_break_enabled", True)
|
||||
monkeypatch.setattr(orch, "_tail_container_logs", AsyncMock(return_value=""))
|
||||
monkeypatch.setattr(
|
||||
@@ -128,10 +126,9 @@ async def test_detects_overload_marker_in_transcript(
|
||||
async def test_agent_writing_about_error_500_does_not_park(
|
||||
orch: AgentOrchestrator, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
# F037: an agent that merely writes about an HTTP error code in its own
|
||||
# notes ("the endpoint returned error 500, retrying") must NOT trip the
|
||||
# overload detector and park the whole Anthropic fleet. Markers must be
|
||||
# specific to the API error formatter, not bare "error NNN".
|
||||
# An agent merely writing about an HTTP error code in its own notes must NOT
|
||||
# trip the detector and park the whole fleet — markers must be specific to
|
||||
# the API error formatter, not bare "error NNN".
|
||||
monkeypatch.setattr(settings, "overload_break_enabled", True)
|
||||
agent_note = (
|
||||
"be-dev-1: the /health endpoint returned error 500 on retry; "
|
||||
@@ -203,11 +200,10 @@ async def test_park_offlines_and_activates_with_kind(
|
||||
async def test_park_registers_waiting_record_so_probe_can_resume(
|
||||
orch: AgentOrchestrator, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
# F035: the probe-resume loop reads _waiting_records filtered by
|
||||
# waiting_for == "rate_limit_lifted" + context.provider. Without a record
|
||||
# here, _parked_agents_for(provider) returns [] and _on_probe_success
|
||||
# resumes nobody — recovery falls to the 600s stale-claim reaper instead of
|
||||
# the probe-success path the parking design relies on.
|
||||
# The probe-resume loop reads _waiting_records filtered by
|
||||
# waiting_for == "rate_limit_lifted" + context.provider; without a record
|
||||
# here recovery falls to the 600s stale-claim reaper instead of the
|
||||
# probe-success path.
|
||||
orch._waiting_records = {}
|
||||
inst = _instance()
|
||||
inst.current_task_id = "task-1"
|
||||
@@ -232,7 +228,7 @@ async def test_park_registers_waiting_record_so_probe_can_resume(
|
||||
async def test_probe_success_respawns_parked_agent(
|
||||
orch: AgentOrchestrator, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
# F035: once the probe succeeds, the parked agent must be respawned via
|
||||
# Once the probe succeeds, the parked agent must be respawned via
|
||||
# resolve_wait — not left stranded for the 600s reaper.
|
||||
orch._waiting_records = {
|
||||
"be-dev-1": WaitingRecord(
|
||||
|
||||
@@ -570,13 +570,11 @@ class TestCEONotificationThreshold:
|
||||
|
||||
|
||||
class TestOrphanProviderFallback:
|
||||
"""F045: an activate() failure in the in-verb ``i_am_blocked(rate_limited)``
|
||||
path leaves agents parked in ``_waiting_records`` but the provider never
|
||||
makes it into the tracker — so the tracker-driven loop never probes it and
|
||||
the parked agents strand in WAITING_LONG forever. The sweep must scan the
|
||||
in-memory records for any ``rate_limit_lifted`` provider the tracker-listed
|
||||
set did NOT cover and probe it via the time-expiry fallback so
|
||||
``_on_probe_success`` can resume them.
|
||||
"""An activate() failure in the ``i_am_blocked(rate_limited)`` path parks
|
||||
agents in ``_waiting_records`` without entering the tracker, so the
|
||||
tracker-driven loop never probes them. The sweep must scan the in-memory
|
||||
records for any ``rate_limit_lifted`` provider the tracker missed and probe
|
||||
it via the time-expiry fallback so ``_on_probe_success`` can resume them.
|
||||
"""
|
||||
|
||||
async def test_orphan_parked_agent_resumed_when_tracker_lacks_provider(
|
||||
|
||||
@@ -82,11 +82,8 @@ async def test_readopt_swallows_probe_errors() -> None:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_readopt_records_container_id_so_health_check_can_see_exit() -> None:
|
||||
# F033: a re-adopted instance registered with container_id=None is skipped by
|
||||
# _check_health (`if instance.container_id is None: continue`), so when the
|
||||
# container later exits the stopped-container handler never runs — the task
|
||||
# is stranded under a phantom ACTIVE instance forever. Re-adopt must capture
|
||||
# the real container id so the health loop can observe the later exit.
|
||||
# Re-adopt must capture the real container id; a None container_id is skipped
|
||||
# by _check_health, stranding the task under a phantom ACTIVE instance.
|
||||
orch = _orch()
|
||||
orch._inspect_container_state = AsyncMock(return_value=(True, 0))
|
||||
orch._resolve_container_id = AsyncMock(return_value="deadbeef1234")
|
||||
|
||||
@@ -1,21 +1,10 @@
|
||||
"""F072 — reaper Docker subprocess calls (``docker inspect`` / ``docker exec``)
|
||||
had no deadline: a hung Docker daemon or a stuck container FS would freeze the
|
||||
single asyncio event loop, because the reaper runs inline before every dispatch
|
||||
tick and shares that loop with every background sweeper (rate-limit probe,
|
||||
self-heal, ci-watch, dep-update, release-manager, grok-auth refresh).
|
||||
|
||||
The fix bounds each call with ``asyncio.wait_for``; on expiry ``proc.kill()`` the
|
||||
child and either raise (``inspect`` / ``resolve_container_id`` — the callers
|
||||
already apply their own fail-direction) or return ``None`` (the gateway probe —
|
||||
inconclusive, the caller declines to act, matching its existing probe-failure
|
||||
contract). The deadlines are generous enough that a legitimate slow docker call
|
||||
is never wrongly aborted. ``_check_health`` is also hardened so one agent's hung
|
||||
inspect skips that agent, not the whole sweep — preserving the per-tick
|
||||
check-all-agents invariant the timeout-then-raise would otherwise break.
|
||||
|
||||
Deterministic: the slow-docker tests patch the timeout constants tiny and use a
|
||||
never-resolving ``communicate``/``wait`` so a bounded fail-close is asserted in
|
||||
well under a second, never relying on real wall-clock timing of the defaults.
|
||||
"""Reaper Docker subprocess calls (``docker inspect`` / ``docker exec``) are
|
||||
bounded with ``asyncio.wait_for`` so a hung Docker daemon can't freeze the shared
|
||||
asyncio event loop (the reaper runs before every dispatch tick). On timeout the
|
||||
child is killed and the call either raises (``inspect`` /
|
||||
``resolve_container_id``) or returns ``None`` (gateway probe — inconclusive).
|
||||
``_check_health`` is hardened per-agent so one hung inspect skips that agent, not
|
||||
the whole sweep, preserving the per-tick check-all-agents invariant.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -1,13 +1,8 @@
|
||||
"""F098: a re-park during probe-success resume must not orphan the agent.
|
||||
|
||||
``resolve_wait`` deletes the waiting record (in-memory + durable) and then calls
|
||||
``spawn_agent`` to respawn the parked agent. If the provider re-parks in the
|
||||
window between the probe-success clear and the spawn (the rate limit lifts then
|
||||
immediately re-limits, or a second provider limit lands), ``spawn_agent`` bails
|
||||
with an OFFLINE instance — the F095 parked-provider short-circuit. The old order
|
||||
deleted the record BEFORE the spawn, so a bail orphaned the agent: no record
|
||||
means the probe-resume loop can never revive it and the spawn gate bails every
|
||||
tick. The record must stay until a container actually launches.
|
||||
"""A re-park during probe-success resume must not orphan the agent. The waiting
|
||||
record must stay until ``spawn_agent`` actually launches a container — deleting
|
||||
it before the spawn lets a provider re-park (``spawn_agent`` bails OFFLINE on
|
||||
the parked-provider short-circuit) leave the agent with no record for the
|
||||
probe-resume loop to revive.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -83,13 +83,9 @@ def test_partition_drops_terminal_and_missing_rows() -> None:
|
||||
|
||||
|
||||
def test_partition_restamps_last_check_to_now_to_avoid_stale_tracing_gap() -> None:
|
||||
# F034: a persisted last_check from BEFORE the restart would make the first
|
||||
# post-restart ``_pm_made_rule_following_retry`` audit lookup
|
||||
# (``since = record.get("last_check")``) match a PRE-restart tracing_gap
|
||||
# row, falsely resetting the breaker on the very first post-restart spawn —
|
||||
# exactly when a fresh strike count should be evaluating current state.
|
||||
# Restore must re-stamp last_check to the restore time so only post-restart
|
||||
# tracing gaps can reset the counter.
|
||||
# Restore must re-stamp last_check to the restore time so a pre-restart
|
||||
# tracing_gap row can't falsely reset the breaker on the first post-restart
|
||||
# spawn.
|
||||
tid = uuid4()
|
||||
stale_check = datetime(2026, 6, 20, tzinfo=UTC)
|
||||
rows = [_row(tid, last_check=stale_check)]
|
||||
@@ -380,7 +376,7 @@ async def test_restart_midloop_continues_identically_to_no_restart() -> None:
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# F096 — fire-and-forget persist commit ordering
|
||||
# fire-and-forget persist commit ordering
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
|
||||
@@ -1,17 +1,7 @@
|
||||
"""F071 — the Secretary non-blocking spawn (``start_secretary_session`` →
|
||||
``_schedule_bg(_spawn_secretary_container_guarded)``) runs ``docker run`` and
|
||||
only registers the instance in ``_instances`` at the END. If shutdown arrives
|
||||
between ``docker run`` and the registration line, the container is started but
|
||||
the orchestrator has no handle to it — ``stop()`` iterates only ``_instances``,
|
||||
so the container is orphaned (leaked, must be cleaned up with ``docker rm``).
|
||||
Worse, the F070 drain can let the spawn coroutine COMPLETE the registration
|
||||
AFTER ``stop()`` already iterated ``_instances``, landing a live container into
|
||||
a shutting-down registry that nothing tears down.
|
||||
|
||||
The fix: after ``docker run`` returns the container id, re-check ``self._running``
|
||||
and, if the orchestrator began shutting down, remove the just-started container
|
||||
and abort WITHOUT registering. The guarded wrapper closes the relay silently
|
||||
(shutdown is not a user-facing failure).
|
||||
"""The Secretary non-blocking spawn registers the instance in ``_instances``
|
||||
only at the END of ``docker run``; if shutdown arrives mid-spawn the container
|
||||
must be removed and the registration aborted, or ``stop()`` (which iterates only
|
||||
``_instances``) leaks an orphaned container into a shutting-down registry.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -38,7 +28,7 @@ def _make_orchestrator() -> AgentOrchestrator:
|
||||
orch._instances = {}
|
||||
orch._bg_tasks = set()
|
||||
orch._running = True
|
||||
# F093: concurrent secretary starts serialize on this lock; the constructor
|
||||
# Concurrent secretary starts serialize on this lock; the constructor
|
||||
# (skipped here) initializes it.
|
||||
orch._secretary_spawn_lock = asyncio.Lock()
|
||||
return orch
|
||||
@@ -170,12 +160,11 @@ async def test_running_spawn_registers_normally(
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# F093 — concurrent Secretary starts must serialize. The Secretary agent id is a
|
||||
# single fixed id, so two concurrent ``spawn_secretary_session`` calls race on
|
||||
# the container name (``docker run --name roboco-agent-secretary``) and the
|
||||
# ``_instances[SECRETARY_AGENT_ID]`` write, orphaning a container + relay. The
|
||||
# spawn body runs under ``_secretary_spawn_lock`` so the second start only begins
|
||||
# once the first has fully registered (so the second's reap-prior sees it).
|
||||
# Concurrent Secretary starts must serialize — the single fixed Secretary agent
|
||||
# id makes two concurrent ``spawn_secretary_session`` calls race on the container
|
||||
# name and the ``_instances[SECRETARY_AGENT_ID]`` write. The spawn body runs
|
||||
# under ``_secretary_spawn_lock`` so the second start only begins once the first
|
||||
# has fully registered.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
|
||||
@@ -1,23 +1,7 @@
|
||||
"""F059: self-heal fix tasks must WAIT for the CEO's Approve-&-Start.
|
||||
|
||||
The module docstring promises the loop 'only NOTIFIES and, at most, OPENS a
|
||||
PENDING task ... the task waits for the CEO's Approve-&-Start and terminates at
|
||||
awaiting_ceo_approval'. The implementation did the opposite: it created the
|
||||
task ``confirmed_by_human=True`` and the orchestrator dispatched it at once —
|
||||
a self-heal fix that re-broke CI would trigger another self-heal cycle, open
|
||||
another auto-dispatched fix, and loop with no CEO gate on dispatch.
|
||||
|
||||
The fix restores the documented gate:
|
||||
* ``_originate`` opens the task ``confirmed_by_human=False`` (held for the CEO).
|
||||
* The orchestrator holds a self-heal task out of dispatch until the CEO
|
||||
approves it (``confirmed_by_human`` flips True via ``approve_and_start``).
|
||||
* ``give_me_work`` (``list_pending_for_agent``) never offers a held task to an
|
||||
already-alive agent.
|
||||
* ``approve_and_start`` is the CEO's start gate — it flips ``confirmed_by_human``
|
||||
True so the held task finally dispatches.
|
||||
|
||||
The 'never self-deploys' guarantee (no merge) is unchanged; only the dispatch
|
||||
gate is restored.
|
||||
"""Self-heal fix tasks must WAIT for the CEO's Approve-&-Start. ``_originate``
|
||||
opens them ``confirmed_by_human=False`` (held); the orchestrator holds them out
|
||||
of dispatch until ``approve_and_start`` flips it True. The 'never self-deploys'
|
||||
guarantee (no merge) is unchanged.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -269,7 +253,7 @@ async def test_list_pending_for_agent_excludes_held_self_heal() -> None:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_pending_for_agent_still_offers_delegated_subtask() -> None:
|
||||
"""Regression guard (F059): the hold is scoped to self-heal. A delegated
|
||||
"""Regression guard: the hold is scoped to self-heal. A delegated
|
||||
subtask (source != self_heal, confirmed_by_human=False — the default for
|
||||
PM-delegated work, where the delegation IS the authorization to start) must
|
||||
STILL be offered via give_me_work. A universal confirmed_by_human filter
|
||||
|
||||
@@ -380,11 +380,10 @@ async def test_reap_releases_on_registry_miss_when_container_gone(
|
||||
async def test_reap_spares_provider_parked_agent_for_probe_resume(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""F035: a provider-parked agent (dead container, OFFLINE, with a
|
||||
"""A provider-parked agent (dead container, OFFLINE, with a
|
||||
``rate_limit_lifted`` WaitingRecord) must NOT be reaped by the stale-claim
|
||||
reaper. The probe-resume loop owns its recovery and respawns it when the
|
||||
provider recovers; reaping would release the claim to pending, and then
|
||||
probe-success would respawn the agent on a task it no longer owns.
|
||||
reaper — reaping would release the claim to pending and probe-success would
|
||||
respawn the agent on a task it no longer owns.
|
||||
"""
|
||||
now = datetime.now(UTC)
|
||||
task_id = uuid4()
|
||||
|
||||
Reference in New Issue
Block a user