[F117] stop the orchestrator in lifespan shutdown BEFORE closing the DB

The lifespan shutdown closed OptimalService + the DB, and only THEN did
bootstrap's finally block call orchestrator.stop() — so stop() ran with
the DB already closed. stop() drains fire-and-forget _bg_tasks writes
(respawn_tracker upserts, audit-log rows) and stop_agent finalizes work
sessions / agent state, all needing the DB still open; closing it first
silently dropped those final writes (the durable PM-respawn counter's
last few strikes, the metrics-bearing audit trail tail).

Move orchestrator.stop() into the lifespan shutdown path, BEFORE
close_optimal_service + close_db, guarded by a new get_orchestrator_or_none()
safe accessor (no crash when no orchestrator is wired — tests,
skip_orchestrator). bootstrap's finally-block stop() becomes an idempotent
safety net: stop() gains a _stopped flag (getattr-guarded so __new__-
constructed test instances still stop) so the double-call is a clean no-op,
not a re-stop of already-stopped agents / re-drain of an empty bg set.
This commit is contained in:
Renn F
2026-06-28 23:10:47 +02:00
parent 1a773e45d7
commit fc01bee39c
5 changed files with 131 additions and 1 deletions
+57
View File
@@ -16,6 +16,7 @@ import pytest
from fastapi import FastAPI
from roboco.api.app import app as default_app
from roboco.api.app import create_app, lifespan
from roboco.api.deps import clear_orchestrator, set_orchestrator
if TYPE_CHECKING:
from collections.abc import AsyncIterator
@@ -144,6 +145,62 @@ async def test_lifespan_startup_and_shutdown_happy_path() -> None:
transcription_mock.stop.assert_awaited_once()
@pytest.mark.asyncio
async def test_lifespan_stops_orchestrator_before_closing_db_and_optimal() -> None:
"""F117: orchestrator.stop() must run BEFORE close_optimal_service / close_db
on shutdown. stop() drains fire-and-forget DB writes (respawn_tracker
upserts, audit-log rows) and stop_agent finalizes work sessions / agent
state — all needing the DB still open. Closing the DB first (the old order,
where only bootstrap's finally called stop() after lifespan had already
closed the DB) silently dropped those final writes."""
order: list[str] = []
def _record(label: str) -> AsyncMock:
async def _fn() -> None:
order.append(label)
return AsyncMock(side_effect=_fn)
transcription_mock = MagicMock()
transcription_mock.start = AsyncMock()
transcription_mock.stop = AsyncMock()
orchestrator_mock = MagicMock()
orchestrator_mock.stop = _record("orchestrator.stop")
set_orchestrator(orchestrator_mock)
try:
with (
patch("roboco.api.app.init_db", new=AsyncMock()),
patch("roboco.api.app.close_db", new=_record("close_db")),
patch(
"roboco.api.app.close_optimal_service",
new=_record("close_optimal_service"),
),
patch(
"roboco.api.app.TranscriptionService", return_value=transcription_mock
),
patch("roboco.api.app.ExtractionService"),
patch("roboco.api.app.ExtractionPipeline"),
patch(
"roboco.api.app.get_optimal_service",
new=AsyncMock(return_value=MagicMock()),
),
):
app = create_app()
async with lifespan(app):
pass
finally:
# Clear the global so it doesn't leak into other tests.
clear_orchestrator()
# orchestrator.stop() ran, and it ran BEFORE close_optimal_service + close_db.
assert "orchestrator.stop" in order
assert order.index("orchestrator.stop") < order.index("close_optimal_service")
assert order.index("orchestrator.stop") < order.index("close_db")
# The DB is still the last thing closed (innermost resource).
assert order.index("close_optimal_service") < order.index("close_db")
@pytest.mark.asyncio
async def test_lifespan_handles_optimal_init_failure_gracefully() -> None:
"""Optimal-service init failure → app.state.optimal=None, no raise."""