"""0.10.0 observability metric layer: cycle-time, bottleneck, rework, scorecard. Seeds an audit_log journey + reworked tasks + rejector-attributed fail events + spawn-session costs against a real Postgres and asserts the reconstructed metrics. The named task.qa_fail / task.pr_fail events must NOT pollute the cycle-time reconstruction (they share a timestamp with the needs_revision row). """ from __future__ import annotations from datetime import UTC, datetime, timedelta from typing import TYPE_CHECKING, Any from uuid import uuid4 import pytest import pytest_asyncio from roboco.db.tables import ( AgentSpawnSessionTable, AgentTable, AuditLogTable, ProjectTable, TaskTable, ) from roboco.models.base import ( AgentRole, AgentStatus, Complexity, TaskNature, TaskStatus, TaskType, Team, ) from roboco.services.metrics import MetricsService if TYPE_CHECKING: from collections.abc import AsyncIterator from sqlalchemy.ext.asyncio import AsyncSession _T0 = datetime(2026, 6, 20, 12, 0, 0, tzinfo=UTC) _EXPECTED_TASKS = 2 # completed tasks seeded per rework / scorecard test _EXPECTED_TOKENS = 1500 # 1000 input + 500 output in the scorecard spawn session def _agent(role: AgentRole, team: Team, slug: str) -> AgentTable: return AgentTable( id=uuid4(), name=slug, slug=slug, role=role, team=team, status=AgentStatus.ACTIVE, model_config={}, system_prompt="x", capabilities=[], permissions={}, metrics={}, ) def _task( project_id: Any, created_by: Any, *, assigned_to: Any = None, revision_count: int = 0, started_hours_ago: int | None = None, ) -> TaskTable: """A COMPLETED backend task completed `now` (in-window), optionally started.""" started = ( datetime.now(UTC) - timedelta(hours=started_hours_ago) if started_hours_ago is not None else None ) return TaskTable( id=uuid4(), title="t", description="d", acceptance_criteria=["ac"], task_type=TaskType.CODE, nature=TaskNature.TECHNICAL, status=TaskStatus.COMPLETED, team=Team.BACKEND, project_id=project_id, created_by=created_by, assigned_to=assigned_to, revision_count=revision_count, estimated_complexity=Complexity.MEDIUM, completed_at=datetime.now(UTC), started_at=started, ) def _audit( task_id: Any, status: str, ts: datetime, *, agent_id: Any = None, event_type: str | None = None, ) -> AuditLogTable: return AuditLogTable( id=uuid4(), event_type=event_type or f"task.{status}", agent_id=agent_id, target_type="task", target_id=task_id, severity="info", details={"to_status": status, "from_status": "prev", "team": "backend"}, timestamp=ts, ) def _spawn( slug: str, *, task_id: str | None, cost: float, tokens_in: int, tokens_out: int ) -> AgentSpawnSessionTable: return AgentSpawnSessionTable( id=uuid4(), agent_slug=slug, team="backend", role="developer", model="claude", task_id=task_id, started_at=datetime.now(UTC) - timedelta(hours=1), tokens_input=tokens_in, tokens_output=tokens_out, estimated_cost_usd=cost, ) @pytest_asyncio.fixture async def obs_setup(db_session: AsyncSession) -> AsyncIterator[dict]: dev = _agent(AgentRole.DEVELOPER, Team.BACKEND, f"be-dev-{uuid4().hex[:6]}") qa = _agent(AgentRole.QA, Team.BACKEND, f"be-qa-{uuid4().hex[:6]}") db_session.add_all([dev, qa]) await db_session.flush() project = ProjectTable( id=uuid4(), name="P", slug=f"p-{uuid4().hex[:6]}", git_url="https://example.com/r.git", assigned_cell=Team.BACKEND, created_by=dev.id, ) db_session.add(project) await db_session.flush() yield { "svc": MetricsService(db_session), "db": db_session, "project_id": project.id, "dev_id": dev.id, "qa_id": qa.id, "dev_slug": dev.slug, } @pytest.mark.asyncio async def test_cycle_time_reconstructs_per_stage_dwell(obs_setup: dict) -> None: db = obs_setup["db"] tid = uuid4() # claimed (60s) -> in_progress (3600s) -> awaiting_qa (120s) -> completed db.add_all( [ _audit(tid, "claimed", _T0), _audit(tid, "in_progress", _T0 + timedelta(seconds=60)), _audit(tid, "awaiting_qa", _T0 + timedelta(seconds=60 + 3600)), _audit(tid, "completed", _T0 + timedelta(seconds=60 + 3600 + 120)), # A named fail event sharing the awaiting_qa timestamp must be ignored. _audit( tid, "needs_revision", _T0 + timedelta(seconds=60 + 3600), event_type="task.qa_fail", ), ] ) await db.flush() stages = {s.status: s for s in await obs_setup["svc"].get_cycle_time_by_stage()} assert stages["claimed"].avg_seconds == pytest.approx(60.0) assert stages["in_progress"].avg_seconds == pytest.approx(3600.0) assert stages["awaiting_qa"].avg_seconds == pytest.approx(120.0) # The named qa_fail event did not create a zero-length needs_revision stage. assert "needs_revision" not in stages @pytest.mark.asyncio async def test_bottleneck_ranks_longest_cumulative_stage(obs_setup: dict) -> None: db = obs_setup["db"] tid = uuid4() db.add_all( [ _audit(tid, "claimed", _T0), _audit(tid, "in_progress", _T0 + timedelta(seconds=60)), _audit(tid, "awaiting_qa", _T0 + timedelta(seconds=60 + 7200)), _audit(tid, "completed", _T0 + timedelta(seconds=60 + 7200 + 30)), ] ) await db.flush() report = await obs_setup["svc"].get_bottleneck_distribution() assert report.worst_stage == "in_progress" assert report.by_stage[0].status == "in_progress" @pytest.mark.asyncio async def test_rework_rate_and_attribution(obs_setup: dict) -> None: db = obs_setup["db"] pid, dev_id, qa_id = ( obs_setup["project_id"], obs_setup["dev_id"], obs_setup["qa_id"], ) clean = _task(pid, dev_id, assigned_to=dev_id) reworked = _task(pid, dev_id, assigned_to=dev_id, revision_count=2) db.add_all([clean, reworked]) await db.flush() # The QA agent bounced the reworked task once (rejector attribution). db.add( _audit( reworked.id, "needs_revision", datetime.now(UTC) - timedelta(hours=1), agent_id=qa_id, event_type="task.qa_fail", ) ) # A spawn session attributes the rework's cost. db.add( _spawn( obs_setup["dev_slug"], task_id=str(reworked.id), cost=0.42, tokens_in=100, tokens_out=50, ) ) await db.flush() report = await obs_setup["svc"].get_rework_metrics(days=30) assert report.total_completed == _EXPECTED_TASKS assert report.total_reworked == 1 assert report.rate == pytest.approx(0.5) assert report.rework_cost_usd == pytest.approx(0.42) qa_row = next(a for a in report.by_agent if a.qa_fails > 0) assert qa_row.qa_fails == 1 @pytest.mark.asyncio async def test_scorecard_agent_and_cell(obs_setup: dict) -> None: db = obs_setup["db"] pid, dev_id = obs_setup["project_id"], obs_setup["dev_id"] db.add_all( [ _task(pid, dev_id, assigned_to=dev_id, started_hours_ago=2), _task( pid, dev_id, assigned_to=dev_id, revision_count=1, started_hours_ago=4 ), ] ) db.add( _spawn( obs_setup["dev_slug"], task_id=None, cost=1.25, tokens_in=1000, tokens_out=500, ) ) await db.flush() card = await obs_setup["svc"].get_scorecard(agent_id=dev_id, days=7) assert card is not None assert card.scope == "agent" assert card.tasks_completed == _EXPECTED_TASKS assert card.rework_rate == pytest.approx(0.5) assert card.tokens == _EXPECTED_TOKENS assert card.cost_usd == pytest.approx(1.25) cell = await obs_setup["svc"].get_scorecard(team=Team.BACKEND, days=7) assert cell is not None assert cell.scope == "cell" assert cell.tasks_completed == _EXPECTED_TASKS assert await obs_setup["svc"].get_scorecard(agent_id=uuid4()) is None