Files
roboco/tests/unit/gateway/test_evidence_populates_files_changed.py
T
d7a1c2d203 fix(db): bound lock waits and idle transactions so a parked coroutine can't wedge the pool (#721)
* fix(db): bound lock waits and idle transactions so a parked coroutine can't wedge the pool

2026-07-29 production incident: verb/evidence handlers and background
dispatch coroutines held an open DB transaction across minutes of git
subprocess work and asyncio lock queues (per-workspace ensure locks).
Early writes in those transactions held tasks/agents row locks, every
other write convoyed behind them, and blocked statements camped on pool
connections until all 30 were waiters — 1000+ QueuePool timeouts per
hour, one transaction open 1h20m.

Two layers:

- get_engine now passes asyncpg server_settings:
  idle_in_transaction_session_timeout (default 120s) kills any session
  parked mid-transaction on non-DB work, releasing its locks and pool
  slot; lock_timeout (default 30s) makes a statement queued on someone
  else's row lock give up instead of holding a connection for the wait.
  Both env-tunable (ROBOCO_DATABASE_IDLE_IN_TRANSACTION_TIMEOUT_MS /
  ROBOCO_DATABASE_LOCK_TIMEOUT_MS), 0 disables. Alembic runs its own
  sync engine and is untouched; best-effort writers (proactive-context
  injection) already swallow errors and now fail in 30s instead of
  camping for an hour.

- ContentActions.evidence commits the request session before its
  fetch/diff git work, so a multi-minute evidence call no longer pins a
  pool connection for the duration (expire_on_commit=False keeps the
  loaded task usable; later reads reopen a transaction on demand).

The deeper restructuring — claim flows committing their transition
before briefing/workspace assembly — is scoped to the existing
evidence-assembly-timeout task and not attempted here.

* fix(db): bound lock waits and idle transactions so a parked coroutine can't wedge the pool

2026-07-29 production incident: verb/evidence handlers and background
dispatch coroutines held an open DB transaction across minutes of git
subprocess work and asyncio lock queues (per-workspace ensure locks).
Early writes in those transactions held tasks/agents row locks, every
other write convoyed behind them, and blocked statements camped on pool
connections until all 30 were waiters — 1000+ QueuePool timeouts per
hour, one transaction open 1h20m.

Two layers:

- get_engine now passes asyncpg server_settings:
  idle_in_transaction_session_timeout (default 20 min) kills any session
  parked mid-transaction on non-DB work, releasing its locks and pool
  slot; lock_timeout (default 60s) makes a statement queued on someone
  else's row lock give up with a clean retryable error instead of
  camping on a pool connection for the wait. Both env-tunable
  (ROBOCO_DATABASE_IDLE_IN_TRANSACTION_TIMEOUT_MS /
  ROBOCO_DATABASE_LOCK_TIMEOUT_MS), 0 disables. The idle default
  deliberately clears the longest LEGITIMATE in-transaction window — a
  cold-workspace claim holds its transaction across the clone (300s
  budget) + dep install (600s budget) under the 900s slow-verb wall —
  so routine claims never trip it while today's 80-minute parked
  transaction dies at 20 min. Alembic's env.py builds its own engine
  and never carries these; best-effort writers (proactive-context
  injection) already swallow errors and now fail in 60s instead of
  camping for an hour.

- ContentActions.evidence ends the request transaction (commit, or
  rollback on a poisoned session) before its fetch/diff git work, so a
  multi-minute evidence call no longer pins a pool connection for the
  duration (expire_on_commit=False keeps the loaded task usable; later
  reads reopen a transaction on demand).

The deeper restructuring — claim flows committing their transition
before briefing/workspace assembly — is scoped to the existing
evidence-assembly-timeout task and not attempted here.

---------

Co-authored-by: Renn F <rennf93@users.noreply.github.com>
2026-07-29 21:22:35 +02:00

217 lines
7.8 KiB
Python

"""Task #154: evidence() must populate files_changed + use full PR diff.
Bug:
ContentActions.evidence() hard-coded ``files_changed=[]`` and called
``git.diff(branch_name=..., base="HEAD~1")``. Result: QA / reviewers
inspecting a real PR saw an empty change list and only the latest
commit's delta, even when GitHub showed a multi-commit change set.
Fix:
Pull files via ``git.list_changed_files(branch_name=...)`` (no base
→ full diff vs parent branch). Pull diff with ``base=None`` so the
full PR diff comes through. Both use git as the authoritative source
instead of the legacy ``work_session.files_modified`` field, which
the gateway ``commit()`` does not populate.
"""
from __future__ import annotations
from unittest.mock import AsyncMock, MagicMock
from uuid import uuid4
import pytest
from roboco.services.gateway.content_actions import ContentActions, ContentActionsDeps
def _deps_for_evidence(
task_svc: AsyncMock,
git_svc: AsyncMock,
workspace_svc: AsyncMock,
evidence_repo: AsyncMock,
) -> ContentActionsDeps:
return ContentActionsDeps(
task=task_svc,
git=git_svc,
a2a=AsyncMock(),
journal=AsyncMock(),
workspace=workspace_svc,
notifications=AsyncMock(),
notification_delivery=AsyncMock(),
evidence_repo=evidence_repo,
)
def _task_with_pr(task_id: object, *, commits: list[str]) -> MagicMock:
return MagicMock(
id=task_id,
status="awaiting_qa",
assigned_to=None,
branch_name="feature/backend/abc12345--def67890",
work_session_id=uuid4(),
commits=commits,
pr_number=20,
pr_url="https://github.com/org/repo/pull/20",
dev_notes="see PR description",
acceptance_criteria_status=[],
)
@pytest.mark.asyncio
async def test_evidence_populates_files_changed_from_git() -> None:
"""The smoke-9 regression: PR #20 has README change on GitHub but
evidence() reports files_changed=[]. The fix queries git directly."""
agent_id = uuid4()
task_id = uuid4()
task_svc = AsyncMock()
task_svc.get.return_value = _task_with_pr(task_id, commits=["abc", "def"])
git_svc = AsyncMock()
git_svc.diff.return_value = "diff --git a/README.md b/README.md\n+added line\n"
git_svc.list_changed_files.return_value = ["README.md", "docs/guide.md"]
workspace_svc = AsyncMock()
evidence_repo = AsyncMock()
evidence_repo.journal_highlights_for_task.return_value = []
ca = ContentActions(
_deps_for_evidence(task_svc, git_svc, workspace_svc, evidence_repo)
)
env = await ca.evidence(agent_id=agent_id, task_id=task_id)
body = env.as_dict()
assert body["error"] is None
assert body["evidence"]["files_changed"] == ["README.md", "docs/guide.md"]
assert "diff --git" in body["evidence"]["pr_diff_summary"]
git_svc.list_changed_files.assert_awaited_once()
@pytest.mark.asyncio
async def test_evidence_uses_full_pr_diff_not_head_minus_one() -> None:
"""git.diff must be called with base=None (full PR diff vs parent),
not base='HEAD~1' (only the last commit)."""
agent_id = uuid4()
task_id = uuid4()
task_svc = AsyncMock()
# Multi-commit branch — the pre-fix code passed base='HEAD~1' when
# task.commits was non-empty, masking earlier commits' changes.
task_svc.get.return_value = _task_with_pr(task_id, commits=["sha1", "sha2", "sha3"])
git_svc = AsyncMock()
git_svc.diff.return_value = "full diff"
git_svc.list_changed_files.return_value = []
workspace_svc = AsyncMock()
evidence_repo = AsyncMock()
evidence_repo.journal_highlights_for_task.return_value = []
ca = ContentActions(
_deps_for_evidence(task_svc, git_svc, workspace_svc, evidence_repo)
)
await ca.evidence(agent_id=agent_id, task_id=task_id)
git_svc.diff.assert_awaited_once()
call_kwargs = git_svc.diff.await_args.kwargs
# Pre-fix bug: kwargs['base'] would be 'HEAD~1' for any multi-commit
# branch. Post-fix: base is omitted (or explicitly None).
base = call_kwargs.get("base")
assert base in (None, ""), (
f"git.diff must use full-PR diff (base=None), got base={base!r}"
)
assert call_kwargs.get("branch_name") == "feature/backend/abc12345--def67890"
@pytest.mark.asyncio
async def test_evidence_populates_journal_highlights() -> None:
"""evidence() must return journal_highlights so QA gets the dev's
decision/reflection context — same as qa.py's claim_review evidence."""
agent_id = uuid4()
task_id = uuid4()
task_svc = AsyncMock()
task_svc.get.return_value = _task_with_pr(task_id, commits=["abc"])
git_svc = AsyncMock()
git_svc.diff.return_value = ""
git_svc.list_changed_files.return_value = []
workspace_svc = AsyncMock()
evidence_repo = AsyncMock()
highlights = [
{"scope": "decision", "title": "Use README format X", "content": "..."},
{"scope": "reflect", "title": "Lesson learned", "content": "..."},
]
evidence_repo.journal_highlights_for_task.return_value = highlights
ca = ContentActions(
_deps_for_evidence(task_svc, git_svc, workspace_svc, evidence_repo)
)
env = await ca.evidence(agent_id=agent_id, task_id=task_id)
body = env.as_dict()
assert body["evidence"]["journal_highlights"] == highlights
evidence_repo.journal_highlights_for_task.assert_awaited_once_with(
task_id, include_ancestors=True
)
@pytest.mark.asyncio
async def test_evidence_commits_session_before_git_work() -> None:
"""2026-07-29 pool exhaustion: evidence() must release its DB transaction
(commit) BEFORE the fetch/diff git work — those can run for minutes on a
cold workspace, and an open transaction pins a pool connection for the
whole duration."""
order: list[str] = []
agent_id = uuid4()
task_id = uuid4()
task_svc = AsyncMock()
task_svc.get.return_value = _task_with_pr(task_id, commits=["abc"])
task_svc.session.commit = AsyncMock(side_effect=lambda: order.append("commit"))
git_svc = AsyncMock()
git_svc.diff.return_value = ""
git_svc.list_changed_files.return_value = []
workspace_svc = AsyncMock()
workspace_svc.fetch_branch_for_inspection = AsyncMock(
side_effect=lambda **_kw: order.append("fetch")
)
evidence_repo = AsyncMock()
evidence_repo.journal_highlights_for_task.return_value = []
ca = ContentActions(
_deps_for_evidence(task_svc, git_svc, workspace_svc, evidence_repo)
)
env = await ca.evidence(agent_id=agent_id, task_id=task_id)
assert env.as_dict()["error"] is None
assert order == ["commit", "fetch"], (
f"session must be committed before git work, got order={order}"
)
@pytest.mark.asyncio
async def test_evidence_no_branch_skips_git_calls() -> None:
"""A task without a branch_name has no PR yet — skip git entirely,
still return a valid envelope with empty files_changed."""
agent_id = uuid4()
task_id = uuid4()
task_svc = AsyncMock()
no_branch = MagicMock(
id=task_id,
status="claimed",
assigned_to=agent_id,
branch_name=None,
work_session_id=None,
commits=[],
pr_number=None,
pr_url=None,
dev_notes=None,
acceptance_criteria_status=[],
)
task_svc.get.return_value = no_branch
git_svc = AsyncMock()
workspace_svc = AsyncMock()
evidence_repo = AsyncMock()
evidence_repo.journal_highlights_for_task.return_value = []
ca = ContentActions(
_deps_for_evidence(task_svc, git_svc, workspace_svc, evidence_repo)
)
env = await ca.evidence(agent_id=agent_id, task_id=task_id)
body = env.as_dict()
assert body["error"] is None
assert body["evidence"]["files_changed"] == []
assert body["evidence"]["pr_diff_summary"] == ""
git_svc.diff.assert_not_awaited()
git_svc.list_changed_files.assert_not_awaited()