From acaf3486e846b1972c46454dd159d9ab7dc39605 Mon Sep 17 00:00:00 2001 From: Renzo F <45401804+rennf93@users.noreply.github.com> Date: Wed, 24 Jun 2026 04:39:40 +0200 Subject: [PATCH] fix: note-tool timeout (background RAG indexing) + feature-flag raw-key display (#252) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - note timeout: JournalService.add_entry awaited RAG indexing inline (despite its "non-blocking" comment); indexing embeds via Ollama, which is CPU-bound, so under concurrent load it slowed enough to time the `note` gateway tool out. The entry is already committed before indexing, so it's best-effort — schedule it fire-and-forget (_schedule_rag_index) so the write returns immediately. A new drain_rag_index_tasks() helper lets tests await the pending index. - feature flags: the "Gateway-health recovery" toggle rendered its raw key `gateway_health_enabled` (the only flag with no human description). Added the blurb and changed the fallback to render nothing rather than leak a raw key. Co-authored-by: Renn F --- CHANGELOG.md | 4 + .../settings/feature-flags-card.tsx | 4 +- roboco/services/journal.py | 119 ++++++++++++------ tests/integration/test_journal_service.py | 3 +- 4 files changed, 90 insertions(+), 40 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c0a69af4..8ba06db0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), - **A dev claiming a new task no longer gets stuck on `BRANCH_MISMATCH`.** Each developer has one persistent clone shared across all their tasks, so a finished or abandoned prior task could leave the clone dirty and sitting on a sibling task's branch. The claim's git work (creating/checking out the new task's branch) runs as a side-effect *after* the claim's DB transition commits — so when the checkout failed on that dirty tree, the task was already marked assigned while the workspace stayed on the wrong branch, and the dev's next commit was rejected with `BRANCH_MISMATCH` (stalling, then blocking, the task). The claim now does a `git reset --hard` to clean the tree before the checkouts. It runs only on a fresh claim (resume short-circuits earlier), so the discarded changes are abandoned cruft from a finished task — never committed work, and never the gitignored `.venv`. +- **The `note` tool no longer times out under load.** Writing a journal entry / note synchronously waited on RAG indexing, which embeds via Ollama — and Ollama is CPU-bound, so under concurrent load that embed slowed enough to time the `note` gateway tool out entirely (despite a "non-blocking" comment on the code). The entry is already persisted before indexing, so indexing is pure best-effort enrichment: it now runs fire-and-forget on the event loop, and the note/journal write returns immediately. + +- **A feature flag stopped showing its raw internal key.** In Settings → Feature Flags, the "Gateway-health recovery" toggle displayed its raw key `gateway_health_enabled` as its description (the only flag missing a human blurb). Added the description, and changed the fallback so a future flag without one renders nothing rather than leaking a snake_case key. + ## [0.10.0] - 2026-06-23 ### Added diff --git a/panel/src/components/settings/feature-flags-card.tsx b/panel/src/components/settings/feature-flags-card.tsx index 0e90574b..45a0f914 100644 --- a/panel/src/components/settings/feature-flags-card.tsx +++ b/panel/src/components/settings/feature-flags-card.tsx @@ -32,6 +32,8 @@ const FLAG_DESCRIPTIONS: Record = { "Enforce a per-project architectural standard (.roboco/conventions.yml): inject the map, attach baseline constraints, and block i_am_done / pr_pass on misplaced definitions or lint suppressions.", rag_auto_update_enabled: "Keep the knowledge base index refreshed automatically.", transcript_prune_enabled: "Run the background sweep that prunes old transcripts.", + gateway_health_enabled: + "Recover an agent whose MCP gateway has broken (it can run no tools) while its container stays up — kill + respawn it instead of shielding it from the reaper forever.", }; export function FeatureFlagsCard() { @@ -87,7 +89,7 @@ export function FeatureFlagsCard() {

- {FLAG_DESCRIPTIONS[flag.key] ?? flag.key} + {FLAG_DESCRIPTIONS[flag.key] ?? ""}

None: + """Await all in-flight background journal RAG index tasks (test helper).""" + await asyncio.gather(*list(_RAG_INDEX_TASKS), return_exceptions=True) + + # Scope mapping is canonical in foundation.policy.journaling. # Derived as string-keyed dict here because the service's call sites pass # scope strings (from the gateway's content_actions layer) not Scope enums. @@ -270,44 +283,28 @@ class JournalService(BaseService): type=entry_create.type, ) - # Index in RAG - non-blocking - try: - optimal = await self._get_optimal_service() - # Convert SQLAlchemy UUIDs to Python UUIDs for the params - agent_id_for_index = ( - require_uuid(journal_row.agent_id) - if journal_row - else entry_create.journal_id - ) - await optimal.index_journal_entry( - IndexJournalEntryParams( - entry_id=require_uuid(entry_row.id), - agent_id=agent_id_for_index, - content=entry_create.content, - entry_type=type_key, - task_id=entry_create.task_id, - tags=entry_create.tags, - ) - ) - - # Also index LEARNING entries to the learnings index for cross-agent sharing - if type_key == JournalEntryType.LEARNING.value: - from roboco.services.optimal_brain.indexes.learnings import ( - RecordLearningParams, - ) - - await optimal.record_learning( - RecordLearningParams( - content=entry_create.content, - category="journal_learning", - agent_id=agent_id_for_index, - task_id=entry_create.task_id, - shareable=not entry_create.is_private, - tags=entry_create.tags, - ) - ) - except Exception as e: - self.log.warning("Failed to index journal entry in RAG", error=str(e)) + # Index in RAG — fire-and-forget. The entry is already committed above, so + # indexing is pure best-effort enrichment; it embeds via Ollama, which is + # CPU-bound and slows under concurrent load. Awaiting it inline (the old + # code, despite the "non-blocking" comment) blocked the journal write and + # made the `note` gateway tool time out under load. Schedule it instead so + # the write returns immediately. + agent_id_for_index = ( + require_uuid(journal_row.agent_id) + if journal_row + else entry_create.journal_id + ) + self._schedule_rag_index( + IndexJournalEntryParams( + entry_id=require_uuid(entry_row.id), + agent_id=agent_id_for_index, + content=entry_create.content, + entry_type=type_key, + task_id=entry_create.task_id, + tags=entry_create.tags, + ), + is_private=entry_create.is_private, + ) return JournalEntry( id=require_uuid(entry_row.id), @@ -324,6 +321,52 @@ class JournalService(BaseService): created_at=entry_row.created_at, ) + def _schedule_rag_index( + self, params: IndexJournalEntryParams, *, is_private: bool + ) -> None: + """Index a journal entry in RAG off the critical path (fire-and-forget). + + Embedding goes through Ollama, which is CPU-bound and slows under load; + awaiting it inline made the `note` gateway tool time out. The entry is + already persisted, so indexing is best-effort enrichment — schedule it on + the event loop and return. Errors are logged, never raised; a strong ref + in ``_RAG_INDEX_TASKS`` keeps the task alive until it finishes. + """ + + async def _index() -> None: + try: + optimal = await self._get_optimal_service() + await optimal.index_journal_entry(params) + if params.entry_type == JournalEntryType.LEARNING.value: + from roboco.services.optimal_brain.indexes.learnings import ( + RecordLearningParams, + ) + + await optimal.record_learning( + RecordLearningParams( + content=params.content, + category="journal_learning", + agent_id=params.agent_id, + task_id=params.task_id, + shareable=not is_private, + tags=params.tags or [], + ) + ) + except Exception as e: + self.log.warning( + "Failed to index journal entry in RAG (best-effort)", + entry_id=str(params.entry_id), + error=str(e), + ) + + try: + task = asyncio.create_task(_index()) + except RuntimeError: + # No running event loop (sync context) — skip best-effort indexing. + return + _RAG_INDEX_TASKS.add(task) + task.add_done_callback(_RAG_INDEX_TASKS.discard) + async def get_entry(self, entry_id: UUID) -> JournalEntry | None: """Get a journal entry by ID.""" result = await self.session.execute( diff --git a/tests/integration/test_journal_service.py b/tests/integration/test_journal_service.py index 0d148a25..b8cd44aa 100644 --- a/tests/integration/test_journal_service.py +++ b/tests/integration/test_journal_service.py @@ -35,7 +35,7 @@ from roboco.models.journal import ( StruggleEntryParams, TaskReflectionParams, ) -from roboco.services.journal import JournalService +from roboco.services.journal import JournalService, drain_rag_index_tasks from sqlalchemy import select, update from sqlalchemy.exc import IntegrityError as _IE @@ -505,6 +505,7 @@ async def test_create_entry_learning_calls_record_learning( ) ) assert entry is not None + await drain_rag_index_tasks() # indexing is fire-and-forget; let it run mock_optimal.record_learning.assert_awaited()