External/roboco
1
0
Fork 0
mirror of https://github.com/rennf93/roboco.git synced 2026-08-03 07:23:24 +02:00
Code Issues Packages Projects Releases Wiki Activity

Added serious RAG capabilties

Browse Source
This commit is contained in:
Renn F
2025-12-30 02:35:58 +01:00
parent 5315e9c72d
commit 0a5988a18f
18 changed files with 2294 additions and 218 deletions
Download Patch File Download Diff File Expand all files Collapse all files
alembic/versions/001_initial_schema.py
+1
View File
@@ -174,6 +174,7 @@ def upgrade() -> None:
sa.Column("self_verified", sa.Boolean(), server_default="false"),
sa.Column("qa_verified", sa.Boolean(), nullable=True),
sa.Column("quick_context", sa.Text(), nullable=True),
sa.Column("proactive_context", postgresql.JSON(), nullable=True),
)
# Add foreign key for agents.current_task_id after tasks table exists
docs/workflows/KNOWLEDGE_BASE.md
+178 -55
View File
@@ -98,18 +98,19 @@ roboco_kb_index_docs(
Record and search error patterns:
```python
# Record an error and how you fixed it
roboco_record_error(
error_type="ConnectionError",
message="Redis connection timed out",
solution="Increased timeout to 30s and added retry logic",
worked=True
# Search for similar errors FIRST
roboco_search_error(
error_message="Redis connection timed out",
context="trying to connect during startup"
)
# Search for similar errors
roboco_search_error(
pattern="ConnectionError",
context="redis timeout"
# Record an error and how you fixed it
roboco_record_error_solution(
error_message="Redis connection timed out",
context="Service startup - Redis wasn't ready yet",
solution="Added retry logic with exponential backoff",
worked=True,
tags=["redis", "startup", "timeout"]
)
```
@@ -120,21 +121,22 @@ roboco_search_error(
Record architectural decisions:
```python
# Record a decision
roboco_record_decision(
topic="Database for session storage",
decision="Use Redis instead of PostgreSQL",
rationale="Need sub-millisecond reads, sessions are ephemeral",
alternatives=["PostgreSQL", "In-memory"],
task_id="uuid-here"
)
# Check if similar decisions exist FIRST
roboco_check_decision(topic="session storage")
# Returns: has_precedent, decisions, recommendation
# Check if similar decisions exist
roboco_decision_check(
topic="session storage",
proposed_approach="Use in-memory cache"
)
# Returns: relevant past decisions to consider
# Record a decision
roboco_record_decision(params={
"topic": "Database for session storage",
"decision": "Use Redis instead of PostgreSQL",
"rationale": "Need sub-millisecond reads, sessions are ephemeral",
"alternatives": [
{"name": "PostgreSQL", "pros": "ACID", "cons": "Too slow"},
{"name": "In-memory", "pros": "Fast", "cons": "No persistence"}
],
"scope": "team", # or "org"
"tags": ["database", "session", "architecture"]
})
```
---
@@ -144,17 +146,26 @@ roboco_decision_check(
Check code against team standards:
```python
# Get applicable standards for a file
roboco_standards_get(
file_path="src/api/routes/users.py",
domain="api"
# Get applicable standards for a domain
roboco_get_standards(
domain="coding", # or "security", "workflow"
language="python" # optional filter
)
# Validate an action against standards
roboco_validate_action(
action="Adding a new API endpoint",
context="User management feature"
action_type="create_endpoint",
context="Adding user management API endpoint"
)
# Returns: allowed, violations, warnings, relevant_standards
# Get code reviewed before committing
roboco_review_code(
code="def handle_auth(token): ...",
file_path="src/api/auth.py",
change_type="modify" # or "add", "delete"
)
# Returns: approved, score (0-100), comments, standards_checked
```
---
@@ -322,52 +333,164 @@ Everything you journal becomes searchable:
## Proactive Context
The system can automatically provide relevant context when you claim a task:
The system automatically provides relevant context when you claim a task:
```python
# Automatic context injection on task claim
# System searches KB for:
# - Similar past tasks
# - Related decisions
# - Relevant standards
# - Past error solutions
# Get context that was injected when task was claimed
roboco_get_proactive_context(
task_id="uuid-here",
force_refresh=False # True to regenerate fresh context
)
# Returns:
# - similar_tasks: Past tasks like this one
# - relevant_learnings: What others learned doing similar work
# - code_patterns: Relevant code examples
# - applicable_standards: Standards that apply
# - recent_decisions: Related architectural decisions
# - known_issues: Issues you should be aware of
# - summary: Human-readable overview
```
This helps you start informed without manual searching.
---
## Code Review Support
## Mentor (Conversational RAG)
Request AI-assisted code review:
Ask the organizational knowledge base for help with follow-up context:
```python
roboco_code_review(
file_path="src/api/routes/users.py",
focus=["security", "performance"]
# First question
response = roboco_ask_mentor(
question="How do I handle authentication in this codebase?",
domain="coding" # optional: coding, security, workflow
)
# Returns: review comments, standards checked, similar past reviews
# Follow-up question (maintains conversation context)
roboco_ask_mentor(
question="What about refresh tokens?",
conversation_id=response["conversation_id"]
)
# Returns: answer, sources, suggested_followups
```
The mentor searches across standards, decisions, learnings, and code patterns.
---
## Index Management
### Check Index Health
```python
roboco_index_status()
# Returns: initialized, indexes with document_count, chunk_count, last_updated
```
### Trigger Reindexing (PM/Developer)
```python
roboco_reindex_all(force=False)
# force=True reindexes even if indexes aren't empty
# Returns: code_files_indexed, docs_files_indexed
```
### Clear an Index (PM only)
```python
roboco_clear_index(index_type="code")
# Valid types: code, documentation, conversations, journals,
# errors, standards, decisions, reviews, learnings
```
---
## Lifecycle Tracking
Task lifecycle events are automatically indexed for pattern analysis:
| Event | What's Tracked |
|-------|----------------|
| `block` | Which task blocked, blocker title |
| `unblock` | When unblocked |
| `pause` | When paused |
| `resume` | When resumed |
| `cancel` | Who cancelled, how many descendants cancelled |
This enables queries like:
- "Which tasks get cancelled most often?"
- "What causes the most blocks?"
- "Which teams have the longest pause durations?"
---
## Tool Quick Reference
### Core Search & Query
| Tool | Purpose | Who Can Use |
|------|---------|-------------|
| `roboco_kb_search` | Semantic search across all indexes | Everyone |
| `roboco_rag_query` | AI-generated answers with citations | Everyone |
| `roboco_kb_stats` | What's indexed (counts by type) | Everyone |
| `roboco_tokens_estimate` | Estimate token count for content | Everyone |
### Indexing & Management
| Tool | Purpose | Who Can Use |
|------|---------|-------------|
| `roboco_kb_search` | Semantic search | Everyone |
| `roboco_rag_query` | AI-generated answers | Everyone |
| `roboco_kb_stats` | What's indexed | Everyone |
| `roboco_kb_index_code` | Index code files | PM, Developer |
| `roboco_kb_index_docs` | Index documentation | PM, Documenter |
| `roboco_tokens_estimate` | Token count | Everyone |
| `roboco_clear_index` | Clear a specific index | PM |
| `roboco_reindex_all` | Trigger full code+docs reindex | PM, Developer |
| `roboco_index_status` | Detailed index health & counts | Everyone |
### Mentor (Conversational RAG)
| Tool | Purpose | Who Can Use |
|------|---------|-------------|
| `roboco_ask_mentor` | Conversational help with follow-ups | Everyone |
### Error Tracking
| Tool | Purpose | Who Can Use |
|------|---------|-------------|
| `roboco_search_error` | Find past error solutions | Everyone |
| `roboco_record_error_solution` | Record how you fixed an error | Everyone |
### Decision Tracking
| Tool | Purpose | Who Can Use |
|------|---------|-------------|
| `roboco_check_decision` | Check for similar past decisions | Everyone |
| `roboco_record_decision` | Record an architectural decision | Everyone |
### Standards & Validation
| Tool | Purpose | Who Can Use |
|------|---------|-------------|
| `roboco_get_standards` | Get applicable standards | Everyone |
| `roboco_validate_action` | Validate action against standards | Everyone |
| `roboco_review_code` | AI-assisted code review | Developer, QA |
### Learning & Knowledge Sharing
| Tool | Purpose | Who Can Use |
|------|---------|-------------|
| `roboco_record_learning` | Record a learning for future agents | Everyone |
| `roboco_search_learnings` | Search learnings from teammates | Everyone |
### Proactive Context
| Tool | Purpose | Who Can Use |
|------|---------|-------------|
| `roboco_get_proactive_context` | Get context injected at task claim | Everyone |
### Journal Tools
| Tool | Purpose | Who Can Use |
|------|---------|-------------|
| `roboco_journal_search` | Search your journal | Everyone |
| `roboco_journal_read_team` | Read team journals | PM, Documenter |
| `roboco_record_error` | Record error & fix | Everyone |
| `roboco_search_error` | Find past errors | Everyone |
| `roboco_record_decision` | Record decision | Everyone |
| `roboco_decision_check` | Check past decisions | Everyone |
| `roboco_standards_get` | Get applicable standards | Everyone |
| `roboco_validate_action` | Validate against standards | Everyone |
| `roboco_record_learning` | Record a learning | Everyone |
| `roboco_code_review` | AI-assisted review | Developer, QA |
roboco/api/routes/optimal.py
+178 -71
View File
@@ -45,6 +45,7 @@ from roboco.api.schemas.optimal import (
ProactiveContextResponse,
PromptTemplateRequest,
PromptTemplateResponse,
RAGHealthResponse,
RAGQueryRequest,
RAGQueryResponse,
RefreshIndexResponse,
@@ -52,6 +53,7 @@ from roboco.api.schemas.optimal import (
SearchRequest,
SearchResponse,
SearchResultResponse,
SingleIndexStatsResponse,
StandardsGetRequest,
StandardsGetResponse,
TokenEstimateRequest,
@@ -363,13 +365,8 @@ async def get_context(
async def get_stats(
agent: CurrentAgentContext,
permissions: PermissionServiceDep,
db: DbSession,
) -> IndexStatsResponse:
"""Get statistics about all indexes."""
from sqlalchemy import func, select
from roboco.db.tables import IndexedDocumentTable
if not permissions.can_perform_kb_action(agent, KBAction.VIEW_STATS):
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
@@ -377,30 +374,127 @@ async def get_stats(
)
service = await get_optimal_service()
stats = await service.get_stats()
# Enhance stats with actual document counts from DB
indexes = stats.get("indexes", {})
for index_type_str in indexes:
# Get actual document count from indexed_documents table
count_query = (
select(func.count())
.select_from(IndexedDocumentTable)
.where(IndexedDocumentTable.index_type == index_type_str)
)
count_result = await db.execute(count_query)
doc_count = count_result.scalar() or 0
# Add document_count (actual files) vs chunk_count (vector entries)
chunk_count = indexes[index_type_str].get("document_count", 0)
indexes[index_type_str] = {
"document_count": doc_count, # Actual files/documents
"chunk_count": chunk_count, # Vector DB entries
}
stats = await service.get_all_index_stats()
return IndexStatsResponse(
initialized=stats.get("initialized", False),
indexes=indexes,
indexes=stats.get("indexes", {}),
)
@router.get("/stats/{index_type}", response_model=SingleIndexStatsResponse)
async def get_single_index_stats(
index_type: str,
agent: CurrentAgentContext,
permissions: PermissionServiceDep,
) -> SingleIndexStatsResponse:
"""Get statistics for a specific index type."""
if not permissions.can_perform_kb_action(agent, KBAction.VIEW_STATS):
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Not authorized to view index statistics",
)
# Validate index type
try:
idx_type = IndexType(index_type)
except ValueError as e:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=f"Invalid index type: {index_type}",
) from e
service = await get_optimal_service()
stats = await service.get_index_stats(idx_type)
return SingleIndexStatsResponse(
index_type=stats["index_type"],
document_count=stats["document_count"],
chunk_count=stats["chunk_count"],
last_updated=stats.get("last_updated"),
)
@router.get("/health", response_model=RAGHealthResponse)
async def rag_health_check() -> RAGHealthResponse:
"""
Check RAG system health.
Tests connectivity to:
- Embedding model (sentence-transformers)
- LLM (Ollama for HyDE)
- Vector store (PostgreSQL/pgvector)
Each test has a 10-second timeout to prevent hanging.
"""
import asyncio
import httpx
from roboco.config import settings
details: dict[str, Any] = {}
embedding_ok = False
llm_ok = False
vector_ok = False
health_timeout = 10.0 # seconds
# Test embedding model with timeout
from roboco.services.optimal_brain.shared_embedder import (
get_shared_embedder,
)
try:
async with asyncio.timeout(health_timeout):
embedder = await get_shared_embedder(model=settings.default_embedding_model)
test_embedding = embedder.embed("health check")
if test_embedding and len(test_embedding) == settings.embedding_dimensions:
embedding_ok = True
details["embedding_model"] = settings.default_embedding_model
details["embedding_dimensions"] = len(test_embedding)
except TimeoutError:
details["embedding_error"] = f"Timeout after {health_timeout}s"
except Exception as e:
details["embedding_error"] = str(e)
# Test LLM (Ollama) - already has timeout via httpx
try:
async with httpx.AsyncClient(timeout=health_timeout) as client:
resp = await client.post(
f"{settings.local_llm_base_url}/chat/completions",
json={
"model": settings.local_llm_model,
"messages": [{"role": "user", "content": "ping"}],
"max_tokens": 5,
},
)
if resp.is_success:
llm_ok = True
details["llm_model"] = settings.local_llm_model
details["llm_base_url"] = settings.local_llm_base_url
except Exception as e:
details["llm_error"] = str(e)
# Test vector store with timeout
try:
async with asyncio.timeout(health_timeout):
service = await get_optimal_service()
stats = await service.get_stats()
if stats.get("initialized"):
vector_ok = True
details["vector_store"] = "connected"
except TimeoutError:
details["vector_store_error"] = f"Timeout after {health_timeout}s"
except Exception as e:
details["vector_store_error"] = str(e)
return RAGHealthResponse(
healthy=embedding_ok and llm_ok and vector_ok,
embedding_status="ok" if embedding_ok else "error",
llm_status="ok" if llm_ok else "error",
vector_store_status="ok" if vector_ok else "error",
details=details,
)
@@ -541,27 +635,34 @@ async def refresh_index(
)
# =============================================================================
# PROMPT TEMPLATE STORAGE
# =============================================================================
@router.post("/kb/reindex")
async def reindex_all(
agent: CurrentAgentContext,
permissions: PermissionServiceDep,
force: bool = False,
) -> dict[str, Any]:
"""
Trigger re-indexing of code and documentation.
Re-scans the codebase and docs directories to update indexes.
This is useful when files have been added/changed outside of normal
workflow or to recover from indexing issues.
class _PromptTemplateStorageHolder:
"""Holder for prompt template storage (would be database in production)."""
Args:
force: If True, reindex even if indexes aren't empty
templates: dict[str, dict[str, Any]] | None = None
Returns:
Count of indexed code files and documentation files
"""
if not permissions.can_perform_kb_action(agent, KBAction.INDEX_CODE):
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Not authorized to trigger reindexing",
)
def _get_prompt_templates() -> dict[str, dict[str, Any]]:
"""Get the prompt templates storage."""
if _PromptTemplateStorageHolder.templates is None:
_PromptTemplateStorageHolder.templates = {}
return _PromptTemplateStorageHolder.templates
def reset_prompt_templates() -> None:
"""Reset prompt templates (for testing)."""
_PromptTemplateStorageHolder.templates = {}
service = await get_optimal_service()
result = await service.auto_index_on_startup(force=force)
return {"status": "reindexed", **result}
# =============================================================================
@@ -583,30 +684,31 @@ async def create_prompt_template(
Templates can include {variables} that get substituted when rendering.
"""
# Any authenticated agent can create prompt templates
template_id = str(uuid4())
created_at = datetime.now(UTC).isoformat()
templates = _get_prompt_templates()
templates[template_id] = {
"id": template_id,
"name": request.name,
"template": request.template,
"description": request.description,
"variables": request.variables,
"category": request.category,
"created_at": created_at,
"created_by": str(agent.agent_id),
}
service = await get_optimal_service()
template = service.create_prompt_template(
{
"id": template_id,
"name": request.name,
"template": request.template,
"description": request.description,
"variables": request.variables,
"category": request.category,
"created_at": created_at,
"created_by": str(agent.agent_id),
}
)
return PromptTemplateResponse(
id=template_id,
name=request.name,
template=request.template,
description=request.description,
variables=request.variables,
category=request.category,
created_at=created_at,
id=template["id"],
name=template["name"],
template=template["template"],
description=template["description"],
variables=template["variables"],
category=template["category"],
created_at=template["created_at"],
)
@@ -616,12 +718,10 @@ async def list_prompt_templates(
category: str | None = None,
) -> list[PromptTemplateResponse]:
"""List all prompt templates, optionally filtered by category."""
# Any authenticated agent can list templates
_ = agent # Used for authentication
templates = list(_get_prompt_templates().values())
if category:
templates = [t for t in templates if t.get("category") == category]
service = await get_optimal_service()
templates = service.list_prompt_templates(category=category)
return [
PromptTemplateResponse(
@@ -652,12 +752,19 @@ async def mentor_ask(
Conversational RAG - use conversation_id for follow-up questions.
"""
mentor = await get_mentor_service()
service = await get_optimal_service()
# Initialize services with proper error handling
try:
mentor = await get_mentor_service()
service = await get_optimal_service()
# Initialize mentor with optimal service if needed
if mentor._optimal_service is None:
await mentor.initialize(service)
# Initialize mentor with optimal service if needed
if mentor._optimal_service is None:
await mentor.initialize(service)
except Exception as e:
raise HTTPException(
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
detail=f"Mentor service initialization failed: {e}",
) from e
response = await mentor.ask(
question=request.question,
roboco/api/schemas/optimal.py
+19
View File
@@ -82,6 +82,25 @@ class IndexStatsResponse(BaseModel):
indexes: dict[str, dict[str, Any]]
class SingleIndexStatsResponse(BaseModel):
"""Statistics for a single index."""
index_type: str
document_count: int
chunk_count: int
last_updated: str | None = None
class RAGHealthResponse(BaseModel):
"""Response from RAG health check."""
healthy: bool
embedding_status: str = Field(..., description="Embedding model status")
llm_status: str = Field(..., description="LLM (HyDE) status")
vector_store_status: str = Field(..., description="Vector store status")
details: dict[str, Any] = Field(default_factory=dict)
class RefreshRequest(BaseModel):
"""Request to refresh an index."""
roboco/config.py
+15 -5
View File
@@ -123,7 +123,7 @@ class Settings(BaseSettings):
rag_chunk_strategy: str = Field(
default="fixed",
pattern="^(fixed|semantic|hierarchical|contextual)$",
description="Chunking strategy (fixed recommended - semantic loads separate model)",
description="Chunking strategy (fixed recommended, semantic loads extra model)",
)
rag_chunk_size: int = Field(default=512, ge=100)
rag_chunk_size_docs: int = Field(
@@ -163,14 +163,24 @@ class Settings(BaseSettings):
openai_api_key: str | None = None # For embeddings
# Default models
default_llm_model: str = "claude-3-opus-20240229"
default_llm_model: str = "claude-opus-4-5-20251101"
default_embedding_model: str = Field(
default="nomic-ai/nomic-embed-text-v1.5",
description="HuggingFace model for local or OpenAI name with API key",
default="BAAI/bge-base-en-v1.5",
description="Embedding model",
)
embedding_dimensions: int = Field(
default=768,
description="Embedding dimensions (768 for nomic-embed, BGE-base)",
description="Embedding dimensions (768 for BGE-base)",
)
# Local LLM for RAG (HyDE, reranking, etc.)
local_llm_model: str = Field(
default="qwen3:8b",
description="Local LLM model for HyDE and RAG operations",
)
local_llm_base_url: str = Field(
default="http://192.168.50.111:11434/v1",
description="Base URL for local LLM (Ollama)",
)
# ==========================================================================
roboco/db/tables.py
+5
View File
@@ -202,6 +202,11 @@ class TaskTable(Base):
# Quick Context
quick_context: Mapped[str | None] = mapped_column(Text, nullable=True)
# Proactive Knowledge Context (injected when task is claimed)
proactive_context: Mapped[dict[str, Any] | None] = mapped_column(
JSON, nullable=True
)
# Relationships
creator: Mapped["AgentTable"] = relationship(
"AgentTable", foreign_keys=[created_by], lazy="joined"
roboco/mcp/optimal_server.py
+152 -6
View File
@@ -814,29 +814,171 @@ def _register_learning_tools(mcp: FastMCP, client: ApiClient) -> None:
}
def _register_index_management_tools(mcp: FastMCP, client: ApiClient) -> None:
"""Register index management tools for administrative operations."""
@mcp.tool()
async def roboco_clear_index(index_type: str) -> dict[str, Any]:
"""
Clear all documents from a specific index.
Use with caution - this permanently deletes indexed content.
Useful for recovering from corrupted indexes or starting fresh.
PERMISSION: Requires CLEAR_INDEX permission.
Args:
index_type: One of: code, documentation, conversations, journals,
errors, standards, decisions, reviews, learnings
Returns:
Confirmation of cleared index
"""
valid_types = {
"code",
"documentation",
"conversations",
"journals",
"errors",
"standards",
"decisions",
"reviews",
"learnings",
}
if index_type not in valid_types:
return format_error_response(
"INVALID_INDEX_TYPE",
f"Invalid index type. Must be one of: {', '.join(sorted(valid_types))}",
)
resp = await client.delete(f"/optimal/kb/{index_type}")
if not resp.ok:
if resp.status_code == http_status.HTTP_403_FORBIDDEN:
return format_error_response(
"NOT_AUTHORIZED",
"You don't have permission to clear indexes",
)
return format_error_response(
"CLEAR_FAILED",
"Failed to clear index",
{"api_error": resp.text},
)
return {"status": "success", "cleared": index_type}
@mcp.tool()
async def roboco_reindex_all(force: bool = False) -> dict[str, Any]:
"""
Trigger re-indexing of code and documentation.
Re-scans the codebase and docs directories to update indexes.
Useful when files have been added/changed outside of normal workflow.
PERMISSION: Requires INDEX_CODE permission.
Args:
force: If True, reindex even if indexes aren't empty
Returns:
Count of indexed code files and documentation files
"""
resp = await client.post(
"/optimal/kb/reindex",
params={"force": str(force).lower()},
)
if not resp.ok:
if resp.status_code == http_status.HTTP_403_FORBIDDEN:
return format_error_response(
"NOT_AUTHORIZED",
"You don't have permission to trigger reindexing",
)
return format_error_response(
"REINDEX_FAILED",
"Failed to trigger reindexing",
{"api_error": resp.text},
)
result = resp.json()
return {
"status": "success",
"code_files_indexed": result.get("code", 0),
"docs_files_indexed": result.get("docs", 0),
}
@mcp.tool()
async def roboco_index_status() -> dict[str, Any]:
"""
Get detailed status of all indexes.
Shows document counts and last update times for each index type.
Useful for monitoring and debugging indexing issues.
Returns:
Status information for each index including document counts
"""
resp = await client.get("/optimal/stats")
if not resp.ok:
return format_error_response(
"STATS_FAILED",
"Failed to get index status",
{"api_error": resp.text},
)
result = resp.json()
return {
"status": "success",
"initialized": result.get("initialized", False),
"indexes": result.get("indexes", {}),
}
def _register_proactive_tools(mcp: FastMCP, client: ApiClient) -> None:
"""Register proactive context tools."""
@mcp.tool()
async def roboco_get_proactive_context(
task_id: str,
force_refresh: bool = False,
) -> dict[str, Any]:
"""
Get proactive context for a task.
Fetches relevant knowledge to help you work on a task:
- Similar past tasks and their learnings
- Relevant code patterns
- Applicable standards
- Recent decisions
- Known issues
First checks for stored context (injected when task was claimed).
Falls back to generating fresh context if not available.
Args:
task_id: UUID of the task to get context for
force_refresh: If True, skip stored context and generate fresh
Returns:
Dictionary with context categories and a summary
"""
# Try to get stored context from task first (unless force_refresh)
if not force_refresh:
task_resp = await client.get(f"/tasks/{task_id}")
if task_resp.ok:
task_data = task_resp.json()
stored_context = task_data.get("proactive_context")
if stored_context and isinstance(stored_context, dict):
# Return stored context with source indicator
return {
"status": "success",
"source": "stored",
"task_id": task_id,
"similar_tasks": stored_context.get("similar_tasks", []),
"relevant_learnings": stored_context.get(
"relevant_learnings", []
),
"code_patterns": stored_context.get("code_patterns", []),
"applicable_standards": stored_context.get(
"applicable_standards", []
),
"recent_decisions": stored_context.get("recent_decisions", []),
"known_issues": stored_context.get("known_issues", []),
"summary": stored_context.get("summary", ""),
}
# Fall back to generating fresh context
payload = {"task_id": task_id}
resp = await client.post("/optimal/context/proactive", json=payload)
@@ -850,6 +992,7 @@ def _register_proactive_tools(mcp: FastMCP, client: ApiClient) -> None:
result = resp.json()
return {
"status": "success",
"source": "fresh",
"task_id": result.get("task_id"),
"similar_tasks": result.get("similar_tasks", []),
"relevant_learnings": result.get("relevant_learnings", []),
@@ -879,6 +1022,9 @@ def create_optimal_mcp_server(agent_id: str) -> FastMCP:
_register_learning_tools(mcp, client)
_register_proactive_tools(mcp, client)
# Register index management tools
_register_index_management_tools(mcp, client)
return mcp
roboco/models/task.py
+6
View File
@@ -210,6 +210,12 @@ class Task(TimestampMixin):
description="2-3 sentences for quick context restoration",
)
# Proactive Knowledge Context (injected when task is claimed)
proactive_context: dict | None = Field(
default=None,
description="RAG context: similar tasks, learnings, patterns, standards",
)
# NOTE: Task state mutations should be performed through TaskService,
# not directly on the model. See roboco/services/task.py for:
# - claim(), start(), block(), pause(), resume()
roboco/services/a2a.py
+13 -5
View File
@@ -395,7 +395,7 @@ class A2AService:
async def cancel_task(self, task_id: str, reason: str | None = None) -> A2ATask:
"""
Cancel a task.
Cancel a task and all non-terminal descendants.
Args:
task_id: Task UUID string
@@ -407,11 +407,15 @@ class A2AService:
Raises:
ValueError: If task not found or already in terminal state
"""
# Import here to avoid circular imports
from roboco.services.task import TaskService
try:
task_uuid = UUID(task_id)
except ValueError as e:
raise ValueError(f"Invalid task ID: {task_id}") from e
# Check task exists and is cancellable before using service
result = await self.session.execute(
select(TaskTable).where(TaskTable.id == task_uuid)
)
@@ -429,17 +433,21 @@ class A2AService:
if status_value in ["completed", "cancelled"]:
raise ValueError(f"Task already in terminal state: {status_value}")
# Cancel the task
task.status = TaskStatus.CANCELLED
# Add reason to notes before cancel
if reason:
reason_text = f"Cancellation reason: {reason}"
if task.dev_notes:
task.dev_notes = f"{task.dev_notes}\n\n{reason_text}"
else:
task.dev_notes = reason_text
await self.session.flush()
await self.session.flush()
await self.session.refresh(task)
# Use TaskService for consistent cancel behavior (cascades to descendants)
task_service = TaskService(self.session)
task = await task_service.cancel(task_uuid)
if task is None:
raise ValueError(f"Failed to cancel task: {task_id}")
logger.info("Cancelled task via A2A", task_id=task_id, reason=reason)
roboco/services/messaging.py
+33 -1
View File
@@ -10,8 +10,9 @@ Comprehensive service for managing communication:
Implements the communication model.
"""
import asyncio
from datetime import UTC, datetime
from typing import ClassVar, cast
from typing import Any, ClassVar, cast
from uuid import UUID
from sqlalchemy import select
@@ -72,6 +73,7 @@ class MessagingService(BaseService):
"""
service_name: ClassVar[str] = "messaging"
_background_tasks: ClassVar[set[asyncio.Task[Any]]] = set()
# =========================================================================
# CHANNEL OPERATIONS (TASK-013)
@@ -817,6 +819,31 @@ class MessagingService(BaseService):
message_id=str(message.id),
)
async def _index_message_async(self, message: MessageTable) -> None:
"""Index message in RAG system (fire-and-forget)."""
from roboco.models.optimal import IndexConversationParams
from roboco.services.optimal import get_optimal_service
try:
optimal = await get_optimal_service()
await optimal.index_conversation(
IndexConversationParams(
content=message.content,
channel_id=message.channel_id,
session_id=message.session_id,
agent_id=message.agent_id,
task_id=message.task_id,
message_type=message.type.value if message.type else None,
)
)
self.log.debug("Message indexed", message_id=str(message.id))
except Exception as e:
self.log.warning(
"Failed to index message",
message_id=str(message.id),
error=str(e),
)
async def send_message(
self,
req: MessageCreateRequest,
@@ -865,6 +892,11 @@ class MessagingService(BaseService):
# Notify mentioned agents via Redis Streams
await self._notify_mentions(message, req.agent_id, channel.slug)
# Index message in RAG (fire-and-forget)
bg_task = asyncio.create_task(self._index_message_async(message))
self._background_tasks.add(bg_task)
bg_task.add_done_callback(self._background_tasks.discard)
if self._check_session_boundaries(session):
await self.close_session(cast("UUID", session.id), "Boundary exceeded")
roboco/services/optimal.py
+349 -40
View File
@@ -86,6 +86,7 @@ class OptimalService:
def __init__(self) -> None:
self._initialized = False
self._plugins: dict[IndexType, BaseIndexPlugin] = {}
self._prompt_templates: dict[str, dict[str, Any]] = {}
async def initialize(self) -> None:
"""Initialize all knowledge base indexes."""
@@ -104,9 +105,76 @@ class OptimalService:
self._initialized = True
logger.info("OptimalService initialization complete")
# Auto-index documentation on startup
# Auto-index code and documentation on startup
await self._auto_index_on_startup()
async def _auto_index_on_startup(self) -> None:
"""
Auto-index code and documentation on startup.
Indexes:
- /roboco/ - Source code files
- /docs/standards/ - Coding, security, workflow standards
- /docs/workflows/ - Agent workflow documentation
This ensures agents can search for code, standards, and workflows
immediately after startup.
"""
await self._auto_index_code()
await self._auto_index_docs()
async def _auto_index_code(self) -> None:
"""Auto-index source code files on startup."""
# Find the roboco source directory (the Python package, not the repo root)
# We want to index roboco/ package, NOT the entire repo (which has .venv)
possible_code_roots = [
Path("/app/roboco"), # Docker: /app is repo root, roboco/ is package
Path(__file__).parent.parent, # Local: optimal.py -> services -> roboco
Path.cwd() / "roboco", # Local: cwd/roboco
]
code_root = None
for path in possible_code_roots:
# Check for __init__.py to confirm it's a Python package (not repo root)
init_file = path / "__init__.py"
if path.exists() and path.is_dir() and init_file.exists():
code_root = path
logger.debug(
"Found code package directory",
path=str(path),
)
break
if code_root is None:
logger.warning(
"Code directory not found",
searched_paths=[str(p) for p in possible_code_roots],
)
return
# Check if code index is empty
code_plugin = self._get_plugin(IndexType.CODE)
code_count = await code_plugin.count()
if code_count > 0:
logger.info(
"Code index already populated",
chunk_count=code_count,
skipping=True,
)
return
logger.info(
"Auto-indexing source code",
directory=str(code_root),
)
try:
count = await self.index_code([str(code_root)], project="roboco")
logger.info("Code auto-indexing complete", files_indexed=count)
except Exception as e:
logger.warning("Code auto-indexing failed", error=str(e))
async def _auto_index_docs(self) -> None:
"""
Auto-index documentation directories on startup.
@@ -114,9 +182,6 @@ class OptimalService:
Indexes:
- /docs/standards/ - Coding, security, workflow standards
- /docs/workflows/ - Agent workflow documentation
This ensures agents can search for standards and workflows
immediately after startup.
"""
# Find the docs directory relative to the project root
possible_docs_roots = [
@@ -222,10 +287,27 @@ class OptimalService:
sources: list[str],
project: str | None = None,
) -> int:
"""Index code files/directories."""
"""Index code files/directories and track in database."""
plugin = self._get_plugin(IndexType.CODE)
if isinstance(plugin, CodeIndexPlugin):
return await plugin.index_sources(sources, project)
count, indexed_files = await plugin.index_sources(sources, project)
# Batch track all indexed files using repository
docs_to_track = [
{
"source": f["source"],
"title": f["title"],
"preview": f.get("preview"),
"metadata": {
"language": f.get("language"),
"file_path": f.get("file_path"),
"project": project,
},
}
for f in indexed_files
]
await self._track_indexed_documents_batch(IndexType.CODE, docs_to_track)
return count
return await plugin.add_sources(sources)
async def index_documentation(
@@ -233,10 +315,29 @@ class OptimalService:
sources: list[str],
project: str | None = None,
) -> int:
"""Index documentation files."""
"""Index documentation files and track in database."""
plugin = self._get_plugin(IndexType.DOCUMENTATION)
if isinstance(plugin, DocsIndexPlugin):
return await plugin.index_sources(sources, project)
count, indexed_files = await plugin.index_sources(sources, project)
# Batch track all indexed files using repository
docs_to_track = [
{
"source": f["source"],
"title": f["title"],
"preview": f.get("preview"),
"metadata": {
"doc_type": f.get("doc_type"),
"file_path": f.get("file_path"),
"project": project,
},
}
for f in indexed_files
]
await self._track_indexed_documents_batch(
IndexType.DOCUMENTATION, docs_to_track
)
return count
return await plugin.add_sources(sources)
async def _track_indexed_document(
@@ -248,43 +349,29 @@ class OptimalService:
metadata: dict | None = None,
) -> None:
"""Track an indexed document in the database for browsing/stats."""
import hashlib
doc = {
"source": source,
"title": title,
"preview": preview,
"metadata": metadata,
}
await self._track_indexed_documents_batch(index_type, [doc])
async def _track_indexed_documents_batch(
self,
index_type: IndexType,
documents: list[dict],
) -> None:
"""Track multiple indexed documents in a single transaction."""
from roboco.db import get_db_context
from roboco.db.tables import IndexedDocumentTable
from roboco.services.repositories import IndexedDocumentRepository
source_hash = hashlib.sha256(source.encode()).hexdigest()
if not documents:
return
async with get_db_context() as db:
from sqlalchemy import select
existing = await db.execute(
select(IndexedDocumentTable).where(
IndexedDocumentTable.index_type == index_type.value,
IndexedDocumentTable.source_hash == source_hash,
)
)
doc = existing.scalar_one_or_none()
if doc:
if title:
doc.title = title
if preview:
doc.preview = preview[:500] if preview else None
if metadata:
doc.extra_data = {**(doc.extra_data or {}), **metadata}
else:
doc = IndexedDocumentTable(
index_type=index_type.value,
source=source,
source_hash=source_hash,
title=title,
preview=preview[:500] if preview else None,
extra_data=metadata or {},
)
db.add(doc)
await db.commit()
repo = IndexedDocumentRepository(db)
await repo.upsert_batch(index_type.value, documents)
async def index_conversation(self, params: IndexConversationParams) -> None:
"""Index a conversation message."""
@@ -677,6 +764,228 @@ class OptimalService:
return stats
async def get_index_stats(self, index_type: IndexType) -> dict[str, Any]:
"""
Get detailed stats for a specific index.
Returns:
Dict with document_count, chunk_count, last_updated
"""
if not self._initialized:
return {"error": "Not initialized"}
from sqlalchemy import func, select
from roboco.db import get_db_context
from roboco.db.tables import IndexedDocumentTable
plugin = self._get_plugin(index_type)
chunk_count = await plugin.count()
# Query DB for document count and last_updated
async with get_db_context() as session:
# Document count
count_query = (
select(func.count())
.select_from(IndexedDocumentTable)
.where(IndexedDocumentTable.index_type == index_type.value)
)
count_result = await session.execute(count_query)
doc_count = count_result.scalar() or 0
# Last updated
last_updated_query = (
select(func.max(IndexedDocumentTable.indexed_at))
.select_from(IndexedDocumentTable)
.where(IndexedDocumentTable.index_type == index_type.value)
)
last_updated_result = await session.execute(last_updated_query)
last_updated = last_updated_result.scalar()
return {
"index_type": index_type.value,
"document_count": doc_count,
"chunk_count": chunk_count,
"last_updated": last_updated.isoformat() if last_updated else None,
}
async def get_all_index_stats(self) -> dict[str, Any]:
"""
Get detailed stats for all indexes including document counts and last_updated.
Returns:
Dict with initialized flag and indexes with full stats
"""
if not self._initialized:
return {"initialized": False, "indexes": {}}
from sqlalchemy import func, select
from roboco.db import get_db_context
from roboco.db.tables import IndexedDocumentTable
async with get_db_context() as session:
stats: dict[str, Any] = {"initialized": True, "indexes": {}}
for index_type, plugin in self._plugins.items():
try:
chunk_count = await plugin.count()
# Document count
count_query = (
select(func.count())
.select_from(IndexedDocumentTable)
.where(IndexedDocumentTable.index_type == index_type.value)
)
count_result = await session.execute(count_query)
doc_count = count_result.scalar() or 0
# Last updated
last_updated_query = (
select(func.max(IndexedDocumentTable.indexed_at))
.select_from(IndexedDocumentTable)
.where(IndexedDocumentTable.index_type == index_type.value)
)
last_updated_result = await session.execute(last_updated_query)
last_updated = last_updated_result.scalar()
stats["indexes"][index_type.value] = {
"document_count": doc_count,
"chunk_count": chunk_count,
"last_updated": (
last_updated.isoformat() if last_updated else None
),
}
except Exception as e:
stats["indexes"][index_type.value] = {"error": str(e)}
return stats
async def auto_index_on_startup(
self,
code_sources: list[str] | None = None,
docs_sources: list[str] | None = None,
force: bool = False,
) -> dict[str, int]:
"""
Auto-index code and docs if indexes are empty.
Called during bootstrap to ensure RAG has content to search.
Args:
code_sources: Paths to index for code (default: auto-detect)
docs_sources: Paths to index for docs (default: auto-detect)
force: Force re-index even if not empty
Returns:
Dict with counts: {"code": N, "docs": M}
"""
if not self._initialized:
await self.initialize()
# Auto-detect paths if not provided
if code_sources is None:
# Try Docker paths first, then local
# ONLY use package directories (with __init__.py), NOT repo roots
for path in ["/app/roboco", "roboco/"]:
p = Path(path)
if p.exists() and (p / "__init__.py").exists():
code_sources = [path]
break
code_sources = code_sources or ["roboco/"]
if docs_sources is None:
# Try Docker paths first, then local
for path in ["/app/docs", "docs/"]:
if Path(path).exists():
docs_sources = [path]
break
docs_sources = docs_sources or ["docs/"]
result = {"code": 0, "docs": 0}
# Check code index
code_plugin = self._get_plugin(IndexType.CODE)
code_count = await code_plugin.count()
if code_count == 0 or force:
logger.info(
"Auto-indexing code",
sources=code_sources,
reason="empty" if code_count == 0 else "forced",
)
result["code"] = await self.index_code(code_sources, project="roboco")
# Check docs index
docs_plugin = self._get_plugin(IndexType.DOCUMENTATION)
docs_count = await docs_plugin.count()
if docs_count == 0 or force:
logger.info(
"Auto-indexing documentation",
sources=docs_sources,
reason="empty" if docs_count == 0 else "forced",
)
result["docs"] = await self.index_documentation(
docs_sources, project="roboco"
)
if result["code"] > 0 or result["docs"] > 0:
logger.info(
"Auto-indexing complete",
code_files=result["code"],
doc_files=result["docs"],
)
else:
logger.info("Indexes already populated, skipping auto-index")
return result
# =========================================================================
# PROMPT TEMPLATE MANAGEMENT
# =========================================================================
def create_prompt_template(self, template_data: dict[str, Any]) -> dict[str, Any]:
"""
Create a reusable prompt template.
Args:
template_data: Dict with id, name, template, description,
variables, category, created_at, created_by
Raises:
ValueError: If template_data is missing required 'id' field
"""
if "id" not in template_data:
raise ValueError("Template data must include 'id' field")
template_id = template_data["id"]
self._prompt_templates[template_id] = template_data
return self._prompt_templates[template_id]
def list_prompt_templates(
self, category: str | None = None
) -> list[dict[str, Any]]:
"""List all prompt templates, optionally filtered by category."""
templates = list(self._prompt_templates.values())
if category:
templates = [t for t in templates if t.get("category") == category]
return templates
def get_prompt_template(self, template_id: str) -> dict[str, Any] | None:
"""Get a prompt template by ID."""
return self._prompt_templates.get(template_id)
def delete_prompt_template(self, template_id: str) -> bool:
"""Delete a prompt template. Returns True if deleted."""
if template_id in self._prompt_templates:
del self._prompt_templates[template_id]
return True
return False
def reset_prompt_templates(self) -> None:
"""Reset all prompt templates (for testing)."""
self._prompt_templates.clear()
async def clear_index(self, index_type: IndexType) -> None:
"""Clear a specific index."""
plugin = self._get_plugin(index_type)
roboco/services/optimal_brain/indexes/base.py
+118 -4
View File
@@ -32,9 +32,9 @@ class IndexConfig:
use_hyde: bool = True
use_hybrid_search: bool = True
use_cross_encoder: bool = False
embedding_model: str = "nomic-ai/nomic-embed-text-v1.5"
embedding_model: str = "BAAI/bge-base-en-v1.5"
llm_model: str = "llama3.2"
llm_base_url: str = "http://localhost:11434/v1"
llm_base_url: str = "http://192.168.50.111:11434/v1"
@classmethod
def from_settings(cls, index_type: IndexType) -> "IndexConfig":
@@ -56,6 +56,8 @@ class IndexConfig:
use_hybrid_search=settings.rag_use_hybrid_search,
use_cross_encoder=settings.rag_use_cross_encoder,
embedding_model=settings.default_embedding_model,
llm_model=settings.local_llm_model,
llm_base_url=settings.local_llm_base_url,
)
@@ -200,7 +202,7 @@ class BaseIndexPlugin(ABC):
# Create store with correct vector dimension for embedding model
# Piragi's factory defaults to 768 for PostgresStore, matching
# nomic-embed-text-v1.5 which produces 768-dimensional embeddings
# the embedding model which produces 768-dimensional embeddings
store = self._create_store_with_dimension()
# Use config with dummy embedding URL to prevent model loading
@@ -238,7 +240,7 @@ class BaseIndexPlugin(ABC):
if store_url.startswith("postgres://") or store_url.startswith("postgresql://"):
from piragi.stores.postgres import PostgresStore
# Get dimension from settings (768 for nomic-embed-text-v1.5)
# Get dimension from settings
vector_dimension = settings.embedding_dimensions
logger.debug(
@@ -407,6 +409,118 @@ class BaseIndexPlugin(ABC):
error=str(e),
)
async def ingest_batch(
self,
documents: list[tuple[str, str | None, dict[str, Any]]],
) -> list[IngestResult]:
"""
Batch ingest multiple documents efficiently.
This method processes all documents together, batching:
- Chunking (fast, ~100ms total)
- Embedding (main bottleneck - batched in groups of 32)
- Storage (single transaction)
For 179 files, this achieves ~10-15x speedup vs sequential ingest().
Args:
documents: List of (content, doc_id, kwargs) tuples
Returns:
List of IngestResult for each document
"""
import asyncio
if not documents:
return []
# Validate and prepare all documents
docs_to_process: list[tuple[Document, str | None, dict[str, Any]]] = []
results: list[IngestResult] = []
for content, doc_id, kwargs in documents:
is_valid, error = self.validate_content(content, **kwargs)
if not is_valid:
results.append(
IngestResult(
doc_id=doc_id or "unknown",
chunk_count=0,
success=False,
error=error,
)
)
continue
metadata = self.prepare_metadata(content, **kwargs)
source = self.build_source_uri(doc_id, **kwargs)
doc = Document(content=content, source=source, metadata=metadata)
docs_to_process.append((doc, doc_id, kwargs))
if not docs_to_process:
return results
# Process all valid documents in batch
ragi_sync = self.ragi._sync
chunk_counts: dict[int, int] = {}
def _batch_process() -> None:
# Chunk ALL documents
all_chunks = []
for idx, (doc, _, _) in enumerate(docs_to_process):
chunks = ragi_sync.chunker.chunk_document(doc)
for chunk in chunks:
chunk.metadata = {**chunk.metadata, **doc.metadata}
all_chunks.extend(chunks)
chunk_counts[idx] = len(chunks)
logger.info(
f"Batch: {len(all_chunks)} chunks from {len(docs_to_process)} docs"
)
if all_chunks:
# Embed ALL chunks (piragi batches internally at 32)
chunks_with_embeddings = ragi_sync.embedder.embed_chunks(all_chunks)
# Store ALL in single transaction
ragi_sync.store.add_chunks(chunks_with_embeddings)
try:
await asyncio.to_thread(_batch_process)
# Build success results
for idx, (doc, doc_id, _) in enumerate(docs_to_process):
results.append(
IngestResult(
doc_id=doc_id or doc.source,
chunk_count=chunk_counts.get(idx, 0),
success=True,
)
)
logger.info(
"Batch ingest complete",
index_type=self.index_type.value,
documents=len(docs_to_process),
total_chunks=sum(chunk_counts.values()),
)
except Exception as e:
logger.error(
"Batch ingest failed",
index_type=self.index_type.value,
error=str(e),
)
# Mark all as failed
for _doc, doc_id, _ in docs_to_process:
results.append(
IngestResult(
doc_id=doc_id or "unknown",
chunk_count=0,
success=False,
error=str(e),
)
)
return results
async def search(
self,
query: str,
roboco/services/optimal_brain/indexes/code.py
+302 -6
View File
@@ -2,12 +2,144 @@
Code Index Plugin
Handles indexing and searching code files and repositories.
Uses a simple line-based chunking strategy instead of sentence-based,
since code doesn't have natural sentence boundaries.
"""
from pathlib import Path
from typing import Any
import structlog
from piragi.types import Chunk
from roboco.models.optimal import IndexType
from roboco.services.optimal_brain.indexes.base import BaseIndexPlugin
from roboco.services.optimal_brain.indexes.base import BaseIndexPlugin, IngestResult
logger = structlog.get_logger()
def chunk_code(
content: str,
source: str,
chunk_size: int = 1500,
chunk_overlap: int = 200,
) -> list[Chunk]:
"""
Chunk code using a simple line-based strategy.
Unlike prose, code doesn't have sentence boundaries. This chunker:
- Splits by lines (respects code structure)
- Uses character-based sizes (not token-based, simpler and faster)
- Tries to break at blank lines or function/class boundaries
Args:
content: Source code content
source: Source file path/URI
chunk_size: Target chunk size in characters (~375 tokens)
chunk_overlap: Overlap between chunks in characters
Returns:
List of Chunk objects
"""
if not content.strip():
return []
lines = content.split("\n")
chunks: list[Chunk] = []
current_chunk_lines: list[str] = []
current_size = 0
chunk_index = 0
for line in lines:
line_size = len(line) + 1 # +1 for newline
# Check if adding this line would exceed chunk size
if current_size + line_size > chunk_size and current_chunk_lines:
# Save current chunk
chunk_text = "\n".join(current_chunk_lines)
chunks.append(
Chunk(
text=chunk_text,
source=source,
chunk_index=chunk_index,
metadata={},
)
)
chunk_index += 1
# Calculate overlap: keep last N characters worth of lines
overlap_lines: list[str] = []
overlap_size = 0
for prev_line in reversed(current_chunk_lines):
if overlap_size + len(prev_line) + 1 > chunk_overlap:
break
overlap_lines.insert(0, prev_line)
overlap_size += len(prev_line) + 1
current_chunk_lines = overlap_lines
current_size = overlap_size
current_chunk_lines.append(line)
current_size += line_size
# Don't forget the last chunk
if current_chunk_lines:
chunk_text = "\n".join(current_chunk_lines)
chunks.append(
Chunk(
text=chunk_text,
source=source,
chunk_index=chunk_index,
metadata={},
)
)
return chunks
# Common code file extensions
CODE_EXTENSIONS = {
".py": "python",
".js": "javascript",
".ts": "typescript",
".tsx": "typescript",
".jsx": "javascript",
".go": "go",
".rs": "rust",
".java": "java",
".c": "c",
".cpp": "cpp",
".h": "c",
".hpp": "cpp",
".rb": "ruby",
".php": "php",
".sh": "shell",
".sql": "sql",
".yaml": "yaml",
".yml": "yaml",
".json": "json",
".toml": "toml",
}
# Directories to skip during indexing
SKIP_DIRECTORIES = {
".git",
".venv",
"venv",
"__pycache__",
".piragi",
"node_modules",
".next",
"dist",
"build",
".mypy_cache",
".pytest_cache",
".ruff_cache",
"htmlcov",
".tox",
"eggs",
"*.egg-info",
}
class CodeIndexPlugin(BaseIndexPlugin):
@@ -46,16 +178,180 @@ class CodeIndexPlugin(BaseIndexPlugin):
async def index_sources(
self,
sources: list[str],
_project: str | None = None,
) -> int:
project: str | None = None,
) -> tuple[int, list[dict[str, Any]]]:
"""
Index code files/directories.
Index code files/directories with batch embedding for performance.
Uses batch processing to embed all files together instead of one-by-one,
achieving 10-15x speedup on large codebases.
Args:
sources: List of file paths, directories, or glob patterns
project: Optional project identifier for filtering
Returns:
Number of documents indexed
Tuple of (count, indexed_files) where indexed_files contains
metadata for each file indexed (for database tracking)
"""
return await self.add_sources(sources)
# Step 1: Collect all files and their contents
files_data: list[dict[str, Any]] = []
for source in sources:
source_path = Path(source)
# Expand glob patterns and directories
if "*" in source:
files = list(Path().glob(source))
elif source_path.is_dir():
files = [
f
for f in source_path.rglob("*")
if f.suffix in CODE_EXTENSIONS
and not any(skip in f.parts for skip in SKIP_DIRECTORIES)
]
elif source_path.exists():
files = [source_path]
else:
logger.warning(f"Source not found: {source}")
continue
logger.info(f"Found {len(files)} code files to index in {source}")
for file_path in files:
if not file_path.is_file():
continue
if file_path.suffix not in CODE_EXTENSIONS:
continue
if any(skip in file_path.parts for skip in SKIP_DIRECTORIES):
continue
try:
content = file_path.read_text(encoding="utf-8")
language = CODE_EXTENSIONS.get(file_path.suffix)
files_data.append(
{
"content": content,
"file_path": file_path,
"language": language,
"project": project or "default",
}
)
except Exception as e:
logger.warning(
"Failed to read code file",
file=str(file_path),
error=str(e),
)
if not files_data:
return 0, []
logger.info(f"Batch processing {len(files_data)} code files")
results = await self._ingest_code_batch(files_data)
count = sum(1 for r in results if r.success)
# Build indexed_files list for database tracking
indexed_files = [
{
"source": str(data["file_path"].absolute()),
"title": data["file_path"].name,
"preview": data["content"][:500] if data["content"] else None,
"language": data["language"],
"file_path": str(data["file_path"]),
}
for data in files_data
]
logger.info(f"Batch indexing complete: {count} files indexed")
return count, indexed_files
async def _ingest_code_batch(
self,
files_data: list[dict[str, Any]],
) -> list[IngestResult]:
"""
Batch ingest code files using line-based chunking.
Unlike the base class ingest_batch which uses piragi's sentence-based
chunker, this method uses a simple line-based chunker that's
appropriate for source code.
Args:
files_data: List of dicts with content, file_path, language, project
Returns:
List of IngestResult for each file
"""
import asyncio
if not files_data:
return []
# Chunk ALL files using line-based chunker (fast, no tokenizer needed)
all_chunks: list[Chunk] = []
chunk_counts: dict[int, int] = {}
for idx, data in enumerate(files_data):
content = str(data["content"])
file_path = str(data["file_path"])
source = self.build_source_uri(file_path, file_path=file_path)
# Use line-based chunking (1500 chars ≈ 375 tokens, with 200 char overlap)
chunks = chunk_code(content, source, chunk_size=1500, chunk_overlap=200)
# Add metadata to chunks
metadata = self.prepare_metadata(
content,
file_path=file_path,
language=data.get("language"),
project=data.get("project", "default"),
)
for chunk in chunks:
chunk.metadata = {**chunk.metadata, **metadata}
all_chunks.extend(chunks)
chunk_counts[idx] = len(chunks)
logger.info(
f"Code batch: {len(all_chunks)} chunks from {len(files_data)} files "
f"(avg {len(all_chunks) / len(files_data):.1f} chunks/file)"
)
if not all_chunks:
return [
IngestResult(doc_id=str(d["file_path"]), chunk_count=0, success=True)
for d in files_data
]
# Embed and store using piragi's internals
ragi_sync = self.ragi._sync
def _embed_and_store() -> None:
chunks_with_embeddings = ragi_sync.embedder.embed_chunks(all_chunks)
ragi_sync.store.add_chunks(chunks_with_embeddings)
try:
await asyncio.to_thread(_embed_and_store)
return [
IngestResult(
doc_id=str(data["file_path"]),
chunk_count=chunk_counts.get(idx, 0),
success=True,
)
for idx, data in enumerate(files_data)
]
except Exception as e:
logger.error("Code batch ingest failed", error=str(e))
return [
IngestResult(
doc_id=str(data["file_path"]),
chunk_count=0,
success=False,
error=str(e),
)
for data in files_data
]
roboco/services/optimal_brain/indexes/docs.py
+132 -5
View File
@@ -4,11 +4,29 @@ Documentation Index Plugin
Handles indexing and searching documentation files (markdown, text, etc.).
"""
from pathlib import Path
from typing import Any
import structlog
from roboco.models.optimal import IndexType
from roboco.services.optimal_brain.indexes.base import BaseIndexPlugin
logger = structlog.get_logger()
# Directories to skip during indexing
SKIP_DIRECTORIES = {
".git",
".venv",
"venv",
"__pycache__",
".piragi",
"node_modules",
".next",
"dist",
"build",
}
class DocsIndexPlugin(BaseIndexPlugin):
"""
@@ -47,16 +65,125 @@ class DocsIndexPlugin(BaseIndexPlugin):
async def index_sources(
self,
sources: list[str],
_project: str | None = None,
) -> int:
project: str | None = None,
) -> tuple[int, list[dict[str, Any]]]:
"""
Index documentation files/directories.
Index documentation files/directories with batch embedding.
Uses batch processing to embed all files together instead of one-by-one,
achieving 10-15x speedup on large documentation sets.
Args:
sources: List of file paths, directories, URLs, or glob patterns
project: Optional project identifier for filtering
Returns:
Number of documents indexed
Tuple of (count, indexed_files) where indexed_files contains
metadata for each file indexed (for database tracking)
"""
return await self.add_sources(sources)
# Step 1: Collect all files and their contents
files_data: list[dict[str, Any]] = []
for source in sources:
source_path = Path(source)
# Expand glob patterns and directories
if "*" in source:
files = list(Path().glob(source))
elif source_path.is_dir():
md_files = [
f
for f in source_path.rglob("*.md")
if not any(skip in f.parts for skip in SKIP_DIRECTORIES)
]
txt_files = [
f
for f in source_path.rglob("*.txt")
if not any(skip in f.parts for skip in SKIP_DIRECTORIES)
]
files = md_files + txt_files
elif source_path.exists():
files = [source_path]
else:
logger.warning(f"Source not found: {source}")
continue
logger.info(f"Found {len(files)} doc files to index in {source}")
for file_path in files:
if not file_path.is_file():
continue
if any(skip in file_path.parts for skip in SKIP_DIRECTORIES):
continue
try:
content = file_path.read_text(encoding="utf-8")
doc_type = self._detect_doc_type(file_path)
files_data.append(
{
"content": content,
"file_path": file_path,
"doc_type": doc_type,
"project": project or "default",
}
)
except Exception as e:
logger.warning(
"Failed to read documentation file",
file=str(file_path),
error=str(e),
)
if not files_data:
return 0, []
logger.info(f"Batch processing {len(files_data)} documentation files")
# Use base class batch ingest for efficient processing
documents: list[tuple[str, str | None, dict[str, Any]]] = [
(
str(data["content"]), # Ensure str type
str(data["file_path"]),
{
"file_path": str(data["file_path"]),
"doc_type": data["doc_type"],
"project": data["project"],
},
)
for data in files_data
]
results = await self.ingest_batch(documents)
count = sum(1 for r in results if r.success)
# Build indexed_files list for database tracking
indexed_files = [
{
"source": str(data["file_path"].absolute()),
"title": data["file_path"]
.stem.replace("-", " ")
.replace("_", " ")
.title(),
"preview": data["content"][:500] if data["content"] else None,
"doc_type": data["doc_type"],
"file_path": str(data["file_path"]),
}
for data in files_data
]
logger.info(f"Batch indexing complete: {count} docs indexed")
return count, indexed_files
def _detect_doc_type(self, file_path: Path) -> str:
"""Detect documentation type from filename."""
name_lower = file_path.stem.lower()
if "readme" in name_lower:
return "readme"
if "api" in name_lower:
return "api"
if "guide" in name_lower or "tutorial" in name_lower:
return "guide"
if "changelog" in name_lower:
return "changelog"
return "general"
roboco/services/optimal_brain/shared_embedder.py
+2 -2
View File
@@ -31,7 +31,7 @@ class _SharedEmbedderHolder:
async def get_shared_embedder(
model: str = "nomic-ai/nomic-embed-text-v1.5",
model: str = "BAAI/bge-base-en-v1.5",
device: str | None = None,
) -> "EmbeddingGenerator":
"""Get or create the shared embedder instance.
@@ -39,7 +39,7 @@ async def get_shared_embedder(
Thread-safe singleton that loads the model only once.
Args:
model: Embedding model name (default: nomic-ai/nomic-embed-text-v1.5)
model: Embedding model name (default: BAAI/bge-base-en-v1.5)
device: Device to use (None = auto-detect)
Returns:
roboco/services/repositories/__init__.py
+2
View File
@@ -6,6 +6,7 @@ for database operations. Reduces boilerplate in services.
"""
from roboco.services.repositories.base import BaseRepository
from roboco.services.repositories.indexed_document import IndexedDocumentRepository
from roboco.services.repositories.query_helpers import (
agent_id_filter,
get_agent_slug,
@@ -19,6 +20,7 @@ from roboco.services.repositories.query_helpers import (
__all__ = [
"BaseRepository",
"IndexedDocumentRepository",
"agent_id_filter",
"get_agent_slug",
"pagination",
roboco/services/repositories/indexed_document.py
+109
View File
@@ -0,0 +1,109 @@
"""
Indexed Document Repository
Repository for managing indexed documents in the knowledge base.
"""
import hashlib
from typing import Any
from sqlalchemy import select
from roboco.db.tables import IndexedDocumentTable
from roboco.services.repositories.base import BaseRepository
class IndexedDocumentRepository(BaseRepository[IndexedDocumentTable]):
"""Repository for indexed document operations."""
model = IndexedDocumentTable
model_name = "IndexedDocument"
async def upsert_batch(
self,
index_type: str,
documents: list[dict[str, Any]],
) -> int:
"""
Upsert multiple documents in a single transaction.
Args:
index_type: The index type (code, documentation, standards, etc.)
documents: List of document dicts with keys:
- source: Source path/URI (required)
- title: Document title
- preview: Content preview (truncated to 500 chars)
- metadata: Additional metadata dict
Returns:
Number of documents upserted
"""
if not documents:
return 0
count = 0
for doc_info in documents:
source = doc_info["source"]
title = doc_info.get("title")
preview = doc_info.get("preview")
metadata = doc_info.get("metadata")
source_hash = hashlib.sha256(source.encode()).hexdigest()
existing = await self.session.execute(
select(IndexedDocumentTable).where(
IndexedDocumentTable.index_type == index_type,
IndexedDocumentTable.source_hash == source_hash,
)
)
doc = existing.scalar_one_or_none()
if doc:
# Update existing
if title:
doc.title = title
if preview:
doc.preview = preview[:500]
if metadata:
doc.extra_data = {**(doc.extra_data or {}), **metadata}
else:
# Insert new
doc = IndexedDocumentTable(
index_type=index_type,
source=source,
source_hash=source_hash,
title=title,
preview=preview[:500] if preview else None,
extra_data=metadata or {},
)
self.session.add(doc)
count += 1
await self.session.flush()
return count
async def get_by_index_type(
self,
index_type: str,
limit: int = 100,
offset: int = 0,
) -> list[IndexedDocumentTable]:
"""Get all documents for an index type."""
return await self.find_by(
IndexedDocumentTable.index_type == index_type,
limit=limit,
offset=offset,
)
async def count_by_index_type(self, index_type: str) -> int:
"""Count documents in an index type."""
return await self.count(IndexedDocumentTable.index_type == index_type)
async def delete_by_index_type(self, index_type: str) -> int:
"""Delete all documents for an index type."""
docs = await self.get_by_index_type(index_type, limit=10000)
for doc in docs:
await self.session.delete(doc)
await self.session.flush()
return len(docs)
roboco/services/task.py
+680 -18
View File
@@ -5,6 +5,7 @@ Provides CRUD operations and lifecycle management for tasks.
Handles status transitions, assignments, and queries.
"""
import asyncio
from datetime import UTC, datetime
from typing import Any, ClassVar, cast
from uuid import UUID
@@ -131,6 +132,7 @@ class TaskService(BaseService):
"""
service_name: ClassVar[str] = "task"
_background_tasks: ClassVar[set[asyncio.Task[None]]] = set()
# =========================================================================
# STATUS TRANSITION HELPER
@@ -522,40 +524,534 @@ class TaskService(BaseService):
await self.session.flush()
# Trigger proactive knowledge injection (fire and forget)
await self._inject_proactive_context(task, agent_id)
# Trigger proactive knowledge injection (fire-and-forget)
bg_task = asyncio.create_task(self._inject_proactive_context(task, agent_id))
self._background_tasks.add(bg_task)
bg_task.add_done_callback(self._background_tasks.discard)
return task
async def _inject_proactive_context(self, task: TaskTable, agent_id: UUID) -> None:
"""Inject proactive knowledge context when task is claimed."""
"""Inject proactive knowledge context when task is claimed.
Runs as a background task, so uses its own database session.
"""
from uuid import UUID as PyUUID
from roboco.db.base import get_session_factory
from roboco.services.proactive import get_proactive_service
task_id = PyUUID(str(task.id))
task_title = task.title
task_description = task.description or ""
try:
from uuid import UUID as PyUUID
from roboco.services.proactive import get_proactive_service
proactive = await get_proactive_service()
# Convert SQLAlchemy UUID to Python UUID
task_uuid = PyUUID(str(task.id))
agent_uuid = PyUUID(str(agent_id))
context = await proactive.on_task_claimed(
task_id=task_uuid,
task_id=task_id,
agent_id=agent_uuid,
task_title=task.title,
task_description=task.description or "",
task_type=None, # TaskTable doesn't have task_type
task_title=task_title,
task_description=task_description,
task_type=None,
)
if context and not context.is_empty():
# Store context in the task using a fresh session
session_factory = get_session_factory()
async with session_factory() as session:
from sqlalchemy import update
await session.execute(
update(TaskTable)
.where(TaskTable.id == task_id)
.values(proactive_context=context.to_dict())
)
await session.commit()
self.log.info(
"Injected proactive context",
task_id=str(task.id),
"Stored proactive context",
task_id=str(task_id),
items=len(context.similar_tasks)
+ len(context.relevant_learnings)
+ len(context.code_patterns),
)
except Exception as e:
# Don't fail the claim if proactive injection fails
# Don't fail - this is fire-and-forget
self.log.warning(
"Failed to inject proactive context",
task_id=str(task.id),
task_id=str(task_id),
error=str(e),
)
# =========================================================================
# RAG AUTO-INDEXING HOOKS (Fire-and-forget background tasks)
# =========================================================================
# Learning extraction thresholds
_DURATION_OVER_RATIO = 1.5 # Flag if task took 1.5x expected time
_DURATION_UNDER_RATIO = 0.3 # Flag if task took less than 30% expected
_MIN_COMMITS_GOOD = 5 # Minimum commits for "good granularity" pattern
_MIN_NOTES_LENGTH = 50 # Minimum notes length to extract learnings
async def _extract_completion_learnings(
self, task: TaskTable, agent_id: UUID | None
) -> None:
"""Extract and record learnings from a completed task (fire-and-forget)."""
from roboco.services.learning import (
LearningType,
RecordLearningParams,
get_learning_service,
)
# Extract data before session detaches
task_id = task.id
task_title = task.title
task_team = task.team.value if task.team else None
started_at = task.started_at
completed_at = task.completed_at
estimated_complexity = task.estimated_complexity
commits = list(task.commits) if task.commits else []
dev_notes = task.dev_notes
qa_notes = task.qa_notes
assigned_to = task.assigned_to
try:
learning_svc = await get_learning_service()
learnings: list[tuple[str, LearningType]] = []
# Determine scope based on team
scope = self._determine_learning_scope(task_team)
# 1. Duration vs estimate insight
if started_at and completed_at:
duration_hours = (completed_at - started_at).total_seconds() / 3600
complexity_hours = {"low": 2.0, "medium": 8.0, "high": 24.0}
complexity_val = (
estimated_complexity.value
if hasattr(estimated_complexity, "value")
else str(estimated_complexity)
)
expected = complexity_hours.get(complexity_val, 8.0)
ratio = duration_hours / expected if expected > 0 else 1.0
if ratio > self._DURATION_OVER_RATIO:
msg = (
f"Task '{task_title}' ({complexity_val}) took "
f"{duration_hours:.1f}h vs expected {expected:.0f}h."
)
learnings.append((msg, LearningType.INSIGHT))
elif ratio < self._DURATION_UNDER_RATIO:
msg = (
f"Task '{task_title}' ({complexity_val}) completed "
f"quickly in {duration_hours:.1f}h."
)
learnings.append((msg, LearningType.INSIGHT))
# 2. Commit pattern analysis
if len(commits) >= self._MIN_COMMITS_GOOD:
msg = f"Good commit granularity on '{task_title}': {len(commits)}."
learnings.append((msg, LearningType.PATTERN))
elif len(commits) == 1:
learnings.append(
(
f"Single commit on '{task_title}'. Try smaller increments.",
LearningType.GOTCHA,
)
)
# 3. Extract from dev_notes
if dev_notes and len(dev_notes) > self._MIN_NOTES_LENGTH:
learnings.append(
(
f"[DEV NOTES] {task_title}: {dev_notes[:500]}",
LearningType.SOLUTION,
)
)
# 4. Extract from qa_notes
if qa_notes and len(qa_notes) > self._MIN_NOTES_LENGTH:
learnings.append(
(
f"[QA FEEDBACK] {task_title}: {qa_notes[:500]}",
LearningType.REVIEW_FEEDBACK,
)
)
# Record all learnings
for content, ltype in learnings:
await learning_svc.record_learning(
RecordLearningParams(
agent_id=assigned_to or agent_id or UUID(int=0),
agent_role="developer",
content=content,
learning_type=ltype,
scope=scope,
task_id=task_id,
tags=["auto-extracted", task_team or "general"],
)
)
if learnings:
self.log.info(
"Extracted completion learnings",
task_id=str(task_id),
count=len(learnings),
)
except Exception as e:
self.log.warning(
"Failed to extract learnings",
task_id=str(task_id),
error=str(e),
)
def _determine_learning_scope(self, team: str | None) -> Any:
"""Map team to learning scope."""
from roboco.services.learning import LearningScope
if team in ("backend", "frontend", "ux_ui"):
return LearningScope.CELL
if team in ("board", "main_pm"):
return LearningScope.ORG
return LearningScope.TEAM
async def _index_code_changes_background(
self, task_id: UUID, commits: list[dict[str, Any]], project: str
) -> None:
"""Index code files from task commits (fire-and-forget)."""
from roboco.services.optimal import get_optimal_service
try:
optimal = await get_optimal_service()
# Extract unique file paths from commits
files: set[str] = set()
for commit in commits:
commit_files = commit.get("files", [])
if isinstance(commit_files, list):
files.update(str(f) for f in commit_files)
if files:
count = await optimal.index_code(list(files), project=project)
self.log.debug(
"Indexed code files",
task_id=str(task_id),
files_count=count,
)
except Exception as e:
self.log.warning(
"Failed to index code",
task_id=str(task_id),
error=str(e),
)
def _extract_decisions_from_notes(
self, notes: str, task_title: str
) -> list[dict[str, str]]:
"""Parse notes for decision patterns."""
decisions = []
decision_patterns = [
"decided to",
"chose",
"decision:",
"went with",
"selected",
"opted for",
"rationale:",
"instead of",
]
notes_lower = notes.lower()
for pattern in decision_patterns:
if pattern in notes_lower:
lines = notes.split(".")
for line in lines:
if pattern in line.lower():
decisions.append(
{
"topic": task_title,
"decision": line.strip()[:300],
"rationale": "Auto-extracted from task notes",
}
)
break
return decisions
async def _index_decisions_background(
self,
task_id: UUID,
task_title: str,
task_team: Team | None,
dev_notes: str | None,
agent_id: UUID | None,
) -> None:
"""Index decisions detected in notes (fire-and-forget)."""
from roboco.models.optimal import IndexDecisionParams
from roboco.services.optimal import get_optimal_service
if not dev_notes:
return
try:
optimal = await get_optimal_service()
decisions = self._extract_decisions_from_notes(dev_notes, task_title)
for decision in decisions:
await optimal.index_decision(
IndexDecisionParams(
topic=decision["topic"],
decision=decision["decision"],
rationale=decision["rationale"],
agent_id=agent_id,
task_id=task_id,
scope="team",
tags=[task_team.value if task_team else "general", "auto"],
)
)
if decisions:
self.log.debug(
"Indexed decisions",
task_id=str(task_id),
count=len(decisions),
)
except Exception as e:
self.log.warning(
"Failed to index decisions",
task_id=str(task_id),
error=str(e),
)
async def _index_docs_background(
self, task_id: UUID, documents: list[dict[str, Any]]
) -> None:
"""Index documentation from completed doc task (fire-and-forget)."""
from roboco.services.optimal import get_optimal_service
try:
optimal = await get_optimal_service()
# Extract doc paths from documents array
doc_paths: list[str] = [
str(d.get("path")) for d in documents if d.get("path")
]
if doc_paths:
count = await optimal.index_documentation(doc_paths, project="roboco")
self.log.debug(
"Indexed docs",
task_id=str(task_id),
docs_count=count,
)
except Exception as e:
self.log.warning(
"Failed to index docs",
task_id=str(task_id),
error=str(e),
)
# =========================================================================
# QA AND ERROR INDEXING HOOKS
# =========================================================================
def _parse_qa_notes(self, qa_notes: str) -> list[dict[str, str]]:
"""Parse QA notes into structured issues."""
issues = []
for raw_line in qa_notes.split("\n"):
stripped = raw_line.strip()
if stripped.startswith(("-", "*", "•")):
issues.append(
{
"severity": "error",
"description": stripped.lstrip("-*• "),
}
)
elif stripped and stripped[0].isdigit() and "." in stripped[:3]:
parts = stripped.split(".", 1)
desc = parts[1].strip() if len(parts) > 1 else stripped
issues.append({"severity": "error", "description": desc})
if not issues and qa_notes.strip():
issues.append({"severity": "error", "description": qa_notes[:500]})
return issues
async def _index_qa_review_background(
self,
task_id: UUID,
quick_context: str | None,
passed: bool,
qa_notes: str,
qa_agent_id: UUID | None,
) -> None:
"""Index QA review (fire-and-forget)."""
from roboco.models.optimal import IndexReviewParams
from roboco.services.optimal import get_optimal_service
try:
optimal = await get_optimal_service()
original_dev = extract_original_developer(quick_context)
await optimal.record_review(
IndexReviewParams(
file_path=f"task/{task_id}",
comments=[
{
"body": qa_notes,
"type": "qa",
"severity": "info" if passed else "error",
}
],
approved=passed,
summary=qa_notes[:500] if qa_notes else "QA Review",
reviewer_id=qa_agent_id,
author_id=UUID(original_dev) if original_dev else None,
task_id=task_id,
)
)
self.log.debug("Indexed QA review", task_id=str(task_id), passed=passed)
except Exception as e:
self.log.warning(
"Failed to index QA review",
task_id=str(task_id),
error=str(e),
)
async def _index_qa_errors_background(
self,
task_id: UUID,
task_title: str,
task_team: Team | None,
qa_notes: str,
) -> None:
"""Index QA failure issues as error patterns (fire-and-forget)."""
from roboco.models.optimal import IndexErrorParams
from roboco.services.optimal import get_optimal_service
try:
optimal = await get_optimal_service()
issues = self._parse_qa_notes(qa_notes)
for issue in issues:
await optimal.index_error(
IndexErrorParams(
error_message=f"QA Failure: {issue['description'][:200]}",
context=f"Task: {task_title}",
solution="",
worked=False,
task_id=task_id,
team=task_team.value if task_team else None,
tags=["qa_failure", issue["severity"]],
)
)
self.log.debug(
"Indexed QA errors",
task_id=str(task_id),
count=len(issues),
)
except Exception as e:
self.log.warning(
"Failed to index QA errors",
task_id=str(task_id),
error=str(e),
)
async def _index_blocker_background(
self,
task_id: UUID,
task_team: Team | None,
blocker_info: dict[str, str],
) -> None:
"""Index blocker as error pattern (fire-and-forget).
Args:
task_id: Task UUID
task_team: Team for categorization
blocker_info: Dict with keys: type, title, reason, what_needed
"""
from roboco.models.optimal import IndexErrorParams
from roboco.services.optimal import get_optimal_service
try:
optimal = await get_optimal_service()
blocker_type = blocker_info.get("type", "unknown")
reason = blocker_info.get("reason", "")
title = blocker_info.get("title", "")
what_needed = blocker_info.get("what_needed", "")
await optimal.index_error(
IndexErrorParams(
error_message=f"Blocker ({blocker_type}): {reason[:200]}",
context=f"Task: {title}\nNeeded: {what_needed}",
solution="",
worked=False,
task_id=task_id,
team=task_team.value if task_team else None,
tags=["blocker", blocker_type.lower()],
)
)
self.log.debug("Indexed blocker", task_id=str(task_id))
except Exception as e:
self.log.warning(
"Failed to index blocker",
task_id=str(task_id),
error=str(e),
)
async def _index_lifecycle_event_background(
self,
task_id: UUID,
event_type: str,
task_title: str,
task_team: Team | None,
details: dict[str, Any] | None = None,
) -> None:
"""Index lifecycle event for pattern analysis (fire-and-forget).
Tracks task state transitions for organizational learning:
- Cancellation patterns (what gets cancelled and why)
- Pause/resume patterns (context switching costs)
- Block/unblock patterns (dependency bottlenecks)
Args:
task_id: Task UUID
event_type: One of: cancel, pause, resume, block, unblock
task_title: Task title for context
task_team: Team for categorization
details: Additional event details
"""
from roboco.services.optimal import IndexType, get_optimal_service
try:
optimal = await get_optimal_service()
details = details or {}
# Build content for indexing
content = f"[{event_type.upper()}] {task_title}"
if details:
content += f"\n{details}"
# Index to journals for lifecycle tracking
await optimal.ingest(
index_type=IndexType.JOURNALS,
content=content,
doc_id=f"lifecycle-{task_id}-{event_type}",
task_id=task_id,
project=task_team.value if task_team else "default",
entry_type="lifecycle",
event_type=event_type,
**details,
)
self.log.debug(
"Indexed lifecycle event",
task_id=str(task_id),
event_type=event_type,
)
except Exception as e:
self.log.warning(
"Failed to index lifecycle event",
task_id=str(task_id),
event_type=event_type,
error=str(e),
)
@@ -651,6 +1147,24 @@ class TaskService(BaseService):
task_id=str(task_id),
blocker_id=str(blocker_task_id),
)
# Index lifecycle event (fire-and-forget)
blocker_title = blocker.title if blocker else "unknown"
bg_task = asyncio.create_task(
self._index_lifecycle_event_background(
task_id=task_id,
event_type="block",
task_title=task.title,
task_team=task.team,
details={
"blocker_task_id": str(blocker_task_id),
"blocker_title": blocker_title,
},
)
)
self._background_tasks.add(bg_task)
bg_task.add_done_callback(self._background_tasks.discard)
return task
async def soft_block(
@@ -700,6 +1214,19 @@ class TaskService(BaseService):
task.status = TaskStatus.BLOCKED
await self.session.flush()
# Index blocker as error pattern (fire-and-forget)
blocker_info = {
"type": blocker_type,
"title": task.title,
"reason": reason,
"what_needed": what_needed,
}
bg_task = asyncio.create_task(
self._index_blocker_background(task.id, task.team, blocker_info)
)
self._background_tasks.add(bg_task)
bg_task.add_done_callback(self._background_tasks.discard)
self.log.info(
"Task soft-blocked",
task_id=str(task_id),
@@ -721,6 +1248,19 @@ class TaskService(BaseService):
await self.session.flush()
self.log.info("Task unblocked", task_id=str(task_id))
# Index lifecycle event (fire-and-forget)
bg_task = asyncio.create_task(
self._index_lifecycle_event_background(
task_id=task_id,
event_type="unblock",
task_title=task.title,
task_team=task.team,
)
)
self._background_tasks.add(bg_task)
bg_task.add_done_callback(self._background_tasks.discard)
return task
async def pause(self, task_id: UUID) -> TaskTable | None:
@@ -736,6 +1276,19 @@ class TaskService(BaseService):
await self.session.flush()
self.log.info("Task paused", task_id=str(task_id))
# Index lifecycle event (fire-and-forget)
bg_task = asyncio.create_task(
self._index_lifecycle_event_background(
task_id=task_id,
event_type="pause",
task_title=task.title,
task_team=task.team,
)
)
self._background_tasks.add(bg_task)
bg_task.add_done_callback(self._background_tasks.discard)
return task
async def resume(self, task_id: UUID) -> TaskTable | None:
@@ -751,6 +1304,19 @@ class TaskService(BaseService):
await self.session.flush()
self.log.info("Task resumed", task_id=str(task_id))
# Index lifecycle event (fire-and-forget)
bg_task = asyncio.create_task(
self._index_lifecycle_event_background(
task_id=task_id,
event_type="resume",
task_title=task.title,
task_team=task.team,
)
)
self._background_tasks.add(bg_task)
bg_task.add_done_callback(self._background_tasks.discard)
return task
async def submit_for_verification(self, task_id: UUID) -> TaskTable | None:
@@ -821,12 +1387,29 @@ class TaskService(BaseService):
if notes:
task.qa_notes = notes
# Store QA agent before clearing assignment
qa_agent_id = task.assigned_to
# Clear assignment so documenter can claim the task
task.assigned_to = None
task.qa_verified = True
task.status = TaskStatus.AWAITING_DOCUMENTATION
await self.session.flush()
# Index positive QA review (fire-and-forget)
bg_task = asyncio.create_task(
self._index_qa_review_background(
task.id,
task.quick_context,
True,
notes or "Passed QA review",
qa_agent_id,
)
)
self._background_tasks.add(bg_task)
bg_task.add_done_callback(self._background_tasks.discard)
self.log.info("Task passed QA", task_id=str(task_id))
return task
@@ -858,6 +1441,9 @@ class TaskService(BaseService):
task.qa_verified = False
task.status = TaskStatus.NEEDS_REVISION
# Store QA agent before reassigning
qa_agent_id = task.assigned_to
# Reassign to original developer so they can work on revisions
original_dev = extract_original_developer(task.quick_context)
if original_dev:
@@ -877,6 +1463,26 @@ class TaskService(BaseService):
await self.session.flush()
# Index negative QA review (fire-and-forget)
review_task = asyncio.create_task(
self._index_qa_review_background(
task.id,
task.quick_context,
False,
notes,
qa_agent_id,
)
)
self._background_tasks.add(review_task)
review_task.add_done_callback(self._background_tasks.discard)
# Index issues as error patterns (fire-and-forget)
error_task = asyncio.create_task(
self._index_qa_errors_background(task.id, task.title, task.team, notes)
)
self._background_tasks.add(error_task)
error_task.add_done_callback(self._background_tasks.discard)
self.log.info("Task failed QA", task_id=str(task_id))
return task
@@ -946,6 +1552,14 @@ class TaskService(BaseService):
task.assigned_to = None
await self.session.flush()
# Index documentation artifacts (fire-and-forget)
if task.documents:
bg_task = asyncio.create_task(
self._index_docs_background(task.id, task.documents)
)
self._background_tasks.add(bg_task)
bg_task.add_done_callback(self._background_tasks.discard)
self.log.info(
"Documentation complete, awaiting PM review",
task_id=str(task_id),
@@ -1108,6 +1722,36 @@ class TaskService(BaseService):
self._validate_and_set_status(task, TaskStatus.COMPLETED, "cell_pm")
await self.session.flush()
# RAG auto-indexing hooks (fire-and-forget)
# 1. Extract completion learnings
bg_task = asyncio.create_task(
self._extract_completion_learnings(task, agent_id)
)
self._background_tasks.add(bg_task)
bg_task.add_done_callback(self._background_tasks.discard)
# 2. Index code changes from commits
if task.commits:
code_task = asyncio.create_task(
self._index_code_changes_background(
task.id,
task.commits,
task.team.value if task.team else "default",
)
)
self._background_tasks.add(code_task)
code_task.add_done_callback(self._background_tasks.discard)
# 3. Detect and index decisions from notes
if task.dev_notes:
decision_task = asyncio.create_task(
self._index_decisions_background(
task.id, task.title, task.team, task.dev_notes, task.assigned_to
)
)
self._background_tasks.add(decision_task)
decision_task.add_done_callback(self._background_tasks.discard)
# Unblock any tasks waiting on this one
await self._unblock_dependents(task_id)
return task
@@ -1121,10 +1765,11 @@ class TaskService(BaseService):
return None
# Cancel all descendants first (children, grandchildren, etc.)
# Skip tasks already in terminal states (completed or cancelled)
descendants = await self.get_all_descendants(task_id)
cancelled_count = 0
for descendant in descendants:
if descendant.status != TaskStatus.CANCELLED:
if descendant.status not in (TaskStatus.COMPLETED, TaskStatus.CANCELLED):
descendant.status = TaskStatus.CANCELLED
cancelled_count += 1
@@ -1138,6 +1783,23 @@ class TaskService(BaseService):
# Validate transition with PM role requirement
self._validate_and_set_status(task, TaskStatus.CANCELLED, agent_role)
await self.session.flush()
# Index lifecycle event (fire-and-forget)
bg_task = asyncio.create_task(
self._index_lifecycle_event_background(
task_id=task_id,
event_type="cancel",
task_title=task.title,
task_team=task.team,
details={
"cancelled_by_role": agent_role,
"descendants_cancelled": cancelled_count,
},
)
)
self._background_tasks.add(bg_task)
bg_task.add_done_callback(self._background_tasks.discard)
return task
async def _unblock_dependents(self, completed_task_id: UUID) -> None: