External/roboco
1
0
Fork 0
mirror of https://github.com/rennf93/roboco.git synced 2026-08-03 07:23:24 +02:00
Code Issues Packages Projects Releases Wiki Activity

fix(prompter): use the local Ollama LLM instead of the Anthropic cloud API

Browse Source 
The Prompter service called AsyncAnthropic, which needs an ANTHROPIC_API_KEY
and 500'd in production ("Anthropic API key not configured"). Align it with the
rest of the system (RAG / HyDE): call the local LLM over the OpenAI-compatible
/chat/completions endpoint (settings.local_llm_*) via httpx — no external key.

- Replace the Anthropic client and _extract_text with a single _create_message
  seam that POSTs to the local LLM and returns the reply text.
- Move the system prompt into an OpenAI-style system message; drop the
  hard-coded Claude model and the per-call model parameter.
- Strip a wrapping markdown code fence before parsing the draft JSON (local
  models often wrap it).
- Repoint the prompter unit and route tests at the new seam (return strings).
This commit is contained in:
Renn F
2026-06-08 17:04:35 +02:00
parent f2df7fc30b
commit ae65bff883
3 changed files with 128 additions and 203 deletions
Download Patch File Download Diff File Expand all files Collapse all files
roboco/services/prompter.py
+48 -42
View File
@@ -2,8 +2,9 @@
Prompter Service
Conversational LLM assistant that helps users draft tasks.
Uses Anthropic Claude for natural-language interaction and
structured JSON draft generation.
Uses the project's local LLM (Ollama, OpenAI-compatible) for
natural-language interaction and structured JSON draft generation —
the same engine as RAG/HyDE, so no external API key is required.
Provides both a session-based approach (DB-persisted) and a
legacy stateless interface for backward compatibility.
@@ -18,8 +19,8 @@ from datetime import UTC, datetime
from typing import TYPE_CHECKING, Any
from uuid import UUID, uuid4
import httpx
import structlog
from anthropic import AsyncAnthropic
from sqlalchemy import select
from roboco.config import settings
@@ -124,26 +125,31 @@ class PrompterService:
def __init__(self, db: AsyncSession | None = None) -> None:
self.log = logger.bind(component="prompter_service")
self._client: AsyncAnthropic | None = None
self._db = db
def _get_client(self) -> AsyncAnthropic:
"""Lazy-init Anthropic client."""
if self._client is None:
api_key = settings.anthropic_api_key
if not api_key:
raise ServiceError("Anthropic API key not configured")
self._client = AsyncAnthropic(api_key=api_key)
return self._client
async def _create_message(
self, *, messages: list[dict[str, str]], max_tokens: int
) -> str:
"""Call the local LLM and return the reply text.
async def _create_message(self, **kwargs: Any) -> Any:
"""Single seam for the Anthropic ``messages.create`` call.
The SDK exposes ``messages`` as a cached_property, so it can't be
patched at the client-class level; tests substitute this method.
Uses the project's local LLM — the same OpenAI-compatible Ollama
endpoint as RAG/HyDE (``settings.local_llm_*``), so no external API key
is required. ``messages`` is an OpenAI-style list (system + turns). This
is the single seam the prompter tests substitute.
"""
client = self._get_client()
return await client.messages.create(**kwargs)
async with httpx.AsyncClient(timeout=120.0) as client:
resp = await client.post(
f"{settings.local_llm_base_url}/chat/completions",
json={
"model": settings.local_llm_model,
"messages": messages,
"max_tokens": max_tokens,
"options": {"num_ctx": 8192},
},
)
resp.raise_for_status()
data = resp.json()
return str(data["choices"][0]["message"]["content"] or "").strip()
@property
def _session(self) -> AsyncSession:
@@ -438,23 +444,22 @@ class PrompterService:
self,
messages: list[dict[str, str]],
context: dict[str, Any] | None = None,
model: str = "claude-3-5-sonnet-20241022",
max_tokens: int = 2048,
) -> dict[str, Any]:
"""Call the LLM for a chat response. Returns {message, draft_ready}."""
user_prompt = _build_chat_prompt(messages, context)
try:
response = await self._create_message(
model=model,
content = await self._create_message(
messages=[
{"role": "system", "content": _PROMPTER_SYSTEM_PROMPT},
{"role": "user", "content": user_prompt},
],
max_tokens=max_tokens,
system=_PROMPTER_SYSTEM_PROMPT,
messages=[{"role": "user", "content": user_prompt}],
)
except Exception as e:
self.log.error("Prompter chat LLM call failed", error=str(e))
raise ServiceError(f"LLM chat failed: {e}") from e
content = _extract_text(response)
if not content:
raise ServiceError("LLM returned empty content")
@@ -467,28 +472,27 @@ class PrompterService:
self,
messages: list[dict[str, str]],
context: dict[str, Any] | None = None,
model: str = "claude-3-5-sonnet-20241022",
max_tokens: int = 4096,
) -> dict[str, Any]:
"""Call the LLM to generate a structured draft. Returns {draft, reasoning}."""
user_prompt = _build_draft_prompt(messages, context)
try:
response = await self._create_message(
model=model,
content = await self._create_message(
messages=[
{"role": "system", "content": _DRAFT_SYSTEM_PROMPT},
{"role": "user", "content": user_prompt},
],
max_tokens=max_tokens,
system=_DRAFT_SYSTEM_PROMPT,
messages=[{"role": "user", "content": user_prompt}],
)
except Exception as e:
self.log.error("Prompter draft LLM call failed", error=str(e))
raise ServiceError(f"LLM draft generation failed: {e}") from e
content = _extract_text(response)
if not content:
raise ServiceError("LLM returned empty content for draft")
try:
draft_data = json.loads(content)
draft_data = json.loads(_strip_code_fences(content))
except json.JSONDecodeError as e:
self.log.warning("Draft JSON parse failed", content_preview=content[:200])
raise ValidationError(
@@ -511,14 +515,12 @@ class PrompterService:
self,
messages: list[dict[str, str]],
context: dict[str, Any] | None = None,
model: str = "claude-3-5-sonnet-20241022",
max_tokens: int = 2048,
) -> dict[str, Any]:
"""Continue a Prompter conversation (stateless)."""
return await self._llm_chat(
messages=messages,
context=context,
model=model,
max_tokens=max_tokens,
)
@@ -526,14 +528,12 @@ class PrompterService:
self,
messages: list[dict[str, str]],
context: dict[str, Any] | None = None,
model: str = "claude-3-5-sonnet-20241022",
max_tokens: int = 4096,
) -> dict[str, Any]:
"""Generate a structured task draft from conversation context (stateless)."""
return await self._llm_draft(
messages=messages,
context=context,
model=model,
max_tokens=max_tokens,
)
@@ -590,12 +590,18 @@ def _build_draft_prompt(
return "\n".join(lines)
def _extract_text(response: Any) -> str:
text_parts: list[str] = []
for block in getattr(response, "content", []):
if hasattr(block, "text"):
text_parts.append(block.text)
return "\n".join(text_parts).strip()
def _strip_code_fences(content: str) -> str:
"""Strip a wrapping markdown code fence (```json ... ```) if present.
Local models often wrap JSON output in a fenced block; drop the opening
fence line and the closing fence so the body parses cleanly as JSON.
"""
text = content.strip()
if text.startswith("```"):
text = text.split("\n", 1)[1] if "\n" in text else text[3:]
if text.rstrip().endswith("```"):
text = text.rstrip()[:-3]
return text.strip()
def _detect_draft_ready(content: str) -> bool:
tests/integration/test_prompter_routes.py
+24 -56
View File
@@ -13,7 +13,7 @@ from __future__ import annotations
import json
from http import HTTPStatus
from typing import TYPE_CHECKING, Any
from unittest.mock import AsyncMock, MagicMock, patch
from unittest.mock import AsyncMock, patch
from uuid import uuid4
import pytest
@@ -163,8 +163,7 @@ async def test_send_message_success(prompter_client: dict) -> None:
session_resp = await client.post("/api/prompter/sessions", json={}, headers=_HDR)
session_id = session_resp.json()["id"]
mock_response = MagicMock()
mock_response.content = [MagicMock(text="Great! Let's gather requirements.")]
mock_response = "Great! Let's gather requirements."
with patch(
"roboco.services.prompter.PrompterService._create_message",
@@ -194,15 +193,9 @@ async def test_send_message_marks_draft_ready(prompter_client: dict) -> None:
session_resp = await client.post("/api/prompter/sessions", json={}, headers=_HDR)
session_id = session_resp.json()["id"]
mock_response = MagicMock()
mock_response.content = [
MagicMock(
text=(
"I have enough information to draft a task now. "
"Ready to draft when you are."
)
)
]
mock_response = (
"I have enough information to draft a task now. Ready to draft when you are."
)
with patch(
"roboco.services.prompter.PrompterService._create_message",
@@ -254,11 +247,9 @@ async def test_get_draft_generates_from_conversation(prompter_client: dict) -> N
"priority": 2,
}
chat_response = MagicMock()
chat_response.content = [MagicMock(text="Tell me more about the requirements.")]
chat_response = "Tell me more about the requirements."
draft_response = MagicMock()
draft_response.content = [MagicMock(text=json.dumps(draft_json))]
draft_response = json.dumps(draft_json)
with patch(
"roboco.services.prompter.PrompterService._create_message",
@@ -308,10 +299,8 @@ async def test_get_draft_cached(prompter_client: dict) -> None:
"estimated_complexity": "medium",
"priority": 2,
}
chat_response = MagicMock()
chat_response.content = [MagicMock(text="Got it.")]
draft_response = MagicMock()
draft_response.content = [MagicMock(text=json.dumps(draft_json))]
chat_response = "Got it."
draft_response = json.dumps(draft_json)
with patch(
"roboco.services.prompter.PrompterService._create_message",
@@ -326,7 +315,7 @@ async def test_get_draft_cached(prompter_client: dict) -> None:
call_count = 0
async def _mock_create(**_kwargs: Any) -> MagicMock:
async def _mock_create(**_kwargs: Any) -> str:
nonlocal call_count
call_count += 1
return draft_response
@@ -382,10 +371,8 @@ async def test_confirm_draft_creates_task(
"priority": 2,
}
chat_response = MagicMock()
chat_response.content = [MagicMock(text="Got it.")]
draft_response = MagicMock()
draft_response.content = [MagicMock(text=json.dumps(draft_json))]
chat_response = "Got it."
draft_response = json.dumps(draft_json)
with patch(
"roboco.services.prompter.PrompterService._create_message",
@@ -438,10 +425,8 @@ async def test_confirm_draft_requires_project_or_product(
"priority": 2,
}
chat_response = MagicMock()
chat_response.content = [MagicMock(text="Got it.")]
draft_response = MagicMock()
draft_response.content = [MagicMock(text=json.dumps(draft_json))]
chat_response = "Got it."
draft_response = json.dumps(draft_json)
with patch(
"roboco.services.prompter.PrompterService._create_message",
@@ -489,10 +474,7 @@ async def test_full_happy_path(
session_id = step1.json()["id"]
# Step 2: Send messages
chat_mock = MagicMock()
chat_mock.content = [
MagicMock(text="Please describe the acceptance criteria for this feature.")
]
chat_mock = "Please describe the acceptance criteria for this feature."
with patch(
"roboco.services.prompter.PrompterService._create_message",
@@ -506,10 +488,7 @@ async def test_full_happy_path(
)
assert step2a.status_code == HTTPStatus.OK
chat_mock2 = MagicMock()
chat_mock2.content = [
MagicMock(text="I have enough information to draft a task now.")
]
chat_mock2 = "I have enough information to draft a task now."
with patch(
"roboco.services.prompter.PrompterService._create_message",
new_callable=AsyncMock,
@@ -538,8 +517,7 @@ async def test_full_happy_path(
"estimated_complexity": "low",
"priority": 2,
}
draft_mock = MagicMock()
draft_mock.content = [MagicMock(text=json.dumps(draft_json))]
draft_mock = json.dumps(draft_json)
with patch(
"roboco.services.prompter.PrompterService._create_message",
@@ -574,8 +552,7 @@ async def test_full_happy_path(
async def test_prompter_chat_success(prompter_client: dict) -> None:
client = prompter_client["client"]
mock_response = MagicMock()
mock_response.content = [MagicMock(text="Great! Let's gather requirements.")]
mock_response = "Great! Let's gather requirements."
with patch(
"roboco.services.prompter.PrompterService._create_message",
@@ -600,15 +577,9 @@ async def test_prompter_chat_success(prompter_client: dict) -> None:
async def test_prompter_chat_draft_ready(prompter_client: dict) -> None:
client = prompter_client["client"]
mock_response = MagicMock()
mock_response.content = [
MagicMock(
text=(
"I have enough information. draft_ready=true."
" Ready to generate a draft."
)
)
]
mock_response = (
"I have enough information. draft_ready=true. Ready to generate a draft."
)
with patch(
"roboco.services.prompter.PrompterService._create_message",
@@ -672,8 +643,7 @@ async def test_prompter_draft_success(prompter_client: dict) -> None:
"priority": 2,
}
mock_response = MagicMock()
mock_response.content = [MagicMock(text=json.dumps(draft_json))]
mock_response = json.dumps(draft_json)
with patch(
"roboco.services.prompter.PrompterService._create_message",
@@ -702,8 +672,7 @@ async def test_prompter_draft_success(prompter_client: dict) -> None:
async def test_prompter_draft_invalid_json_from_llm(prompter_client: dict) -> None:
client = prompter_client["client"]
mock_response = MagicMock()
mock_response.content = [MagicMock(text="not valid json")]
mock_response = "not valid json"
with patch(
"roboco.services.prompter.PrompterService._create_message",
@@ -732,8 +701,7 @@ async def test_prompter_draft_schema_mismatch(prompter_client: dict) -> None:
"description": "too short",
}
mock_response = MagicMock()
mock_response.content = [MagicMock(text=json.dumps(bad_draft))]
mock_response = json.dumps(bad_draft)
with patch(
"roboco.services.prompter.PrompterService._create_message",
tests/unit/services/test_prompter.py
+56 -105
View File
@@ -8,7 +8,7 @@ from __future__ import annotations
import json
from typing import Any
from unittest.mock import AsyncMock, MagicMock, patch
from unittest.mock import AsyncMock, patch
from uuid import uuid4
import pytest
@@ -21,7 +21,6 @@ from roboco.services.prompter import (
_build_draft_prompt,
_build_reasoning,
_detect_draft_ready,
_extract_text,
get_prompter_service,
)
@@ -53,30 +52,6 @@ def test_detect_draft_ready_negative() -> None:
assert not _detect_draft_ready(text), f"Expected False for: {text!r}"
def test_extract_text_with_blocks() -> None:
block1 = MagicMock()
block1.text = "Hello, "
block2 = MagicMock()
block2.text = "world!"
response = MagicMock()
response.content = [block1, block2]
result = _extract_text(response)
assert result == "Hello, \nworld!"
def test_extract_text_empty_response() -> None:
response = MagicMock()
response.content = []
assert _extract_text(response) == ""
def test_extract_text_no_text_attr() -> None:
block = MagicMock(spec=[]) # no 'text' attribute
response = MagicMock()
response.content = [block]
assert _extract_text(response) == ""
def test_build_chat_prompt_basic() -> None:
messages = [
{"role": "user", "content": "I need a feature"},
@@ -138,14 +113,12 @@ def test_get_prompter_service_raises_without_db_for_session_methods() -> None:
async def test_chat_success_with_mock_llm() -> None:
service = get_prompter_service()
mock_response = MagicMock()
mock_response.content = [MagicMock(text="Great, let's continue!")]
with patch.object(service, "_get_client") as mock_get_client:
mock_client = AsyncMock()
mock_client.messages.create = AsyncMock(return_value=mock_response)
mock_get_client.return_value = mock_client
with patch.object(
service,
"_create_message",
new_callable=AsyncMock,
return_value="Great, let's continue!",
):
result = await service.chat(
messages=[{"role": "user", "content": "I need a feature"}]
)
@@ -158,16 +131,12 @@ async def test_chat_success_with_mock_llm() -> None:
async def test_chat_draft_ready_signal() -> None:
service = get_prompter_service()
mock_response = MagicMock()
mock_response.content = [
MagicMock(text="I have enough information. Ready to draft.")
]
with patch.object(service, "_get_client") as mock_get_client:
mock_client = AsyncMock()
mock_client.messages.create = AsyncMock(return_value=mock_response)
mock_get_client.return_value = mock_client
with patch.object(
service,
"_create_message",
new_callable=AsyncMock,
return_value="I have enough information. Ready to draft.",
):
result = await service.chat(
messages=[{"role": "user", "content": "I need a feature"}]
)
@@ -179,31 +148,29 @@ async def test_chat_draft_ready_signal() -> None:
async def test_chat_raises_on_empty_response() -> None:
service = get_prompter_service()
mock_response = MagicMock()
mock_response.content = [] # Empty content blocks
with patch.object(service, "_get_client") as mock_get_client:
mock_client = AsyncMock()
mock_client.messages.create = AsyncMock(return_value=mock_response)
mock_get_client.return_value = mock_client
with pytest.raises(ServiceError, match="LLM returned empty content"):
await service.chat(messages=[{"role": "user", "content": "Hello"}])
with (
patch.object(
service, "_create_message", new_callable=AsyncMock, return_value=""
),
pytest.raises(ServiceError, match="LLM returned empty content"),
):
await service.chat(messages=[{"role": "user", "content": "Hello"}])
@pytest.mark.asyncio
async def test_chat_raises_on_llm_error() -> None:
service = get_prompter_service()
with patch.object(service, "_get_client") as mock_get_client:
mock_client = AsyncMock()
mock_client.messages.create = AsyncMock(
side_effect=Exception("API unavailable")
)
mock_get_client.return_value = mock_client
with pytest.raises(ServiceError, match="LLM chat failed"):
await service.chat(messages=[{"role": "user", "content": "Hello"}])
with (
patch.object(
service,
"_create_message",
new_callable=AsyncMock,
side_effect=Exception("API unavailable"),
),
pytest.raises(ServiceError, match="LLM chat failed"),
):
await service.chat(messages=[{"role": "user", "content": "Hello"}])
@pytest.mark.asyncio
@@ -220,14 +187,12 @@ async def test_draft_success_with_mock_llm() -> None:
"estimated_complexity": "medium",
}
mock_response = MagicMock()
mock_response.content = [MagicMock(text=json.dumps(draft_data))]
with patch.object(service, "_get_client") as mock_get_client:
mock_client = AsyncMock()
mock_client.messages.create = AsyncMock(return_value=mock_response)
mock_get_client.return_value = mock_client
with patch.object(
service,
"_create_message",
new_callable=AsyncMock,
return_value=json.dumps(draft_data),
):
result = await service.draft(
messages=[{"role": "user", "content": "I need a login feature"}]
)
@@ -242,46 +207,32 @@ async def test_draft_success_with_mock_llm() -> None:
async def test_draft_raises_on_invalid_json() -> None:
service = get_prompter_service()
mock_response = MagicMock()
mock_response.content = [MagicMock(text="Not JSON at all")]
with patch.object(service, "_get_client") as mock_get_client:
mock_client = AsyncMock()
mock_client.messages.create = AsyncMock(return_value=mock_response)
mock_get_client.return_value = mock_client
with pytest.raises(ValidationError, match="not valid JSON"):
await service.draft(messages=[{"role": "user", "content": "Hello"}])
with (
patch.object(
service,
"_create_message",
new_callable=AsyncMock,
return_value="Not JSON at all",
),
pytest.raises(ValidationError, match="not valid JSON"),
):
await service.draft(messages=[{"role": "user", "content": "Hello"}])
@pytest.mark.asyncio
async def test_draft_raises_on_llm_error() -> None:
service = get_prompter_service()
with patch.object(service, "_get_client") as mock_get_client:
mock_client = AsyncMock()
mock_client.messages.create = AsyncMock(
side_effect=Exception("API unavailable")
)
mock_get_client.return_value = mock_client
with pytest.raises(ServiceError, match="LLM draft generation failed"):
await service.draft(messages=[{"role": "user", "content": "Hello"}])
# =============================================================================
# API key validation
# =============================================================================
def test_get_client_raises_without_api_key() -> None:
service = get_prompter_service()
service._client = None # Force fresh init
with patch("roboco.services.prompter.settings") as mock_settings:
mock_settings.anthropic_api_key = None
with pytest.raises(ServiceError, match="Anthropic API key not configured"):
service._get_client()
with (
patch.object(
service,
"_create_message",
new_callable=AsyncMock,
side_effect=Exception("API unavailable"),
),
pytest.raises(ServiceError, match="LLM draft generation failed"),
):
await service.draft(messages=[{"role": "user", "content": "Hello"}])
# =============================================================================