From ae65bff88307758b99ef5065ab9eb7afaff217aa Mon Sep 17 00:00:00 2001 From: Renn F Date: Mon, 8 Jun 2026 17:04:35 +0200 Subject: [PATCH] fix(prompter): use the local Ollama LLM instead of the Anthropic cloud API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Prompter service called AsyncAnthropic, which needs an ANTHROPIC_API_KEY and 500'd in production ("Anthropic API key not configured"). Align it with the rest of the system (RAG / HyDE): call the local LLM over the OpenAI-compatible /chat/completions endpoint (settings.local_llm_*) via httpx — no external key. - Replace the Anthropic client and _extract_text with a single _create_message seam that POSTs to the local LLM and returns the reply text. - Move the system prompt into an OpenAI-style system message; drop the hard-coded Claude model and the per-call model parameter. - Strip a wrapping markdown code fence before parsing the draft JSON (local models often wrap it). - Repoint the prompter unit and route tests at the new seam (return strings). --- roboco/services/prompter.py | 90 ++++++------ tests/integration/test_prompter_routes.py | 80 ++++------- tests/unit/services/test_prompter.py | 161 ++++++++-------------- 3 files changed, 128 insertions(+), 203 deletions(-) diff --git a/roboco/services/prompter.py b/roboco/services/prompter.py index 24c8ffff..30df872f 100644 --- a/roboco/services/prompter.py +++ b/roboco/services/prompter.py @@ -2,8 +2,9 @@ Prompter Service Conversational LLM assistant that helps users draft tasks. -Uses Anthropic Claude for natural-language interaction and -structured JSON draft generation. +Uses the project's local LLM (Ollama, OpenAI-compatible) for +natural-language interaction and structured JSON draft generation — +the same engine as RAG/HyDE, so no external API key is required. Provides both a session-based approach (DB-persisted) and a legacy stateless interface for backward compatibility. @@ -18,8 +19,8 @@ from datetime import UTC, datetime from typing import TYPE_CHECKING, Any from uuid import UUID, uuid4 +import httpx import structlog -from anthropic import AsyncAnthropic from sqlalchemy import select from roboco.config import settings @@ -124,26 +125,31 @@ class PrompterService: def __init__(self, db: AsyncSession | None = None) -> None: self.log = logger.bind(component="prompter_service") - self._client: AsyncAnthropic | None = None self._db = db - def _get_client(self) -> AsyncAnthropic: - """Lazy-init Anthropic client.""" - if self._client is None: - api_key = settings.anthropic_api_key - if not api_key: - raise ServiceError("Anthropic API key not configured") - self._client = AsyncAnthropic(api_key=api_key) - return self._client + async def _create_message( + self, *, messages: list[dict[str, str]], max_tokens: int + ) -> str: + """Call the local LLM and return the reply text. - async def _create_message(self, **kwargs: Any) -> Any: - """Single seam for the Anthropic ``messages.create`` call. - - The SDK exposes ``messages`` as a cached_property, so it can't be - patched at the client-class level; tests substitute this method. + Uses the project's local LLM — the same OpenAI-compatible Ollama + endpoint as RAG/HyDE (``settings.local_llm_*``), so no external API key + is required. ``messages`` is an OpenAI-style list (system + turns). This + is the single seam the prompter tests substitute. """ - client = self._get_client() - return await client.messages.create(**kwargs) + async with httpx.AsyncClient(timeout=120.0) as client: + resp = await client.post( + f"{settings.local_llm_base_url}/chat/completions", + json={ + "model": settings.local_llm_model, + "messages": messages, + "max_tokens": max_tokens, + "options": {"num_ctx": 8192}, + }, + ) + resp.raise_for_status() + data = resp.json() + return str(data["choices"][0]["message"]["content"] or "").strip() @property def _session(self) -> AsyncSession: @@ -438,23 +444,22 @@ class PrompterService: self, messages: list[dict[str, str]], context: dict[str, Any] | None = None, - model: str = "claude-3-5-sonnet-20241022", max_tokens: int = 2048, ) -> dict[str, Any]: """Call the LLM for a chat response. Returns {message, draft_ready}.""" user_prompt = _build_chat_prompt(messages, context) try: - response = await self._create_message( - model=model, + content = await self._create_message( + messages=[ + {"role": "system", "content": _PROMPTER_SYSTEM_PROMPT}, + {"role": "user", "content": user_prompt}, + ], max_tokens=max_tokens, - system=_PROMPTER_SYSTEM_PROMPT, - messages=[{"role": "user", "content": user_prompt}], ) except Exception as e: self.log.error("Prompter chat LLM call failed", error=str(e)) raise ServiceError(f"LLM chat failed: {e}") from e - content = _extract_text(response) if not content: raise ServiceError("LLM returned empty content") @@ -467,28 +472,27 @@ class PrompterService: self, messages: list[dict[str, str]], context: dict[str, Any] | None = None, - model: str = "claude-3-5-sonnet-20241022", max_tokens: int = 4096, ) -> dict[str, Any]: """Call the LLM to generate a structured draft. Returns {draft, reasoning}.""" user_prompt = _build_draft_prompt(messages, context) try: - response = await self._create_message( - model=model, + content = await self._create_message( + messages=[ + {"role": "system", "content": _DRAFT_SYSTEM_PROMPT}, + {"role": "user", "content": user_prompt}, + ], max_tokens=max_tokens, - system=_DRAFT_SYSTEM_PROMPT, - messages=[{"role": "user", "content": user_prompt}], ) except Exception as e: self.log.error("Prompter draft LLM call failed", error=str(e)) raise ServiceError(f"LLM draft generation failed: {e}") from e - content = _extract_text(response) if not content: raise ServiceError("LLM returned empty content for draft") try: - draft_data = json.loads(content) + draft_data = json.loads(_strip_code_fences(content)) except json.JSONDecodeError as e: self.log.warning("Draft JSON parse failed", content_preview=content[:200]) raise ValidationError( @@ -511,14 +515,12 @@ class PrompterService: self, messages: list[dict[str, str]], context: dict[str, Any] | None = None, - model: str = "claude-3-5-sonnet-20241022", max_tokens: int = 2048, ) -> dict[str, Any]: """Continue a Prompter conversation (stateless).""" return await self._llm_chat( messages=messages, context=context, - model=model, max_tokens=max_tokens, ) @@ -526,14 +528,12 @@ class PrompterService: self, messages: list[dict[str, str]], context: dict[str, Any] | None = None, - model: str = "claude-3-5-sonnet-20241022", max_tokens: int = 4096, ) -> dict[str, Any]: """Generate a structured task draft from conversation context (stateless).""" return await self._llm_draft( messages=messages, context=context, - model=model, max_tokens=max_tokens, ) @@ -590,12 +590,18 @@ def _build_draft_prompt( return "\n".join(lines) -def _extract_text(response: Any) -> str: - text_parts: list[str] = [] - for block in getattr(response, "content", []): - if hasattr(block, "text"): - text_parts.append(block.text) - return "\n".join(text_parts).strip() +def _strip_code_fences(content: str) -> str: + """Strip a wrapping markdown code fence (```json ... ```) if present. + + Local models often wrap JSON output in a fenced block; drop the opening + fence line and the closing fence so the body parses cleanly as JSON. + """ + text = content.strip() + if text.startswith("```"): + text = text.split("\n", 1)[1] if "\n" in text else text[3:] + if text.rstrip().endswith("```"): + text = text.rstrip()[:-3] + return text.strip() def _detect_draft_ready(content: str) -> bool: diff --git a/tests/integration/test_prompter_routes.py b/tests/integration/test_prompter_routes.py index 12897838..2942ad16 100644 --- a/tests/integration/test_prompter_routes.py +++ b/tests/integration/test_prompter_routes.py @@ -13,7 +13,7 @@ from __future__ import annotations import json from http import HTTPStatus from typing import TYPE_CHECKING, Any -from unittest.mock import AsyncMock, MagicMock, patch +from unittest.mock import AsyncMock, patch from uuid import uuid4 import pytest @@ -163,8 +163,7 @@ async def test_send_message_success(prompter_client: dict) -> None: session_resp = await client.post("/api/prompter/sessions", json={}, headers=_HDR) session_id = session_resp.json()["id"] - mock_response = MagicMock() - mock_response.content = [MagicMock(text="Great! Let's gather requirements.")] + mock_response = "Great! Let's gather requirements." with patch( "roboco.services.prompter.PrompterService._create_message", @@ -194,15 +193,9 @@ async def test_send_message_marks_draft_ready(prompter_client: dict) -> None: session_resp = await client.post("/api/prompter/sessions", json={}, headers=_HDR) session_id = session_resp.json()["id"] - mock_response = MagicMock() - mock_response.content = [ - MagicMock( - text=( - "I have enough information to draft a task now. " - "Ready to draft when you are." - ) - ) - ] + mock_response = ( + "I have enough information to draft a task now. Ready to draft when you are." + ) with patch( "roboco.services.prompter.PrompterService._create_message", @@ -254,11 +247,9 @@ async def test_get_draft_generates_from_conversation(prompter_client: dict) -> N "priority": 2, } - chat_response = MagicMock() - chat_response.content = [MagicMock(text="Tell me more about the requirements.")] + chat_response = "Tell me more about the requirements." - draft_response = MagicMock() - draft_response.content = [MagicMock(text=json.dumps(draft_json))] + draft_response = json.dumps(draft_json) with patch( "roboco.services.prompter.PrompterService._create_message", @@ -308,10 +299,8 @@ async def test_get_draft_cached(prompter_client: dict) -> None: "estimated_complexity": "medium", "priority": 2, } - chat_response = MagicMock() - chat_response.content = [MagicMock(text="Got it.")] - draft_response = MagicMock() - draft_response.content = [MagicMock(text=json.dumps(draft_json))] + chat_response = "Got it." + draft_response = json.dumps(draft_json) with patch( "roboco.services.prompter.PrompterService._create_message", @@ -326,7 +315,7 @@ async def test_get_draft_cached(prompter_client: dict) -> None: call_count = 0 - async def _mock_create(**_kwargs: Any) -> MagicMock: + async def _mock_create(**_kwargs: Any) -> str: nonlocal call_count call_count += 1 return draft_response @@ -382,10 +371,8 @@ async def test_confirm_draft_creates_task( "priority": 2, } - chat_response = MagicMock() - chat_response.content = [MagicMock(text="Got it.")] - draft_response = MagicMock() - draft_response.content = [MagicMock(text=json.dumps(draft_json))] + chat_response = "Got it." + draft_response = json.dumps(draft_json) with patch( "roboco.services.prompter.PrompterService._create_message", @@ -438,10 +425,8 @@ async def test_confirm_draft_requires_project_or_product( "priority": 2, } - chat_response = MagicMock() - chat_response.content = [MagicMock(text="Got it.")] - draft_response = MagicMock() - draft_response.content = [MagicMock(text=json.dumps(draft_json))] + chat_response = "Got it." + draft_response = json.dumps(draft_json) with patch( "roboco.services.prompter.PrompterService._create_message", @@ -489,10 +474,7 @@ async def test_full_happy_path( session_id = step1.json()["id"] # Step 2: Send messages - chat_mock = MagicMock() - chat_mock.content = [ - MagicMock(text="Please describe the acceptance criteria for this feature.") - ] + chat_mock = "Please describe the acceptance criteria for this feature." with patch( "roboco.services.prompter.PrompterService._create_message", @@ -506,10 +488,7 @@ async def test_full_happy_path( ) assert step2a.status_code == HTTPStatus.OK - chat_mock2 = MagicMock() - chat_mock2.content = [ - MagicMock(text="I have enough information to draft a task now.") - ] + chat_mock2 = "I have enough information to draft a task now." with patch( "roboco.services.prompter.PrompterService._create_message", new_callable=AsyncMock, @@ -538,8 +517,7 @@ async def test_full_happy_path( "estimated_complexity": "low", "priority": 2, } - draft_mock = MagicMock() - draft_mock.content = [MagicMock(text=json.dumps(draft_json))] + draft_mock = json.dumps(draft_json) with patch( "roboco.services.prompter.PrompterService._create_message", @@ -574,8 +552,7 @@ async def test_full_happy_path( async def test_prompter_chat_success(prompter_client: dict) -> None: client = prompter_client["client"] - mock_response = MagicMock() - mock_response.content = [MagicMock(text="Great! Let's gather requirements.")] + mock_response = "Great! Let's gather requirements." with patch( "roboco.services.prompter.PrompterService._create_message", @@ -600,15 +577,9 @@ async def test_prompter_chat_success(prompter_client: dict) -> None: async def test_prompter_chat_draft_ready(prompter_client: dict) -> None: client = prompter_client["client"] - mock_response = MagicMock() - mock_response.content = [ - MagicMock( - text=( - "I have enough information. draft_ready=true." - " Ready to generate a draft." - ) - ) - ] + mock_response = ( + "I have enough information. draft_ready=true. Ready to generate a draft." + ) with patch( "roboco.services.prompter.PrompterService._create_message", @@ -672,8 +643,7 @@ async def test_prompter_draft_success(prompter_client: dict) -> None: "priority": 2, } - mock_response = MagicMock() - mock_response.content = [MagicMock(text=json.dumps(draft_json))] + mock_response = json.dumps(draft_json) with patch( "roboco.services.prompter.PrompterService._create_message", @@ -702,8 +672,7 @@ async def test_prompter_draft_success(prompter_client: dict) -> None: async def test_prompter_draft_invalid_json_from_llm(prompter_client: dict) -> None: client = prompter_client["client"] - mock_response = MagicMock() - mock_response.content = [MagicMock(text="not valid json")] + mock_response = "not valid json" with patch( "roboco.services.prompter.PrompterService._create_message", @@ -732,8 +701,7 @@ async def test_prompter_draft_schema_mismatch(prompter_client: dict) -> None: "description": "too short", } - mock_response = MagicMock() - mock_response.content = [MagicMock(text=json.dumps(bad_draft))] + mock_response = json.dumps(bad_draft) with patch( "roboco.services.prompter.PrompterService._create_message", diff --git a/tests/unit/services/test_prompter.py b/tests/unit/services/test_prompter.py index d288e896..3586db74 100644 --- a/tests/unit/services/test_prompter.py +++ b/tests/unit/services/test_prompter.py @@ -8,7 +8,7 @@ from __future__ import annotations import json from typing import Any -from unittest.mock import AsyncMock, MagicMock, patch +from unittest.mock import AsyncMock, patch from uuid import uuid4 import pytest @@ -21,7 +21,6 @@ from roboco.services.prompter import ( _build_draft_prompt, _build_reasoning, _detect_draft_ready, - _extract_text, get_prompter_service, ) @@ -53,30 +52,6 @@ def test_detect_draft_ready_negative() -> None: assert not _detect_draft_ready(text), f"Expected False for: {text!r}" -def test_extract_text_with_blocks() -> None: - block1 = MagicMock() - block1.text = "Hello, " - block2 = MagicMock() - block2.text = "world!" - response = MagicMock() - response.content = [block1, block2] - result = _extract_text(response) - assert result == "Hello, \nworld!" - - -def test_extract_text_empty_response() -> None: - response = MagicMock() - response.content = [] - assert _extract_text(response) == "" - - -def test_extract_text_no_text_attr() -> None: - block = MagicMock(spec=[]) # no 'text' attribute - response = MagicMock() - response.content = [block] - assert _extract_text(response) == "" - - def test_build_chat_prompt_basic() -> None: messages = [ {"role": "user", "content": "I need a feature"}, @@ -138,14 +113,12 @@ def test_get_prompter_service_raises_without_db_for_session_methods() -> None: async def test_chat_success_with_mock_llm() -> None: service = get_prompter_service() - mock_response = MagicMock() - mock_response.content = [MagicMock(text="Great, let's continue!")] - - with patch.object(service, "_get_client") as mock_get_client: - mock_client = AsyncMock() - mock_client.messages.create = AsyncMock(return_value=mock_response) - mock_get_client.return_value = mock_client - + with patch.object( + service, + "_create_message", + new_callable=AsyncMock, + return_value="Great, let's continue!", + ): result = await service.chat( messages=[{"role": "user", "content": "I need a feature"}] ) @@ -158,16 +131,12 @@ async def test_chat_success_with_mock_llm() -> None: async def test_chat_draft_ready_signal() -> None: service = get_prompter_service() - mock_response = MagicMock() - mock_response.content = [ - MagicMock(text="I have enough information. Ready to draft.") - ] - - with patch.object(service, "_get_client") as mock_get_client: - mock_client = AsyncMock() - mock_client.messages.create = AsyncMock(return_value=mock_response) - mock_get_client.return_value = mock_client - + with patch.object( + service, + "_create_message", + new_callable=AsyncMock, + return_value="I have enough information. Ready to draft.", + ): result = await service.chat( messages=[{"role": "user", "content": "I need a feature"}] ) @@ -179,31 +148,29 @@ async def test_chat_draft_ready_signal() -> None: async def test_chat_raises_on_empty_response() -> None: service = get_prompter_service() - mock_response = MagicMock() - mock_response.content = [] # Empty content blocks - - with patch.object(service, "_get_client") as mock_get_client: - mock_client = AsyncMock() - mock_client.messages.create = AsyncMock(return_value=mock_response) - mock_get_client.return_value = mock_client - - with pytest.raises(ServiceError, match="LLM returned empty content"): - await service.chat(messages=[{"role": "user", "content": "Hello"}]) + with ( + patch.object( + service, "_create_message", new_callable=AsyncMock, return_value="" + ), + pytest.raises(ServiceError, match="LLM returned empty content"), + ): + await service.chat(messages=[{"role": "user", "content": "Hello"}]) @pytest.mark.asyncio async def test_chat_raises_on_llm_error() -> None: service = get_prompter_service() - with patch.object(service, "_get_client") as mock_get_client: - mock_client = AsyncMock() - mock_client.messages.create = AsyncMock( - side_effect=Exception("API unavailable") - ) - mock_get_client.return_value = mock_client - - with pytest.raises(ServiceError, match="LLM chat failed"): - await service.chat(messages=[{"role": "user", "content": "Hello"}]) + with ( + patch.object( + service, + "_create_message", + new_callable=AsyncMock, + side_effect=Exception("API unavailable"), + ), + pytest.raises(ServiceError, match="LLM chat failed"), + ): + await service.chat(messages=[{"role": "user", "content": "Hello"}]) @pytest.mark.asyncio @@ -220,14 +187,12 @@ async def test_draft_success_with_mock_llm() -> None: "estimated_complexity": "medium", } - mock_response = MagicMock() - mock_response.content = [MagicMock(text=json.dumps(draft_data))] - - with patch.object(service, "_get_client") as mock_get_client: - mock_client = AsyncMock() - mock_client.messages.create = AsyncMock(return_value=mock_response) - mock_get_client.return_value = mock_client - + with patch.object( + service, + "_create_message", + new_callable=AsyncMock, + return_value=json.dumps(draft_data), + ): result = await service.draft( messages=[{"role": "user", "content": "I need a login feature"}] ) @@ -242,46 +207,32 @@ async def test_draft_success_with_mock_llm() -> None: async def test_draft_raises_on_invalid_json() -> None: service = get_prompter_service() - mock_response = MagicMock() - mock_response.content = [MagicMock(text="Not JSON at all")] - - with patch.object(service, "_get_client") as mock_get_client: - mock_client = AsyncMock() - mock_client.messages.create = AsyncMock(return_value=mock_response) - mock_get_client.return_value = mock_client - - with pytest.raises(ValidationError, match="not valid JSON"): - await service.draft(messages=[{"role": "user", "content": "Hello"}]) + with ( + patch.object( + service, + "_create_message", + new_callable=AsyncMock, + return_value="Not JSON at all", + ), + pytest.raises(ValidationError, match="not valid JSON"), + ): + await service.draft(messages=[{"role": "user", "content": "Hello"}]) @pytest.mark.asyncio async def test_draft_raises_on_llm_error() -> None: service = get_prompter_service() - with patch.object(service, "_get_client") as mock_get_client: - mock_client = AsyncMock() - mock_client.messages.create = AsyncMock( - side_effect=Exception("API unavailable") - ) - mock_get_client.return_value = mock_client - - with pytest.raises(ServiceError, match="LLM draft generation failed"): - await service.draft(messages=[{"role": "user", "content": "Hello"}]) - - -# ============================================================================= -# API key validation -# ============================================================================= - - -def test_get_client_raises_without_api_key() -> None: - service = get_prompter_service() - service._client = None # Force fresh init - - with patch("roboco.services.prompter.settings") as mock_settings: - mock_settings.anthropic_api_key = None - with pytest.raises(ServiceError, match="Anthropic API key not configured"): - service._get_client() + with ( + patch.object( + service, + "_create_message", + new_callable=AsyncMock, + side_effect=Exception("API unavailable"), + ), + pytest.raises(ServiceError, match="LLM draft generation failed"), + ): + await service.draft(messages=[{"role": "user", "content": "Hello"}]) # =============================================================================