mirror of
https://github.com/rennf93/roboco.git
synced 2026-08-03 07:23:24 +02:00
Renn F
de82e06b3c
Recall no longer depends on a per-query HyDE LLM call — it comes from the index. Each chunks_<type> table gets a generated `tsv` column + GIN index (migration 031; the engine CREATE TABLE matches so fresh tables get it too). VectorStore.hybrid_search fuses pgvector cosine with Postgres full-text in one query: score = min(1, cosine + 0.3 * normalized_ts_rank). A vector-only match keeps its cosine score (so decisions/reviewer thresholds are unchanged), a keyword match adds a bounded boost (the recall win), and a keyword-only match stays low. Empty/garbage query text degrades to pure vector. HyDE is removed from the search hot path: _compute_query_embedding now embeds the query directly, and _generate_hyde_passage / rag_use_hyde / IndexConfig.use_hyde are deleted. So a search is one local embed + one indexed SQL — no LLM round-trip. The raw query text is threaded through the embed-once + concurrent fan-out (search_with_embedding(embedding, query_text)). Verified live via a real pgvector round-trip: vector ranking + keyword boost + [0,1] scores + empty-query fallback all correct. Adds wiring + fan-out unit tests; the fusion SQL itself is verified live (needs pgvector, not gated in CI).
95 lines
2.9 KiB
Python
95 lines
2.9 KiB
Python
"""Add a full-text (tsvector) column + GIN index to every RAG chunk table.
|
|
|
|
Hybrid retrieval fuses pgvector cosine similarity with Postgres native
|
|
full-text search (keyword/BM25-style) so recall no longer depends on a
|
|
per-query HyDE LLM call. Each ``chunks_<index_type>`` table gets a generated
|
|
``tsv`` column (``to_tsvector('english', content)``, auto-maintained on
|
|
insert/update) and a GIN index over it.
|
|
|
|
The work runs inside a plpgsql ``DO`` block guarding on ``information_schema``,
|
|
so it is idempotent, offline-renderable (``alembic --sql``), and safe whether a
|
|
table is present, already has ``tsv``, or is absent (e.g. ``chunks_code``).
|
|
Depends on 030 having reshaped the column to ``content``.
|
|
|
|
Revision ID: 031_rag_chunks_fulltext
|
|
Revises: 030_rag_chunks_content_schema
|
|
Create Date: 2026-06-15
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from alembic import op
|
|
|
|
revision = "031_rag_chunks_fulltext"
|
|
down_revision = "030_rag_chunks_content_schema"
|
|
branch_labels = None
|
|
depends_on = None
|
|
|
|
CHUNK_TABLES = (
|
|
"chunks_code",
|
|
"chunks_documentation",
|
|
"chunks_conversations",
|
|
"chunks_journals",
|
|
"chunks_errors",
|
|
"chunks_standards",
|
|
"chunks_decisions",
|
|
"chunks_reviews",
|
|
"chunks_learnings",
|
|
)
|
|
|
|
_TABLES_SQL = ", ".join(f"'{name}'" for name in CHUNK_TABLES)
|
|
|
|
|
|
def upgrade() -> None:
|
|
op.execute(
|
|
f"""
|
|
DO $$
|
|
DECLARE
|
|
t text;
|
|
BEGIN
|
|
FOREACH t IN ARRAY ARRAY[{_TABLES_SQL}] LOOP
|
|
IF EXISTS (
|
|
SELECT 1 FROM information_schema.columns
|
|
WHERE table_schema = 'public'
|
|
AND table_name = t AND column_name = 'content'
|
|
) THEN
|
|
IF NOT EXISTS (
|
|
SELECT 1 FROM information_schema.columns
|
|
WHERE table_schema = 'public'
|
|
AND table_name = t AND column_name = 'tsv'
|
|
) THEN
|
|
EXECUTE format(
|
|
'ALTER TABLE %I ADD COLUMN tsv tsvector '
|
|
'GENERATED ALWAYS AS '
|
|
'(to_tsvector(''english'', content)) STORED',
|
|
t
|
|
);
|
|
END IF;
|
|
EXECUTE format(
|
|
'CREATE INDEX IF NOT EXISTS %I ON %I USING gin (tsv)',
|
|
t || '_tsv_idx', t
|
|
);
|
|
END IF;
|
|
END LOOP;
|
|
END $$;
|
|
"""
|
|
)
|
|
|
|
|
|
def downgrade() -> None:
|
|
op.execute(
|
|
f"""
|
|
DO $$
|
|
DECLARE
|
|
t text;
|
|
BEGIN
|
|
FOREACH t IN ARRAY ARRAY[{_TABLES_SQL}] LOOP
|
|
EXECUTE format('DROP INDEX IF EXISTS %I', t || '_tsv_idx');
|
|
EXECUTE format(
|
|
'ALTER TABLE IF EXISTS %I DROP COLUMN IF EXISTS tsv', t
|
|
);
|
|
END LOOP;
|
|
END $$;
|
|
"""
|
|
)
|