mirror of
https://github.com/rennf93/roboco.git
synced 2026-08-03 07:23:24 +02:00
feat(vault): Obsidian vault V2 — janitor, archival, weekly report, KB ingest, Bases + sync runbook (#482)
* feat(vault): V2 — create-seam + drift janitor, archival, weekly org-report, KB ingest, Bases views + sync runbook Implements the vault V2 canonical spec end to end (the splice guard shipped separately and is reused at KB-ingest time): - materialize-on-create: TaskService.create writes each task's note best-effort from the moment it exists; the transition-touch stops no-oping on live work - drift janitor (services/vault_janitor.py + hourly _vault_janitor_loop): daily changed-task re-projection, random drift sample, archival pass — restart-proof via RoboCo/_meta/.janitor_state.json, 200/cycle caps, per-item isolation, processed-only resume markers, self-repairing state file - archival: vault_archive_days (30, 0=off) moves old terminal tasks' notes to RoboCo/Archive/<year>/Tasks/<project>/ — one write_task code path for janitor and rebuild, id8 lookup across Tasks/+Archive/, alias links keep moves safe - weekly org-report: VaultWriter.write_org_report renders Reports/<ISO-week>.md from MetricsService/UsageService (numbers duplicated into frontmatter for trend queries), once per ISO week, with a best-effort CEO notification - KB ingest: IndexType.VAULT_NOTES + VaultNotesIndexPlugin + _vault_kb_loop embed the CEO's RoboCo/Notes into the RAG corpus — injection guard as a hard gate (flagged notes quarantined with an idempotent callout), traversal- and symlink-contained at both config and engine layers, content-hash dedup, 50-ingest/cycle cap, frontmatter stripped; reaches roboco_kb_search, the mentor default domain, claim-time briefings (kind vault_note), and the panel KB browser; no migration (chunks table auto-creates; migration 030's CHUNK_TABLES tuple appended per the chunks_playbooks precedent) - Bases views (Task Board.base, Reports.base — schema verified against the Obsidian docs) + the Mac sync runbook vault asset - config/flags/compose: vault_archive_days, vault_report_enabled (flags card), vault_kb_enabled (flags card; NAS compose arms it, registry ships it off), vault_kb_dirs (+ overlap/traversal validator), vault_kb_interval_seconds - e2e smoke (tests/e2e_smoke/test_vault_v2.py): real create-seam, real janitor cycle incl. archival + state, real KB engine + real guard * docs: vault V2 sweep — map, RAG corpus, CLAUDE.md - docs/map/vault.md: V1+V2 — janitor/archival/report/KB data flows, new files, config, health posture - docs/map/orchestrator.md + task-service.md: the two new loops, the create seam, the three janitor queries - docs/rag/architecture/obsidian-vault.md: agent-facing what-changed (notes from creation, archive link-safety, CEO notes retrievable, weekly report) - docs/rag/architecture/config-reference.md: the five new settings - CLAUDE.md: vault paragraph covers V1+V2; flags-card list mentions the vault report/KB flags --------- Co-authored-by: Renn F <rennf93@users.noreply.github.com>
This commit is contained in:
Renzo F
@@ -0,0 +1,196 @@
|
||||
"""Real-DB tests for the vault janitor's TaskService queries.
|
||||
|
||||
``list_updated_since`` / ``list_archive_candidates`` / ``sample_stale_tasks``
|
||||
carry the janitor's resume-marker contract (COALESCE timestamps, ascending
|
||||
order, half-open archive window) — SQL semantics mocks can't prove. Follows
|
||||
the ``test_audit_real_query.py`` pattern: real Postgres via the session-scoped
|
||||
test DB (local: ROBOCO_TEST_DB_PORT=55432 ROBOCO_TEST_DB_USER=renzof).
|
||||
|
||||
Foreign rows from other tests may share the DB, so every assertion is scoped
|
||||
to this module's seeded ids rather than exact result sets.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
import pytest
|
||||
from roboco.db.tables import AgentTable, TaskTable
|
||||
from roboco.models.base import (
|
||||
AgentRole,
|
||||
AgentStatus,
|
||||
TaskStatus,
|
||||
TaskType,
|
||||
Team,
|
||||
)
|
||||
from roboco.services.task import TaskService
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
|
||||
async def _seed_agent(session: AsyncSession) -> UUID:
|
||||
"""``tasks.created_by`` is a NOT NULL FK to ``agents.id``."""
|
||||
agent = AgentTable(
|
||||
id=uuid4(),
|
||||
name="Vault Query Test Agent",
|
||||
slug=f"vault-query-{uuid4().hex[:8]}",
|
||||
role=AgentRole.DEVELOPER,
|
||||
team=None,
|
||||
status=AgentStatus.ACTIVE,
|
||||
model_config={},
|
||||
system_prompt="vault query test",
|
||||
capabilities=[],
|
||||
permissions={},
|
||||
metrics={},
|
||||
)
|
||||
session.add(agent)
|
||||
await session.flush()
|
||||
return UUID(str(agent.id))
|
||||
|
||||
|
||||
async def _seed_task(session: AsyncSession, created_by: UUID, **cols: Any) -> UUID:
|
||||
"""Seed one task; ``cols`` are timestamp/status column overrides."""
|
||||
task = TaskTable(
|
||||
id=uuid4(),
|
||||
title="vault query seed",
|
||||
description="seed",
|
||||
acceptance_criteria=["seeded"],
|
||||
status=cols.pop("status", TaskStatus.IN_PROGRESS),
|
||||
priority=2,
|
||||
task_type=TaskType.CODE,
|
||||
team=Team.BACKEND,
|
||||
created_by=created_by,
|
||||
**cols,
|
||||
)
|
||||
session.add(task)
|
||||
await session.flush()
|
||||
return UUID(str(task.id))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_changed_set_and_sample_set_are_complementary(
|
||||
db_session: AsyncSession,
|
||||
) -> None:
|
||||
"""For a given ``since``: touched-after rows appear in the changed set and
|
||||
never in the stale sample; touched-before rows the reverse. COALESCE puts
|
||||
a never-updated (updated_at NULL) row on its created_at."""
|
||||
agent_id = await _seed_agent(db_session)
|
||||
now = datetime.now(UTC)
|
||||
since = now - timedelta(days=1)
|
||||
old_updated = await _seed_task(
|
||||
db_session,
|
||||
agent_id,
|
||||
created_at=now - timedelta(days=10),
|
||||
updated_at=now - timedelta(days=3),
|
||||
)
|
||||
old_never_updated = await _seed_task(
|
||||
db_session, agent_id, created_at=now - timedelta(days=3)
|
||||
)
|
||||
new_updated = await _seed_task(
|
||||
db_session,
|
||||
agent_id,
|
||||
created_at=now - timedelta(days=10),
|
||||
updated_at=now - timedelta(hours=1),
|
||||
)
|
||||
new_created = await _seed_task(
|
||||
db_session, agent_id, created_at=now - timedelta(hours=1)
|
||||
)
|
||||
svc = TaskService(db_session)
|
||||
|
||||
changed_ids = {t.id for t in await svc.list_updated_since(since, limit=10_000)}
|
||||
stale_ids = {t.id for t in await svc.sample_stale_tasks(since, limit=100_000)}
|
||||
|
||||
assert {new_updated, new_created} <= changed_ids
|
||||
assert {old_updated, old_never_updated}.isdisjoint(changed_ids)
|
||||
assert {old_updated, old_never_updated} <= stale_ids
|
||||
assert {new_updated, new_created}.isdisjoint(stale_ids)
|
||||
assert changed_ids.isdisjoint(stale_ids)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_archive_candidates_window_boundaries(
|
||||
db_session: AsyncSession,
|
||||
) -> None:
|
||||
"""[after, before): after-inclusive, before-exclusive, terminal-only;
|
||||
a terminal row with NULL completed_at falls back to updated_at."""
|
||||
agent_id = await _seed_agent(db_session)
|
||||
now = datetime.now(UTC)
|
||||
after = now - timedelta(days=100)
|
||||
before = now - timedelta(days=30)
|
||||
created = now - timedelta(days=200)
|
||||
|
||||
at_after = await _seed_task(
|
||||
db_session,
|
||||
agent_id,
|
||||
status=TaskStatus.COMPLETED,
|
||||
created_at=created,
|
||||
completed_at=after,
|
||||
)
|
||||
inside = await _seed_task(
|
||||
db_session,
|
||||
agent_id,
|
||||
status=TaskStatus.CANCELLED,
|
||||
created_at=created,
|
||||
completed_at=now - timedelta(days=60),
|
||||
)
|
||||
inside_no_completed_at = await _seed_task(
|
||||
db_session,
|
||||
agent_id,
|
||||
status=TaskStatus.COMPLETED,
|
||||
created_at=created,
|
||||
updated_at=now - timedelta(days=60),
|
||||
)
|
||||
at_before = await _seed_task(
|
||||
db_session,
|
||||
agent_id,
|
||||
status=TaskStatus.COMPLETED,
|
||||
created_at=created,
|
||||
completed_at=before,
|
||||
)
|
||||
non_terminal_inside = await _seed_task(
|
||||
db_session,
|
||||
agent_id,
|
||||
status=TaskStatus.IN_PROGRESS,
|
||||
created_at=created,
|
||||
completed_at=now - timedelta(days=60),
|
||||
)
|
||||
svc = TaskService(db_session)
|
||||
|
||||
ids = {t.id for t in await svc.list_archive_candidates(after, before, limit=10_000)}
|
||||
|
||||
assert {at_after, inside, inside_no_completed_at} <= ids
|
||||
assert at_before not in ids
|
||||
assert non_terminal_inside not in ids
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_updated_since_pagination_is_complete_and_ascending(
|
||||
db_session: AsyncSession,
|
||||
) -> None:
|
||||
"""Paging with a small limit visits every row exactly once, oldest first
|
||||
(the capped drain's resume contract)."""
|
||||
agent_id = await _seed_agent(db_session)
|
||||
base = datetime.now(UTC) + timedelta(days=365) # beyond any foreign row
|
||||
seeded = [
|
||||
await _seed_task(
|
||||
db_session,
|
||||
agent_id,
|
||||
created_at=base + timedelta(minutes=i),
|
||||
)
|
||||
for i in range(5)
|
||||
]
|
||||
svc = TaskService(db_session)
|
||||
|
||||
pages: list[UUID] = []
|
||||
offset = 0
|
||||
while True:
|
||||
page = await svc.list_updated_since(base, limit=2, offset=offset)
|
||||
if not page:
|
||||
break
|
||||
pages.extend(UUID(str(t.id)) for t in page)
|
||||
offset += len(page)
|
||||
|
||||
assert pages == seeded # complete, no dupes, ascending touched-order
|
||||
Reference in New Issue
Block a user