Feat: transcript retention (#123)

* feat(retention): prune old agent transcripts + panel-tunable setting

Agents write a {session-id}.jsonl per spawn under ~/.claude/projects; nothing
ever deleted them, so the operator's bind-mounted ~/.claude grew without bound.

Add a throttled orchestrator sweep that prunes agent-owned transcripts (the
shared -app dir + per-workspace dirs) older than a retention window — and ONLY
agent-owned dirs, never the operator's own Claude sessions (proven by the
temp-dir selection tests). The window is panel-tunable: a new system_settings
key-value table (migration 027) holds transcript_retention_days, read via
SettingsService with the roboco.config default (14d) as the fallback, exposed
through GET/PUT /api/settings. Panel wiring follows.

* feat(panel): add a panel-tunable transcript retention control

Wire the settings page to the /api/settings backend: a settings API client and
a self-contained Transcript Retention card (React Query) that loads
transcript_retention_days and saves it back, with client-side validation. The
existing settings controls are unchanged.

---------

Co-authored-by: Renn F <rennf93@users.noreply.github.com>
This commit is contained in:
Renzo F
2026-06-12 23:11:01 +02:00
committed by GitHub
co-authored by Renn F
parent 718d7dd83e
commit fbbb7b3251
15 changed files with 619 additions and 0 deletions
+45
View File
@@ -0,0 +1,45 @@
"""Add system_settings — runtime-editable, panel-tunable settings.
A key-value store for operator settings that must persist across restarts and be
editable from the panel. First user: ``transcript_retention_days``, the window
for the agent-transcript prune sweep. Seeded to 14 so the prune has a value
before anyone touches the panel; code defaults in ``roboco.config`` are the
fallback when a key is absent.
Revision ID: 027_system_settings
Revises: 026_token_usage_tables
Create Date: 2026-06-12
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
revision = "027_system_settings"
down_revision = "026_token_usage_tables"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
"system_settings",
sa.Column("key", sa.String(length=100), primary_key=True, nullable=False),
sa.Column("value", sa.Text(), nullable=False),
sa.Column(
"updated_at",
sa.DateTime(timezone=True),
nullable=False,
server_default=sa.text("now()"),
),
)
# Seed the retention window so the prune sweep has a value pre-panel.
op.execute(
"INSERT INTO system_settings (key, value, updated_at) "
"VALUES ('transcript_retention_days', '14', now())"
)
def downgrade() -> None:
op.drop_table("system_settings")
@@ -26,6 +26,7 @@ import {
} from "lucide-react"; } from "lucide-react";
import { toast } from "sonner"; import { toast } from "sonner";
import { API_URL, WS_URL } from "@/lib/constants"; import { API_URL, WS_URL } from "@/lib/constants";
import { TranscriptRetentionCard } from "@/components/settings/transcript-retention-card";
export default function SettingsPage() { export default function SettingsPage() {
const { theme, setTheme } = useTheme(); const { theme, setTheme } = useTheme();
@@ -183,6 +184,9 @@ export default function SettingsPage() {
</CardContent> </CardContent>
</Card> </Card>
{/* Transcript Retention (panel-tunable; persisted server-side) */}
<TranscriptRetentionCard />
{/* Connection Info */} {/* Connection Info */}
<Card> <Card>
<CardHeader> <CardHeader>
@@ -0,0 +1,93 @@
"use client";
import { useEffect, useState } from "react";
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
import { settingsApi } from "@/lib/api";
import {
Card,
CardContent,
CardDescription,
CardHeader,
CardTitle,
} from "@/components/ui/card";
import { Label } from "@/components/ui/label";
import { Input } from "@/components/ui/input";
import { Button } from "@/components/ui/button";
import { HardDrive, Save } from "lucide-react";
import { toast } from "sonner";
const RETENTION_KEY = "transcript_retention_days";
const DEFAULT_RETENTION = "14";
export function TranscriptRetentionCard() {
const queryClient = useQueryClient();
const [days, setDays] = useState<string>(DEFAULT_RETENTION);
const { data: settings, isLoading } = useQuery({
queryKey: ["settings"],
queryFn: settingsApi.getAll,
});
useEffect(() => {
const stored = settings?.[RETENTION_KEY];
if (stored !== undefined) {
setDays(stored);
}
}, [settings]);
const saveMutation = useMutation({
mutationFn: (value: string) => settingsApi.update(RETENTION_KEY, value),
onSuccess: () => {
queryClient.invalidateQueries({ queryKey: ["settings"] });
toast.success("Transcript retention updated");
},
onError: (error) => {
toast.error(
`Failed to save: ${error instanceof Error ? error.message : "Unknown error"}`,
);
},
});
const handleSave = () => {
const parsed = Number(days);
if (!Number.isInteger(parsed) || parsed < 1) {
toast.error("Retention must be a whole number of days (at least 1)");
return;
}
saveMutation.mutate(String(parsed));
};
return (
<Card>
<CardHeader>
<CardTitle className="flex items-center gap-2">
<HardDrive className="h-5 w-5" />
Transcript Retention
</CardTitle>
<CardDescription>
How long agent transcripts are kept before the background sweep prunes
them. Only agent-owned transcripts are pruned your own Claude
sessions are never touched.
</CardDescription>
</CardHeader>
<CardContent className="space-y-4">
<div className="space-y-2">
<Label htmlFor="transcript-retention-days">Retention window (days)</Label>
<Input
id="transcript-retention-days"
type="number"
min={1}
value={days}
disabled={isLoading}
onChange={(e) => setDays(e.target.value)}
className="max-w-[160px]"
/>
</div>
<Button onClick={handleSave} disabled={saveMutation.isPending || isLoading}>
<Save className="h-4 w-4 mr-2" />
{saveMutation.isPending ? "Saving..." : "Save"}
</Button>
</CardContent>
</Card>
);
}
+1
View File
@@ -13,3 +13,4 @@ export { gitApi } from "./git";
export { a2aApi } from "./a2a"; export { a2aApi } from "./a2a";
export { streamApi } from "./stream"; export { streamApi } from "./stream";
export { groupsApi } from "./groups"; export { groupsApi } from "./groups";
export { settingsApi } from "./settings";
+18
View File
@@ -0,0 +1,18 @@
import api from "./client";
export interface SettingsResponse {
settings: Record<string, string>;
}
export const settingsApi = {
// GET /api/settings — all runtime-editable settings as a flat key→value map.
getAll: async (): Promise<Record<string, string>> => {
const { data } = await api.get<SettingsResponse>("/settings");
return data.settings;
},
// PUT /api/settings/{key} — persist one setting; returns the full updated map.
update: async (key: string, value: string): Promise<Record<string, string>> => {
const { data } = await api.put<SettingsResponse>(`/settings/${key}`, { value });
return data.settings;
},
};
+7
View File
@@ -33,6 +33,7 @@ from roboco.api.routes.project import router as project_router
from roboco.api.routes.prompter_live import router as prompter_live_router from roboco.api.routes.prompter_live import router as prompter_live_router
from roboco.api.routes.provider import router as provider_router from roboco.api.routes.provider import router as provider_router
from roboco.api.routes.sessions import router as sessions_router from roboco.api.routes.sessions import router as sessions_router
from roboco.api.routes.settings import router as settings_router
from roboco.api.routes.stream import router as stream_router from roboco.api.routes.stream import router as stream_router
from roboco.api.routes.system import router as system_router from roboco.api.routes.system import router as system_router
from roboco.api.routes.tasks import router as tasks_router from roboco.api.routes.tasks import router as tasks_router
@@ -221,6 +222,12 @@ def create_app() -> FastAPI:
tags=["Sessions"], tags=["Sessions"],
) )
app.include_router(
settings_router,
prefix=f"{api_prefix}/settings",
tags=["Settings"],
)
app.include_router( app.include_router(
messages_router, messages_router,
prefix=f"{api_prefix}/messages", prefix=f"{api_prefix}/messages",
+37
View File
@@ -0,0 +1,37 @@
"""System settings API — read and update runtime-editable settings.
Backs the panel settings page. Values persist in the ``system_settings`` table
and are read by the backend (e.g. the transcript-retention prune sweep) with a
``roboco.config`` default as the fallback when a key is unset.
"""
from fastapi import APIRouter, HTTPException, status
from roboco.api.deps import DbSession
from roboco.api.schemas.settings import SettingsResponse, SettingUpdate
from roboco.services.settings import SettingValidationError, get_settings_service
router = APIRouter()
@router.get("", response_model=SettingsResponse)
async def list_settings(db: DbSession) -> SettingsResponse:
"""Return every stored runtime-editable setting."""
return SettingsResponse(settings=await get_settings_service(db).all())
@router.put("/{key}", response_model=SettingsResponse)
async def update_setting(
key: str, data: SettingUpdate, db: DbSession
) -> SettingsResponse:
"""Validate and persist a single setting, returning the full updated map."""
service = get_settings_service(db)
try:
await service.set(key, data.value)
except SettingValidationError as exc:
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=str(exc)
) from exc
# Write route commits explicitly (get_db auto-commit is unreliable).
await db.commit()
return SettingsResponse(settings=await service.all())
+17
View File
@@ -0,0 +1,17 @@
"""Schemas for the system-settings API."""
from __future__ import annotations
from pydantic import BaseModel, Field
class SettingUpdate(BaseModel):
"""Body for PUT /settings/{key} — the new value (stored as text)."""
value: str = Field(..., description="New value for the setting, stored as text")
class SettingsResponse(BaseModel):
"""All runtime-editable settings as a flat key→value map."""
settings: dict[str, str] = Field(default_factory=dict)
+31
View File
@@ -244,6 +244,37 @@ class Settings(BaseSettings):
), ),
) )
# ==========================================================================
# Transcript retention (agent Claude Code transcripts under ~/.claude)
# ==========================================================================
transcript_retention_days: int = Field(
default=14,
ge=1,
description=(
"Default retention window, in days, for agent Claude Code "
"transcripts (the *.jsonl files agents write under "
"~/.claude/projects). A background sweep prunes agent-owned "
"transcripts older than this. Panel-editable: a stored "
"`transcript_retention_days` system setting overrides this default "
"when present; this is the fallback used before one is set."
),
)
transcript_prune_enabled: bool = Field(
default=True,
description=(
"Whether the orchestrator background sweep prunes old agent "
"transcripts. Disable to keep every transcript indefinitely."
),
)
transcript_prune_interval_seconds: int = Field(
default=3600,
ge=300,
description=(
"Minimum seconds between transcript-retention prune passes. The "
"prune is age-based (days), so it need not run more than hourly."
),
)
# ========================================================================== # ==========================================================================
# Git command execution # Git command execution
# ========================================================================== # ==========================================================================
+21
View File
@@ -1736,6 +1736,27 @@ class ProviderConfigTable(Base):
__table_args__ = (Index("ix_provider_configs_enabled", "enabled"),) __table_args__ = (Index("ix_provider_configs_enabled", "enabled"),)
class SystemSettingTable(Base):
"""Key-value store for runtime-editable, panel-tunable system settings.
Operator-tunable values that must persist across restarts and be editable
from the panel (first user: ``transcript_retention_days``). One row per key;
the value is stored as text and parsed by the reader. Code defaults in
``roboco.config`` are the fallback when a key has no row yet.
"""
__tablename__ = "system_settings"
key: Mapped[str] = mapped_column(String(100), primary_key=True)
value: Mapped[str] = mapped_column(Text, nullable=False)
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True),
default=lambda: datetime.now(UTC),
onupdate=lambda: datetime.now(UTC),
nullable=False,
)
class ModelAssignmentTable(Base): class ModelAssignmentTable(Base):
"""SQLAlchemy table for (scope, provider, model) routing rows. """SQLAlchemy table for (scope, provider, model) routing rows.
+61
View File
@@ -561,6 +561,8 @@ class AgentOrchestrator:
self._health_task: asyncio.Task | None = None self._health_task: asyncio.Task | None = None
self._dispatcher_task: asyncio.Task | None = None self._dispatcher_task: asyncio.Task | None = None
self._sweeper_task: asyncio.Task | None = None self._sweeper_task: asyncio.Task | None = None
# Last time the transcript-retention prune ran (throttled in the sweep).
self._last_transcript_prune: datetime | None = None
# Rate-limit probe loop: 30-second interval, scans Redis for all # Rate-limit probe loop: 30-second interval, scans Redis for all
# rate-limited providers and resolves waiting agents on success. # rate-limited providers and resolves waiting agents on success.
self._rate_limit_probe_task: asyncio.Task | None = None self._rate_limit_probe_task: asyncio.Task | None = None
@@ -3915,6 +3917,65 @@ Start by:
await self._sweep_token_snapshots() await self._sweep_token_snapshots()
await self._sweep_daily_rollup() await self._sweep_daily_rollup()
# Prune old agent transcripts (throttled internally to ~hourly) so the
# operator's bind-mounted ~/.claude doesn't grow without bound.
await self._sweep_transcript_retention()
async def _sweep_transcript_retention(self) -> None:
"""Prune agent transcripts older than the retention window.
Throttled to ``settings.transcript_prune_interval_seconds``. Reads the
window from the ``system_settings`` store (panel-editable), falling back
to ``settings.transcript_retention_days``. Only agent-owned project dirs
(``-app`` + per-workspace dirs) are touched never the operator's own
Claude sessions. Best-effort: any failure is logged, never raised.
"""
if not settings.transcript_prune_enabled:
return
now = datetime.now(UTC)
last = self._last_transcript_prune
if (
last is not None
and (now - last).total_seconds()
< settings.transcript_prune_interval_seconds
):
return
self._last_transcript_prune = now
retention_days = settings.transcript_retention_days
with contextlib.suppress(Exception):
from roboco.db.base import get_session_factory
from roboco.services.settings import get_settings_service
session_factory = get_session_factory()
async with session_factory() as db:
retention_days = await get_settings_service(db).get_int(
"transcript_retention_days", settings.transcript_retention_days
)
from roboco.runtime.transcript_retention import select_prunable_transcripts
projects_root = Path.home() / ".claude" / "projects"
cutoff = (now - timedelta(days=retention_days)).timestamp()
prunable = select_prunable_transcripts(
projects_root, settings.workspaces_root, cutoff
)
pruned = 0
for transcript in prunable:
try:
transcript.unlink()
pruned += 1
except OSError as exc:
logger.debug(
"Transcript prune failed", path=str(transcript), error=str(exc)
)
if pruned:
logger.info(
"Pruned old agent transcripts",
count=pruned,
retention_days=retention_days,
)
@staticmethod @staticmethod
async def _fetch_budget_status( async def _fetch_budget_status(
client: httpx.AsyncClient, url: str, agent_id: str client: httpx.AsyncClient, url: str, agent_id: str
+61
View File
@@ -0,0 +1,61 @@
"""Agent transcript retention — select old Claude Code transcripts to prune.
Agents write a ``{session-id}.jsonl`` transcript per spawn under
``~/.claude/projects/<encoded-cwd>/``. Review/coordinate roles share the
``-app`` dir; authoring roles get a per-workspace dir under the workspaces
root. Nothing ever deleted them, so the host ``~/.claude`` grows unbounded and
it is the operator's *real* ``~/.claude`` via the agent bind mount. This module
selects agent-owned transcripts older than the retention window, and ONLY
agent-owned dirs, never the operator's own Claude sessions.
The selection is a pure function so the deletion target is unit-testable against
a temp dir before it ever runs on a real home directory.
"""
from __future__ import annotations
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from pathlib import Path
def is_agent_owned_dir(dir_name: str, workspaces_root: str) -> bool:
"""True if a ``~/.claude/projects`` subdir was written by a spawned agent.
Agents run either at the image WORKDIR ``/app`` (review/coordinate roles
the shared ``-app`` dir) or in a per-agent clone under the workspaces root
(authoring roles). Claude Code encodes the cwd into the dir name by replacing
``/`` with ``-``, so those dirs are ``-app`` and anything starting with the
encoded workspaces root (e.g. ``-data-workspaces``). The operator's own
sessions live under their real cwd (``-Users-``, ``-home-``) and are never
matched.
"""
if dir_name == "-app":
return True
encoded_root = workspaces_root.rstrip("/").replace("/", "-")
return bool(encoded_root) and dir_name.startswith(encoded_root)
def select_prunable_transcripts(
projects_root: Path, workspaces_root: str, cutoff_epoch: float
) -> list[Path]:
"""Agent-owned ``*.jsonl`` transcripts last modified before ``cutoff_epoch``.
Only files inside agent-owned project dirs are considered; directories and
the operator's own session dirs are never returned. Missing/unreadable
entries are skipped rather than raising.
"""
if not projects_root.is_dir():
return []
prunable: list[Path] = []
for child in sorted(projects_root.iterdir()):
if not child.is_dir() or not is_agent_owned_dir(child.name, workspaces_root):
continue
for transcript in child.glob("*.jsonl"):
try:
if transcript.is_file() and transcript.stat().st_mtime < cutoff_epoch:
prunable.append(transcript)
except OSError:
continue
return prunable
+90
View File
@@ -0,0 +1,90 @@
"""System settings service — runtime-editable, panel-tunable config.
Reads/writes the ``system_settings`` key-value table. Values are stored as text
and parsed by typed accessors. Code defaults in ``roboco.config`` are the
fallback used when a key has no row yet. Only keys in ``KNOWN_SETTINGS`` are
writable, each with a validator, so the panel can't persist junk.
"""
from __future__ import annotations
from typing import TYPE_CHECKING
from sqlalchemy import select
from roboco.db.tables import SystemSettingTable
from roboco.services.base import BaseService
if TYPE_CHECKING:
from sqlalchemy.ext.asyncio import AsyncSession
class SettingValidationError(ValueError):
"""Raised when a setting key is unknown or its value is invalid."""
def _validate_retention_days(value: str) -> None:
try:
days = int(value)
except ValueError as exc:
raise SettingValidationError(
"transcript_retention_days must be an integer"
) from exc
if days < 1:
raise SettingValidationError("transcript_retention_days must be >= 1")
# Writable settings: key -> validator. Keys absent here are rejected on write so
# the panel can only persist values the backend understands.
_VALIDATORS = {
"transcript_retention_days": _validate_retention_days,
}
def validate_setting(key: str, value: str) -> None:
"""Raise SettingValidationError if ``key`` is not writable or ``value`` invalid."""
validator = _VALIDATORS.get(key)
if validator is None:
raise SettingValidationError(f"Unknown or read-only setting: {key}")
validator(value)
class SettingsService(BaseService):
"""CRUD for the ``system_settings`` key-value store."""
async def get(self, key: str) -> str | None:
"""Return the stored value for ``key``, or None if unset."""
result = await self.session.execute(
select(SystemSettingTable.value).where(SystemSettingTable.key == key)
)
return result.scalar_one_or_none()
async def get_int(self, key: str, default: int) -> int:
"""Return ``key`` parsed as int, or ``default`` if unset/unparseable."""
raw = await self.get(key)
if raw is None:
return default
try:
return int(raw)
except ValueError:
return default
async def set(self, key: str, value: str) -> None:
"""Validate then upsert ``key`` = ``value``. Caller commits."""
validate_setting(key, value)
existing = await self.session.get(SystemSettingTable, key)
if existing is None:
self.session.add(SystemSettingTable(key=key, value=value))
else:
existing.value = value
await self.session.flush()
async def all(self) -> dict[str, str]:
"""Return every stored setting as a ``{key: value}`` map."""
result = await self.session.execute(select(SystemSettingTable))
return {row.key: row.value for row in result.scalars().all()}
def get_settings_service(session: AsyncSession) -> SettingsService:
"""Construct a SettingsService bound to ``session``."""
return SettingsService(session)
@@ -0,0 +1,77 @@
"""Tests for transcript-retention selection — what the prune deletes, safely.
The prune is destructive (it unlinks files on the operator's real ~/.claude), so
the selection logic is proven here against a temp dir before it ever runs live.
"""
from __future__ import annotations
import os
import time
from typing import TYPE_CHECKING
from roboco.runtime.transcript_retention import (
is_agent_owned_dir,
select_prunable_transcripts,
)
if TYPE_CHECKING:
from pathlib import Path
WORKSPACES = "/data/workspaces"
_DAY = 86400
def _touch(path: Path, age_days: float) -> Path:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text("{}\n")
mtime = time.time() - age_days * _DAY
os.utime(path, (mtime, mtime))
return path
def test_is_agent_owned_dir_matches_app_and_workspaces() -> None:
assert is_agent_owned_dir("-app", WORKSPACES) is True
assert is_agent_owned_dir("-data-workspaces-roboco-be-dev-1", WORKSPACES) is True
# The operator's own sessions are never agent-owned.
assert is_agent_owned_dir("-Users-renzof-Documents-foo", WORKSPACES) is False
assert is_agent_owned_dir("-home-renzof-code", WORKSPACES) is False
def test_is_agent_owned_dir_handles_root_slash_safely() -> None:
# A pathological "/" workspaces root must not make every dir agent-owned.
assert is_agent_owned_dir("-Users-renzof-secret", "/") is False
assert is_agent_owned_dir("-app", "/") is True
def test_select_prunes_only_old_agent_transcripts(tmp_path: Path) -> None:
projects = tmp_path / "projects"
old_app = _touch(projects / "-app" / "old.jsonl", age_days=30)
fresh_app = _touch(projects / "-app" / "fresh.jsonl", age_days=1)
old_ws = _touch(
projects / "-data-workspaces-roboco-be-dev-1" / "old.jsonl", age_days=30
)
# The operator's own old session — MUST be preserved.
human_old = _touch(
projects / "-Users-renzof-Documents-thing" / "old.jsonl", age_days=99
)
cutoff = time.time() - 14 * _DAY
prunable = set(select_prunable_transcripts(projects, WORKSPACES, cutoff))
assert old_app in prunable
assert old_ws in prunable
assert fresh_app not in prunable # too new
assert human_old not in prunable # operator's own session — never pruned
def test_select_ignores_non_jsonl_and_directories(tmp_path: Path) -> None:
projects = tmp_path / "projects"
_touch(projects / "-app" / "keep.txt", age_days=30) # not a transcript
(projects / "-app" / "subdir.jsonl").mkdir(parents=True) # a dir named like one
cutoff = time.time() - 14 * _DAY
assert select_prunable_transcripts(projects, WORKSPACES, cutoff) == []
def test_select_handles_missing_projects_root(tmp_path: Path) -> None:
assert select_prunable_transcripts(tmp_path / "nope", WORKSPACES, time.time()) == []
@@ -0,0 +1,56 @@
"""Tests for SettingsService + setting validation."""
from __future__ import annotations
from typing import Any
import pytest
from roboco.services.settings import (
SettingValidationError,
get_settings_service,
validate_setting,
)
def test_validate_setting_rejects_unknown_key() -> None:
with pytest.raises(SettingValidationError):
validate_setting("not_a_real_setting", "x")
def test_validate_retention_requires_positive_int() -> None:
validate_setting("transcript_retention_days", "7") # ok, no raise
with pytest.raises(SettingValidationError):
validate_setting("transcript_retention_days", "0")
with pytest.raises(SettingValidationError):
validate_setting("transcript_retention_days", "abc")
_DEFAULT_RETENTION = 14
_NEW_RETENTION = 30
@pytest.mark.asyncio
async def test_get_int_returns_default_when_unset(db_session: Any) -> None:
svc = get_settings_service(db_session)
assert (
await svc.get_int("transcript_retention_days", _DEFAULT_RETENTION)
== _DEFAULT_RETENTION
)
@pytest.mark.asyncio
async def test_set_then_get_roundtrips(db_session: Any) -> None:
svc = get_settings_service(db_session)
await svc.set("transcript_retention_days", str(_NEW_RETENTION))
assert (
await svc.get_int("transcript_retention_days", _DEFAULT_RETENTION)
== _NEW_RETENTION
)
assert (await svc.all())["transcript_retention_days"] == str(_NEW_RETENTION)
@pytest.mark.asyncio
async def test_set_rejects_invalid_value(db_session: Any) -> None:
svc = get_settings_service(db_session)
with pytest.raises(SettingValidationError):
await svc.set("transcript_retention_days", "-5")