Finished reorganization

This commit is contained in:
Renn F
2025-12-14 03:40:48 +01:00
parent 855536eb18
commit 348795c50b
48 changed files with 2491 additions and 1988 deletions
+1 -66
View File
@@ -4,72 +4,7 @@ TOON Metrics
Tracks token savings and usage statistics for TOON vs JSON serialization.
"""
from dataclasses import dataclass, field
from datetime import UTC, datetime
@dataclass
class ToonMetrics:
"""
Metrics for tracking TOON serialization efficiency.
Tracks character counts (as proxy for tokens) for JSON vs TOON
to measure actual savings in production.
"""
json_chars: int = 0
toon_chars: int = 0
encode_count: int = 0
decode_count: int = 0
decode_fallback_count: int = 0
started_at: datetime = field(default_factory=lambda: datetime.now(UTC))
@property
def savings_percent(self) -> float:
"""Calculate percentage of characters saved using TOON."""
if self.json_chars == 0:
return 0.0
return (1 - self.toon_chars / self.json_chars) * 100
@property
def fallback_rate(self) -> float:
"""Calculate rate of fallback to JSON decoding."""
if self.decode_count == 0:
return 0.0
return (self.decode_fallback_count / self.decode_count) * 100
def record_encode(self, json_chars: int, toon_chars: int) -> None:
"""Record an encode operation with character counts."""
self.json_chars += json_chars
self.toon_chars += toon_chars
self.encode_count += 1
def record_decode(self, used_fallback: bool = False) -> None:
"""Record a decode operation."""
self.decode_count += 1
if used_fallback:
self.decode_fallback_count += 1
def to_dict(self) -> dict:
"""Convert metrics to dictionary for logging/reporting."""
return {
"json_chars": self.json_chars,
"toon_chars": self.toon_chars,
"savings_percent": round(self.savings_percent, 2),
"encode_count": self.encode_count,
"decode_count": self.decode_count,
"fallback_rate": round(self.fallback_rate, 2),
"started_at": self.started_at.isoformat(),
}
def reset(self) -> None:
"""Reset all metrics."""
self.json_chars = 0
self.toon_chars = 0
self.encode_count = 0
self.decode_count = 0
self.decode_fallback_count = 0
self.started_at = datetime.now(UTC)
from roboco.models.llm import ToonMetrics
# Global metrics holder
+2 -10
View File
@@ -19,25 +19,17 @@ Usage:
"""
import json
from dataclasses import dataclass
from typing import Any
import structlog
import toon
from pydantic import BaseModel
from roboco.models.llm import ToonConfig
logger = structlog.get_logger()
@dataclass
class ToonConfig:
"""Configuration for TOON encoding."""
delimiter: str = ","
indent: int = 2
include_length: bool = True
class ToonAdapter:
"""
Adapter for TOON serialization at LLM boundaries.