From 31fa95478d8c08a27a0685319539cbff5a9cca2c Mon Sep 17 00:00:00 2001 From: Julius Brussee Date: Fri, 1 May 2026 01:14:53 +0200 Subject: [PATCH] =?UTF-8?q?feat:=20/caveman-stats=20=E2=80=94=20real=20ses?= =?UTF-8?q?sion=20token=20usage=20+=20savings=20estimate?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New slash command that reads the active Claude Code session JSONL (.claude/projects/**/*.jsonl), sums output_tokens and cache_read_input_tokens from assistant turns, and shows an estimated savings figure when the active mode is 'full'. Real numbers, not the model's guess: Caveman Stats ────────────────────────────────── Session: ...projects/my-app/abc123.jsonl Turns: 47 ────────────────────────────────── Output tokens: 3,210 Cache-read tokens: 128,400 ────────────────────────────────── Est. without caveman: 9,171 Est. tokens saved: 5,961 (~65%) Savings est. from benchmarks/ (mean per-task). Actual varies by task. Implementation: * hooks/caveman-stats.js — script. Run directly with `node hooks/caveman-stats.js`, or via `--session-file `. Falls back to most-recent JSONL under .claude/projects/ when no session file is passed. * hooks/caveman-mode-tracker.js — `/caveman-stats` triggers an execFileSync call to caveman-stats.js with the hook's transcript_path, and the output is returned via `decision: "block"` so the user sees the stats inline without a model round-trip. * install.sh / install.ps1 / uninstall.{sh,ps1} include caveman-stats.js in HOOK_FILES. * skills/caveman-stats/SKILL.md (+ plugin mirror) for skill listing. * README install matrix and Caveman Skills section updated. Compression ratio (0.65) is the mean per-task figure from benchmarks/results/*.json (avg_savings: 65 across 10 tasks). Only 'full' mode has measured data — lite/ultra/wenyan show no estimate. Tests: 6 passing in tests/test_caveman_stats.js covering direct invocation, full-mode estimate math, non-full skip, no-session error, mode-tracker block behavior, and flag preservation. Closes #305 (re-implementation; takes the design from @DeeptimaanB but rewritten against current main). Co-Authored-By: Deeptimaan Banerjee Co-Authored-By: Claude Opus 4.7 (1M context) --- README.md | 22 +++ hooks/caveman-mode-tracker.js | 20 +++ hooks/caveman-stats.js | 113 +++++++++++++++ hooks/install.ps1 | 2 +- hooks/install.sh | 2 +- hooks/uninstall.ps1 | 2 +- hooks/uninstall.sh | 2 +- plugins/caveman/skills/caveman-stats/SKILL.md | 10 ++ skills/caveman-stats/SKILL.md | 10 ++ tests/test_caveman_stats.js | 131 ++++++++++++++++++ 10 files changed, 310 insertions(+), 4 deletions(-) create mode 100644 hooks/caveman-stats.js create mode 100644 plugins/caveman/skills/caveman-stats/SKILL.md create mode 100644 skills/caveman-stats/SKILL.md create mode 100644 tests/test_caveman_stats.js diff --git a/README.md b/README.md index 514a5be..b3c8f77 100644 --- a/README.md +++ b/README.md @@ -159,6 +159,7 @@ Auto-activation is built in for Claude Code, Gemini CLI, and the repo-local Code | caveman-review | Y | — | Y | Y | Y | Y | Y | | caveman-compress | Y | Y | Y | Y | Y | Y | Y | | caveman-help | Y | — | Y | Y | Y | Y | Y | +| caveman-stats | Y | — | — | — | — | — | — | > [!NOTE] > Auto-activation works differently per agent: Claude Code uses SessionStart hooks, this repo's Codex dogfood setup uses `.codex/hooks.json`, Gemini uses context files. Cursor/Windsurf/Cline/Copilot can be made always-on, but `npx skills add` installs only the skill, not the repo rule/instruction files. @@ -264,6 +265,7 @@ Auto-activates via `GEMINI.md` context file. Also ships custom Gemini commands: - `/caveman` — switch intensity level (lite/full/ultra/wenyan) - `/caveman-commit` — generate terse commit message - `/caveman-review` — one-line code review +- `/caveman-stats` — real token usage + estimated savings (reads session log) @@ -371,6 +373,26 @@ Level stick until you change it or session end. `/caveman-help` — quick-reference card. All modes, skills, commands, one command away. +### caveman-stats + +`/caveman-stats` — real token usage for current session + estimated savings. Reads the Claude Code session JSONL directly so the numbers are not the model's guess. Savings estimate uses the 65% mean per-task figure from `benchmarks/`; only `full` mode has measured data. + +``` +Caveman Stats +────────────────────────────────── +Session: ...projects/my-app/abc123.jsonl +Turns: 47 +────────────────────────────────── +Output tokens: 3,210 +Cache-read tokens: 128,400 +────────────────────────────────── +Est. without caveman: 9,171 +Est. tokens saved: 5,961 (~65%) +Savings est. from benchmarks/ (mean per-task). Actual varies by task. +``` + +Claude Code only — needs the hook system to read the session transcript. + ### caveman-compress `/caveman:compress ` — caveman make Claude *speak* with fewer tokens. **Compress** make Claude *read* fewer tokens. diff --git a/hooks/caveman-mode-tracker.js b/hooks/caveman-mode-tracker.js index 3badba7..4dbeb62 100644 --- a/hooks/caveman-mode-tracker.js +++ b/hooks/caveman-mode-tracker.js @@ -5,6 +5,7 @@ const fs = require('fs'); const path = require('path'); const os = require('os'); +const { execFileSync } = require('child_process'); const { getDefaultMode, safeWriteFlag, readFlag, VALID_MODES } = require('./caveman-config'); // Modes handled by their own slash commands (/caveman-commit, etc.) — not @@ -34,6 +35,25 @@ process.stdin.on('end', () => { } } + // /caveman-stats — block the prompt and inject stats output as the + // hook's reason. The script reads the active session log, so we pass + // transcript_path through when Claude Code provides it. + if (prompt === '/caveman-stats' || prompt === '/caveman:caveman-stats') { + try { + const statsPath = path.join(__dirname, 'caveman-stats.js'); + const argv = [statsPath]; + if (data.transcript_path) argv.push('--session-file', data.transcript_path); + const out = execFileSync(process.execPath, argv, { encoding: 'utf8', timeout: 5000 }); + process.stdout.write(JSON.stringify({ decision: 'block', reason: out.trim() })); + } catch (e) { + process.stdout.write(JSON.stringify({ + decision: 'block', + reason: 'caveman-stats: could not run stats script.\nTry manually: node hooks/caveman-stats.js' + })); + } + return; + } + // Match /caveman commands if (prompt.startsWith('/caveman')) { const parts = prompt.split(/\s+/); diff --git a/hooks/caveman-stats.js b/hooks/caveman-stats.js new file mode 100644 index 0000000..7cdb383 --- /dev/null +++ b/hooks/caveman-stats.js @@ -0,0 +1,113 @@ +#!/usr/bin/env node +// caveman-stats — read the active Claude Code session log, print real token +// usage plus an estimated savings figure from the benchmark in benchmarks/. +// +// Run directly: node hooks/caveman-stats.js +// Inside Claude: /caveman-stats triggers this via the UserPromptSubmit hook. +// Hook integration passes --session-file so we always read +// the active session, not whichever JSONL was modified most recently. + +const fs = require('fs'); +const path = require('path'); +const os = require('os'); +const { readFlag } = require('./caveman-config'); + +// Mean per-task savings from benchmarks/results/*.json (avg_savings: 65 across +// 10 tasks, sonnet-4-20250514). Only 'full' has measured data; lite/ultra/ +// wenyan modes show no estimate. +const COMPRESSION = { 'full': 0.65 }; + +function findRecentSession(claudeDir) { + const projectsDir = path.join(claudeDir, 'projects'); + let entries; + try { entries = fs.readdirSync(projectsDir, { withFileTypes: true }); } + catch { return null; } + + let best = null; + const stack = entries.map(e => path.join(projectsDir, e.name)); + while (stack.length) { + const p = stack.pop(); + let st; + try { st = fs.statSync(p); } catch { continue; } + if (st.isDirectory()) { + try { + for (const child of fs.readdirSync(p)) stack.push(path.join(p, child)); + } catch {} + } else if (p.endsWith('.jsonl') && (!best || st.mtimeMs > best.mtime)) { + best = { file: p, mtime: st.mtimeMs }; + } + } + return best ? best.file : null; +} + +function parseSession(filePath) { + let raw; + try { raw = fs.readFileSync(filePath, 'utf8'); } + catch { return { outputTokens: 0, cacheReadTokens: 0, turns: 0 }; } + + let outputTokens = 0; + let cacheReadTokens = 0; + let turns = 0; + for (const line of raw.split('\n')) { + if (!line.trim()) continue; + let entry; + try { entry = JSON.parse(line); } catch { continue; } + const usage = entry.type === 'assistant' && entry.message && entry.message.usage; + if (!usage) continue; + outputTokens += usage.output_tokens || 0; + cacheReadTokens += usage.cache_read_input_tokens || 0; + turns++; + } + return { outputTokens, cacheReadTokens, turns }; +} + +function main() { + const args = process.argv.slice(2); + const i = args.indexOf('--session-file'); + const sessionFileArg = i !== -1 ? args[i + 1] : null; + + const claudeDir = process.env.CLAUDE_CONFIG_DIR || path.join(os.homedir(), '.claude'); + const sessionFile = sessionFileArg || findRecentSession(claudeDir); + + if (!sessionFile) { + process.stderr.write('caveman-stats: no Claude Code session found.\n'); + process.exit(1); + } + + const { outputTokens, cacheReadTokens, turns } = parseSession(sessionFile); + const mode = readFlag(path.join(claudeDir, '.caveman-active')); + const ratio = COMPRESSION[mode] != null ? COMPRESSION[mode] : null; + const sep = '──────────────────────────────────'; + const shortPath = sessionFile.length > 45 ? '...' + sessionFile.slice(-45) : sessionFile; + + if (turns === 0) { + process.stdout.write(`\nCaveman Stats\n${sep}\nNo conversation yet — stats available after first response.\n${sep}\n`); + return; + } + + let savings; + let footer = ''; + if (ratio !== null) { + const estNormal = Math.round(outputTokens / (1 - ratio)); + const estSaved = estNormal - outputTokens; + savings = `Est. without caveman: ${estNormal.toLocaleString()}\n` + + `Est. tokens saved: ${estSaved.toLocaleString()} (~${Math.round(ratio * 100)}%)`; + footer = 'Savings est. from benchmarks/ (mean per-task). Actual varies by task.'; + } else if (mode && mode !== 'off') { + savings = `No savings estimate for '${mode}' mode — only 'full' has benchmark data.`; + } else { + savings = 'Caveman not active this session.'; + } + + process.stdout.write( + `\nCaveman Stats\n${sep}\n` + + `Session: ${shortPath}\n` + + `Turns: ${turns}\n${sep}\n` + + `Output tokens: ${outputTokens.toLocaleString()}\n` + + `Cache-read tokens: ${cacheReadTokens.toLocaleString()}\n${sep}\n` + + `${savings}\n` + + (footer ? footer + '\n' : '') + ); +} + +main(); diff --git a/hooks/install.ps1 b/hooks/install.ps1 index b0f9dfe..997c2cc 100644 --- a/hooks/install.ps1 +++ b/hooks/install.ps1 @@ -24,7 +24,7 @@ $HooksDir = Join-Path $ClaudeDir "hooks" $Settings = Join-Path $ClaudeDir "settings.json" $RepoUrl = "https://raw.githubusercontent.com/JuliusBrussee/caveman/main/hooks" -$HookFiles = @("package.json", "caveman-config.js", "caveman-activate.js", "caveman-mode-tracker.js", "caveman-statusline.sh", "caveman-statusline.ps1") +$HookFiles = @("package.json", "caveman-config.js", "caveman-activate.js", "caveman-mode-tracker.js", "caveman-stats.js", "caveman-statusline.sh", "caveman-statusline.ps1") # Resolve source — works from repo clone or remote $ScriptDir = if ($PSScriptRoot) { $PSScriptRoot } else { $null } diff --git a/hooks/install.sh b/hooks/install.sh index 304a7a2..557a3f8 100755 --- a/hooks/install.sh +++ b/hooks/install.sh @@ -37,7 +37,7 @@ HOOKS_DIR="$CLAUDE_DIR/hooks" SETTINGS="$CLAUDE_DIR/settings.json" REPO_URL="https://raw.githubusercontent.com/JuliusBrussee/caveman/main/hooks" -HOOK_FILES=("package.json" "caveman-config.js" "caveman-activate.js" "caveman-mode-tracker.js" "caveman-statusline.sh") +HOOK_FILES=("package.json" "caveman-config.js" "caveman-activate.js" "caveman-mode-tracker.js" "caveman-stats.js" "caveman-statusline.sh") # Resolve source — works from repo clone or curl pipe SCRIPT_DIR="" diff --git a/hooks/uninstall.ps1 b/hooks/uninstall.ps1 index f75cb4a..bfdceaf 100644 --- a/hooks/uninstall.ps1 +++ b/hooks/uninstall.ps1 @@ -11,7 +11,7 @@ $HooksDir = Join-Path $ClaudeDir "hooks" $Settings = Join-Path $ClaudeDir "settings.json" $FlagFile = Join-Path $ClaudeDir ".caveman-active" -$HookFiles = @("package.json", "caveman-config.js", "caveman-activate.js", "caveman-mode-tracker.js", "caveman-statusline.sh", "caveman-statusline.ps1") +$HookFiles = @("package.json", "caveman-config.js", "caveman-activate.js", "caveman-mode-tracker.js", "caveman-stats.js", "caveman-statusline.sh", "caveman-statusline.ps1") # Detect if caveman is installed as a plugin $PluginInstalled = $false diff --git a/hooks/uninstall.sh b/hooks/uninstall.sh index a2b7c05..cc34061 100755 --- a/hooks/uninstall.sh +++ b/hooks/uninstall.sh @@ -10,7 +10,7 @@ HOOKS_DIR="$CLAUDE_DIR/hooks" SETTINGS="$CLAUDE_DIR/settings.json" FLAG_FILE="$CLAUDE_DIR/.caveman-active" -HOOK_FILES=("package.json" "caveman-config.js" "caveman-activate.js" "caveman-mode-tracker.js" "caveman-statusline.sh") +HOOK_FILES=("package.json" "caveman-config.js" "caveman-activate.js" "caveman-mode-tracker.js" "caveman-stats.js" "caveman-statusline.sh") # Detect if caveman is installed as a plugin (check plugin cache) PLUGIN_INSTALLED=0 diff --git a/plugins/caveman/skills/caveman-stats/SKILL.md b/plugins/caveman/skills/caveman-stats/SKILL.md new file mode 100644 index 0000000..a7348ac --- /dev/null +++ b/plugins/caveman/skills/caveman-stats/SKILL.md @@ -0,0 +1,10 @@ +--- +name: caveman-stats +description: > + Show real token usage and estimated savings for the current session. + Reads directly from the Claude Code session log — no AI estimation. + Triggers on /caveman-stats. Output is injected by the mode-tracker hook; + the model itself does not compute the numbers. +--- + +This skill is delivered by `hooks/caveman-stats.js` (read by `hooks/caveman-mode-tracker.js` on `/caveman-stats`). The model does not need to do anything when this skill fires — the hook returns `decision: "block"` with the formatted stats as the reason. The user sees the numbers immediately. diff --git a/skills/caveman-stats/SKILL.md b/skills/caveman-stats/SKILL.md new file mode 100644 index 0000000..a7348ac --- /dev/null +++ b/skills/caveman-stats/SKILL.md @@ -0,0 +1,10 @@ +--- +name: caveman-stats +description: > + Show real token usage and estimated savings for the current session. + Reads directly from the Claude Code session log — no AI estimation. + Triggers on /caveman-stats. Output is injected by the mode-tracker hook; + the model itself does not compute the numbers. +--- + +This skill is delivered by `hooks/caveman-stats.js` (read by `hooks/caveman-mode-tracker.js` on `/caveman-stats`). The model does not need to do anything when this skill fires — the hook returns `decision: "block"` with the formatted stats as the reason. The user sees the numbers immediately. diff --git a/tests/test_caveman_stats.js b/tests/test_caveman_stats.js new file mode 100644 index 0000000..28a71e4 --- /dev/null +++ b/tests/test_caveman_stats.js @@ -0,0 +1,131 @@ +#!/usr/bin/env node +// Tests for /caveman-stats — direct script invocation and via mode tracker. +// Run: node tests/test_caveman_stats.js + +const fs = require('fs'); +const path = require('path'); +const os = require('os'); +const assert = require('assert'); +const { execFileSync } = require('child_process'); + +const ROOT = path.resolve(__dirname, '..'); +const STATS = path.join(ROOT, 'hooks', 'caveman-stats.js'); +const TRACKER = path.join(ROOT, 'hooks', 'caveman-mode-tracker.js'); + +let passed = 0; +let failed = 0; + +function test(name, fn) { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'caveman-stats-test-')); + try { + fn(tmp); + passed++; + console.log(` ✓ ${name}`); + } catch (e) { + failed++; + console.error(` ✗ ${name}\n ${e.message}`); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } +} + +function makeSession(dir, lines) { + const projDir = path.join(dir, '.claude', 'projects', 'p'); + fs.mkdirSync(projDir, { recursive: true }); + const sessFile = path.join(projDir, 's.jsonl'); + fs.writeFileSync(sessFile, lines.map(l => JSON.stringify(l)).join('\n')); + return sessFile; +} + +console.log('caveman-stats tests\n'); + +test('reads --session-file directly and sums output tokens', (tmp) => { + const sess = makeSession(tmp, [ + { type: 'assistant', message: { usage: { output_tokens: 100, cache_read_input_tokens: 200 } } }, + { type: 'user', message: { content: 'hi' } }, + { type: 'assistant', message: { usage: { output_tokens: 50, cache_read_input_tokens: 50 } } }, + ]); + const out = execFileSync(process.execPath, [STATS, '--session-file', sess], { + encoding: 'utf8', + env: { ...process.env, CLAUDE_CONFIG_DIR: path.join(tmp, '.claude') }, + }); + assert.match(out, /Turns:\s+2/); + assert.match(out, /Output tokens:\s+150/); + assert.match(out, /Cache-read tokens:\s+250/); +}); + +test('shows full-mode savings estimate when flag is full', (tmp) => { + const sess = makeSession(tmp, [ + { type: 'assistant', message: { usage: { output_tokens: 350 } } }, + ]); + const claudeDir = path.join(tmp, '.claude'); + fs.writeFileSync(path.join(claudeDir, '.caveman-active'), 'full'); + const out = execFileSync(process.execPath, [STATS, '--session-file', sess], { + encoding: 'utf8', + env: { ...process.env, CLAUDE_CONFIG_DIR: claudeDir }, + }); + // 350 / 0.35 = 1000, saved = 650, ~65% + assert.match(out, /Est\. without caveman:\s+1,000/); + assert.match(out, /Est\. tokens saved:\s+650 \(~65%\)/); +}); + +test('skips estimate for non-full modes', (tmp) => { + const sess = makeSession(tmp, [ + { type: 'assistant', message: { usage: { output_tokens: 100 } } }, + ]); + const claudeDir = path.join(tmp, '.claude'); + fs.writeFileSync(path.join(claudeDir, '.caveman-active'), 'ultra'); + const out = execFileSync(process.execPath, [STATS, '--session-file', sess], { + encoding: 'utf8', + env: { ...process.env, CLAUDE_CONFIG_DIR: claudeDir }, + }); + assert.match(out, /No savings estimate for 'ultra' mode/); +}); + +test('reports no-session when no .jsonl exists', (tmp) => { + fs.mkdirSync(path.join(tmp, '.claude', 'projects'), { recursive: true }); + let err = null; + try { + execFileSync(process.execPath, [STATS], { + encoding: 'utf8', + env: { ...process.env, CLAUDE_CONFIG_DIR: path.join(tmp, '.claude') }, + }); + } catch (e) { err = e; } + assert.ok(err, 'should exit non-zero'); + assert.match(err.stderr, /no Claude Code session found/); +}); + +test('mode tracker handles /caveman-stats with decision block', (tmp) => { + const sess = makeSession(tmp, [ + { type: 'assistant', message: { usage: { output_tokens: 100 } } }, + ]); + const claudeDir = path.join(tmp, '.claude'); + fs.writeFileSync(path.join(claudeDir, '.caveman-active'), 'full'); + const out = execFileSync(process.execPath, [TRACKER], { + encoding: 'utf8', + env: { ...process.env, CLAUDE_CONFIG_DIR: claudeDir, HOME: tmp }, + input: JSON.stringify({ prompt: '/caveman-stats', transcript_path: sess }), + }); + const parsed = JSON.parse(out); + assert.strictEqual(parsed.decision, 'block'); + assert.match(parsed.reason, /Caveman Stats/); + assert.match(parsed.reason, /Output tokens:\s+100/); +}); + +test('mode tracker preserves caveman flag when /caveman-stats fires', (tmp) => { + const sess = makeSession(tmp, [ + { type: 'assistant', message: { usage: { output_tokens: 50 } } }, + ]); + const claudeDir = path.join(tmp, '.claude'); + fs.writeFileSync(path.join(claudeDir, '.caveman-active'), 'full'); + execFileSync(process.execPath, [TRACKER], { + encoding: 'utf8', + env: { ...process.env, CLAUDE_CONFIG_DIR: claudeDir, HOME: tmp }, + input: JSON.stringify({ prompt: '/caveman-stats', transcript_path: sess }), + }); + // The flag must still say 'full' — the stats command must not change mode. + assert.strictEqual(fs.readFileSync(path.join(claudeDir, '.caveman-active'), 'utf8'), 'full'); +}); + +console.log(`\n${passed} passed, ${failed} failed`); +process.exit(failed ? 1 : 0);