External/caveman
1
0
Fork 0
mirror of https://github.com/JuliusBrussee/caveman.git synced 2026-08-11 13:21:09 +02:00
Code Issues Packages Projects Releases Wiki Activity

fix(stats): budget-saved % was output reduction mislabeled as usage share

Browse Source 
The limit-headroom meter (8a5ab60) printed saved/(saved+output) as
'Session budget saved: ~X% of your usage this session' and 'Est. budget
saved: ~X% of tracked usage'. That ratio is algebraically the output
reduction (always ~65% in full mode) with an output-only denominator —
input + cache tokens, which dominate agentic sessions and count against
Pro/Max limits, were excluded. docs/HONEST-NUMBERS.md on this same
branch says real session-level totals land ~14-21% and below zero on
terse workloads, so the label overstated limit relief.

Fix: say only what the math computes.
- Session view: drop the budget line; the saved line now reads
  '(~X% of output)' and the footer states input/cache usage is
  unchanged.
- Lifetime view: relabel to 'Est. output reduction: ~X% (output tokens
  only, est.)'.
- budgetSavedPct -> outputReductionPct with a comment forbidding
  usage/budget relabeling; tests assert no usage/budget claim appears.
- Hook checksum refreshed.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_011kmm2umRGb5nLxdimrwweZ
This commit is contained in:
Julius Brussee
2026-07-02 15:47:02 +02:00
co-authored by Claude Fable 5
parent e6cce23ed6
commit 6919dc2c4f
3 changed files with 51 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/hooks/caveman-stats.js
+21 -21
View File
@@ -182,14 +182,15 @@ function aggregateHistory(historyPath, sinceMs) {
return { sessions: latestPerSession.size, outputTokens, estSavedTokens, estSavedUsd };
}
// Limit-headroom meter. Subscription users (Claude Code Pro/Max) spend a
// 5-hour/weekly usage limit, not dollars — for them the meaningful number is
// what share of their would-be usage caveman freed. Computed ONLY from token
// counts we actually have: saved / (saved + used). We never assume a plan's
// limit size (Anthropic doesn't publish token quotas), so this is a share of
// usage, never "% of your weekly limit". Returns a rounded percent, or null
// when there is nothing measured to divide.
function budgetSavedPct(savedTokens, usedTokens) {
// Output-reduction share: saved / (saved + used) = the fraction of the
// would-be OUTPUT tokens that caveman avoided. That is the only ratio we can
// honestly compute from output counts alone. It is NOT a share of session or
// limit usage — input + cache tokens dominate agentic sessions, count against
// Pro/Max limits, and are not reduced by caveman, so real limit relief is far
// smaller (docs/HONEST-NUMBERS.md: session-level totals land ~14–21%, below
// zero on terse workloads). Never label this "usage" or "budget". Returns a
// rounded percent, or null when there is nothing measured to divide.
function outputReductionPct(savedTokens, usedTokens) {
if (!Number.isFinite(savedTokens) || !Number.isFinite(usedTokens)) return null;
if (savedTokens <= 0 || usedTokens < 0) return null;
const total = savedTokens + usedTokens;
@@ -211,9 +212,9 @@ function formatHistory({ sessions, outputTokens, estSavedTokens, estSavedUsd, si
return `\nCaveman Stats — Lifetime${window}\n${sep}\nNo sessions logged yet — run /caveman-stats inside any session to start tracking.\n${sep}\n`;
}
const usdLine = estSavedUsd > 0 ? `Est. saved (USD): ~${formatUsd(estSavedUsd)}\n` : '';
const pct = budgetSavedPct(estSavedTokens, outputTokens);
const pct = outputReductionPct(estSavedTokens, outputTokens);
const budgetLine = pct !== null
? `Est. budget saved: ~${pct}% of tracked usage (est.)\n`
? `Est. output reduction: ~${pct}% (output tokens only, est.)\n`
: '';
return `\nCaveman Stats — Lifetime${window}\n${sep}\n` +
`Sessions: ${sessions.toLocaleString()}\n${sep}\n` +
@@ -269,17 +270,16 @@ function formatStats({ outputTokens, cacheReadTokens, turns, mode, model, sessio
} else {
footer = 'Savings est. from benchmarks/ (mean per-task). Actual varies by task.';
}
// Limit-headroom framing for subscription (Pro/Max) users — see
// budgetSavedPct. USD stays above for API users.
const pct = budgetSavedPct(estSaved, outputTokens);
let budgetLine = '';
if (pct !== null) {
budgetLine = `Session budget saved: ~${pct}% of your usage this session (est.)\n`;
footer += ' Budget % = est. saved / (saved + used) tokens; no plan-limit size assumed.';
}
// No "% of your usage/budget" line here on purpose: from output tokens
// alone the only computable ratio is the output reduction already shown
// on the line above, and input + cache tokens (which dominate agentic
// sessions and count against Pro/Max limits) are untouched by caveman —
// any session-usage % would overstate real limit relief. See
// docs/HONEST-NUMBERS.md.
footer += ' Reduction is of output tokens only; input/cache usage is unchanged.';
savings = (`Est. without caveman: ${estNormal.toLocaleString()}\n` +
`Est. tokens saved: ${estSaved.toLocaleString()} (~${Math.round(ratio * 100)}%)\n` +
usdLine + budgetLine).replace(/\n$/, '');
`Est. tokens saved: ${estSaved.toLocaleString()} (~${Math.round(ratio * 100)}% of output)\n` +
usdLine).replace(/\n$/, '');
} else if (mode && mode !== 'off') {
savings = `No savings estimate for '${mode}' mode — only 'full' has benchmark data.`;
} else {
@@ -376,5 +376,5 @@ if (require.main === module) main();
module.exports = {
formatStats, formatShare, formatHistory, aggregateHistory, parseDuration, deriveSavings,
parseSession, priceForModel, formatUsd, COMPRESSION, MODEL_OUTPUT_PRICE_PER_M,
findCompressedPairs, summarizeCompressed, humanizeTokens, budgetSavedPct,
findCompressedPairs, summarizeCompressed, humanizeTokens, outputReductionPct,
};
src/hooks/checksums.sha256
+1 -1
View File
@@ -2,7 +2,7 @@
c0b77891d1f8aaef3a7fbf76533ba63a42eaa96431fb7ece611ccff7474de2ba caveman-config.js
52ba51f7c51749fda1e65c63060cec03ffad36c35856c01068c0f63fa33e5b40 caveman-activate.js
a5a532fd57203aa96c5c6eeeef647df2770869343f31be009ba9da318da95301 caveman-mode-tracker.js
c841fa940e016f2a3000f814d51d9f96b0a37cc5cbaa01b63cb4f9820640960c caveman-stats.js
c8d4a43399dc07457c38c15fa141343d8901879dee6293b248ab23ad8036ec83 caveman-stats.js
d2deff457d0a5d8e1848193e6af6a68a0ebdba4fbdf250889400d5ea231e088f caveman-statusline.sh
1690c639f05940cbff39e0383a27053898b30c224aa651043db29b2842cb524a caveman-statusline.ps1
9f2601e8551653609f0b9d700c9bd75bffa43747d2db18ffe83fb73de5dc3607 cavecrew-model-overrides.js
tests/test_caveman_stats.js
+29 -35
View File
@@ -66,7 +66,7 @@ test('shows full-mode savings estimate when flag is full', (tmp) => {
});
// 350 / 0.35 = 1000, saved = 650, ~65%
assert.match(out, /Est\. without caveman:\s+1,000/);
assert.match(out, /Est\. tokens saved:\s+650 \(~65%\)/);
assert.match(out, /Est\. tokens saved:\s+650 \(~65% of output\)/);
});
test('skips estimate for non-full modes', (tmp) => {
@@ -154,7 +154,7 @@ test('omits USD line when model is unknown', (tmp) => {
env: { ...process.env, CLAUDE_CONFIG_DIR: claudeDir },
});
// Token estimate still appears, USD line does not.
assert.match(out, /Est\. tokens saved:\s+650 \(~65%\)/);
assert.match(out, /Est\. tokens saved:\s+650 \(~65% of output\)/);
assert.doesNotMatch(out, /Est\. saved \(USD\)/);
});
@@ -460,20 +460,23 @@ test('mode tracker forwards --share to stats script', (tmp) => {
assert.match(parsed.reason, /^🪨 Saved 650 output tokens/);
});
// ── Limit-headroom meter (subscription users spend usage limit, not $) ─────
// ── Output-reduction share (never a "usage"/"budget" claim) ────────────────
// saved/(saved+used) from output tokens is the OUTPUT reduction — input and
// cache tokens dominate real sessions and are untouched, so printing it as a
// share of usage/budget would overstate limit relief (docs/HONEST-NUMBERS.md).
test('budgetSavedPct = saved / (saved + used), null when nothing saved', () => {
const { budgetSavedPct } = require(STATS);
assert.strictEqual(budgetSavedPct(650, 350), 65);
assert.strictEqual(budgetSavedPct(1, 3), 25);
assert.strictEqual(budgetSavedPct(0, 350), null); // no measured savings → no claim
assert.strictEqual(budgetSavedPct(-5, 350), null);
assert.strictEqual(budgetSavedPct(650, -1), null);
assert.strictEqual(budgetSavedPct(NaN, 350), null);
assert.strictEqual(budgetSavedPct(650, Infinity), null);
test('outputReductionPct = saved / (saved + used), null when nothing saved', () => {
const { outputReductionPct } = require(STATS);
assert.strictEqual(outputReductionPct(650, 350), 65);
assert.strictEqual(outputReductionPct(1, 3), 25);
assert.strictEqual(outputReductionPct(0, 350), null); // no measured savings → no claim
assert.strictEqual(outputReductionPct(-5, 350), null);
assert.strictEqual(outputReductionPct(650, -1), null);
assert.strictEqual(outputReductionPct(NaN, 350), null);
assert.strictEqual(outputReductionPct(650, Infinity), null);
});
test('shows session budget saved % (est.) alongside tokens when savings measured', (tmp) => {
test('session view never claims a % of usage/budget — only output reduction', (tmp) => {
const sess = makeSession(tmp, [
{ type: 'assistant', message: { model: 'claude-sonnet-4-7', usage: { output_tokens: 350 } } },
]);
@@ -483,29 +486,18 @@ test('shows session budget saved % (est.) alongside tokens when savings measured
encoding: 'utf8',
env: { ...process.env, CLAUDE_CONFIG_DIR: claudeDir },
});
// saved 650 / (650 saved + 350 used) = 65%
assert.match(out, /Session budget saved:\s+~65% of your usage this session \(est\.\)/);
// The reduction is labeled as output-only, never a share of session usage.
assert.match(out, /Est\. tokens saved:\s+650 \(~65% of output\)/);
assert.ok(!/budget|of your usage|of tracked usage/i.test(out),
'must not relabel output reduction as a usage/budget share');
// Dollars stay for API users.
assert.match(out, /Est\. saved \(USD\):/);
// Honesty: the % must be labeled as estimate math, never a plan-limit claim.
assert.match(out, /no plan-limit size assumed/);
// Footer must state the reduction excludes input/cache usage.
assert.match(out, /output tokens only; input\/cache usage is unchanged/);
assert.ok(!/weekly limit|5-hour limit/i.test(out), 'must not fabricate Anthropic quota sizes');
});
test('omits budget line when no savings estimate exists (lite mode)', (tmp) => {
const sess = makeSession(tmp, [
{ type: 'assistant', message: { usage: { output_tokens: 100 } } },
]);
const claudeDir = path.join(tmp, '.claude');
fs.writeFileSync(path.join(claudeDir, '.caveman-active'), 'lite');
const out = execFileSync(process.execPath, [STATS, '--session-file', sess], {
encoding: 'utf8',
env: { ...process.env, CLAUDE_CONFIG_DIR: claudeDir },
});
assert.ok(!/budget saved/i.test(out), 'no measured savings → no budget % claim');
});
test('--all lifetime output includes est. budget saved % when savings tracked', (tmp) => {
test('--all lifetime output labels the % as output reduction, not usage', (tmp) => {
const claudeDir = path.join(tmp, '.claude');
fs.mkdirSync(claudeDir, { recursive: true });
const history = [
@@ -520,11 +512,13 @@ test('--all lifetime output includes est. budget saved % when savings tracked',
encoding: 'utf8',
env: { ...process.env, CLAUDE_CONFIG_DIR: claudeDir },
});
// saved 1000 / (1000 saved + 1000 used) = 50%
assert.match(out, /Est\. budget saved:\s+~50% of tracked usage \(est\.\)/);
// saved 1000 / (1000 saved + 1000 used-output) = 50% of would-be output
assert.match(out, /Est\. output reduction:\s+~50% \(output tokens only, est\.\)/);
assert.ok(!/budget|of your usage|of tracked usage/i.test(out),
'must not relabel output reduction as a usage/budget share');
});
test('--all lifetime output omits budget line when nothing saved', (tmp) => {
test('--all lifetime output omits reduction line when nothing saved', (tmp) => {
const claudeDir = path.join(tmp, '.claude');
fs.mkdirSync(claudeDir, { recursive: true });
fs.writeFileSync(
@@ -535,7 +529,7 @@ test('--all lifetime output omits budget line when nothing saved', (tmp) => {
encoding: 'utf8',
env: { ...process.env, CLAUDE_CONFIG_DIR: claudeDir },
});
assert.ok(!/budget saved/i.test(out), 'zero savings → honest zero, no % line');
assert.ok(!/output reduction|budget/i.test(out), 'zero savings → honest zero, no % line');
});
console.log(`\n${passed} passed, ${failed} failed`);