mirror of
https://github.com/block/buzz.git
synced 2026-08-18 06:50:31 +02:00
fix(agent-usage): fall back to ≈in+out display when totalTokens null
No real publisher emits totalTokens today (0 of 1,934 prod reports
do). The UI was keying its flagship visuals off that field, producing
'No usage reported' over millions of real tokens.
Four display surfaces fixed:
- Daily bars header: '≈ N tokens' when knownTotal null but i/o known
- Daily bar heights: 'approx' bar kind scaled to i/o sum (bg-primary/70)
instead of the hatched unknown baseline
- Per-agent row: trailing text '≈ N' + progress bar from i/o approx
- Focused view 'Total tokens' stat: '≈ N,NNN' exact display
PARTIAL badge: no longer keys on totalTokens.incomplete (permanently
true); now keys on inputTokens/outputTokens incompleteness only, so
honest partial rows still badge but the permanently-null-total case
does not.
Caveat paragraph: gated on agent.hasUnknownUsage || invalidReportCount
> 0 only, not on the permanent totals-unknown state. Removed the false
persistent-error banner from every real prod user's focused view.
All approximations use ≈ prefix at every callsite — the wire contract
is unchanged; this is display only.
New lib export: deriveApproxTotal (5 unit tests). sumKnownBucketTotals
now returns approxTotal alongside knownTotal (2 new unit tests + 2
existing tests updated). e2e seeds updated: mockAgentUsage default
now uses {inputTokens, outputTokens, totalTokens: null} — the real
prod shape — so the suite can no longer pass while prod renders empty.
Co-authored-by: Will Pfleger <pfleger.will@gmail.com>
Signed-off-by: Will Pfleger <pfleger.will@gmail.com>
This commit is contained in:
co-authored by
Will Pfleger
parent
0a5b0701b0
commit
cbfbe1ed05
@@ -4,6 +4,7 @@ import test from "node:test";
|
||||
import {
|
||||
bigintRatio,
|
||||
buildLocalDayBoundaries,
|
||||
deriveApproxTotal,
|
||||
deriveUsageIngressTrailing,
|
||||
formatCoverageDate,
|
||||
formatEstimatedCostUsd,
|
||||
@@ -333,6 +334,44 @@ test("bigintRatio clamps part to [0, whole]", () => {
|
||||
assert.equal(bigintRatio(200n, 100n), 1);
|
||||
});
|
||||
|
||||
// ── deriveApproxTotal ─────────────────────────────────────────────────────────
|
||||
|
||||
test("deriveApproxTotal returns null when genuine total is known (no approximation needed)", () => {
|
||||
const usage = reportedUsage({
|
||||
inputTokens: usageField({ value: "800" }),
|
||||
outputTokens: usageField({ value: "200" }),
|
||||
totalTokens: usageField({ value: "1100" }),
|
||||
});
|
||||
assert.equal(deriveApproxTotal(usage), null);
|
||||
});
|
||||
|
||||
test("deriveApproxTotal sums input and output when total is null", () => {
|
||||
const usage = reportedUsage({
|
||||
inputTokens: usageField({ value: "800" }),
|
||||
outputTokens: usageField({ value: "200" }),
|
||||
});
|
||||
assert.equal(deriveApproxTotal(usage), 1000n);
|
||||
});
|
||||
|
||||
test("deriveApproxTotal returns input alone when output is null", () => {
|
||||
const usage = reportedUsage({
|
||||
inputTokens: usageField({ value: "500" }),
|
||||
});
|
||||
assert.equal(deriveApproxTotal(usage), 500n);
|
||||
});
|
||||
|
||||
test("deriveApproxTotal returns output alone when input is null", () => {
|
||||
const usage = reportedUsage({
|
||||
outputTokens: usageField({ value: "300" }),
|
||||
});
|
||||
assert.equal(deriveApproxTotal(usage), 300n);
|
||||
});
|
||||
|
||||
test("deriveApproxTotal returns null when total, input, and output are all null", () => {
|
||||
const usage = reportedUsage();
|
||||
assert.equal(deriveApproxTotal(usage), null);
|
||||
});
|
||||
|
||||
// ── sortAgentsByKnownTotal / sortModelsByKnownTotal ─────────────────────────
|
||||
|
||||
test("sortAgentsByKnownTotal ranks known totals descending", () => {
|
||||
@@ -452,6 +491,7 @@ test("sumKnownBucketTotals returns knownTotal null and partial false for an all-
|
||||
bucket({ reportCount: 0 }),
|
||||
]);
|
||||
assert.equal(result.knownTotal, null);
|
||||
assert.equal(result.approxTotal, null);
|
||||
assert.equal(result.partial, false);
|
||||
});
|
||||
|
||||
@@ -467,6 +507,7 @@ test("sumKnownBucketTotals sums all known totals when every bucket is fully know
|
||||
}),
|
||||
]);
|
||||
assert.equal(result.knownTotal, 300n);
|
||||
assert.equal(result.approxTotal, null); // genuine total is known, no approx needed
|
||||
assert.equal(result.partial, false);
|
||||
});
|
||||
|
||||
@@ -499,6 +540,38 @@ test("sumKnownBucketTotals marks partial true when any bucket has reports but nu
|
||||
assert.equal(result.partial, true);
|
||||
});
|
||||
|
||||
test("sumKnownBucketTotals returns approxTotal from i/o sum when all bucket totals are null but i/o is known", () => {
|
||||
// Real-world case: no publisher emits totalTokens, but i/o are always present.
|
||||
const result = sumKnownBucketTotals([
|
||||
bucket({
|
||||
usage: reportedUsage({
|
||||
inputTokens: usageField({ value: "800" }),
|
||||
outputTokens: usageField({ value: "200" }),
|
||||
}),
|
||||
reportCount: 1,
|
||||
}),
|
||||
bucket({
|
||||
usage: reportedUsage({
|
||||
inputTokens: usageField({ value: "400" }),
|
||||
outputTokens: usageField({ value: "100" }),
|
||||
}),
|
||||
reportCount: 1,
|
||||
}),
|
||||
]);
|
||||
assert.equal(result.knownTotal, null);
|
||||
assert.equal(result.approxTotal, 1500n); // (800+200) + (400+100)
|
||||
assert.equal(result.partial, true); // null totals with reports → partial
|
||||
});
|
||||
|
||||
test("sumKnownBucketTotals returns null approxTotal when even i/o is unavailable", () => {
|
||||
const result = sumKnownBucketTotals([
|
||||
bucket({ usage: reportedUsage(), reportCount: 1, hasUnknownUsage: true }),
|
||||
]);
|
||||
assert.equal(result.knownTotal, null);
|
||||
assert.equal(result.approxTotal, null);
|
||||
assert.equal(result.partial, true);
|
||||
});
|
||||
|
||||
// ── deriveUsageIngressTrailing ────────────────────────────────────────────────
|
||||
|
||||
function baseSeries(overrides = {}) {
|
||||
|
||||
@@ -189,6 +189,30 @@ export function bigintRatio(part: bigint, whole: bigint): number {
|
||||
return Number(permille) / 1000;
|
||||
}
|
||||
|
||||
// ── Display approximation (A2 presentation layer) ────────────────────────────
|
||||
|
||||
/**
|
||||
* Derive a display approximation of the total by summing known input and output
|
||||
* token counts. Returns `null` if neither field is known. This is a *display
|
||||
* label* only — it is NEVER written to the wire or stored; NIP-AM's
|
||||
* "MUST NOT derive total = input + output" governs published/stored data only.
|
||||
* Callers MUST prefix the result with `≈` so the approximation is honest.
|
||||
*/
|
||||
export function deriveApproxTotal(usage: {
|
||||
inputTokens: UsageField;
|
||||
outputTokens: UsageField;
|
||||
totalTokens: UsageField;
|
||||
}): bigint | null {
|
||||
if (parseTokenCount(usage.totalTokens.value) !== null) {
|
||||
// Genuine total is known — callers should use it directly; no approximation needed.
|
||||
return null;
|
||||
}
|
||||
const input = parseTokenCount(usage.inputTokens.value);
|
||||
const output = parseTokenCount(usage.outputTokens.value);
|
||||
if (input === null && output === null) return null;
|
||||
return (input ?? 0n) + (output ?? 0n);
|
||||
}
|
||||
|
||||
// ── Ranking (A2: known lower-bound totals rank; null totals list after) ─────
|
||||
|
||||
type Ranked<T> = { item: T; totalTokens: bigint | null };
|
||||
@@ -316,16 +340,24 @@ export function deriveUsageIngressTrailing(series: AgentUsageSeries): string {
|
||||
* sum cannot include) — never silently reported as a complete figure.
|
||||
* Returns `knownTotal: null` only when every bucket has zero reports (a
|
||||
* true empty window, not partial data).
|
||||
*
|
||||
* Also returns `approxTotal`: when `knownTotal` is null but at least one
|
||||
* bucket has known input/output, this is the sum of all known i/o fields
|
||||
* (labeled `≈` at the call site). Null when even i/o is unavailable.
|
||||
* The approximation is a display-only convenience — never stored or wired.
|
||||
*/
|
||||
export function sumKnownBucketTotals(
|
||||
buckets: readonly AgentUsageSeriesBucket[],
|
||||
): {
|
||||
knownTotal: bigint | null;
|
||||
approxTotal: bigint | null;
|
||||
partial: boolean;
|
||||
} {
|
||||
let sum = 0n;
|
||||
let sawKnown = false;
|
||||
let partial = false;
|
||||
let approxSum = 0n;
|
||||
let sawApprox = false;
|
||||
|
||||
for (const bucket of buckets) {
|
||||
const total = bucket.usage.totalTokens;
|
||||
@@ -337,7 +369,18 @@ export function sumKnownBucketTotals(
|
||||
if (isPartialField(total) || (bucket.reportCount > 0 && known === null)) {
|
||||
partial = true;
|
||||
}
|
||||
if (known === null) {
|
||||
const approx = deriveApproxTotal(bucket.usage);
|
||||
if (approx !== null) {
|
||||
approxSum += approx;
|
||||
sawApprox = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { knownTotal: sawKnown ? sum : null, partial };
|
||||
return {
|
||||
knownTotal: sawKnown ? sum : null,
|
||||
approxTotal: !sawKnown && sawApprox ? approxSum : null,
|
||||
partial,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import { cn } from "@/shared/lib/cn";
|
||||
import type { AgentUsageSeriesBucket } from "@/shared/api/tauriArchive";
|
||||
import {
|
||||
bigintRatio,
|
||||
deriveApproxTotal,
|
||||
formatTokenCountCompact,
|
||||
isPartialField,
|
||||
parseTokenCount,
|
||||
@@ -38,6 +39,10 @@ function dateLabelOf(unixSeconds: number): string {
|
||||
* day (the field is `null` because nothing happened), which is a different
|
||||
* state from `hasUnknownUsage` (activity happened but the total could not
|
||||
* be fully counted) even though both leave `usage.totalTokens.value` null.
|
||||
*
|
||||
* When no genuine total is available but i/o counts are known, the bar falls
|
||||
* back to an approx i/o sum (`kind: "approx"`) rather than the hatched
|
||||
* unknown baseline — the bar height becomes meaningful and is labeled `≈`.
|
||||
*/
|
||||
function deriveBarState(bucket: AgentUsageSeriesBucket) {
|
||||
const total = bucket.usage.totalTokens;
|
||||
@@ -51,20 +56,29 @@ function deriveBarState(bucket: AgentUsageSeriesBucket) {
|
||||
knownTokens: 0n,
|
||||
};
|
||||
}
|
||||
if (known === null) {
|
||||
if (known !== null) {
|
||||
const partial = isPartialField(total);
|
||||
return {
|
||||
accessibleLabel: `${dateLabel} · unknown usage`,
|
||||
kind: "unknown" as const,
|
||||
knownTokens: null,
|
||||
accessibleLabel: `${dateLabel} · ${formatTokenCountCompact(known)} reported tokens${
|
||||
partial ? " (partial)" : ""
|
||||
}`,
|
||||
kind: partial ? ("partial" as const) : ("known" as const),
|
||||
knownTokens: known,
|
||||
};
|
||||
}
|
||||
// Genuine total unknown — try i/o approximation before falling back to hatched.
|
||||
const approx = deriveApproxTotal(bucket.usage);
|
||||
if (approx !== null) {
|
||||
return {
|
||||
accessibleLabel: `${dateLabel} · ≈ ${formatTokenCountCompact(approx)} tokens (approx)`,
|
||||
kind: "approx" as const,
|
||||
knownTokens: approx,
|
||||
};
|
||||
}
|
||||
const partial = isPartialField(total);
|
||||
return {
|
||||
accessibleLabel: `${dateLabel} · ${formatTokenCountCompact(known)} reported tokens${
|
||||
partial ? " (partial)" : ""
|
||||
}`,
|
||||
kind: partial ? ("partial" as const) : ("known" as const),
|
||||
knownTokens: known,
|
||||
accessibleLabel: `${dateLabel} · unknown usage`,
|
||||
kind: "unknown" as const,
|
||||
knownTokens: null,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -86,7 +100,11 @@ export function AgentUsageDailyBars({
|
||||
() =>
|
||||
buckets.reduce<bigint>((max, bucket) => {
|
||||
const total = parseTokenCount(bucket.usage.totalTokens.value);
|
||||
return total !== null && total > max ? total : max;
|
||||
if (total !== null) return total > max ? total : max;
|
||||
// Fall back to the i/o approximation so bars scale correctly when
|
||||
// no bucket reports a genuine total.
|
||||
const approx = deriveApproxTotal(bucket.usage);
|
||||
return approx !== null && approx > max ? approx : max;
|
||||
}, 0n),
|
||||
[buckets],
|
||||
);
|
||||
@@ -136,7 +154,9 @@ function DailyBar({
|
||||
? "—"
|
||||
: kind === "partial"
|
||||
? `≥${formatTokenCountCompact(knownTokens ?? 0n)}`
|
||||
: formatTokenCountCompact(knownTokens ?? 0n);
|
||||
: kind === "approx"
|
||||
? `≈${formatTokenCountCompact(knownTokens ?? 0n)}`
|
||||
: formatTokenCountCompact(knownTokens ?? 0n);
|
||||
|
||||
return (
|
||||
<div
|
||||
@@ -162,9 +182,11 @@ function DailyBar({
|
||||
"w-full rounded-t-sm",
|
||||
kind === "partial"
|
||||
? "bg-primary/50"
|
||||
: kind === "empty"
|
||||
? "bg-muted/40"
|
||||
: "bg-primary",
|
||||
: kind === "approx"
|
||||
? "bg-primary/70"
|
||||
: kind === "empty"
|
||||
? "bg-muted/40"
|
||||
: "bg-primary",
|
||||
)}
|
||||
role="img"
|
||||
style={{ height: knownHeightPx }}
|
||||
|
||||
@@ -18,6 +18,7 @@ import {
|
||||
formatEstimatedCostUsd,
|
||||
formatTokenCountCompact,
|
||||
formatTokenCountExact,
|
||||
deriveApproxTotal,
|
||||
isPartialField,
|
||||
isUnknownField,
|
||||
parseTokenCount,
|
||||
@@ -206,13 +207,27 @@ function AgentUsageFocusedTotals({
|
||||
const { estimatedCostUsd, inputTokens, outputTokens, totalTokens } =
|
||||
agent.usage;
|
||||
const models = sortModelsByKnownTotal(agent.models);
|
||||
// Show the caveat paragraph only when there are genuinely invalid/excluded
|
||||
// rows or truly unknown i/o deltas. `agent.hasUnknownUsage` is true when
|
||||
// deltaReliable is false (some intervals couldn't be computed). The
|
||||
// `coverage.invalidReportCount > 0` path covers rows excluded from buckets.
|
||||
// We do NOT trigger on totalTokens.value being null: that's the permanent
|
||||
// state for all real publishers today (no harness emits a total yet), and
|
||||
// showing the caveat permanently would make it read as a persistent error.
|
||||
const explainPartial =
|
||||
agent.hasUnknownUsage || coverage.invalidReportCount > 0;
|
||||
|
||||
// Approximation for the Total tokens stat when no genuine total is available.
|
||||
const approxTotal = deriveApproxTotal(agent.usage);
|
||||
|
||||
return (
|
||||
<Card className="space-y-4 p-6" data-testid="agent-usage-focused-totals">
|
||||
<div className="grid grid-cols-2 gap-4 sm:grid-cols-4">
|
||||
<TokenStat field={totalTokens} label="Total tokens" />
|
||||
<ApproxTokenStat
|
||||
approxTotal={approxTotal}
|
||||
field={totalTokens}
|
||||
label="Total tokens"
|
||||
/>
|
||||
<TokenStat field={inputTokens} label="Input tokens" />
|
||||
<TokenStat field={outputTokens} label="Output tokens" />
|
||||
<UsageStat
|
||||
@@ -314,6 +329,32 @@ function TokenStat({
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Total-tokens stat that falls back to an `≈` approximation (in+out) when
|
||||
* the genuine total is unavailable. The `≈` prefix keeps the approximation
|
||||
* honest without hiding that real token activity was counted.
|
||||
*/
|
||||
function ApproxTokenStat({
|
||||
approxTotal,
|
||||
field,
|
||||
label,
|
||||
}: {
|
||||
approxTotal: bigint | null;
|
||||
field: { value: string | null; incomplete: boolean };
|
||||
label: string;
|
||||
}) {
|
||||
const parsed = parseTokenCount(field.value);
|
||||
const display =
|
||||
parsed !== null
|
||||
? formatTokenCountExact(parsed)
|
||||
: approxTotal !== null
|
||||
? `≈ ${formatTokenCountExact(approxTotal)}`
|
||||
: null;
|
||||
return (
|
||||
<UsageStat display={display} isPartial={isPartialField(field)} label={label} />
|
||||
);
|
||||
}
|
||||
|
||||
function UsageStat({
|
||||
display,
|
||||
isPartial,
|
||||
|
||||
@@ -21,6 +21,7 @@ import { Tabs, TabsList, TabsTrigger } from "@/shared/ui/tabs";
|
||||
import { useAgentUsageSeries } from "../hooks";
|
||||
import {
|
||||
bigintRatio,
|
||||
deriveApproxTotal,
|
||||
formatCoverageDate,
|
||||
formatTokenCountCompact,
|
||||
isPartialField,
|
||||
@@ -154,13 +155,17 @@ function AgentUsageCard({
|
||||
series.coverage.invalidReportCount > 0;
|
||||
|
||||
// Relative bars are decorative (aria-hidden, per plan) — scale each agent's
|
||||
// known total against the largest known total in the current window so the
|
||||
// sorted-by-total list also reads as a bar chart.
|
||||
// known total (or i/o approximation) against the largest such value in the
|
||||
// current window so the sorted-by-total list also reads as a bar chart.
|
||||
const maxKnownTotal = React.useMemo(
|
||||
() =>
|
||||
agents.reduce<bigint>((max, agent) => {
|
||||
const total = parseTokenCount(agent.usage.totalTokens.value);
|
||||
return total !== null && total > max ? total : max;
|
||||
if (total !== null) return total > max ? total : max;
|
||||
// Fall back to the i/o approximation so bars are still visible when
|
||||
// no agent reports a genuine total.
|
||||
const approx = deriveApproxTotal(agent.usage);
|
||||
return approx !== null && approx > max ? approx : max;
|
||||
}, 0n),
|
||||
[agents],
|
||||
);
|
||||
@@ -179,9 +184,11 @@ function AgentUsageCard({
|
||||
<span className="text-sm text-muted-foreground">
|
||||
{overallTotal.knownTotal !== null
|
||||
? `${formatTokenCountCompact(overallTotal.knownTotal)} tokens`
|
||||
: hasInvalidOnlyInWindow
|
||||
? "Usage uncountable"
|
||||
: "No usage reported"}
|
||||
: overallTotal.approxTotal !== null
|
||||
? `≈ ${formatTokenCountCompact(overallTotal.approxTotal)} tokens`
|
||||
: hasInvalidOnlyInWindow
|
||||
? "Usage uncountable"
|
||||
: "No usage reported"}
|
||||
{overallTotal.partial || hasInvalidOnlyInWindow ? (
|
||||
<Badge className="ml-2" variant="outline">
|
||||
Partial
|
||||
@@ -276,10 +283,18 @@ function AgentUsageRow({
|
||||
(isPartialField(agent.usage.inputTokens) ||
|
||||
isPartialField(agent.usage.outputTokens));
|
||||
|
||||
// Approximate total for display when no genuine total is available.
|
||||
const approxTotal =
|
||||
knownTotal === null ? deriveApproxTotal(agent.usage) : null;
|
||||
// Effective value for the relative bar width — prefer genuine total, then approx.
|
||||
const barTotal = knownTotal ?? approxTotal;
|
||||
|
||||
const trailing =
|
||||
knownTotal !== null
|
||||
? formatTokenCountCompact(knownTotal)
|
||||
: formatIndependentFields(agent);
|
||||
: approxTotal !== null
|
||||
? `≈ ${formatTokenCountCompact(approxTotal)}`
|
||||
: formatIndependentFields(agent);
|
||||
|
||||
return (
|
||||
<button
|
||||
@@ -298,13 +313,13 @@ function AgentUsageRow({
|
||||
<span className="block truncate text-sm font-medium text-foreground">
|
||||
{label}
|
||||
</span>
|
||||
{!unknown ? (
|
||||
{!unknown || approxTotal !== null ? (
|
||||
<Progress
|
||||
aria-hidden="true"
|
||||
className="mt-1.5 h-1.5"
|
||||
value={
|
||||
knownTotal !== null && maxKnownTotal > 0n
|
||||
? bigintRatio(knownTotal, maxKnownTotal) * 100
|
||||
barTotal !== null && maxKnownTotal > 0n
|
||||
? bigintRatio(barTotal, maxKnownTotal) * 100
|
||||
: null
|
||||
}
|
||||
/>
|
||||
|
||||
@@ -53,7 +53,8 @@ function mockAgentUsage(
|
||||
hasUnknownUsage: false,
|
||||
models: [],
|
||||
reportCount: 1,
|
||||
usage: reportedUsage({ totalTokens: "1500" }),
|
||||
// Real-world shape: no publisher emits totalTokens; i/o are always present.
|
||||
usage: reportedUsage({ inputTokens: "1200", outputTokens: "300" }),
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -65,7 +65,11 @@ function mockAgentUsage(
|
||||
hasUnknownUsage: false,
|
||||
models: [],
|
||||
reportCount: 1,
|
||||
usage: reportedUsage({ totalTokens: "1500" }),
|
||||
// Real-world shape: no publisher emits totalTokens today; i/o are always
|
||||
// present. Seeding this as the default ensures the suite exercises the
|
||||
// "approx ≈" display path that prod users see, not just the happy-path
|
||||
// seed that previously masked the "No usage reported" regression.
|
||||
usage: reportedUsage({ inputTokens: "1200", outputTokens: "300" }),
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user