Switch eval plot to dot+whiskers (median, IQR, min/max)

This commit is contained in:
sebastianbreguel
2026-04-08 18:32:06 -04:00
parent 7de87db9e6
commit 784df686b4
3 changed files with 108 additions and 53 deletions
+107 -52
View File
@@ -1,6 +1,6 @@
"""
Generate an intuitive bar chart comparing skill output length vs the
terse control arm — side by side, so the visual gap IS the gain.
Generate a dot plot with whiskers showing how much shorter each skill
makes Claude's answers compared to a plain "Answer concisely." control.
Reads evals/snapshots/results.json and writes:
- evals/snapshots/results.html (interactive plotly)
@@ -12,6 +12,7 @@ Run: uv run --with tiktoken --with plotly --with kaleido python evals/plot.py
from __future__ import annotations
import json
import statistics
from pathlib import Path
import plotly.graph_objects as go
@@ -32,94 +33,148 @@ def main() -> None:
arms = data["arms"]
meta = data.get("metadata", {})
terse_total = sum(count(o) for o in arms["__terse__"])
terse_tokens = [count(o) for o in arms["__terse__"]]
rows = []
for skill, outputs in arms.items():
if skill in ("__baseline__", "__terse__"):
continue
skill_total = sum(count(o) for o in outputs)
saved = terse_total - skill_total
pct = (saved / terse_total) * 100 if terse_total else 0.0
skill_tokens = [count(o) for o in outputs]
# positive % = shorter than control (good)
savings = [
(1 - (s / t)) * 100 if t else 0.0
for s, t in zip(skill_tokens, terse_tokens)
]
savings_sorted = sorted(savings)
n = len(savings_sorted)
q1 = savings_sorted[n // 4]
q3 = savings_sorted[(3 * n) // 4]
rows.append(
{"skill": skill, "skill_total": skill_total, "saved": saved, "pct": pct}
{
"skill": skill,
"median": statistics.median(savings),
"mean": statistics.mean(savings),
"min": min(savings),
"max": max(savings),
"q1": q1,
"q3": q3,
}
)
rows.sort(
key=lambda r: r["pct"]
) # ascending → biggest gain at top in horizontal bar
rows.sort(key=lambda r: r["median"]) # ascending → best at top in horizontal
names = [r["skill"] for r in rows]
skill_totals = [r["skill_total"] for r in rows]
saved = [r["saved"] for r in rows]
pcts = [r["pct"] for r in rows]
medians = [r["median"] for r in rows]
mins = [r["min"] for r in rows]
maxs = [r["max"] for r in rows]
q1s = [r["q1"] for r in rows]
q3s = [r["q3"] for r in rows]
fig = go.Figure()
# the part the skill still uses (what you pay)
# min/max thin whisker (full range, faded)
for name, lo, hi in zip(names, mins, maxs):
fig.add_trace(
go.Scatter(
x=[lo, hi],
y=[name, name],
mode="lines",
line=dict(color="rgba(80,80,80,0.35)", width=2),
showlegend=False,
hoverinfo="skip",
)
)
# IQR thicker whisker (q1 to q3, where the middle half lives)
for name, lo, hi in zip(names, q1s, q3s):
fig.add_trace(
go.Scatter(
x=[lo, hi],
y=[name, name],
mode="lines",
line=dict(color="#2c3e50", width=6),
showlegend=False,
hoverinfo="skip",
)
)
# median dot
fig.add_trace(
go.Bar(
go.Scatter(
x=medians,
y=names,
x=skill_totals,
orientation="h",
name="tokens used",
marker=dict(color="#4c78a8"),
text=[f"{t}" for t in skill_totals],
textposition="inside",
insidetextanchor="middle",
textfont=dict(color="white", size=13),
hovertemplate="<b>%{y}</b><br>tokens used: %{x}<extra></extra>",
mode="markers",
marker=dict(
symbol="circle",
size=14,
color="#2ca02c",
line=dict(color="white", width=2),
),
name="median",
hovertemplate="<b>%{y}</b><br>median: %{x:.1f}%<extra></extra>",
)
)
# the part the skill saved (the win), stacked on top
# right-aligned label column outside the data area
fig.add_trace(
go.Bar(
go.Scatter(
x=[105] * len(names),
y=names,
x=saved,
orientation="h",
name="tokens saved",
marker=dict(color="#2ca02c"),
text=[f"−{s} ({p:.0f}% less)" for s, p in zip(saved, pcts)],
textposition="inside",
insidetextanchor="middle",
textfont=dict(color="white", size=13),
hovertemplate="<b>%{y}</b><br>tokens saved: %{x} (%{customdata:.0f}%)<extra></extra>",
customdata=pcts,
mode="text",
text=[f"<b>{m:+.0f}%</b>" for m in medians],
textposition="middle left",
textfont=dict(size=14, color="#2c3e50"),
showlegend=False,
hoverinfo="skip",
)
)
# reference line: where the terse control sits (= 100% of what you'd pay without the skill)
# zero line — "no effect"
fig.add_vline(
x=terse_total,
line=dict(color="black", width=2, dash="dash"),
annotation_text=f" no skill = {terse_total} tokens",
annotation_position="top right",
x=0,
line=dict(color="black", width=1.5, dash="dash"),
annotation_text=" no effect",
annotation_position="top",
annotation_font=dict(size=11, color="black"),
)
fig.update_layout(
title=dict(
text=f"<b>How much shorter does each skill make Claude's answers?</b><br>"
f"<sub>{meta.get('model', '?')} · {meta.get('n_prompts', '?')} prompts · "
f"compared against a plain <i>'Answer concisely.'</i> baseline</sub>",
f"<sub>Compared against the same model with system prompt = "
f"<i>'Answer concisely.'</i><br>"
f"{meta.get('model', '?')} · n={meta.get('n_prompts', '?')} prompts · "
f"single run per arm</sub>",
x=0.5,
xanchor="center",
),
barmode="stack",
xaxis=dict(
title="Total output tokens across all prompts",
title="← longer · output tokens vs control · shorter →",
ticksuffix="%",
zeroline=False,
gridcolor="rgba(0,0,0,0.08)",
range=[0, terse_total * 1.15],
range=[-30, 110],
),
yaxis=dict(title=""),
yaxis=dict(title="", automargin=True),
plot_bgcolor="white",
height=420,
height=440,
width=950,
margin=dict(l=120, r=80, t=100, b=70),
legend=dict(
orientation="h", yanchor="bottom", y=-0.22, xanchor="center", x=0.5
),
margin=dict(l=120, r=80, t=120, b=110),
showlegend=False,
annotations=[
dict(
x=0.5,
y=-0.28,
xref="paper",
yref="paper",
showarrow=False,
font=dict(size=11, color="#555"),
text=(
"<b>green dot</b> = median across prompts · "
"<b>thick bar</b> = IQR (middle 50%) · "
"<b>thin line</b> = min / max"
),
)
],
)
fig.write_html(HTML_OUT)