mirror of
https://github.com/JuliusBrussee/caveman.git
synced 2026-08-11 13:21:09 +02:00
Switch eval plot to dot+whiskers (median, IQR, min/max)
This commit is contained in:
+107
-52
@@ -1,6 +1,6 @@
|
||||
"""
|
||||
Generate an intuitive bar chart comparing skill output length vs the
|
||||
terse control arm — side by side, so the visual gap IS the gain.
|
||||
Generate a dot plot with whiskers showing how much shorter each skill
|
||||
makes Claude's answers compared to a plain "Answer concisely." control.
|
||||
|
||||
Reads evals/snapshots/results.json and writes:
|
||||
- evals/snapshots/results.html (interactive plotly)
|
||||
@@ -12,6 +12,7 @@ Run: uv run --with tiktoken --with plotly --with kaleido python evals/plot.py
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import statistics
|
||||
from pathlib import Path
|
||||
|
||||
import plotly.graph_objects as go
|
||||
@@ -32,94 +33,148 @@ def main() -> None:
|
||||
arms = data["arms"]
|
||||
meta = data.get("metadata", {})
|
||||
|
||||
terse_total = sum(count(o) for o in arms["__terse__"])
|
||||
terse_tokens = [count(o) for o in arms["__terse__"]]
|
||||
|
||||
rows = []
|
||||
for skill, outputs in arms.items():
|
||||
if skill in ("__baseline__", "__terse__"):
|
||||
continue
|
||||
skill_total = sum(count(o) for o in outputs)
|
||||
saved = terse_total - skill_total
|
||||
pct = (saved / terse_total) * 100 if terse_total else 0.0
|
||||
skill_tokens = [count(o) for o in outputs]
|
||||
# positive % = shorter than control (good)
|
||||
savings = [
|
||||
(1 - (s / t)) * 100 if t else 0.0
|
||||
for s, t in zip(skill_tokens, terse_tokens)
|
||||
]
|
||||
savings_sorted = sorted(savings)
|
||||
n = len(savings_sorted)
|
||||
q1 = savings_sorted[n // 4]
|
||||
q3 = savings_sorted[(3 * n) // 4]
|
||||
rows.append(
|
||||
{"skill": skill, "skill_total": skill_total, "saved": saved, "pct": pct}
|
||||
{
|
||||
"skill": skill,
|
||||
"median": statistics.median(savings),
|
||||
"mean": statistics.mean(savings),
|
||||
"min": min(savings),
|
||||
"max": max(savings),
|
||||
"q1": q1,
|
||||
"q3": q3,
|
||||
}
|
||||
)
|
||||
|
||||
rows.sort(
|
||||
key=lambda r: r["pct"]
|
||||
) # ascending → biggest gain at top in horizontal bar
|
||||
rows.sort(key=lambda r: r["median"]) # ascending → best at top in horizontal
|
||||
names = [r["skill"] for r in rows]
|
||||
skill_totals = [r["skill_total"] for r in rows]
|
||||
saved = [r["saved"] for r in rows]
|
||||
pcts = [r["pct"] for r in rows]
|
||||
medians = [r["median"] for r in rows]
|
||||
mins = [r["min"] for r in rows]
|
||||
maxs = [r["max"] for r in rows]
|
||||
q1s = [r["q1"] for r in rows]
|
||||
q3s = [r["q3"] for r in rows]
|
||||
|
||||
fig = go.Figure()
|
||||
|
||||
# the part the skill still uses (what you pay)
|
||||
# min/max thin whisker (full range, faded)
|
||||
for name, lo, hi in zip(names, mins, maxs):
|
||||
fig.add_trace(
|
||||
go.Scatter(
|
||||
x=[lo, hi],
|
||||
y=[name, name],
|
||||
mode="lines",
|
||||
line=dict(color="rgba(80,80,80,0.35)", width=2),
|
||||
showlegend=False,
|
||||
hoverinfo="skip",
|
||||
)
|
||||
)
|
||||
|
||||
# IQR thicker whisker (q1 to q3, where the middle half lives)
|
||||
for name, lo, hi in zip(names, q1s, q3s):
|
||||
fig.add_trace(
|
||||
go.Scatter(
|
||||
x=[lo, hi],
|
||||
y=[name, name],
|
||||
mode="lines",
|
||||
line=dict(color="#2c3e50", width=6),
|
||||
showlegend=False,
|
||||
hoverinfo="skip",
|
||||
)
|
||||
)
|
||||
|
||||
# median dot
|
||||
fig.add_trace(
|
||||
go.Bar(
|
||||
go.Scatter(
|
||||
x=medians,
|
||||
y=names,
|
||||
x=skill_totals,
|
||||
orientation="h",
|
||||
name="tokens used",
|
||||
marker=dict(color="#4c78a8"),
|
||||
text=[f"{t}" for t in skill_totals],
|
||||
textposition="inside",
|
||||
insidetextanchor="middle",
|
||||
textfont=dict(color="white", size=13),
|
||||
hovertemplate="<b>%{y}</b><br>tokens used: %{x}<extra></extra>",
|
||||
mode="markers",
|
||||
marker=dict(
|
||||
symbol="circle",
|
||||
size=14,
|
||||
color="#2ca02c",
|
||||
line=dict(color="white", width=2),
|
||||
),
|
||||
name="median",
|
||||
hovertemplate="<b>%{y}</b><br>median: %{x:.1f}%<extra></extra>",
|
||||
)
|
||||
)
|
||||
|
||||
# the part the skill saved (the win), stacked on top
|
||||
# right-aligned label column outside the data area
|
||||
fig.add_trace(
|
||||
go.Bar(
|
||||
go.Scatter(
|
||||
x=[105] * len(names),
|
||||
y=names,
|
||||
x=saved,
|
||||
orientation="h",
|
||||
name="tokens saved",
|
||||
marker=dict(color="#2ca02c"),
|
||||
text=[f"−{s} ({p:.0f}% less)" for s, p in zip(saved, pcts)],
|
||||
textposition="inside",
|
||||
insidetextanchor="middle",
|
||||
textfont=dict(color="white", size=13),
|
||||
hovertemplate="<b>%{y}</b><br>tokens saved: %{x} (%{customdata:.0f}%)<extra></extra>",
|
||||
customdata=pcts,
|
||||
mode="text",
|
||||
text=[f"<b>{m:+.0f}%</b>" for m in medians],
|
||||
textposition="middle left",
|
||||
textfont=dict(size=14, color="#2c3e50"),
|
||||
showlegend=False,
|
||||
hoverinfo="skip",
|
||||
)
|
||||
)
|
||||
|
||||
# reference line: where the terse control sits (= 100% of what you'd pay without the skill)
|
||||
# zero line — "no effect"
|
||||
fig.add_vline(
|
||||
x=terse_total,
|
||||
line=dict(color="black", width=2, dash="dash"),
|
||||
annotation_text=f" no skill = {terse_total} tokens",
|
||||
annotation_position="top right",
|
||||
x=0,
|
||||
line=dict(color="black", width=1.5, dash="dash"),
|
||||
annotation_text=" no effect",
|
||||
annotation_position="top",
|
||||
annotation_font=dict(size=11, color="black"),
|
||||
)
|
||||
|
||||
fig.update_layout(
|
||||
title=dict(
|
||||
text=f"<b>How much shorter does each skill make Claude's answers?</b><br>"
|
||||
f"<sub>{meta.get('model', '?')} · {meta.get('n_prompts', '?')} prompts · "
|
||||
f"compared against a plain <i>'Answer concisely.'</i> baseline</sub>",
|
||||
f"<sub>Compared against the same model with system prompt = "
|
||||
f"<i>'Answer concisely.'</i><br>"
|
||||
f"{meta.get('model', '?')} · n={meta.get('n_prompts', '?')} prompts · "
|
||||
f"single run per arm</sub>",
|
||||
x=0.5,
|
||||
xanchor="center",
|
||||
),
|
||||
barmode="stack",
|
||||
xaxis=dict(
|
||||
title="Total output tokens across all prompts",
|
||||
title="← longer · output tokens vs control · shorter →",
|
||||
ticksuffix="%",
|
||||
zeroline=False,
|
||||
gridcolor="rgba(0,0,0,0.08)",
|
||||
range=[0, terse_total * 1.15],
|
||||
range=[-30, 110],
|
||||
),
|
||||
yaxis=dict(title=""),
|
||||
yaxis=dict(title="", automargin=True),
|
||||
plot_bgcolor="white",
|
||||
height=420,
|
||||
height=440,
|
||||
width=950,
|
||||
margin=dict(l=120, r=80, t=100, b=70),
|
||||
legend=dict(
|
||||
orientation="h", yanchor="bottom", y=-0.22, xanchor="center", x=0.5
|
||||
),
|
||||
margin=dict(l=120, r=80, t=120, b=110),
|
||||
showlegend=False,
|
||||
annotations=[
|
||||
dict(
|
||||
x=0.5,
|
||||
y=-0.28,
|
||||
xref="paper",
|
||||
yref="paper",
|
||||
showarrow=False,
|
||||
font=dict(size=11, color="#555"),
|
||||
text=(
|
||||
"<b>green dot</b> = median across prompts · "
|
||||
"<b>thick bar</b> = IQR (middle 50%) · "
|
||||
"<b>thin line</b> = min / max"
|
||||
),
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
fig.write_html(HTML_OUT)
|
||||
|
||||
Reference in New Issue
Block a user