diff --git a/evals/plot.py b/evals/plot.py
index 85c9fe8..42aab23 100644
--- a/evals/plot.py
+++ b/evals/plot.py
@@ -1,6 +1,6 @@
"""
-Generate a dot plot with whiskers showing how much shorter each skill
-makes Claude's answers compared to a plain "Answer concisely." control.
+Generate a boxplot showing the distribution of token compression per
+skill, compared against a plain "Answer concisely." control.
Reads evals/snapshots/results.json and writes:
- evals/snapshots/results.html (interactive plotly)
@@ -40,150 +40,103 @@ def main() -> None:
if skill in ("__baseline__", "__terse__"):
continue
skill_tokens = [count(o) for o in outputs]
- # positive % = shorter than control (good)
savings = [
(1 - (s / t)) * 100 if t else 0.0
for s, t in zip(skill_tokens, terse_tokens)
]
- savings_sorted = sorted(savings)
- n = len(savings_sorted)
- q1 = savings_sorted[n // 4]
- q3 = savings_sorted[(3 * n) // 4]
rows.append(
- {
- "skill": skill,
- "median": statistics.median(savings),
- "mean": statistics.mean(savings),
- "min": min(savings),
- "max": max(savings),
- "q1": q1,
- "q3": q3,
- }
+ {"skill": skill, "savings": savings, "median": statistics.median(savings)}
)
- rows.sort(key=lambda r: r["median"]) # ascending → best at top in horizontal
- names = [r["skill"] for r in rows]
- medians = [r["median"] for r in rows]
- mins = [r["min"] for r in rows]
- maxs = [r["max"] for r in rows]
- q1s = [r["q1"] for r in rows]
- q3s = [r["q3"] for r in rows]
-
- # vertical orientation: best skill on the left, x = skill, y = % shorter
- rows.sort(key=lambda r: -r["median"]) # descending so best is leftmost
- names = [r["skill"] for r in rows]
- medians = [r["median"] for r in rows]
- mins = [r["min"] for r in rows]
- maxs = [r["max"] for r in rows]
- q1s = [r["q1"] for r in rows]
- q3s = [r["q3"] for r in rows]
+ rows.sort(key=lambda r: -r["median"]) # best first
fig = go.Figure()
- # min/max thin whisker (full range, faded)
- for name, lo, hi in zip(names, mins, maxs):
+ for row in rows:
fig.add_trace(
- go.Scatter(
- x=[name, name],
- y=[lo, hi],
- mode="lines",
- line=dict(color="rgba(80,80,80,0.35)", width=2),
- showlegend=False,
- hoverinfo="skip",
+ go.Box(
+ y=row["savings"],
+ name=row["skill"],
+ boxpoints="all",
+ jitter=0.4,
+ pointpos=0,
+ marker=dict(color="#2ca02c", size=7, opacity=0.7),
+ line=dict(color="#2c3e50", width=2),
+ fillcolor="rgba(76, 120, 168, 0.25)",
+ boxmean=True,
+ hovertemplate="%{x}
%{y:.1f}%",
)
)
- # IQR thicker whisker (q1 to q3)
- for name, lo, hi in zip(names, q1s, q3s):
- fig.add_trace(
- go.Scatter(
- x=[name, name],
- y=[lo, hi],
- mode="lines",
- line=dict(color="#2c3e50", width=8),
- showlegend=False,
- hoverinfo="skip",
- )
- )
-
- # median dot with label on top
- fig.add_trace(
- go.Scatter(
- x=names,
- y=medians,
- mode="markers+text",
- marker=dict(
- symbol="circle",
- size=18,
- color="#2ca02c",
- line=dict(color="white", width=2),
- ),
- cliponaxis=False,
- name="median",
- hovertemplate="%{x}
median: %{y:.1f}%",
- )
- )
-
# zero line — "no effect"
fig.add_hline(
y=0,
line=dict(color="black", width=1.5, dash="dash"),
- annotation_text="no effect",
- annotation_position="right",
+ annotation_text="no effect (= same length as control)",
+ annotation_position="top right",
annotation_font=dict(size=11, color="black"),
)
+ # median labels above each box
+ for row in rows:
+ fig.add_annotation(
+ x=row["skill"],
+ y=max(row["savings"]),
+ text=f"{row['median']:+.0f}%",
+ showarrow=False,
+ yshift=22,
+ font=dict(size=16, color="#2c3e50"),
+ )
+
fig.update_layout(
title=dict(
text=f"How much shorter does each skill make Claude's answers?
"
- f"Compared against the same model with system prompt = "
+ f"Distribution of per-prompt savings vs system prompt = "
f"'Answer concisely.'
"
f"{meta.get('model', '?')} · n={meta.get('n_prompts', '?')} prompts · "
f"single run per arm",
x=0.5,
xanchor="center",
),
- xaxis=dict(
- title="", automargin=True, categoryorder="array", categoryarray=names
- ),
+ xaxis=dict(title="", automargin=True),
yaxis=dict(
- title="↑ shorter · output tokens vs control · longer ↓",
+ title="↑ shorter · vs control · longer ↓",
ticksuffix="%",
zeroline=False,
gridcolor="rgba(0,0,0,0.08)",
- range=[-30, 110],
+ range=[-30, 115],
),
plot_bgcolor="white",
height=560,
width=980,
- margin=dict(l=140, r=120, t=120, b=130),
+ margin=dict(l=140, r=80, t=120, b=120),
showlegend=False,
annotations=[
dict(
x=0.5,
- y=-0.28,
+ y=-0.22,
xref="paper",
yref="paper",
showarrow=False,
font=dict(size=11, color="#555"),
text=(
- "green dot = median across prompts · "
- "thick bar = IQR (middle 50%) · "
- "thin line = min / max"
+ "box = IQR (middle 50%) · "
+ "line in box = median · "
+ "dashed line = mean · "
+ "green dots = individual prompts"
),
)
],
)
- # value labels above each whisker top, added AFTER update_layout so they
- # don't get overwritten by the layout-level annotations list
- for name, m, hi in zip(names, medians, maxs):
+ # re-add labels after update_layout (which would otherwise wipe them)
+ for row in rows:
fig.add_annotation(
- x=name,
- y=hi,
- text=f"{m:+.0f}%",
+ x=row["skill"],
+ y=max(row["savings"]),
+ text=f"{row['median']:+.0f}%",
showarrow=False,
- yshift=18,
+ yshift=22,
font=dict(size=16, color="#2c3e50"),
)
diff --git a/evals/snapshots/results.html b/evals/snapshots/results.html
index dbd77c7..6b872db 100644
--- a/evals/snapshots/results.html
+++ b/evals/snapshots/results.html
@@ -3883,6 +3883,6 @@ maplibre-gl/dist/maplibre-gl.js:
window.Plotly = Plotly;
return Plotly;
-}));
+}));