hallmark/eval/report.mjs

#!/usr/bin/env node
// report.mjs: assemble eval/index.html, a static self-contained gallery of
// every eval run: one row per brief, one card per arm-pack cell, plus a
// per-arm summary table at the top compared against fable-baseline.
//
// Reads: briefs.json, models.json, runs/*/*/{run,score,judge}.json, _shots/.
// Writes: eval/index.html (dark neutral shell, system-ui, no external assets).
//
// Usage: node eval/report.mjs

import { existsSync, readdirSync, statSync, readFileSync, writeFileSync } from "node:fs";
import { fileURLToPath } from "node:url";
import path from "node:path";

const ROOT = path.dirname(fileURLToPath(import.meta.url));
const RUNS = path.join(ROOT, "runs");
const OUT = path.join(ROOT, "index.html");

const AXES = ["philosophy", "hierarchy", "execution", "specificity", "restraint", "variety"];
const BASELINE_ARM = "fable-baseline";

function readJson(p, fallback = null) {
  try {
    return JSON.parse(readFileSync(p, "utf8"));
  } catch {
    return fallback;
  }
}

function esc(s) {
  return String(s ?? "")
    .replaceAll("&", "&")
    .replaceAll("<", "&lt;")
    .replaceAll(">", "&gt;")
    .replaceAll('"', "&quot;");
}

function splitCell(cell, armIds) {
  // Run dirs are <armId>-<pack>; arm ids contain hyphens, so match the
  // longest known arm id prefix. Falls back to the whole name.
  const sorted = [...armIds].sort((a, b) => b.length - a.length);
  for (const id of sorted) {
    if (cell === id) return { armId: id, pack: "default" };
    if (cell.startsWith(id + "-")) return { armId: id, pack: cell.slice(id.length + 1) };
  }
  return { armId: cell, pack: "default" };
}

function fmtDuration(sec) {
  if (sec == null) return null;
  const s = Math.round(sec);
  return s >= 60 ? `${Math.floor(s / 60)}m ${s % 60}s` : `${s}s`;
}

function fmtTok(run) {
  const p = run?.promptTok;
  const c = run?.completionTok;
  if (p == null && c == null) return null;
  const total = (p ?? 0) + (c ?? 0);
  return total >= 1000 ? `${(total / 1000).toFixed(total >= 10000 ? 0 : 1)}k` : String(total);
}

function mean(nums) {
  const v = nums.filter((n) => typeof n === "number" && Number.isFinite(n));
  return v.length ? v.reduce((a, b) => a + b, 0) / v.length : null;
}

function judgeAxesMean(judge) {
  if (!judge?.axes) return null;
  return mean(AXES.map((a) => judge.axes[a]));
}

function collect() {
  const briefs = readJson(path.join(ROOT, "briefs.json"), []);
  const models = readJson(path.join(ROOT, "models.json"), { arms: [] });
  const armIds = models.arms.map((a) => a.id);
  const armOrder = new Map(armIds.map((id, i) => [id, i]));

  const cells = []; // flat list of run cells
  if (existsSync(RUNS)) {
    for (const briefId of readdirSync(RUNS).sort()) {
      const briefDir = path.join(RUNS, briefId);
      if (!statSync(briefDir).isDirectory()) continue;
      for (const cell of readdirSync(briefDir).sort()) {
        const runDir = path.join(briefDir, cell);
        if (!statSync(runDir).isDirectory()) continue;
        const { armId, pack } = splitCell(cell, armIds);
        const base = `${briefId}-${cell}`;
        cells.push({
          briefId,
          cell,
          armId,
          pack,
          href: `./runs/${briefId}/${cell}/index.html`,
          hero: existsSync(path.join(ROOT, "_shots", `${base}-hero.png`))
            ? `./_shots/${base}-hero.png`
            : null,
          run: readJson(path.join(runDir, "run.json")),
          score: readJson(path.join(runDir, "score.json")),
          judge: readJson(path.join(runDir, "judge.json")),
        });
      }
    }
  }
  cells.sort((a, b) => {
    const ao = armOrder.get(a.armId) ?? 99;
    const bo = armOrder.get(b.armId) ?? 99;
    return ao - bo || a.pack.localeCompare(b.pack);
  });
  return { briefs, models, cells };
}

function summarize(cells, briefCount) {
  // One summary row per arm-pack column.
  const byCol = new Map();
  for (const c of cells) {
    const key = `${c.armId}${c.pack}`;
    if (!byCol.has(key)) byCol.set(key, { armId: c.armId, pack: c.pack, cells: [] });
    byCol.get(key).cells.push(c);
  }
  const rows = [];
  for (const col of byCol.values()) {
    const ok = col.cells.filter((c) => c.run?.ok !== false && c.hero !== null).length;
    const fails = mean(
      col.cells.map((c) => (c.score && !c.score.unavailable ? c.score.summary?.fails : null))
    );
    const floorClean = col.cells.filter((c) => c.score?.floorClean === true).length;
    const judgeMean = mean(col.cells.map((c) => c.judge?.overall));
    rows.push({ ...col, ok, fails, floorClean, judgeMean, n: briefCount });
  }
  const baseline = rows.find((r) => r.armId === BASELINE_ARM) || null;
  return { rows, baseline };
}

function chip(cls, html) {
  return `<span class="chip ${cls}">${html}</span>`;
}

function cardHtml(c) {
  const chips = [];
  if (c.run) {
    const d = fmtDuration(c.run.durationSec);
    const t = fmtTok(c.run);
    const parts = [d, t ? `${t} tok` : null, c.run.loc != null ? `${c.run.loc} LOC` : null]
      .filter(Boolean)
      .join(" &middot; ");
    if (parts) chips.push(chip("run", parts));
    if (c.run.ok === false) chips.push(chip("bad", "gen failed"));
    if (c.run.nudged) chips.push(chip("warn", "nudged"));
  }
  if (c.score) {
    if (c.score.unavailable) chips.push(chip("mute", "sloplint n/a"));
    else {
      const f = c.score.summary?.fails ?? "?";
      const w = c.score.summary?.warns ?? "?";
      chips.push(chip(f === 0 ? "good" : "bad", `${f} FAIL / ${w} WARN`));
      if (c.score.floorClean === true) chips.push(chip("good", "floor clean"));
      else if (c.score.floorClean === false) chips.push(chip("bad", "floor broken"));
    }
  }
  if (c.judge) {
    if (c.judge.error) chips.push(chip("mute", "judge error"));
    else {
      const am = judgeAxesMean(c.judge);
      chips.push(
        chip(
          "judge",
          `judge ${c.judge.overall ?? "?"}/10${am != null ? ` &middot; axes ${am.toFixed(1)}` : ""}`
        )
      );
    }
  }
  const thumb = c.hero
    ? `<img src="${esc(c.hero)}" alt="Hero of ${esc(c.briefId)} ${esc(c.cell)}" loading="lazy" />`
    : `<div class="noshot">no shot</div>`;
  return `<a class="card" href="${esc(c.href)}" target="_blank" rel="noopener">
  <div class="thumb">${thumb}</div>
  <div class="meta">
    <div class="arm">${esc(c.armId)}<span class="pack">${esc(c.pack)}</span></div>
    <div class="chips">${chips.join("")}</div>
  </div>
</a>`;
}

function summaryTableHtml(summary) {
  const { rows, baseline } = summary;
  if (!rows.length) return `<p class="empty">No runs yet. Run run-matrix.py first.</p>`;
  const delta = (v, b, invert = false) => {
    if (v == null || b == null) return "";
    const d = v - b;
    if (Math.abs(d) < 0.005) return `<span class="delta same">=</span>`;
    const good = invert ? d < 0 : d > 0;
    const sign = d > 0 ? "+" : "";
    return `<span class="delta ${good ? "up" : "down"}">${sign}${d.toFixed(1)}</span>`;
  };
  const tr = rows
    .map((r) => {
      const isBase = baseline && r === baseline;
      return `<tr${isBase ? ' class="base"' : ""}>
  <td>${esc(r.armId)}<span class="pack">${esc(r.pack)}</span>${isBase ? ' <span class="tag">baseline</span>' : ""}</td>
  <td>${r.ok}/${r.n}</td>
  <td>${r.fails != null ? r.fails.toFixed(1) : "n/a"} ${isBase ? "" : delta(r.fails, baseline?.fails, true)}</td>
  <td>${r.floorClean}/${r.n}</td>
  <td>${r.judgeMean != null ? r.judgeMean.toFixed(1) : "n/a"} ${isBase ? "" : delta(r.judgeMean, baseline?.judgeMean)}</td>
</tr>`;
    })
    .join("\n");
  return `<table class="summary">
<thead><tr><th>arm</th><th>artifacts</th><th>mean FAILs</th><th>floor clean</th><th>judge mean</th></tr></thead>
<tbody>${tr}</tbody>
</table>`;
}

function main() {
  const { briefs, cells } = collect();
  const briefCount = briefs.length || 6;
  const summary = summarize(cells, briefCount);

  const rowsHtml = briefs
    .map((b) => {
      const briefCells = cells.filter((c) => c.briefId === b.id);
      const cards = briefCells.length
        ? briefCells.map(cardHtml).join("\n")
        : `<p class="empty">no runs for this brief</p>`;
      return `<section class="brief">
  <header>
    <h2>${esc(b.id)} <span class="bname">${esc(b.name)}</span></h2>
    <p class="btext">${esc(b.brief)}</p>
    <p class="bexpect">expect: ${esc(b.expect?.scope)}. ${esc(b.expect?.notes)}</p>
  </header>
  <div class="grid">${cards}</div>
</section>`;
    })
    .join("\n");

  const html = `<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>Hallmark eval matrix</title>
<style>
:root{--bg:#101014;--panel:#17171d;--panel2:#1d1d25;--line:#2a2a34;--ink:#e8e8ee;--mute:#8b8b98;--good:#4cc38a;--bad:#e5484d;--warn:#d8a03d;--acc:#8a8af0}
*{box-sizing:border-box;margin:0;padding:0}
body{background:var(--bg);color:var(--ink);font:15px/1.5 system-ui,-apple-system,sans-serif;padding:2.5rem clamp(1rem,4vw,3rem) 5rem}
h1{font-size:1.6rem;letter-spacing:-.01em}
.sub{color:var(--mute);margin:.35rem 0 2rem;font-size:.9rem}
table.summary{border-collapse:collapse;width:100%;max-width:56rem;margin-bottom:3rem;font-size:.86rem;font-variant-numeric:tabular-nums}
.summary th,.summary td{text-align:left;padding:.5rem .8rem;border-bottom:1px solid var(--line)}
.summary th{color:var(--mute);font-weight:600;font-size:.72rem;text-transform:uppercase;letter-spacing:.08em}
.summary tr.base td{background:var(--panel)}
.tag{font-size:.64rem;text-transform:uppercase;letter-spacing:.08em;color:var(--acc);border:1px solid var(--acc);border-radius:999px;padding:.1rem .45rem;margin-left:.4rem}
.pack{color:var(--mute);font-size:.78em;margin-left:.45rem}
.delta{font-size:.78em;margin-left:.3rem}
.delta.up{color:var(--good)}.delta.down{color:var(--bad)}.delta.same{color:var(--mute)}
.brief{margin-bottom:3.2rem}
.brief h2{font-size:1.05rem;margin-bottom:.2rem}
.bname{color:var(--mute);font-weight:400}
.btext{color:var(--ink);opacity:.85;font-size:.9rem;max-width:60rem}
.bexpect{color:var(--mute);font-size:.78rem;margin:.2rem 0 1rem;max-width:60rem}
.grid{display:grid;grid-template-columns:repeat(auto-fill,minmax(300px,1fr));gap:1rem}
.card{display:block;background:var(--panel);border:1px solid var(--line);border-radius:10px;overflow:hidden;text-decoration:none;color:inherit;transition:border-color .15s ease,transform .15s ease}
.card:hover{border-color:var(--acc);transform:translateY(-2px)}
.thumb{aspect-ratio:16/10;background:var(--panel2);overflow:hidden}
.thumb img{width:100%;height:100%;object-fit:cover;object-position:left top;display:block}
.noshot{width:100%;height:100%;display:flex;align-items:center;justify-content:center;color:var(--mute);font-size:.8rem}
.meta{padding:.7rem .8rem .8rem;border-top:1px solid var(--line)}
.arm{font-weight:600;font-size:.86rem;margin-bottom:.45rem}
.chips{display:flex;flex-wrap:wrap;gap:.35rem}
.chip{font-size:.68rem;padding:.2rem .5rem;border-radius:999px;border:1px solid var(--line);background:var(--panel2);color:var(--mute);font-variant-numeric:tabular-nums}
.chip.good{color:var(--good);border-color:color-mix(in srgb,var(--good) 40%,var(--line))}
.chip.bad{color:var(--bad);border-color:color-mix(in srgb,var(--bad) 40%,var(--line))}
.chip.warn{color:var(--warn);border-color:color-mix(in srgb,var(--warn) 40%,var(--line))}
.chip.judge{color:var(--acc);border-color:color-mix(in srgb,var(--acc) 40%,var(--line))}
.empty{color:var(--mute);font-size:.85rem;padding:.6rem 0}
@media (prefers-reduced-motion:reduce){.card,.card:hover{transition:none;transform:none}}
</style>
</head>
<body>
<h1>Hallmark eval matrix</h1>
<p class="sub">Generated ${esc(new Date().toISOString())} &middot; ${cells.length} run cells &middot; click a card to open the artifact</p>
${summaryTableHtml(summary)}
${rowsHtml}
</body>
</html>
`;
  writeFileSync(OUT, html);
  console.log(`wrote ${OUT} (${cells.length} cells, ${summary.rows.length} arm columns)`);
}

main();