#!/usr/bin/env node // report.mjs: assemble eval/index.html, a static self-contained gallery of // every eval run: one row per brief, one card per arm-pack cell, plus a // per-arm summary table at the top compared against fable-baseline. // // Reads: briefs.json, models.json, runs/*/*/{run,score,judge}.json, _shots/. // Writes: eval/index.html (dark neutral shell, system-ui, no external assets). // // Usage: node eval/report.mjs import { existsSync, readdirSync, statSync, readFileSync, writeFileSync } from "node:fs"; import { fileURLToPath } from "node:url"; import path from "node:path"; const ROOT = path.dirname(fileURLToPath(import.meta.url)); const RUNS = path.join(ROOT, "runs"); const OUT = path.join(ROOT, "index.html"); const AXES = ["philosophy", "hierarchy", "execution", "specificity", "restraint", "variety"]; const BASELINE_ARM = "fable-baseline"; function readJson(p, fallback = null) { try { return JSON.parse(readFileSync(p, "utf8")); } catch { return fallback; } } function esc(s) { return String(s ?? "") .replaceAll("&", "&") .replaceAll("<", "<") .replaceAll(">", ">") .replaceAll('"', """); } function splitCell(cell, armIds) { // Run dirs are -; arm ids contain hyphens, so match the // longest known arm id prefix. Falls back to the whole name. const sorted = [...armIds].sort((a, b) => b.length - a.length); for (const id of sorted) { if (cell === id) return { armId: id, pack: "default" }; if (cell.startsWith(id + "-")) return { armId: id, pack: cell.slice(id.length + 1) }; } return { armId: cell, pack: "default" }; } function fmtDuration(sec) { if (sec == null) return null; const s = Math.round(sec); return s >= 60 ? `${Math.floor(s / 60)}m ${s % 60}s` : `${s}s`; } function fmtTok(run) { const p = run?.promptTok; const c = run?.completionTok; if (p == null && c == null) return null; const total = (p ?? 0) + (c ?? 0); return total >= 1000 ? `${(total / 1000).toFixed(total >= 10000 ? 0 : 1)}k` : String(total); } function mean(nums) { const v = nums.filter((n) => typeof n === "number" && Number.isFinite(n)); return v.length ? v.reduce((a, b) => a + b, 0) / v.length : null; } function judgeAxesMean(judge) { if (!judge?.axes) return null; return mean(AXES.map((a) => judge.axes[a])); } function collect() { const briefs = readJson(path.join(ROOT, "briefs.json"), []); const models = readJson(path.join(ROOT, "models.json"), { arms: [] }); const armIds = models.arms.map((a) => a.id); const armOrder = new Map(armIds.map((id, i) => [id, i])); const cells = []; // flat list of run cells if (existsSync(RUNS)) { for (const briefId of readdirSync(RUNS).sort()) { const briefDir = path.join(RUNS, briefId); if (!statSync(briefDir).isDirectory()) continue; for (const cell of readdirSync(briefDir).sort()) { const runDir = path.join(briefDir, cell); if (!statSync(runDir).isDirectory()) continue; const { armId, pack } = splitCell(cell, armIds); const base = `${briefId}-${cell}`; cells.push({ briefId, cell, armId, pack, href: `./runs/${briefId}/${cell}/index.html`, hero: existsSync(path.join(ROOT, "_shots", `${base}-hero.png`)) ? `./_shots/${base}-hero.png` : null, run: readJson(path.join(runDir, "run.json")), score: readJson(path.join(runDir, "score.json")), judge: readJson(path.join(runDir, "judge.json")), }); } } } cells.sort((a, b) => { const ao = armOrder.get(a.armId) ?? 99; const bo = armOrder.get(b.armId) ?? 99; return ao - bo || a.pack.localeCompare(b.pack); }); return { briefs, models, cells }; } function summarize(cells, briefCount) { // One summary row per arm-pack column. const byCol = new Map(); for (const c of cells) { const key = `${c.armId}${c.pack}`; if (!byCol.has(key)) byCol.set(key, { armId: c.armId, pack: c.pack, cells: [] }); byCol.get(key).cells.push(c); } const rows = []; for (const col of byCol.values()) { const ok = col.cells.filter((c) => c.run?.ok !== false && c.hero !== null).length; const fails = mean( col.cells.map((c) => (c.score && !c.score.unavailable ? c.score.summary?.fails : null)) ); const floorClean = col.cells.filter((c) => c.score?.floorClean === true).length; const judgeMean = mean(col.cells.map((c) => c.judge?.overall)); rows.push({ ...col, ok, fails, floorClean, judgeMean, n: briefCount }); } const baseline = rows.find((r) => r.armId === BASELINE_ARM) || null; return { rows, baseline }; } function chip(cls, html) { return `${html}`; } function cardHtml(c) { const chips = []; if (c.run) { const d = fmtDuration(c.run.durationSec); const t = fmtTok(c.run); const parts = [d, t ? `${t} tok` : null, c.run.loc != null ? `${c.run.loc} LOC` : null] .filter(Boolean) .join(" · "); if (parts) chips.push(chip("run", parts)); if (c.run.ok === false) chips.push(chip("bad", "gen failed")); if (c.run.nudged) chips.push(chip("warn", "nudged")); } if (c.score) { if (c.score.unavailable) chips.push(chip("mute", "sloplint n/a")); else { const f = c.score.summary?.fails ?? "?"; const w = c.score.summary?.warns ?? "?"; chips.push(chip(f === 0 ? "good" : "bad", `${f} FAIL / ${w} WARN`)); if (c.score.floorClean === true) chips.push(chip("good", "floor clean")); else if (c.score.floorClean === false) chips.push(chip("bad", "floor broken")); } } if (c.judge) { if (c.judge.error) chips.push(chip("mute", "judge error")); else { const am = judgeAxesMean(c.judge); chips.push( chip( "judge", `judge ${c.judge.overall ?? "?"}/10${am != null ? ` · axes ${am.toFixed(1)}` : ""}` ) ); } } const thumb = c.hero ? ` Hero of ${esc(c.briefId)} ${esc(c.cell)}

Hero of ${esc(c.briefId)} ${esc(c.cell)}

` : `

no shot

`; return `

${esc(c.armId)}${esc(c.pack)}

${chips.join("")}

`; } function summaryTableHtml(summary) { const { rows, baseline } = summary; if (!rows.length) return `

No runs yet. Run run-matrix.py first.

`; const delta = (v, b, invert = false) => { if (v == null || b == null) return ""; const d = v - b; if (Math.abs(d) < 0.005) return `=`; const good = invert ? d < 0 : d > 0; const sign = d > 0 ? "+" : ""; return `${sign}${d.toFixed(1)}`; }; const tr = rows .map((r) => { const isBase = baseline && r === baseline; return ` ${esc(r.armId)}${esc(r.pack)}${isBase ? ' baseline' : ""} ${r.ok}/${r.n} ${r.fails != null ? r.fails.toFixed(1) : "n/a"} ${isBase ? "" : delta(r.fails, baseline?.fails, true)} ${r.floorClean}/${r.n} ${r.judgeMean != null ? r.judgeMean.toFixed(1) : "n/a"} ${isBase ? "" : delta(r.judgeMean, baseline?.judgeMean)} `; }) .join("\n"); return `${tr}

arm	artifacts	mean FAILs	floor clean	judge mean

`; } function main() { const { briefs, cells } = collect(); const briefCount = briefs.length || 6; const summary = summarize(cells, briefCount); const rowsHtml = briefs .map((b) => { const briefCells = cells.filter((c) => c.briefId === b.id); const cards = briefCells.length ? briefCells.map(cardHtml).join("\n") : `

no runs for this brief

`; return `

${esc(b.id)} ${esc(b.name)}

${esc(b.brief)}

expect: ${esc(b.expect?.scope)}. ${esc(b.expect?.notes)}

${cards}

`; }) .join("\n"); const html = ` Hallmark eval matrix

Hallmark eval matrix

Generated ${esc(new Date().toISOString())} · ${cells.length} run cells · click a card to open the artifact

${summaryTableHtml(summary)} ${rowsHtml} `; writeFileSync(OUT, html); console.log(`wrote ${OUT} (${cells.length} cells, ${summary.rows.length} arm columns)`); } main();