diff --git a/scripts/run-evals-test.js b/scripts/run-evals-test.js new file mode 100644 index 0000000..489d117 --- /dev/null +++ b/scripts/run-evals-test.js @@ -0,0 +1,143 @@ +#!/usr/bin/env node + +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); +const test = require('node:test'); + +const RUNNER = path.join(__dirname, 'run-evals.js'); + +function writeJson(file, value) { + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, `${JSON.stringify(value, null, 2)}\n`); +} + +function writeSkill(root, name, description) { + const dir = path.join(root, 'skills', name); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync( + path.join(dir, 'SKILL.md'), + `---\nname: ${name}\ndescription: ${description}\n---\n\n# ${name}\n`, + ); +} + +function behavioralEval(files = ['project/context.txt']) { + return { + id: 1, + prompt: 'Inspect the attached project and complete the requested work.', + expected_output: 'A verified result grounded in the attached project', + files, + expectations: ['The attached project is inspected before reporting a result'], + }; +} + +function completeCase(skillName, positivePrompt, topK = 1, files) { + return { + skill_name: skillName, + trigger: { + positive: [1, 2, 3].map(() => ({ prompt: positivePrompt, top_k: topK })), + negative: [ + { prompt: 'unrelated banana request' }, + { prompt: 'unrelated orange request' }, + ], + }, + evals: [behavioralEval(files)], + }; +} + +function makeSandbox() { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-skills-run-evals-test-')); + fs.mkdirSync(path.join(root, 'scripts'), { recursive: true }); + fs.mkdirSync(path.join(root, 'evals', 'cases'), { recursive: true }); + fs.mkdirSync(path.join(root, 'evals', 'fixtures', 'project'), { recursive: true }); + fs.copyFileSync(RUNNER, path.join(root, 'scripts', 'run-evals.js')); + fs.writeFileSync(path.join(root, 'evals', 'fixtures', 'project', 'context.txt'), 'fixture\n'); + return root; +} + +function run(root, args = []) { + return spawnSync(process.execPath, [path.join(root, 'scripts', 'run-evals.js'), ...args], { + cwd: root, + encoding: 'utf8', + }); +} + +test('fails when a skill has no eval case file', () => { + const root = makeSandbox(); + writeSkill(root, 'alpha-skill', 'Handles alpha widgets. Use when changing alpha widgets.'); + + const result = run(root); + + assert.equal(result.status, 1, result.stdout + result.stderr); + assert.match(result.stdout, /no eval case file/); +}); + +test('fails when an eval case is below the required minimums', () => { + const root = makeSandbox(); + writeSkill(root, 'alpha-skill', 'Handles alpha widgets. Use when changing alpha widgets.'); + writeJson(path.join(root, 'evals', 'cases', 'alpha-skill.json'), { + skill_name: 'alpha-skill', + trigger: { + positive: [{ prompt: 'change alpha widget', top_k: 1 }], + negative: [], + }, + evals: [behavioralEval()], + }); + + const result = run(root); + + assert.equal(result.status, 1, result.stdout + result.stderr); + assert.match(result.stdout, /below required minimums/); +}); + +test('fails when a behavioral eval references a missing fixture', () => { + const root = makeSandbox(); + writeSkill(root, 'alpha-skill', 'Handles alpha widgets. Use when changing alpha widgets.'); + writeJson( + path.join(root, 'evals', 'cases', 'alpha-skill.json'), + completeCase('alpha-skill', 'change alpha widget', 1, ['missing/project.txt']), + ); + + const result = run(root); + + assert.equal(result.status, 1, result.stdout + result.stderr); + assert.match(result.stdout, /fixture not found/); +}); + +test('enforces the configured rank-1 floor', () => { + const root = makeSandbox(); + writeSkill(root, 'alpha-skill', 'Handles widget work. Use when implementing widget changes.'); + writeSkill( + root, + 'beta-skill', + 'Diagnoses urgent widget failures in production. Use when repairing urgent widget failures.', + ); + writeJson( + path.join(root, 'evals', 'cases', 'alpha-skill.json'), + completeCase('alpha-skill', 'urgent widget failure production', 2), + ); + writeJson( + path.join(root, 'evals', 'cases', 'beta-skill.json'), + completeCase('beta-skill', 'repair urgent widget failure', 1), + ); + + const passing = run(root, ['--min-rank1', '50']); + const failing = run(root, ['--min-rank1', '60']); + + assert.equal(passing.status, 0, passing.stdout + passing.stderr); + assert.equal(failing.status, 1, failing.stdout + failing.stderr); + assert.match(failing.stdout, /below required 60%/); +}); + +test('rejects an invalid rank-1 floor', () => { + const root = makeSandbox(); + + const result = run(root, ['--min-rank1', '101']); + + assert.equal(result.status, 1, result.stdout + result.stderr); + assert.match(result.stderr, /--min-rank1 must be a number from 0 to 100/); +}); diff --git a/scripts/run-evals.js b/scripts/run-evals.js index 9272dc1..c9a9ba5 100644 --- a/scripts/run-evals.js +++ b/scripts/run-evals.js @@ -12,7 +12,9 @@ * overlapping skills drifting in. * - Coverage + schema: every case file maps to a real skill, skill_name * matches, and behavioral evals follow the skill-creator evals.json shape. - * Skills without a case file are reported as warnings (not errors, yet). + * Every skill must have a complete case file backed by real fixtures. + * - Rank-1 ratchet: --min-rank1 fails when routing quality drops + * below the checked-in CI baseline. * Tier 3 (opt-in, costs tokens, never in CI): * node scripts/run-evals.js --behavioral [--dry-run] * Runs each behavioral eval through headless `claude` in a throwaway @@ -47,7 +49,7 @@ const GRADER_TIMEOUT_MS = 5 * 60 * 1000; // tokens; review this list if your fixtures invoke anything unusual. const EXECUTOR_TOOLS = 'Read,Glob,Grep,Edit,Write,Bash'; -// Documented minimums per case file (evals/README.md). Warning-level for now. +// Required minimums per case file (evals/README.md). const MIN_POSITIVE = 3; const MIN_NEGATIVE = 2; const MIN_EVALS = 1; @@ -193,7 +195,7 @@ function resolveFixturePath(root, rel) { // ---------- tier 2 ---------- -function runDeterministic() { +function runDeterministic(minRank1) { const skills = loadSkills(); const cases = loadCases(); const corpus = buildCorpus(skills); @@ -210,8 +212,8 @@ function runDeterministic() { // Coverage for (const s of skills) { if (!cases.some((c) => c.file === `${s.name}.json`)) { - console.log(` ⚠ ${s.name}: no eval case file (evals/cases/${s.name}.json)`); - warnings++; + console.log(` ✗ ${s.name}: no eval case file (evals/cases/${s.name}.json)`); + errors++; } } @@ -246,6 +248,29 @@ function runDeterministic() { console.log(` ✗ ${c.file}: eval id=${ev.id} does not match evals.json schema`); errors++; } + if (!Array.isArray(ev.files) || ev.files.length === 0 || !ev.files.every((x) => typeof x === 'string')) { + console.log(` ✗ ${c.file}: eval id=${ev.id} needs a non-empty files[] fixture list`); + errors++; + } else { + for (const rel of ev.files) { + let fixture; + try { + fixture = resolveFixturePath(FIXTURES_DIR, rel); + } catch (e) { + console.log(` ✗ ${c.file}: eval id=${ev.id} has invalid fixture path "${rel}" — ${e.message}`); + errors++; + continue; + } + if (!fs.existsSync(fixture)) { + console.log(` ✗ ${c.file}: eval id=${ev.id} fixture not found: evals/fixtures/${rel}`); + errors++; + } + } + } + if (ev.trust_level === 'provisional') { + console.log(` ✗ ${c.file}: eval id=${ev.id} is still provisional; add real fixtures before trusting it`); + errors++; + } } // Trigger: positive @@ -303,13 +328,13 @@ function runDeterministic() { if (ok) passed++; } - // Documented minimums (warning-level during the transition window) + // Required minimums const pc = (d.trigger?.positive || []).length; const nc = (d.trigger?.negative || []).length; const ec = (d.evals || []).length; if (pc < MIN_POSITIVE || nc < MIN_NEGATIVE || ec < MIN_EVALS) { - console.log(` ⚠ ${expected}: below documented minimums (${pc} positive/${nc} negative/${ec} behavioral; need ${MIN_POSITIVE}/${MIN_NEGATIVE}/${MIN_EVALS})`); - warnings++; + console.log(` ✗ ${expected}: below required minimums (${pc} positive/${nc} negative/${ec} behavioral; need ${MIN_POSITIVE}/${MIN_NEGATIVE}/${MIN_EVALS})`); + errors++; } } @@ -330,7 +355,12 @@ function runDeterministic() { } } - const rate = positives ? ((rank1 / positives) * 100).toFixed(0) : 'n/a'; + const rank1Rate = positives ? (rank1 / positives) * 100 : 0; + const rate = positives ? rank1Rate.toFixed(0) : 'n/a'; + if (minRank1 !== null && (!positives || rank1Rate < minRank1)) { + console.log(` ✗ trigger rank-1 rate ${rate}% is below required ${minRank1}%`); + errors++; + } console.log(`\n${passed} checks passed — ${errors} error(s), ${warnings} warning(s)`); console.log(`trigger rank-1 rate: ${rate}% (${rank1}/${positives} positive prompts rank their skill first)`); console.log(errors ? 'FAILED' : 'PASSED'); @@ -352,6 +382,14 @@ function materializeWorkspace(ev) { fs.mkdirSync(path.dirname(dest), { recursive: true }); fs.cpSync(src, dest, { recursive: true }); } + // Give workflow-oriented evals a real baseline to inspect, modify, diff, and + // commit. A local identity keeps this deterministic and never leaves the + // throwaway workspace. + execFileSync('git', ['init', '--quiet'], { cwd: workspace }); + execFileSync('git', ['config', 'user.name', 'Skill Eval'], { cwd: workspace }); + execFileSync('git', ['config', 'user.email', 'skill-eval@example.invalid'], { cwd: workspace }); + execFileSync('git', ['add', '--all'], { cwd: workspace }); + execFileSync('git', ['commit', '--quiet', '-m', 'fixture baseline'], { cwd: workspace }); return workspace; } @@ -389,8 +427,10 @@ function runBehavioral(skillName, dryRun) { for (const ev of d.evals) { const fixtures = (ev.files || []).length; - if (ev.trust_level === 'provisional' || !fixtures) { - console.log(` note: eval ${ev.id} is provisional (${fixtures ? 'flagged' : 'no fixtures'}) — results are a sanity check, not evidence`); + if (!fixtures) { + console.error(`eval ${ev.id} has no fixtures; run the deterministic eval gate first`); + failures++; + continue; } if (dryRun) { console.log(`[dry-run] eval ${ev.id}: workspace + ${fixtures} fixture(s); claude -p --verbose --output-format stream-json --permission-mode acceptEdits --allowedTools ${EXECUTOR_TOOLS} --append-system-prompt <${skillName}/SKILL.md> < prompt-on-stdin`); @@ -442,8 +482,22 @@ function runBehavioral(skillName, dryRun) { const args = process.argv.slice(2); const bIdx = args.indexOf('--behavioral'); +const rankIdx = args.indexOf('--min-rank1'); +let minRank1 = null; +if (rankIdx !== -1) { + const raw = args[rankIdx + 1]; + minRank1 = Number(raw); + if (raw === undefined || raw === '' || !Number.isFinite(minRank1) || minRank1 < 0 || minRank1 > 100) { + console.error('--min-rank1 must be a number from 0 to 100'); + process.exit(1); + } +} if (bIdx !== -1) { + if (minRank1 !== null) { + console.error('--min-rank1 applies only to deterministic evals'); + process.exit(1); + } runBehavioral(args[bIdx + 1], args.includes('--dry-run')); } else { - runDeterministic(); + runDeterministic(minRank1); }