test: ratchet deterministic skill eval gates

2026-08-12 18:07:26 +02:00 · 2026-07-10 14:53:37 +08:00
parent 6bcfeb9dae
commit 4290228323
2 changed files with 209 additions and 12 deletions
@@ -0,0 +1,143 @@
+#!/usr/bin/env node
+
+'use strict';
+
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+const { spawnSync } = require('node:child_process');
+const test = require('node:test');
+
+const RUNNER = path.join(__dirname, 'run-evals.js');
+
+function writeJson(file, value) {
+  fs.mkdirSync(path.dirname(file), { recursive: true });
+  fs.writeFileSync(file, `${JSON.stringify(value, null, 2)}\n`);
+}
+
+function writeSkill(root, name, description) {
+  const dir = path.join(root, 'skills', name);
+  fs.mkdirSync(dir, { recursive: true });
+  fs.writeFileSync(
+    path.join(dir, 'SKILL.md'),
+    `---\nname: ${name}\ndescription: ${description}\n---\n\n# ${name}\n`,
+  );
+}
+
+function behavioralEval(files = ['project/context.txt']) {
+  return {
+    id: 1,
+    prompt: 'Inspect the attached project and complete the requested work.',
+    expected_output: 'A verified result grounded in the attached project',
+    files,
+    expectations: ['The attached project is inspected before reporting a result'],
+  };
+}
+
+function completeCase(skillName, positivePrompt, topK = 1, files) {
+  return {
+    skill_name: skillName,
+    trigger: {
+      positive: [1, 2, 3].map(() => ({ prompt: positivePrompt, top_k: topK })),
+      negative: [
+        { prompt: 'unrelated banana request' },
+        { prompt: 'unrelated orange request' },
+      ],
+    },
+    evals: [behavioralEval(files)],
+  };
+}
+
+function makeSandbox() {
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-skills-run-evals-test-'));
+  fs.mkdirSync(path.join(root, 'scripts'), { recursive: true });
+  fs.mkdirSync(path.join(root, 'evals', 'cases'), { recursive: true });
+  fs.mkdirSync(path.join(root, 'evals', 'fixtures', 'project'), { recursive: true });
+  fs.copyFileSync(RUNNER, path.join(root, 'scripts', 'run-evals.js'));
+  fs.writeFileSync(path.join(root, 'evals', 'fixtures', 'project', 'context.txt'), 'fixture\n');
+  return root;
+}
+
+function run(root, args = []) {
+  return spawnSync(process.execPath, [path.join(root, 'scripts', 'run-evals.js'), ...args], {
+    cwd: root,
+    encoding: 'utf8',
+  });
+}
+
+test('fails when a skill has no eval case file', () => {
+  const root = makeSandbox();
+  writeSkill(root, 'alpha-skill', 'Handles alpha widgets. Use when changing alpha widgets.');
+
+  const result = run(root);
+
+  assert.equal(result.status, 1, result.stdout + result.stderr);
+  assert.match(result.stdout, /no eval case file/);
+});
+
+test('fails when an eval case is below the required minimums', () => {
+  const root = makeSandbox();
+  writeSkill(root, 'alpha-skill', 'Handles alpha widgets. Use when changing alpha widgets.');
+  writeJson(path.join(root, 'evals', 'cases', 'alpha-skill.json'), {
+    skill_name: 'alpha-skill',
+    trigger: {
+      positive: [{ prompt: 'change alpha widget', top_k: 1 }],
+      negative: [],
+    },
+    evals: [behavioralEval()],
+  });
+
+  const result = run(root);
+
+  assert.equal(result.status, 1, result.stdout + result.stderr);
+  assert.match(result.stdout, /below required minimums/);
+});
+
+test('fails when a behavioral eval references a missing fixture', () => {
+  const root = makeSandbox();
+  writeSkill(root, 'alpha-skill', 'Handles alpha widgets. Use when changing alpha widgets.');
+  writeJson(
+    path.join(root, 'evals', 'cases', 'alpha-skill.json'),
+    completeCase('alpha-skill', 'change alpha widget', 1, ['missing/project.txt']),
+  );
+
+  const result = run(root);
+
+  assert.equal(result.status, 1, result.stdout + result.stderr);
+  assert.match(result.stdout, /fixture not found/);
+});
+
+test('enforces the configured rank-1 floor', () => {
+  const root = makeSandbox();
+  writeSkill(root, 'alpha-skill', 'Handles widget work. Use when implementing widget changes.');
+  writeSkill(
+    root,
+    'beta-skill',
+    'Diagnoses urgent widget failures in production. Use when repairing urgent widget failures.',
+  );
+  writeJson(
+    path.join(root, 'evals', 'cases', 'alpha-skill.json'),
+    completeCase('alpha-skill', 'urgent widget failure production', 2),
+  );
+  writeJson(
+    path.join(root, 'evals', 'cases', 'beta-skill.json'),
+    completeCase('beta-skill', 'repair urgent widget failure', 1),
+  );
+
+  const passing = run(root, ['--min-rank1', '50']);
+  const failing = run(root, ['--min-rank1', '60']);
+
+  assert.equal(passing.status, 0, passing.stdout + passing.stderr);
+  assert.equal(failing.status, 1, failing.stdout + failing.stderr);
+  assert.match(failing.stdout, /below required 60%/);
+});
+
+test('rejects an invalid rank-1 floor', () => {
+  const root = makeSandbox();
+
+  const result = run(root, ['--min-rank1', '101']);
+
+  assert.equal(result.status, 1, result.stdout + result.stderr);
+  assert.match(result.stderr, /--min-rank1 must be a number from 0 to 100/);
+});
@@ -12,7 +12,9 @@
 *       overlapping skills drifting in.
 *     - Coverage + schema: every case file maps to a real skill, skill_name
 *       matches, and behavioral evals follow the skill-creator evals.json shape.
- *       Skills without a case file are reported as warnings (not errors, yet).
+ *       Every skill must have a complete case file backed by real fixtures.
+ *     - Rank-1 ratchet: --min-rank1 <pct> fails when routing quality drops
+ *       below the checked-in CI baseline.
 *   Tier 3 (opt-in, costs tokens, never in CI):
 *     node scripts/run-evals.js --behavioral <skill> [--dry-run]
 *     Runs each behavioral eval through headless `claude` in a throwaway
@@ -47,7 +49,7 @@ const GRADER_TIMEOUT_MS = 5 * 60 * 1000;
 // tokens; review this list if your fixtures invoke anything unusual.
 const EXECUTOR_TOOLS = 'Read,Glob,Grep,Edit,Write,Bash';

-// Documented minimums per case file (evals/README.md). Warning-level for now.
+// Required minimums per case file (evals/README.md).
 const MIN_POSITIVE = 3;
 const MIN_NEGATIVE = 2;
 const MIN_EVALS = 1;
@@ -193,7 +195,7 @@ function resolveFixturePath(root, rel) {

 // ---------- tier 2 ----------

-function runDeterministic() {
+function runDeterministic(minRank1) {
  const skills = loadSkills();
  const cases = loadCases();
  const corpus = buildCorpus(skills);
@@ -210,8 +212,8 @@ function runDeterministic() {
  // Coverage
  for (const s of skills) {
    if (!cases.some((c) => c.file === `${s.name}.json`)) {
-      console.log(`  ⚠  ${s.name}: no eval case file (evals/cases/${s.name}.json)`);
-      warnings++;
+      console.log(`  ✗  ${s.name}: no eval case file (evals/cases/${s.name}.json)`);
+      errors++;
    }
  }

@@ -246,6 +248,29 @@ function runDeterministic() {
        console.log(`  ✗  ${c.file}: eval id=${ev.id} does not match evals.json schema`);
        errors++;
      }
+      if (!Array.isArray(ev.files) || ev.files.length === 0 || !ev.files.every((x) => typeof x === 'string')) {
+        console.log(`  ✗  ${c.file}: eval id=${ev.id} needs a non-empty files[] fixture list`);
+        errors++;
+      } else {
+        for (const rel of ev.files) {
+          let fixture;
+          try {
+            fixture = resolveFixturePath(FIXTURES_DIR, rel);
+          } catch (e) {
+            console.log(`  ✗  ${c.file}: eval id=${ev.id} has invalid fixture path "${rel}" — ${e.message}`);
+            errors++;
+            continue;
+          }
+          if (!fs.existsSync(fixture)) {
+            console.log(`  ✗  ${c.file}: eval id=${ev.id} fixture not found: evals/fixtures/${rel}`);
+            errors++;
+          }
+        }
+      }
+      if (ev.trust_level === 'provisional') {
+        console.log(`  ✗  ${c.file}: eval id=${ev.id} is still provisional; add real fixtures before trusting it`);
+        errors++;
+      }
    }

    // Trigger: positive
@@ -303,13 +328,13 @@ function runDeterministic() {
      if (ok) passed++;
    }

-    // Documented minimums (warning-level during the transition window)
+    // Required minimums
    const pc = (d.trigger?.positive || []).length;
    const nc = (d.trigger?.negative || []).length;
    const ec = (d.evals || []).length;
    if (pc < MIN_POSITIVE || nc < MIN_NEGATIVE || ec < MIN_EVALS) {
-      console.log(`  ⚠  ${expected}: below documented minimums (${pc} positive/${nc} negative/${ec} behavioral; need ${MIN_POSITIVE}/${MIN_NEGATIVE}/${MIN_EVALS})`);
-      warnings++;
+      console.log(`  ✗  ${expected}: below required minimums (${pc} positive/${nc} negative/${ec} behavioral; need ${MIN_POSITIVE}/${MIN_NEGATIVE}/${MIN_EVALS})`);
+      errors++;
    }
  }

@@ -330,7 +355,12 @@ function runDeterministic() {
    }
  }

-  const rate = positives ? ((rank1 / positives) * 100).toFixed(0) : 'n/a';
+  const rank1Rate = positives ? (rank1 / positives) * 100 : 0;
+  const rate = positives ? rank1Rate.toFixed(0) : 'n/a';
+  if (minRank1 !== null && (!positives || rank1Rate < minRank1)) {
+    console.log(`  ✗  trigger rank-1 rate ${rate}% is below required ${minRank1}%`);
+    errors++;
+  }
  console.log(`\n${passed} checks passed — ${errors} error(s), ${warnings} warning(s)`);
  console.log(`trigger rank-1 rate: ${rate}% (${rank1}/${positives} positive prompts rank their skill first)`);
  console.log(errors ? 'FAILED' : 'PASSED');
@@ -352,6 +382,14 @@ function materializeWorkspace(ev) {
    fs.mkdirSync(path.dirname(dest), { recursive: true });
    fs.cpSync(src, dest, { recursive: true });
  }
+  // Give workflow-oriented evals a real baseline to inspect, modify, diff, and
+  // commit. A local identity keeps this deterministic and never leaves the
+  // throwaway workspace.
+  execFileSync('git', ['init', '--quiet'], { cwd: workspace });
+  execFileSync('git', ['config', 'user.name', 'Skill Eval'], { cwd: workspace });
+  execFileSync('git', ['config', 'user.email', 'skill-eval@example.invalid'], { cwd: workspace });
+  execFileSync('git', ['add', '--all'], { cwd: workspace });
+  execFileSync('git', ['commit', '--quiet', '-m', 'fixture baseline'], { cwd: workspace });
  return workspace;
 }

@@ -389,8 +427,10 @@ function runBehavioral(skillName, dryRun) {

  for (const ev of d.evals) {
    const fixtures = (ev.files || []).length;
-    if (ev.trust_level === 'provisional' || !fixtures) {
-      console.log(`  note: eval ${ev.id} is provisional (${fixtures ? 'flagged' : 'no fixtures'}) — results are a sanity check, not evidence`);
+    if (!fixtures) {
+      console.error(`eval ${ev.id} has no fixtures; run the deterministic eval gate first`);
+      failures++;
+      continue;
    }
    if (dryRun) {
      console.log(`[dry-run] eval ${ev.id}: workspace + ${fixtures} fixture(s); claude -p --verbose --output-format stream-json --permission-mode acceptEdits --allowedTools ${EXECUTOR_TOOLS} --append-system-prompt <${skillName}/SKILL.md> < prompt-on-stdin`);
@@ -442,8 +482,22 @@ function runBehavioral(skillName, dryRun) {

 const args = process.argv.slice(2);
 const bIdx = args.indexOf('--behavioral');
+const rankIdx = args.indexOf('--min-rank1');
+let minRank1 = null;
+if (rankIdx !== -1) {
+  const raw = args[rankIdx + 1];
+  minRank1 = Number(raw);
+  if (raw === undefined || raw === '' || !Number.isFinite(minRank1) || minRank1 < 0 || minRank1 > 100) {
+    console.error('--min-rank1 must be a number from 0 to 100');
+    process.exit(1);
+  }
+}
 if (bIdx !== -1) {
+  if (minRank1 !== null) {
+    console.error('--min-rank1 applies only to deterministic evals');
+    process.exit(1);
+  }
  runBehavioral(args[bIdx + 1], args.includes('--dry-run'));
 } else {
-  runDeterministic();
+  runDeterministic(minRank1);
 }