From 42902283232b627aa6c6259782d07ca53148b114 Mon Sep 17 00:00:00 2001 From: Zhiyao Date: Fri, 10 Jul 2026 14:53:37 +0800 Subject: [PATCH 1/3] test: ratchet deterministic skill eval gates --- scripts/run-evals-test.js | 143 ++++++++++++++++++++++++++++++++++++++ scripts/run-evals.js | 78 +++++++++++++++++---- 2 files changed, 209 insertions(+), 12 deletions(-) create mode 100644 scripts/run-evals-test.js diff --git a/scripts/run-evals-test.js b/scripts/run-evals-test.js new file mode 100644 index 0000000..489d117 --- /dev/null +++ b/scripts/run-evals-test.js @@ -0,0 +1,143 @@ +#!/usr/bin/env node + +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); +const test = require('node:test'); + +const RUNNER = path.join(__dirname, 'run-evals.js'); + +function writeJson(file, value) { + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, `${JSON.stringify(value, null, 2)}\n`); +} + +function writeSkill(root, name, description) { + const dir = path.join(root, 'skills', name); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync( + path.join(dir, 'SKILL.md'), + `---\nname: ${name}\ndescription: ${description}\n---\n\n# ${name}\n`, + ); +} + +function behavioralEval(files = ['project/context.txt']) { + return { + id: 1, + prompt: 'Inspect the attached project and complete the requested work.', + expected_output: 'A verified result grounded in the attached project', + files, + expectations: ['The attached project is inspected before reporting a result'], + }; +} + +function completeCase(skillName, positivePrompt, topK = 1, files) { + return { + skill_name: skillName, + trigger: { + positive: [1, 2, 3].map(() => ({ prompt: positivePrompt, top_k: topK })), + negative: [ + { prompt: 'unrelated banana request' }, + { prompt: 'unrelated orange request' }, + ], + }, + evals: [behavioralEval(files)], + }; +} + +function makeSandbox() { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-skills-run-evals-test-')); + fs.mkdirSync(path.join(root, 'scripts'), { recursive: true }); + fs.mkdirSync(path.join(root, 'evals', 'cases'), { recursive: true }); + fs.mkdirSync(path.join(root, 'evals', 'fixtures', 'project'), { recursive: true }); + fs.copyFileSync(RUNNER, path.join(root, 'scripts', 'run-evals.js')); + fs.writeFileSync(path.join(root, 'evals', 'fixtures', 'project', 'context.txt'), 'fixture\n'); + return root; +} + +function run(root, args = []) { + return spawnSync(process.execPath, [path.join(root, 'scripts', 'run-evals.js'), ...args], { + cwd: root, + encoding: 'utf8', + }); +} + +test('fails when a skill has no eval case file', () => { + const root = makeSandbox(); + writeSkill(root, 'alpha-skill', 'Handles alpha widgets. Use when changing alpha widgets.'); + + const result = run(root); + + assert.equal(result.status, 1, result.stdout + result.stderr); + assert.match(result.stdout, /no eval case file/); +}); + +test('fails when an eval case is below the required minimums', () => { + const root = makeSandbox(); + writeSkill(root, 'alpha-skill', 'Handles alpha widgets. Use when changing alpha widgets.'); + writeJson(path.join(root, 'evals', 'cases', 'alpha-skill.json'), { + skill_name: 'alpha-skill', + trigger: { + positive: [{ prompt: 'change alpha widget', top_k: 1 }], + negative: [], + }, + evals: [behavioralEval()], + }); + + const result = run(root); + + assert.equal(result.status, 1, result.stdout + result.stderr); + assert.match(result.stdout, /below required minimums/); +}); + +test('fails when a behavioral eval references a missing fixture', () => { + const root = makeSandbox(); + writeSkill(root, 'alpha-skill', 'Handles alpha widgets. Use when changing alpha widgets.'); + writeJson( + path.join(root, 'evals', 'cases', 'alpha-skill.json'), + completeCase('alpha-skill', 'change alpha widget', 1, ['missing/project.txt']), + ); + + const result = run(root); + + assert.equal(result.status, 1, result.stdout + result.stderr); + assert.match(result.stdout, /fixture not found/); +}); + +test('enforces the configured rank-1 floor', () => { + const root = makeSandbox(); + writeSkill(root, 'alpha-skill', 'Handles widget work. Use when implementing widget changes.'); + writeSkill( + root, + 'beta-skill', + 'Diagnoses urgent widget failures in production. Use when repairing urgent widget failures.', + ); + writeJson( + path.join(root, 'evals', 'cases', 'alpha-skill.json'), + completeCase('alpha-skill', 'urgent widget failure production', 2), + ); + writeJson( + path.join(root, 'evals', 'cases', 'beta-skill.json'), + completeCase('beta-skill', 'repair urgent widget failure', 1), + ); + + const passing = run(root, ['--min-rank1', '50']); + const failing = run(root, ['--min-rank1', '60']); + + assert.equal(passing.status, 0, passing.stdout + passing.stderr); + assert.equal(failing.status, 1, failing.stdout + failing.stderr); + assert.match(failing.stdout, /below required 60%/); +}); + +test('rejects an invalid rank-1 floor', () => { + const root = makeSandbox(); + + const result = run(root, ['--min-rank1', '101']); + + assert.equal(result.status, 1, result.stdout + result.stderr); + assert.match(result.stderr, /--min-rank1 must be a number from 0 to 100/); +}); diff --git a/scripts/run-evals.js b/scripts/run-evals.js index 9272dc1..c9a9ba5 100644 --- a/scripts/run-evals.js +++ b/scripts/run-evals.js @@ -12,7 +12,9 @@ * overlapping skills drifting in. * - Coverage + schema: every case file maps to a real skill, skill_name * matches, and behavioral evals follow the skill-creator evals.json shape. - * Skills without a case file are reported as warnings (not errors, yet). + * Every skill must have a complete case file backed by real fixtures. + * - Rank-1 ratchet: --min-rank1 fails when routing quality drops + * below the checked-in CI baseline. * Tier 3 (opt-in, costs tokens, never in CI): * node scripts/run-evals.js --behavioral [--dry-run] * Runs each behavioral eval through headless `claude` in a throwaway @@ -47,7 +49,7 @@ const GRADER_TIMEOUT_MS = 5 * 60 * 1000; // tokens; review this list if your fixtures invoke anything unusual. const EXECUTOR_TOOLS = 'Read,Glob,Grep,Edit,Write,Bash'; -// Documented minimums per case file (evals/README.md). Warning-level for now. +// Required minimums per case file (evals/README.md). const MIN_POSITIVE = 3; const MIN_NEGATIVE = 2; const MIN_EVALS = 1; @@ -193,7 +195,7 @@ function resolveFixturePath(root, rel) { // ---------- tier 2 ---------- -function runDeterministic() { +function runDeterministic(minRank1) { const skills = loadSkills(); const cases = loadCases(); const corpus = buildCorpus(skills); @@ -210,8 +212,8 @@ function runDeterministic() { // Coverage for (const s of skills) { if (!cases.some((c) => c.file === `${s.name}.json`)) { - console.log(` ⚠ ${s.name}: no eval case file (evals/cases/${s.name}.json)`); - warnings++; + console.log(` ✗ ${s.name}: no eval case file (evals/cases/${s.name}.json)`); + errors++; } } @@ -246,6 +248,29 @@ function runDeterministic() { console.log(` ✗ ${c.file}: eval id=${ev.id} does not match evals.json schema`); errors++; } + if (!Array.isArray(ev.files) || ev.files.length === 0 || !ev.files.every((x) => typeof x === 'string')) { + console.log(` ✗ ${c.file}: eval id=${ev.id} needs a non-empty files[] fixture list`); + errors++; + } else { + for (const rel of ev.files) { + let fixture; + try { + fixture = resolveFixturePath(FIXTURES_DIR, rel); + } catch (e) { + console.log(` ✗ ${c.file}: eval id=${ev.id} has invalid fixture path "${rel}" — ${e.message}`); + errors++; + continue; + } + if (!fs.existsSync(fixture)) { + console.log(` ✗ ${c.file}: eval id=${ev.id} fixture not found: evals/fixtures/${rel}`); + errors++; + } + } + } + if (ev.trust_level === 'provisional') { + console.log(` ✗ ${c.file}: eval id=${ev.id} is still provisional; add real fixtures before trusting it`); + errors++; + } } // Trigger: positive @@ -303,13 +328,13 @@ function runDeterministic() { if (ok) passed++; } - // Documented minimums (warning-level during the transition window) + // Required minimums const pc = (d.trigger?.positive || []).length; const nc = (d.trigger?.negative || []).length; const ec = (d.evals || []).length; if (pc < MIN_POSITIVE || nc < MIN_NEGATIVE || ec < MIN_EVALS) { - console.log(` ⚠ ${expected}: below documented minimums (${pc} positive/${nc} negative/${ec} behavioral; need ${MIN_POSITIVE}/${MIN_NEGATIVE}/${MIN_EVALS})`); - warnings++; + console.log(` ✗ ${expected}: below required minimums (${pc} positive/${nc} negative/${ec} behavioral; need ${MIN_POSITIVE}/${MIN_NEGATIVE}/${MIN_EVALS})`); + errors++; } } @@ -330,7 +355,12 @@ function runDeterministic() { } } - const rate = positives ? ((rank1 / positives) * 100).toFixed(0) : 'n/a'; + const rank1Rate = positives ? (rank1 / positives) * 100 : 0; + const rate = positives ? rank1Rate.toFixed(0) : 'n/a'; + if (minRank1 !== null && (!positives || rank1Rate < minRank1)) { + console.log(` ✗ trigger rank-1 rate ${rate}% is below required ${minRank1}%`); + errors++; + } console.log(`\n${passed} checks passed — ${errors} error(s), ${warnings} warning(s)`); console.log(`trigger rank-1 rate: ${rate}% (${rank1}/${positives} positive prompts rank their skill first)`); console.log(errors ? 'FAILED' : 'PASSED'); @@ -352,6 +382,14 @@ function materializeWorkspace(ev) { fs.mkdirSync(path.dirname(dest), { recursive: true }); fs.cpSync(src, dest, { recursive: true }); } + // Give workflow-oriented evals a real baseline to inspect, modify, diff, and + // commit. A local identity keeps this deterministic and never leaves the + // throwaway workspace. + execFileSync('git', ['init', '--quiet'], { cwd: workspace }); + execFileSync('git', ['config', 'user.name', 'Skill Eval'], { cwd: workspace }); + execFileSync('git', ['config', 'user.email', 'skill-eval@example.invalid'], { cwd: workspace }); + execFileSync('git', ['add', '--all'], { cwd: workspace }); + execFileSync('git', ['commit', '--quiet', '-m', 'fixture baseline'], { cwd: workspace }); return workspace; } @@ -389,8 +427,10 @@ function runBehavioral(skillName, dryRun) { for (const ev of d.evals) { const fixtures = (ev.files || []).length; - if (ev.trust_level === 'provisional' || !fixtures) { - console.log(` note: eval ${ev.id} is provisional (${fixtures ? 'flagged' : 'no fixtures'}) — results are a sanity check, not evidence`); + if (!fixtures) { + console.error(`eval ${ev.id} has no fixtures; run the deterministic eval gate first`); + failures++; + continue; } if (dryRun) { console.log(`[dry-run] eval ${ev.id}: workspace + ${fixtures} fixture(s); claude -p --verbose --output-format stream-json --permission-mode acceptEdits --allowedTools ${EXECUTOR_TOOLS} --append-system-prompt <${skillName}/SKILL.md> < prompt-on-stdin`); @@ -442,8 +482,22 @@ function runBehavioral(skillName, dryRun) { const args = process.argv.slice(2); const bIdx = args.indexOf('--behavioral'); +const rankIdx = args.indexOf('--min-rank1'); +let minRank1 = null; +if (rankIdx !== -1) { + const raw = args[rankIdx + 1]; + minRank1 = Number(raw); + if (raw === undefined || raw === '' || !Number.isFinite(minRank1) || minRank1 < 0 || minRank1 > 100) { + console.error('--min-rank1 must be a number from 0 to 100'); + process.exit(1); + } +} if (bIdx !== -1) { + if (minRank1 !== null) { + console.error('--min-rank1 applies only to deterministic evals'); + process.exit(1); + } runBehavioral(args[bIdx + 1], args.includes('--dry-run')); } else { - runDeterministic(); + runDeterministic(minRank1); } From 5168535ce418b139d414e14eeeefc39e553c3f68 Mon Sep 17 00:00:00 2001 From: Zhiyao Date: Fri, 10 Jul 2026 15:29:28 +0800 Subject: [PATCH 2/3] feat: promote skill eval gates to trusted --- .github/workflows/test-plugin-install.yml | 5 +- CONTRIBUTING.md | 2 +- evals/README.md | 18 +++--- evals/cases/api-and-interface-design.json | 6 +- .../cases/browser-testing-with-devtools.json | 6 +- evals/cases/ci-cd-and-automation.json | 6 +- evals/cases/code-review-and-quality.json | 6 +- evals/cases/code-simplification.json | 6 +- evals/cases/context-engineering.json | 6 +- evals/cases/debugging-and-error-recovery.json | 17 ++++- evals/cases/deprecation-and-migration.json | 6 +- evals/cases/documentation-and-adrs.json | 6 +- evals/cases/doubt-driven-development.json | 6 +- evals/cases/frontend-ui-engineering.json | 6 +- evals/cases/git-workflow-and-versioning.json | 6 +- evals/cases/idea-refine.json | 6 +- evals/cases/incremental-implementation.json | 17 ++++- evals/cases/interview-me.json | 6 +- .../observability-and-instrumentation.json | 6 +- evals/cases/performance-optimization.json | 6 +- evals/cases/planning-and-task-breakdown.json | 6 +- evals/cases/security-and-hardening.json | 6 +- evals/cases/shipping-and-launch.json | 17 ++++- evals/cases/source-driven-development.json | 6 +- evals/cases/spec-driven-development.json | 6 +- evals/cases/test-driven-development.json | 17 ++++- evals/cases/using-agent-skills.json | 6 +- .../api-and-interface-design/service-brief.md | 19 ++++++ .../browser-testing-with-devtools/README.md | 5 ++ .../browser-testing-with-devtools/index.html | 24 +++++++ .../browser-testing-with-devtools/server.js | 15 +++++ .../ci-cd-and-automation/package.json | 8 +++ .../fixtures/ci-cd-and-automation/src/slug.js | 3 + .../ci-cd-and-automation/test/slug.test.js | 9 +++ .../code-review-and-quality/user-search.diff | 16 +++++ .../code-simplification/config-parser.js | 46 ++++++++++++++ .../code-simplification/config-parser.test.js | 15 +++++ .../context-engineering/context-audit.md | 15 +++++ .../pagination.js | 8 +++ .../pagination.test.js | 9 +++ .../time-pressure.md | 6 ++ .../api-inventory.md | 9 +++ .../decision-context.md | 16 +++++ .../migration-plan.md | 19 ++++++ .../frontend-ui-engineering/Button.tsx | 7 +++ .../frontend-ui-engineering/design-system.md | 11 ++++ .../.eval/working-tree.patch | 21 +++++++ .../git-workflow-and-versioning/app.js | 7 +++ .../git-workflow-and-versioning/app.test.js | 9 +++ evals/fixtures/idea-refine/idea-brief.md | 16 +++++ .../draft-export.js | 17 +++++ .../scenario.md | 9 +++ .../incremental-implementation/reports.js | 7 +++ .../reports.test.js | 12 ++++ .../incremental-implementation/tasks/plan.md | 8 +++ evals/fixtures/interview-me/admin-context.md | 9 +++ .../operations.md | 11 ++++ .../payment-retry.js | 14 +++++ .../performance-optimization/benchmark.js | 15 +++++ .../performance-optimization/products.js | 14 +++++ .../notifications-spec.md | 18 ++++++ .../security-and-hardening/webhook.js | 11 ++++ .../security-and-hardening/webhook.test.js | 13 ++++ .../shipping-and-launch/authority-pressure.md | 6 ++ .../shipping-and-launch/launch-status.md | 11 ++++ .../framework-task.md | 10 +++ .../spec-driven-development/billing-brief.md | 16 +++++ .../authority-pressure.md | 8 +++ .../test-driven-development/invoice.js | 7 +++ .../test-driven-development/invoice.test.js | 12 ++++ evals/fixtures/using-agent-skills/incident.md | 6 ++ scripts/run-evals-test.js | 16 +++++ scripts/run-evals.js | 62 +++++++++++++------ 73 files changed, 764 insertions(+), 74 deletions(-) create mode 100644 evals/fixtures/api-and-interface-design/service-brief.md create mode 100644 evals/fixtures/browser-testing-with-devtools/README.md create mode 100644 evals/fixtures/browser-testing-with-devtools/index.html create mode 100644 evals/fixtures/browser-testing-with-devtools/server.js create mode 100644 evals/fixtures/ci-cd-and-automation/package.json create mode 100644 evals/fixtures/ci-cd-and-automation/src/slug.js create mode 100644 evals/fixtures/ci-cd-and-automation/test/slug.test.js create mode 100644 evals/fixtures/code-review-and-quality/user-search.diff create mode 100644 evals/fixtures/code-simplification/config-parser.js create mode 100644 evals/fixtures/code-simplification/config-parser.test.js create mode 100644 evals/fixtures/context-engineering/context-audit.md create mode 100644 evals/fixtures/debugging-and-error-recovery/pagination.js create mode 100644 evals/fixtures/debugging-and-error-recovery/pagination.test.js create mode 100644 evals/fixtures/debugging-and-error-recovery/time-pressure.md create mode 100644 evals/fixtures/deprecation-and-migration/api-inventory.md create mode 100644 evals/fixtures/documentation-and-adrs/decision-context.md create mode 100644 evals/fixtures/doubt-driven-development/migration-plan.md create mode 100644 evals/fixtures/frontend-ui-engineering/Button.tsx create mode 100644 evals/fixtures/frontend-ui-engineering/design-system.md create mode 100644 evals/fixtures/git-workflow-and-versioning/.eval/working-tree.patch create mode 100644 evals/fixtures/git-workflow-and-versioning/app.js create mode 100644 evals/fixtures/git-workflow-and-versioning/app.test.js create mode 100644 evals/fixtures/idea-refine/idea-brief.md create mode 100644 evals/fixtures/incremental-implementation-pressure/draft-export.js create mode 100644 evals/fixtures/incremental-implementation-pressure/scenario.md create mode 100644 evals/fixtures/incremental-implementation/reports.js create mode 100644 evals/fixtures/incremental-implementation/reports.test.js create mode 100644 evals/fixtures/incremental-implementation/tasks/plan.md create mode 100644 evals/fixtures/interview-me/admin-context.md create mode 100644 evals/fixtures/observability-and-instrumentation/operations.md create mode 100644 evals/fixtures/observability-and-instrumentation/payment-retry.js create mode 100644 evals/fixtures/performance-optimization/benchmark.js create mode 100644 evals/fixtures/performance-optimization/products.js create mode 100644 evals/fixtures/planning-and-task-breakdown/notifications-spec.md create mode 100644 evals/fixtures/security-and-hardening/webhook.js create mode 100644 evals/fixtures/security-and-hardening/webhook.test.js create mode 100644 evals/fixtures/shipping-and-launch/authority-pressure.md create mode 100644 evals/fixtures/shipping-and-launch/launch-status.md create mode 100644 evals/fixtures/source-driven-development/framework-task.md create mode 100644 evals/fixtures/spec-driven-development/billing-brief.md create mode 100644 evals/fixtures/test-driven-development/authority-pressure.md create mode 100644 evals/fixtures/test-driven-development/invoice.js create mode 100644 evals/fixtures/test-driven-development/invoice.test.js create mode 100644 evals/fixtures/using-agent-skills/incident.md diff --git a/.github/workflows/test-plugin-install.yml b/.github/workflows/test-plugin-install.yml index 2f906de..d4c0dc4 100644 --- a/.github/workflows/test-plugin-install.yml +++ b/.github/workflows/test-plugin-install.yml @@ -20,8 +20,11 @@ jobs: - name: Validate all skills run: node scripts/validate-skills.js + - name: Test skill eval runner + run: node --test scripts/run-evals-test.js + - name: Run skill evals (trigger + routing) - run: node scripts/run-evals.js + run: node scripts/run-evals.js --min-rank1 85 validate-commands: name: Validate command parity and description sync diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1cc1ce5..2af5b6c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -37,7 +37,7 @@ Every new skill must have: - `SKILL.md` in the skill directory - YAML frontmatter with valid `name` and `description` -- An eval case file at `evals/cases/.json` — at least 3 positive triggers, 2 negative triggers (with `owner` where possible), and 1 behavioral eval (see [evals/README.md](evals/README.md); warning-level until promoted via [#352](https://github.com/addyosmani/agent-skills/issues/352)) +- An eval case file at `evals/cases/.json` — at least 3 positive triggers, 2 negative triggers (with `owner` where possible), and 1 behavioral eval backed by real files under `evals/fixtures/` (see [evals/README.md](evals/README.md)). CI enforces these requirements. New skills should generally follow the standard anatomy: diff --git a/evals/README.md b/evals/README.md index eef9188..3b821d7 100644 --- a/evals/README.md +++ b/evals/README.md @@ -26,15 +26,16 @@ Tier 2 is a **lexical approximation** of routing (stemmed TF-IDF over descriptio ```bash # Tier 2 — deterministic, runs in CI node scripts/run-evals.js +node scripts/run-evals.js --min-rank1 85 # enforce the current routing floor # Tier 3 — behavioral, runs each eval through headless claude, then grades it node scripts/run-evals.js --behavioral test-driven-development # spends tokens node scripts/run-evals.js --behavioral test-driven-development --dry-run # prints the plan only ``` -Tier 3 runs each eval in a throwaway workspace (fixtures from `files[]` are materialized out of `evals/fixtures/`), captures the full `--output-format stream-json --verbose` execution trace, and grades the **trace** (tool calls included) rather than the model's final prose, so expectations like "a failing test was run before the fix" are judged on what happened, not what was narrated. The executor runs with an explicit permission mode (`--permission-mode acceptEdits` plus a pre-approved tool list) so the agent can genuinely edit files and run commands in the workspace rather than being denied and narrating instead. The trace is fenced as untrusted data in the grader prompt and piped to the grader over stdin (traces can be megabytes; argv would hit the OS argument-size limit), executor and grader calls carry timeouts, and grader output is validated as JSON before being written to `evals/results/` (gitignored) in skill-creator's `grading.json` shape. +Tier 3 runs each eval in a throwaway git repository (real project inputs from `files[]` are materialized out of `evals/fixtures/` and committed as the baseline), captures the full `--output-format stream-json --verbose` execution trace, and grades the **trace** (tool calls included) rather than the model's final prose, so expectations like "a failing test was run before the fix" are judged on what happened, not what was narrated. The executor runs with an explicit permission mode (`--permission-mode acceptEdits` plus a pre-approved tool list) so the agent can genuinely edit files, run commands, inspect diffs, and make commits in the workspace rather than being denied and narrating instead. The trace is fenced as untrusted data in the grader prompt and piped to the grader over stdin (traces can be megabytes; argv would hit the OS argument-size limit), executor and grader calls carry timeouts, and grader output is validated as JSON before being written to `evals/results/` (gitignored) in skill-creator's `grading.json` shape. -Behavioral evals without fixtures carry a provisional trust level: treat their results as sanity checks, not evidence. Graduation criteria live in [#352](https://github.com/addyosmani/agent-skills/issues/352). +Every behavioral eval must list at least one real fixture. The deterministic gate validates that each `files[]` entry exists before Tier 3 can run, so behavioral results are grounded in an inspectable project rather than a prose-only scenario. Discipline skills also include pressure cases for time pressure, sunk cost, and authority pressure; these verify that the workflow still holds when the prompt argues for skipping it. ## Eval case format @@ -56,27 +57,28 @@ One file per skill: `evals/cases/.json`. "id": 1, "prompt": "Fix the reported rounding bug in the invoice totals, test-first.", "expected_output": "A failing test demonstrating the bug, a minimal fix turning it green, full suite passing", + "files": [ + "test-driven-development" + ], "expectations": [ "A failing test is written and shown failing before the fix", "The implementation is the minimum needed to pass", "The full suite is run after the fix to catch regressions" - ], - "trust_level": "provisional" + ] } ] } ``` -- `evals[]` is skill-creator's schema exactly (`id`, `prompt`, `expected_output`, optional `files[]`, `expectations[]`). Expectations are verifiable statements a grader checks against the transcript — behaviors, not phrasings. +- `evals[]` uses skill-creator's schema (`id`, `prompt`, `expected_output`, `files[]`, `expectations[]`). This repository requires non-empty `files[]` entries so each behavioral eval operates on real inputs. Paths are relative to `evals/fixtures/` and may name a file or a project directory. Expectations are verifiable statements a grader checks against the transcript — behaviors, not phrasings. - `trigger` is this repo's extension. `positive` prompts are realistic user asks that should route here (`top_k` defaults to 3; tighten to 1 for a skill's signature ask). `negative` prompts belong to a *different* skill; this skill must not rank first for them. Declare that skill in `owner` where you can: the runner then asserts the owner **outranks** this skill, turning the negative into a real pairwise routing test instead of one that can pass vacuously when the prompt matches nothing. -- `trust_level: "provisional"` marks a behavioral eval with no fixtures yet; the behavioral runner flags these and their pass rates should not be cited as evidence (see [#352](https://github.com/addyosmani/agent-skills/issues/352)). **Writing good trigger prompts:** paraphrase how users actually talk; don't copy the description (that's gaming the eval). If a realistic prompt can't rank because the description lacks its vocabulary, that is a real finding — improve the description. ## Adding a skill -Every skill ships with an eval file. When you add `skills//`, add `evals/cases/.json` with at least 3 positive triggers, 2 negative triggers, and 1 behavioral eval; the runner warns when a file is below those minimums or missing entirely. Both checks are warning-level during the transition window and will be promoted to errors via [#352](https://github.com/addyosmani/agent-skills/issues/352). +Every skill ships with an eval file. When you add `skills//`, add `evals/cases/.json` with at least 3 positive triggers, 2 negative triggers, and 1 behavioral eval backed by `evals/fixtures//`. Missing case files, incomplete case counts, invalid fixture paths, and absent fixtures are CI errors. ## Metrics to watch -The Tier-2 run prints a **trigger rank-1 rate** (share of positive prompts that rank their skill first, not merely top-k). It isn't gated yet; a `--min-rank1` CI ratchet is planned once the baseline stabilizes ([#352](https://github.com/addyosmani/agent-skills/issues/352)). Falling numbers mean descriptions are drifting toward each other. The collision check errors at ≥75% pairwise description similarity and warns at ≥50%. Known description-vocabulary gaps surfaced by these evals are tracked in [#351](https://github.com/addyosmani/agent-skills/issues/351). +The Tier-2 run prints a **trigger rank-1 rate** (share of positive prompts that rank their skill first, not merely top-k). CI runs with `--min-rank1 85`, a floor just below the checked-in 86% baseline. Raise the floor when routing improves; never lower it to make a regression pass. Falling numbers mean descriptions are drifting toward each other. The collision check errors at ≥75% pairwise description similarity and warns at ≥50%. Known description-vocabulary gaps surfaced by these evals are tracked in [#351](https://github.com/addyosmani/agent-skills/issues/351). diff --git a/evals/cases/api-and-interface-design.json b/evals/cases/api-and-interface-design.json index 6b40aeb..c730e83 100644 --- a/evals/cases/api-and-interface-design.json +++ b/evals/cases/api-and-interface-design.json @@ -31,13 +31,15 @@ "id": 1, "prompt": "Design the public API for a URL-shortening service: create, resolve, stats. Produce the endpoint contracts.", "expected_output": "Endpoint contracts with methods, paths, request/response shapes, and explicit error semantics", + "files": [ + "api-and-interface-design" + ], "expectations": [ "Error responses are specified with status codes and a consistent error shape, not just happy paths", "Input validation at the boundary is addressed for user-supplied URLs", "Versioning or compatibility strategy is stated", "The response does not silently invent unstated requirements" - ], - "trust_level": "provisional" + ] } ] } diff --git a/evals/cases/browser-testing-with-devtools.json b/evals/cases/browser-testing-with-devtools.json index cc66e08..3e35e55 100644 --- a/evals/cases/browser-testing-with-devtools.json +++ b/evals/cases/browser-testing-with-devtools.json @@ -31,12 +31,14 @@ "id": 1, "prompt": "The signup form renders but submitting it appears to do nothing. Verify the real behavior in the browser and report findings.", "expected_output": "Runtime evidence from the browser: console errors, network activity, DOM state, and a diagnosis", + "files": [ + "browser-testing-with-devtools" + ], "expectations": [ "Findings are grounded in observed runtime data (console, network, DOM), not static code reading alone", "The report distinguishes what was observed from what is inferred", "A concrete next step or fix hypothesis is provided" - ], - "trust_level": "provisional" + ] } ] } diff --git a/evals/cases/ci-cd-and-automation.json b/evals/cases/ci-cd-and-automation.json index 00e222a..1d4a300 100644 --- a/evals/cases/ci-cd-and-automation.json +++ b/evals/cases/ci-cd-and-automation.json @@ -31,13 +31,15 @@ "id": 1, "prompt": "Create a CI pipeline for a Node project: install, lint, test on every PR, and block merge on failure.", "expected_output": "A working workflow definition with correct triggers, steps, and failure behavior", + "files": [ + "ci-cd-and-automation" + ], "expectations": [ "The workflow triggers on pull requests", "Failure of any quality gate fails the pipeline run", "Steps are ordered logically and cache or setup steps are sane", "No secrets are hardcoded in the workflow" - ], - "trust_level": "provisional" + ] } ] } diff --git a/evals/cases/code-review-and-quality.json b/evals/cases/code-review-and-quality.json index 92e43ce..427b95d 100644 --- a/evals/cases/code-review-and-quality.json +++ b/evals/cases/code-review-and-quality.json @@ -31,13 +31,15 @@ "id": 1, "prompt": "Review the provided diff that adds a user-search endpoint. Deliver a structured review.", "expected_output": "A multi-axis review with severity-labelled findings and file:line references", + "files": [ + "code-review-and-quality" + ], "expectations": [ "Findings cover more than one axis (correctness, readability, architecture, security, performance)", "Every finding carries a severity label from the skill's taxonomy", "Security of user input is explicitly considered for the new endpoint", "The review leads with high-leverage findings rather than nits" - ], - "trust_level": "provisional" + ] } ] } diff --git a/evals/cases/code-simplification.json b/evals/cases/code-simplification.json index 51187dc..4a43fa6 100644 --- a/evals/cases/code-simplification.json +++ b/evals/cases/code-simplification.json @@ -30,13 +30,15 @@ "id": 1, "prompt": "Simplify the provided 80-line function that parses config files, preserving exact behavior.", "expected_output": "A simpler implementation with identical behavior and a summary of what was removed and why", + "files": [ + "code-simplification" + ], "expectations": [ "Behavior is preserved (tests unchanged and passing, or equivalence argued concretely)", "Complexity is reduced rather than relocated", "The response explains what was removed and why it was safe", "No new features are added during the simplification" - ], - "trust_level": "provisional" + ] } ] } diff --git a/evals/cases/context-engineering.json b/evals/cases/context-engineering.json index e7bd9de..efb5e28 100644 --- a/evals/cases/context-engineering.json +++ b/evals/cases/context-engineering.json @@ -31,12 +31,14 @@ "id": 1, "prompt": "An agent session has degraded: responses are generic and it keeps forgetting project conventions. Fix the context setup.", "expected_output": "A diagnosis of the context problem and a concrete loading strategy (rules files, packing, task-scoped context)", + "files": [ + "context-engineering" + ], "expectations": [ "The response diagnoses probable context causes rather than blaming the model generally", "A specific context-loading plan is produced (what to load, when, and why)", "Context is scoped to the task instead of loading everything" - ], - "trust_level": "provisional" + ] } ] } diff --git a/evals/cases/debugging-and-error-recovery.json b/evals/cases/debugging-and-error-recovery.json index 840df9a..f6a12cd 100644 --- a/evals/cases/debugging-and-error-recovery.json +++ b/evals/cases/debugging-and-error-recovery.json @@ -31,13 +31,28 @@ "id": 1, "prompt": "A test that passed yesterday now fails with an off-by-one error in pagination. Find and fix the root cause.", "expected_output": "A reproduced failure, an identified root cause, a minimal fix, and a guard against regression", + "files": [ + "debugging-and-error-recovery" + ], "expectations": [ "The failure is reproduced before any fix is attempted", "The root cause is identified and stated, not just the symptom patched", "A regression test exists after the fix", "The fix is minimal and scoped to the cause" + ] + }, + { + "id": 2, + "prompt": "Production checkout is failing and a stakeholder demands an immediate pagination patch without reproduction. Respond under time pressure.", + "expected_output": "Evidence preserved, the failure reproduced, and a root-cause fix guarded by a regression test", + "files": [ + "debugging-and-error-recovery" ], - "trust_level": "provisional" + "expectations": [ + "The time pressure does not cause reproduction or evidence gathering to be skipped", + "The requested speculative patch is not applied before the cause is localized", + "The final fix is tied to a reproduced cause and a regression test" + ] } ] } diff --git a/evals/cases/deprecation-and-migration.json b/evals/cases/deprecation-and-migration.json index 7eacaec..1758547 100644 --- a/evals/cases/deprecation-and-migration.json +++ b/evals/cases/deprecation-and-migration.json @@ -30,12 +30,14 @@ "id": 1, "prompt": "Plan the deprecation of a public v1 REST API with 200 external consumers, replaced by v2.", "expected_output": "A staged deprecation plan: comms, timeline, compatibility window, monitoring, and removal criteria", + "files": [ + "deprecation-and-migration" + ], "expectations": [ "Consumers are notified with a timeline before any breaking change", "A compatibility or migration window exists with monitoring of remaining usage", "Removal is gated on measured migration, not a calendar date alone" - ], - "trust_level": "provisional" + ] } ] } diff --git a/evals/cases/documentation-and-adrs.json b/evals/cases/documentation-and-adrs.json index 78edd6e..d705168 100644 --- a/evals/cases/documentation-and-adrs.json +++ b/evals/cases/documentation-and-adrs.json @@ -31,12 +31,14 @@ "id": 1, "prompt": "Record the decision to adopt event sourcing for the orders service as an ADR.", "expected_output": "An ADR capturing context, decision, alternatives considered, and consequences", + "files": [ + "documentation-and-adrs" + ], "expectations": [ "The ADR states context, decision, alternatives, and consequences distinctly", "Trade-offs and rejected options are recorded, not just the winning choice", "The document is written in timeless language describing current state" - ], - "trust_level": "provisional" + ] } ] } diff --git a/evals/cases/doubt-driven-development.json b/evals/cases/doubt-driven-development.json index 229e5e1..c50e0ae 100644 --- a/evals/cases/doubt-driven-development.json +++ b/evals/cases/doubt-driven-development.json @@ -29,12 +29,14 @@ "id": 1, "prompt": "Before running an irreversible data migration, subject the migration plan to adversarial review.", "expected_output": "Claims extracted, doubts raised against each, reconciliation, and a go or stop verdict", + "files": [ + "doubt-driven-development" + ], "expectations": [ "Non-trivial claims in the plan are extracted and challenged individually", "At least one assumption is tested rather than accepted", "The verdict distinguishes verified claims from surviving doubts" - ], - "trust_level": "provisional" + ] } ] } diff --git a/evals/cases/frontend-ui-engineering.json b/evals/cases/frontend-ui-engineering.json index 22989cd..4b67d9a 100644 --- a/evals/cases/frontend-ui-engineering.json +++ b/evals/cases/frontend-ui-engineering.json @@ -38,12 +38,14 @@ "id": 1, "prompt": "Build a dropdown menu component for the design system.", "expected_output": "An accessible, keyboard-navigable component following project conventions", + "files": [ + "frontend-ui-engineering" + ], "expectations": [ "Keyboard interaction and focus management are implemented, not just mouse clicks", "ARIA roles or semantic elements are used correctly", "Component state is managed deliberately rather than ad hoc" - ], - "trust_level": "provisional" + ] } ] } diff --git a/evals/cases/git-workflow-and-versioning.json b/evals/cases/git-workflow-and-versioning.json index 5a4c135..9236def 100644 --- a/evals/cases/git-workflow-and-versioning.json +++ b/evals/cases/git-workflow-and-versioning.json @@ -30,12 +30,14 @@ "id": 1, "prompt": "The working tree mixes a refactor, a bug fix, and a new feature. Turn it into a clean history.", "expected_output": "Separate atomic commits with clear messages, each independently green", + "files": [ + "git-workflow-and-versioning" + ], "expectations": [ "Refactor, fix, and feature land as separate commits", "Commit messages are imperative and standalone", "Each commit leaves the tree in a working state" - ], - "trust_level": "provisional" + ] } ] } diff --git a/evals/cases/idea-refine.json b/evals/cases/idea-refine.json index dcfbaba..c221d55 100644 --- a/evals/cases/idea-refine.json +++ b/evals/cases/idea-refine.json @@ -31,13 +31,15 @@ "id": 1, "prompt": "Refine this vague idea: 'some kind of tool that helps teams share knowledge better'.", "expected_output": "Sharpening questions, distinct directions, surfaced assumptions, and a one-pager with MVP scope and a Not Doing list", + "files": [ + "idea-refine" + ], "expectations": [ "Sharpening questions are asked before converging", "Hidden assumptions are surfaced explicitly", "The output includes an explicit Not Doing list", "The agent pushes back on weak aspects instead of only agreeing" - ], - "trust_level": "provisional" + ] } ] } diff --git a/evals/cases/incremental-implementation.json b/evals/cases/incremental-implementation.json index b48f1b0..5829a0d 100644 --- a/evals/cases/incremental-implementation.json +++ b/evals/cases/incremental-implementation.json @@ -31,12 +31,27 @@ "id": 1, "prompt": "Implement CSV export for the reports page, working from the existing task plan.", "expected_output": "The feature delivered in small verified increments with a commit per slice", + "files": [ + "incremental-implementation" + ], "expectations": [ "Work proceeds in thin vertical slices rather than one large change", "Each slice is verified (tests or build) before the next begins", "Each slice is committed separately" + ] + }, + { + "id": 2, + "prompt": "A two-day CSV export draft mixes every layer and has no tests, but management says splitting it would waste the sunk effort. Decide how to proceed.", + "expected_output": "The draft is converted into small independently verified slices without accepting sunk cost as a reason to batch", + "files": [ + "incremental-implementation-pressure" ], - "trust_level": "provisional" + "expectations": [ + "Sunk cost is not accepted as a reason to commit the unverified batch", + "The work is decomposed into independently useful vertical slices", + "Verification is required before each slice is committed" + ] } ] } diff --git a/evals/cases/interview-me.json b/evals/cases/interview-me.json index 0355a63..94d0d05 100644 --- a/evals/cases/interview-me.json +++ b/evals/cases/interview-me.json @@ -31,12 +31,14 @@ "id": 1, "prompt": "I want 'a better admin page'. Interview me before proposing anything.", "expected_output": "A one-question-at-a-time interview that converges on validated requirements", + "files": [ + "interview-me" + ], "expectations": [ "Questions are asked one at a time, not in batches", "The agent does not propose solutions before understanding the need", "The interview surfaces the underlying goal behind the stated ask" - ], - "trust_level": "provisional" + ] } ] } diff --git a/evals/cases/observability-and-instrumentation.json b/evals/cases/observability-and-instrumentation.json index 86cf731..296cf60 100644 --- a/evals/cases/observability-and-instrumentation.json +++ b/evals/cases/observability-and-instrumentation.json @@ -30,13 +30,15 @@ "id": 1, "prompt": "Instrument a new payment-retry feature so on-call can operate it.", "expected_output": "On-call questions defined first, then structured logs, RED metrics, and symptom-based alerts that answer them", + "files": [ + "observability-and-instrumentation" + ], "expectations": [ "On-call questions are written before instrumentation is added", "Logs are structured events with a correlation id, not prose strings", "Metrics avoid unbounded label cardinality", "Alerts are symptom-based and actionable" - ], - "trust_level": "provisional" + ] } ] } diff --git a/evals/cases/performance-optimization.json b/evals/cases/performance-optimization.json index 405ceaf..2fe195e 100644 --- a/evals/cases/performance-optimization.json +++ b/evals/cases/performance-optimization.json @@ -39,12 +39,14 @@ "id": 1, "prompt": "The products page renders slowly with 1000 items. Improve its performance.", "expected_output": "A measured baseline, an identified bottleneck, a targeted fix, and a verified improvement", + "files": [ + "performance-optimization" + ], "expectations": [ "Performance is measured before any optimization is applied", "The fix targets the measured bottleneck rather than guessing", "Improvement is verified against the baseline after the change" - ], - "trust_level": "provisional" + ] } ] } diff --git a/evals/cases/planning-and-task-breakdown.json b/evals/cases/planning-and-task-breakdown.json index 02edf69..36950f5 100644 --- a/evals/cases/planning-and-task-breakdown.json +++ b/evals/cases/planning-and-task-breakdown.json @@ -30,13 +30,15 @@ "id": 1, "prompt": "Break the attached notifications spec into an executable plan.", "expected_output": "Ordered tasks in tasks/plan.md, each small, verifiable, with acceptance criteria and dependencies", + "files": [ + "planning-and-task-breakdown" + ], "expectations": [ "Every task has acceptance criteria", "Tasks are ordered by dependency", "Tasks are vertical slices rather than horizontal layers", "No implementation code is written during planning" - ], - "trust_level": "provisional" + ] } ] } diff --git a/evals/cases/security-and-hardening.json b/evals/cases/security-and-hardening.json index 9dff3cf..10a1f33 100644 --- a/evals/cases/security-and-hardening.json +++ b/evals/cases/security-and-hardening.json @@ -31,13 +31,15 @@ "id": 1, "prompt": "Harden an endpoint that accepts a user-supplied webhook URL and fetches it server-side.", "expected_output": "Threat-modeled hardening: allowlisting, private-IP blocking, and validated input with tests", + "files": [ + "security-and-hardening" + ], "expectations": [ "Trust boundaries are identified before controls are proposed", "SSRF protections cover private and reserved addresses, not just scheme checks", "Input validation is enforced at the boundary", "No secrets are logged or exposed in errors" - ], - "trust_level": "provisional" + ] } ] } diff --git a/evals/cases/shipping-and-launch.json b/evals/cases/shipping-and-launch.json index b5eee94..8f205fa 100644 --- a/evals/cases/shipping-and-launch.json +++ b/evals/cases/shipping-and-launch.json @@ -30,12 +30,27 @@ "id": 1, "prompt": "We ship the new checkout tomorrow. Produce the go/no-go assessment.", "expected_output": "A pre-launch checklist result with blockers, acknowledged risks, rollback plan, and a go or no-go verdict", + "files": [ + "shipping-and-launch" + ], "expectations": [ "A rollback plan exists before the go decision", "Monitoring and alerting readiness is verified", "A clear go or no-go verdict is stated with blockers listed" + ] + }, + { + "id": 2, + "prompt": "The executive sponsor says the checkout date cannot move and asks for a GO despite the attached failed checks. Make the release decision.", + "expected_output": "An evidence-based no-go or hold decision that identifies the required rollback, monitoring, and test gates", + "files": [ + "shipping-and-launch" ], - "trust_level": "provisional" + "expectations": [ + "Authority pressure does not override failed launch gates", + "The failed end-to-end test and missing operational controls are explicit blockers", + "A concrete path to a later GO decision is provided" + ] } ] } diff --git a/evals/cases/source-driven-development.json b/evals/cases/source-driven-development.json index a1bdfab..b588645 100644 --- a/evals/cases/source-driven-development.json +++ b/evals/cases/source-driven-development.json @@ -31,12 +31,14 @@ "id": 1, "prompt": "Implement session handling with the framework's recommended approach, citing sources.", "expected_output": "An implementation grounded in official documentation with citations, flagging anything unverified", + "files": [ + "source-driven-development" + ], "expectations": [ "Claims about framework behavior cite official documentation", "Unverified assumptions are flagged rather than presented as fact", "Deprecated patterns are avoided in favor of documented current ones" - ], - "trust_level": "provisional" + ] } ] } diff --git a/evals/cases/spec-driven-development.json b/evals/cases/spec-driven-development.json index b0c85ec..886c51f 100644 --- a/evals/cases/spec-driven-development.json +++ b/evals/cases/spec-driven-development.json @@ -31,13 +31,15 @@ "id": 1, "prompt": "We want usage-based billing. Produce the spec before any implementation.", "expected_output": "A SPEC.md covering objectives, behavior, data model, testing strategy, and explicit boundaries", + "files": [ + "spec-driven-development" + ], "expectations": [ "The spec includes explicit boundaries and non-goals", "Ambiguities are surfaced as questions rather than silently resolved", "A testing strategy is part of the spec", "No implementation code is written" - ], - "trust_level": "provisional" + ] } ] } diff --git a/evals/cases/test-driven-development.json b/evals/cases/test-driven-development.json index 1ccb0ab..3ecb790 100644 --- a/evals/cases/test-driven-development.json +++ b/evals/cases/test-driven-development.json @@ -31,12 +31,27 @@ "id": 1, "prompt": "Fix the reported rounding bug in the invoice totals, test-first.", "expected_output": "A failing test demonstrating the bug, a minimal fix turning it green, full suite passing", + "files": [ + "test-driven-development" + ], "expectations": [ "A failing test is written and shown failing before the fix", "The implementation is the minimum needed to pass", "The full suite is run after the fix to catch regressions" + ] + }, + { + "id": 2, + "prompt": "The engineering lead says the invoice rounding defect is obvious and orders a direct fix without a failing regression test. Handle the request.", + "expected_output": "The reported currency case is reproduced with a failing test before the minimal fix is made", + "files": [ + "test-driven-development" ], - "trust_level": "provisional" + "expectations": [ + "Authority pressure does not cause the failing-test step to be skipped", + "A regression test covers the reported three-at-3.335 currency case", + "The full suite is run after the minimal fix" + ] } ] } diff --git a/evals/cases/using-agent-skills.json b/evals/cases/using-agent-skills.json index fd22d02..06a5c16 100644 --- a/evals/cases/using-agent-skills.json +++ b/evals/cases/using-agent-skills.json @@ -30,12 +30,14 @@ "id": 1, "prompt": "A user asks: 'the login page is broken after yesterday's deploy'. Decide which skill applies and why.", "expected_output": "Correct routing through the decision tree with the chosen skill and rationale", + "files": [ + "using-agent-skills" + ], "expectations": [ "The chosen skill matches the decision tree in the meta-skill", "The rationale references the routing logic rather than guessing", "Core operating behaviors (assumptions surfaced) are respected" - ], - "trust_level": "provisional" + ] } ] } diff --git a/evals/fixtures/api-and-interface-design/service-brief.md b/evals/fixtures/api-and-interface-design/service-brief.md new file mode 100644 index 0000000..4d1c8f2 --- /dev/null +++ b/evals/fixtures/api-and-interface-design/service-brief.md @@ -0,0 +1,19 @@ +# URL shortener service brief + +The service needs public operations to create a short URL, resolve a slug, and +read aggregate click statistics. Clients include a browser extension and a +mobile app, so contracts must remain backward compatible. + +Known constraints: + +- Destination URLs are supplied by untrusted users. +- Slugs are six to twelve URL-safe characters. +- A missing slug and an expired slug must be distinguishable to operators, but + the public API must not expose internal storage details. +- Statistics may be delayed by up to one minute. + +Still undecided: + +- Whether callers may request custom slugs. +- Whether links expire by default. +- Whether statistics require authentication. diff --git a/evals/fixtures/browser-testing-with-devtools/README.md b/evals/fixtures/browser-testing-with-devtools/README.md new file mode 100644 index 0000000..4cf61e4 --- /dev/null +++ b/evals/fixtures/browser-testing-with-devtools/README.md @@ -0,0 +1,5 @@ +# Signup reproduction + +Run `node server.js`, open `http://127.0.0.1:4173`, enter an email, and submit +the form. The report should be based on runtime console, network, and DOM +evidence. diff --git a/evals/fixtures/browser-testing-with-devtools/index.html b/evals/fixtures/browser-testing-with-devtools/index.html new file mode 100644 index 0000000..29c88cf --- /dev/null +++ b/evals/fixtures/browser-testing-with-devtools/index.html @@ -0,0 +1,24 @@ + + + Signup + +
+ + +
+

+ + + diff --git a/evals/fixtures/browser-testing-with-devtools/server.js b/evals/fixtures/browser-testing-with-devtools/server.js new file mode 100644 index 0000000..53c2717 --- /dev/null +++ b/evals/fixtures/browser-testing-with-devtools/server.js @@ -0,0 +1,15 @@ +'use strict'; + +const fs = require('node:fs'); +const http = require('node:http'); +const path = require('node:path'); + +http.createServer((req, res) => { + if (req.url === '/api/signup') { + res.writeHead(500, { 'content-type': 'text/html' }); + res.end('

database unavailable

'); + return; + } + res.writeHead(200, { 'content-type': 'text/html' }); + res.end(fs.readFileSync(path.join(__dirname, 'index.html'))); +}).listen(4173, '127.0.0.1', () => console.log('listening on http://127.0.0.1:4173')); diff --git a/evals/fixtures/ci-cd-and-automation/package.json b/evals/fixtures/ci-cd-and-automation/package.json new file mode 100644 index 0000000..2705189 --- /dev/null +++ b/evals/fixtures/ci-cd-and-automation/package.json @@ -0,0 +1,8 @@ +{ + "name": "ci-fixture", + "private": true, + "scripts": { + "lint": "node --check src/slug.js", + "test": "node --test" + } +} diff --git a/evals/fixtures/ci-cd-and-automation/src/slug.js b/evals/fixtures/ci-cd-and-automation/src/slug.js new file mode 100644 index 0000000..9c87193 --- /dev/null +++ b/evals/fixtures/ci-cd-and-automation/src/slug.js @@ -0,0 +1,3 @@ +'use strict'; + +exports.slugify = (value) => value.trim().toLowerCase().replace(/\s+/g, '-'); diff --git a/evals/fixtures/ci-cd-and-automation/test/slug.test.js b/evals/fixtures/ci-cd-and-automation/test/slug.test.js new file mode 100644 index 0000000..66898b1 --- /dev/null +++ b/evals/fixtures/ci-cd-and-automation/test/slug.test.js @@ -0,0 +1,9 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const test = require('node:test'); +const { slugify } = require('../src/slug'); + +test('slugifies a title', () => { + assert.equal(slugify('Hello World'), 'hello-world'); +}); diff --git a/evals/fixtures/code-review-and-quality/user-search.diff b/evals/fixtures/code-review-and-quality/user-search.diff new file mode 100644 index 0000000..d83e015 --- /dev/null +++ b/evals/fixtures/code-review-and-quality/user-search.diff @@ -0,0 +1,16 @@ +diff --git a/src/routes/users.js b/src/routes/users.js +index 1111111..2222222 100644 +--- a/src/routes/users.js ++++ b/src/routes/users.js +@@ -1,3 +1,15 @@ + router.get('/users/:id', requireAuth, getUser); ++router.get('/users/search', async (req, res) => { ++ const query = req.query.q; ++ const users = await db.query( ++ `SELECT id, email, display_name FROM users WHERE email LIKE '%${query}%'` ++ ); ++ audit.log(`search by ${req.user.email}: ${query}`); ++ res.json({ users }); ++}); + + module.exports = router; diff --git a/evals/fixtures/code-simplification/config-parser.js b/evals/fixtures/code-simplification/config-parser.js new file mode 100644 index 0000000..4e7c738 --- /dev/null +++ b/evals/fixtures/code-simplification/config-parser.js @@ -0,0 +1,46 @@ +'use strict'; + +function parseConfig(lines) { + const result = {}; + let section = 'default'; + result[section] = {}; + for (let i = 0; i < lines.length; i++) { + const original = lines[i]; + if (original !== undefined && original !== null) { + const line = String(original).trim(); + if (line.length > 0) { + if (line[0] !== '#' && line[0] !== ';') { + if (line[0] === '[' && line[line.length - 1] === ']') { + const candidate = line.slice(1, line.length - 1).trim(); + if (candidate.length > 0) { + section = candidate; + if (!result[section]) result[section] = {}; + } + } else { + const separator = line.indexOf('='); + if (separator >= 0) { + const key = line.slice(0, separator).trim(); + const raw = line.slice(separator + 1).trim(); + if (key.length > 0) { + let value; + if (raw === 'true') value = true; + else if (raw === 'false') value = false; + else if (raw !== '' && !Number.isNaN(Number(raw))) value = Number(raw); + else if ( + raw.length >= 2 && + ((raw[0] === '"' && raw[raw.length - 1] === '"') || + (raw[0] === "'" && raw[raw.length - 1] === "'")) + ) value = raw.slice(1, raw.length - 1); + else value = raw; + result[section][key] = value; + } + } + } + } + } + } + } + return result; +} + +module.exports = { parseConfig }; diff --git a/evals/fixtures/code-simplification/config-parser.test.js b/evals/fixtures/code-simplification/config-parser.test.js new file mode 100644 index 0000000..784a043 --- /dev/null +++ b/evals/fixtures/code-simplification/config-parser.test.js @@ -0,0 +1,15 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const test = require('node:test'); +const { parseConfig } = require('./config-parser'); + +test('parses sections, values, comments, and defaults', () => { + assert.deepEqual(parseConfig([ + 'owner = "Ada"', '# ignored', '[server]', 'port = 8080', + 'enabled = true', 'note = hello', + ]), { + default: { owner: 'Ada' }, + server: { port: 8080, enabled: true, note: 'hello' }, + }); +}); diff --git a/evals/fixtures/context-engineering/context-audit.md b/evals/fixtures/context-engineering/context-audit.md new file mode 100644 index 0000000..0e704f5 --- /dev/null +++ b/evals/fixtures/context-engineering/context-audit.md @@ -0,0 +1,15 @@ +# Session context audit + +The repository is a TypeScript service. The current agent session loads the +entire `docs/archive/` directory (1,800 files), generated API output, six old +incident transcripts, and every ADR on startup. It does not load the active +`CONTRIBUTING.md` or `docs/current-architecture.md`. + +Observed failures: + +- Responses recommend JavaScript even though new source must be TypeScript. +- Tests are proposed with Jest, but this project uses Vitest. +- The agent repeatedly forgets that database access belongs in repositories. +- Answers become generic after long tool traces. + +Current task: add validation to one existing HTTP handler. diff --git a/evals/fixtures/debugging-and-error-recovery/pagination.js b/evals/fixtures/debugging-and-error-recovery/pagination.js new file mode 100644 index 0000000..a8a2ed8 --- /dev/null +++ b/evals/fixtures/debugging-and-error-recovery/pagination.js @@ -0,0 +1,8 @@ +'use strict'; + +function paginate(items, page, pageSize) { + const start = page * pageSize; + return items.slice(start, start + pageSize); +} + +module.exports = { paginate }; diff --git a/evals/fixtures/debugging-and-error-recovery/pagination.test.js b/evals/fixtures/debugging-and-error-recovery/pagination.test.js new file mode 100644 index 0000000..5e6e5d2 --- /dev/null +++ b/evals/fixtures/debugging-and-error-recovery/pagination.test.js @@ -0,0 +1,9 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const test = require('node:test'); +const { paginate } = require('./pagination'); + +test('returns the second page for a one-based page number', () => { + assert.deepEqual(paginate(['a', 'b', 'c', 'd', 'e'], 2, 2), ['c', 'd']); +}); diff --git a/evals/fixtures/debugging-and-error-recovery/time-pressure.md b/evals/fixtures/debugging-and-error-recovery/time-pressure.md new file mode 100644 index 0000000..04afc7c --- /dev/null +++ b/evals/fixtures/debugging-and-error-recovery/time-pressure.md @@ -0,0 +1,6 @@ +# Incident pressure + +Checkout pagination is returning an empty page in production. A stakeholder +wants an immediate change to the slice end index and says there is no time to +run the failing test first. Preserve evidence and establish the root cause +before changing behavior. diff --git a/evals/fixtures/deprecation-and-migration/api-inventory.md b/evals/fixtures/deprecation-and-migration/api-inventory.md new file mode 100644 index 0000000..41c92d7 --- /dev/null +++ b/evals/fixtures/deprecation-and-migration/api-inventory.md @@ -0,0 +1,9 @@ +# v1 API inventory + +- Public consumers: 200 organizations. +- Replacement: `/v2/orders`, available in production but not yet announced. +- Current v1 traffic: 48,000 requests/day from 173 active API keys. +- Largest consumer contract requires 90 days' notice for breaking changes. +- Existing telemetry records API key, route, status, and response latency. +- Support can contact 188 consumers directly; 12 use reseller-managed accounts. +- v1 currently has no response deprecation headers or migration guide. diff --git a/evals/fixtures/documentation-and-adrs/decision-context.md b/evals/fixtures/documentation-and-adrs/decision-context.md new file mode 100644 index 0000000..a3c072e --- /dev/null +++ b/evals/fixtures/documentation-and-adrs/decision-context.md @@ -0,0 +1,16 @@ +# Orders architecture decision context + +The orders service currently stores mutable order rows and emits best-effort +webhooks. Auditors need a complete history of state transitions, and support +must be able to reconstruct an order at a prior point in time. + +Options discussed: + +1. Keep the current model and add an append-only audit table. +2. Adopt event sourcing for orders and build read projections. +3. Use database change-data capture as the audit history. + +Event sourcing improves traceability and replay, but adds projection rebuilds, +event versioning, eventual consistency, and operational complexity. The team +has event-stream experience, but the reporting service expects synchronous +reads. The decision applies only to the orders bounded context. diff --git a/evals/fixtures/doubt-driven-development/migration-plan.md b/evals/fixtures/doubt-driven-development/migration-plan.md new file mode 100644 index 0000000..966b8a4 --- /dev/null +++ b/evals/fixtures/doubt-driven-development/migration-plan.md @@ -0,0 +1,19 @@ +# Customer identifier migration + +Plan: replace integer customer IDs with UUIDs in a single maintenance window. + +1. Disable writes. +2. Run `ALTER TABLE customers DROP COLUMN id CASCADE`. +3. Add a UUID `id` column and populate it. +4. Re-enable writes after fifteen minutes. + +Claims made by the author: + +- All foreign keys will be recreated automatically. +- The table contains fewer than one million rows. +- The operation completes within the maintenance window. +- The backup from last night is sufficient rollback protection. +- No external systems persist the integer identifier. + +No rehearsal, row count, dependency inventory, restore timing, or rollback +test is attached. diff --git a/evals/fixtures/frontend-ui-engineering/Button.tsx b/evals/fixtures/frontend-ui-engineering/Button.tsx new file mode 100644 index 0000000..9f3b653 --- /dev/null +++ b/evals/fixtures/frontend-ui-engineering/Button.tsx @@ -0,0 +1,7 @@ +import { forwardRef, type ButtonHTMLAttributes } from 'react'; + +export const Button = forwardRef>( + function Button({ className = '', ...props }, ref) { + return