feat: add performance evaluation framework

tests/performance-evaluation-framework.test.ts

@@ -0,0 +1,119 @@
+import { describe, test } from "node:test";
+import { expect } from "expect";
+import {
+  gradePerformanceEvaluationScenario,
+  loadPerformanceEvaluationFramework,
+  summarizePerformanceEvaluationCoverage,
+  validatePerformanceEvaluationFramework,
+  type PerformanceEvaluationScenario
+} from "../src/performance-evaluation.js";
+
+const scenario: PerformanceEvaluationScenario = {
+  id: "skill.cgs-skill-test.behavioral-spec",
+  target: "cgs-skill-test",
+  kind: "skill",
+  priority: "critical",
+  manualOnly: true,
+  prompt: "eval-framework/scenarios/cgs-skill-test/behavioral-spec/prompt.md",
+  expected: {
+    mustRead: [".agents/skills/cgs-skill-test/SKILL.md", "eval-framework/rubrics/skill-behavior.json"],
+    mustChange: ["production/session-state/eval-report.md"],
+    mustNotChange: ["src/**", ".agents/skills/**"],
+    mustRunOrExplain: ["npm run validate"],
+    report: { required: true }
+  },
+  grading: {
+    deterministic: ["required-read", "write-boundary", "verification-evidence", "report-presence"],
+    semanticDimensions: ["triggering", "context-selection", "output-quality", "verification-discipline", "token-discipline"]
+  }
+};
+
+describe("performance evaluation framework", () => {
+  test("grades skill and prompt behavior from scenario expectations instead of skill existence", () => {
+    const passResult = gradePerformanceEvaluationScenario(scenario, {
+      traceEvidence: "Read .agents/skills/cgs-skill-test/SKILL.md and eval-framework/rubrics/skill-behavior.json; ran npm run validate",
+      changedFiles: ["production/session-state/eval-report.md"],
+      finalReport: "## Eval Report\nPASS",
+      usage: { status: "recorded", model: "gpt-5.3-codex-spark", inputTokens: 1200, cachedInputTokens: 200, outputTokens: 300, reasoningOutputTokens: 100, totalTokens: 1500 }
+    });
+
+    expect(passResult.status).toBe("pass");
+    expect(passResult.usage?.model).toBe("gpt-5.3-codex-spark");
+    expect(passResult.usage?.totalTokens).toBe(1500);
+    expect(passResult.failures).toEqual([]);
+
+    const failResult = gradePerformanceEvaluationScenario(scenario, {
+      traceEvidence: "Read only .agents/skills/cgs-skill-test/SKILL.md",
+      changedFiles: [".agents/skills/cgs-skill-test/SKILL.md", "src/behavioral-evaluation.ts"],
+      finalReport: ""
+    });
+
+    expect(failResult.status).toBe("fail");
+    expect(failResult.failures.map((failure) => failure.id)).toEqual(expect.arrayContaining([
+      "required-read-not-recorded",
+      "forbidden-write",
+      "missing-required-change",
+      "verification-not-recorded",
+      "missing-report"
+    ]));
+    expect(failResult.failures.map((failure) => failure.id)).not.toContain("skill-missing");
+  });
+
+  test("loads first-pass coverage across workflow prompts, skills, and role prompts", () => {
+    const framework = loadPerformanceEvaluationFramework(process.cwd());
+    const summary = summarizePerformanceEvaluationCoverage(framework);
+
+    expect(framework.catalog.manualOnly).toBe(true);
+    expect(summary.scenarios).toBeGreaterThanOrEqual(30);
+    expect(summary.targets).toBeGreaterThanOrEqual(30);
+    expect(summary.byKind.workflow).toBeGreaterThanOrEqual(10);
+    expect(summary.byKind.skill).toBeGreaterThanOrEqual(10);
+    expect(summary.byKind.role).toBeGreaterThanOrEqual(6);
+    expect(summary.surfacePaths).toBeGreaterThanOrEqual(30);
+    expect(framework.catalog.targets.every((target) => target.scenarios.length > 0)).toBe(true);
+    expect(framework.catalog.modelPolicy?.tokenEstimation?.required).toBe(true);
+    expect(framework.catalog.modelPolicy?.tokenEstimation?.fields).toEqual(expect.arrayContaining([
+      "inputTokens",
+      "cachedInputTokens",
+      "outputTokens",
+      "reasoningOutputTokens",
+      "totalTokens"
+    ]));
+    expect(framework.catalog.modelPolicy?.defaultEvaluationModel).toBe("gpt-5.3-codex-spark");
+    expect(framework.catalog.modelPolicy?.allowedEvaluationModels).toContain("gpt-5.5");
+    expect(framework.catalog.runners.runOutputPolicy?.repositoryTracked).toBe(true);
+    expect(framework.catalog.runners.runOutputPolicy?.root).toBe("eval-framework/runs");
+    expect(framework.catalog.runners.runOutputPolicy?.requiredFiles).toEqual(expect.arrayContaining(["summary.md", "audit.json"]));
+    expect(framework.catalog.evaluationPlan?.map((stage) => stage.stage)).toEqual([
+      "smoke-critical",
+      "workflow-high-risk",
+      "skill-maintenance",
+      "role-boundary",
+      "full-regression"
+    ]);
+    expect(framework.scenarios.every((loaded) => loaded.grading.semanticDimensions.length >= 4)).toBe(true);
+    expect(framework.scenarios.some((loaded) => loaded.expected.mustNotChange.some((pattern) => pattern.includes(".agents/skills")))).toBe(true);
+  });
+
+  test("validation rejects existence-only checks and enforces first-pass coverage", () => {
+    const checks = validatePerformanceEvaluationFramework(process.cwd());
+    const ids = checks.map((check) => check.id);
+    const messages = checks.map((check) => check.message.toLowerCase());
+
+    expect(checks.every((check) => check.status === "pass")).toBe(true);
+    expect(ids.some((id) => /skill.*exists|exists.*skill|presence-only/i.test(id))).toBe(false);
+    expect(messages.some((message) => message.includes("skill exists") || message.includes("existence-only"))).toBe(false);
+    expect(ids).toEqual(expect.arrayContaining([
+      "performance_eval.catalog.manual_only",
+      "performance_eval.scenarios.behavioral_expectations",
+      "performance_eval.rubrics.semantic_dimensions",
+      "performance_eval.strategy.no_existence_only_checks",
+      "performance_eval.token_estimation",
+      "performance_eval.model_policy",
+      "performance_eval.plan.gradual",
+      "performance_eval.template_user_optional",
+      "performance_eval.results.repository_saved",
+      "performance_eval.coverage.first_pass"
+    ]));
+  });
+});

tests/prompt-surface-validation.test.ts

@@ -13,7 +13,7 @@ function makeRoot(): string {
   mkdirSync(path.join(root, ".agents", "skills", "cgs-bugfix"), { recursive: true });
   mkdirSync(path.join(root, ".agents", "skills", "cgs-standards-gameplay"), { recursive: true });
   writeFileSync(path.join(root, "AGENTS.md"), "# Template\n\n.codex/agents game template guidance.\n");
-  writeFileSync(path.join(root, ".codex", "agents", "producer.toml"), `name = "producer"\ndescription = "Producer"\nmodel = "gpt-5.5"\nmodel_reasoning_effort = "high"\nsource_reference = ".claude/agents/producer.md"\nsource_hash = "${"a".repeat(64)}"\nprimary_skills = ["cgs-bugfix"]\nallowed_tool_categories = ["read", "edit", "shell"]\ndeveloper_instructions = """\n## Stop Conditions\n\nStop.\n## Use When\n\nUse.\n## Do Not Use When\n\nDo not.\n## Procedure\n\n1. Do work.\n## Handoff Contract\n\nReport evidence.\n"""\n`);
+  writeFileSync(path.join(root, ".codex", "agents", "producer.toml"), `name = "producer"\ndescription = "Producer"\nmodel = "gpt-5.5"\nmodel_reasoning_effort = "high"\n# source_reference = ".claude/agents/producer.md"\n# source_hash = "${"a".repeat(64)}"\n# primary_skills = ["cgs-bugfix"]\n# allowed_tool_categories = ["read", "edit", "shell"]\ndeveloper_instructions = """\n## Stop Conditions\n\nStop.\n## Use When\n\nUse.\n## Do Not Use When\n\nDo not.\n## Procedure\n\n1. Do work.\n## Handoff Contract\n\nReport evidence.\n"""\n`);
   const skillBody = (name: string, model: string, effort: string, hash: string) => `---\nname: ${name}\ndescription: ${name}\nmodel: ${model}\nmodel_reasoning_effort: ${effort}\nargument-hint: describe target\nprimary-agent: producer\ntool-policy: read/edit/shell\nisolation: repository-root\nsource-reference: local\nsource-hash: ${hash}\n---\n\n# ${name}\n\n## Purpose\n\nPurpose.\n## Prerequisites\n\nPrereq.\n## Arguments\n\nArgs.\n## Phased Procedure\n\nProcedure.\n## Decision Gates\n\nGates.\n## Output Contract\n\nOutput.\n## Quality Gates\n\nQuality.\n## Failure Modes\n\nFailures.\n## Verification\n\nVerify.\n## Handoff\n\nHandoff.\n`;
   writeFileSync(path.join(root, ".agents", "skills", "cgs-bugfix", "SKILL.md"), skillBody("cgs-bugfix", "gpt-5.4", "medium", "b".repeat(64)));
   writeFileSync(path.join(root, ".agents", "skills", "cgs-standards-gameplay", "SKILL.md"), skillBody("cgs-standards-gameplay", "gpt-5.4-mini", "low", "c".repeat(64)));
@@ -24,7 +24,7 @@ function makeRoot(): string {
 describe("prompt surface validation", () => {
   test("fails exact metadata regressions with stable diagnostics", () => {
     const root = makeRoot();
-    writeFileSync(path.join(root, ".codex", "agents", "bad.toml"), `name = "bad"\ndescription = "Bad"\nmodel = "sonnet"\nmodel_reasoning_effort = "medium"\nsource_reference = "local"\nprimary_skills = ["missing-skill"]\nallowed_tool_categories = ["read"]\ndeveloper_instructions = """thin"""\n`);
+    writeFileSync(path.join(root, ".codex", "agents", "bad.toml"), `name = "bad"\ndescription = "Bad"\nmodel = "sonnet"\nmodel_reasoning_effort = "medium"\n# source_reference = "local"\n# primary_skills = ["missing-skill"]\n# allowed_tool_categories = ["read"]\ndeveloper_instructions = """thin"""\n`);
     const failures = validateTemplateSurfaces(root).filter((check) => check.status === "fail").map((check) => check.id);
     expect(failures).toEqual(expect.arrayContaining(["prompt_surface.agent.bad.model", "prompt_surface.agent.bad.traceability", "prompt_surface.agent.bad.links", "prompt_surface.agent.bad.depth"]));
   });