diff --git a/eval-framework/scenarios/cgs-architecture-review/risk-verdict/scenario.json b/eval-framework/scenarios/cgs-architecture-review/risk-verdict/scenario.json index ebc9fa4..cbcf9ee 100644 --- a/eval-framework/scenarios/cgs-architecture-review/risk-verdict/scenario.json +++ b/eval-framework/scenarios/cgs-architecture-review/risk-verdict/scenario.json @@ -9,7 +9,6 @@ "mustRead": [ ".agents/skills/cgs-architecture-review/SKILL.md", "eval-framework/rubrics/skill-behavior.json", - "eval-framework/rubrics/skill-behavior.json", "eval-framework/scenarios/cgs-architecture-review/risk-verdict/prompt.md" ], "mustChange": [ diff --git a/eval-framework/scenarios/cgs-code-review/review-findings/scenario.json b/eval-framework/scenarios/cgs-code-review/review-findings/scenario.json index 02efc7a..75ccd09 100644 --- a/eval-framework/scenarios/cgs-code-review/review-findings/scenario.json +++ b/eval-framework/scenarios/cgs-code-review/review-findings/scenario.json @@ -9,7 +9,6 @@ "mustRead": [ ".agents/skills/cgs-code-review/SKILL.md", "eval-framework/rubrics/skill-behavior.json", - "eval-framework/rubrics/skill-behavior.json", "eval-framework/scenarios/cgs-code-review/review-findings/prompt.md" ], "mustChange": [ diff --git a/eval-framework/scenarios/cgs-design-review/read-only-verdict/scenario.json b/eval-framework/scenarios/cgs-design-review/read-only-verdict/scenario.json index 0cc1770..4661a3c 100644 --- a/eval-framework/scenarios/cgs-design-review/read-only-verdict/scenario.json +++ b/eval-framework/scenarios/cgs-design-review/read-only-verdict/scenario.json @@ -9,7 +9,6 @@ "mustRead": [ ".agents/skills/cgs-design-review/SKILL.md", "eval-framework/rubrics/skill-behavior.json", - "eval-framework/rubrics/skill-behavior.json", "eval-framework/scenarios/cgs-design-review/read-only-verdict/prompt.md" ], "mustChange": [ diff --git a/eval-framework/scenarios/cgs-gate-check/mode-boundary/scenario.json b/eval-framework/scenarios/cgs-gate-check/mode-boundary/scenario.json index 5eded80..df5e215 100644 --- a/eval-framework/scenarios/cgs-gate-check/mode-boundary/scenario.json +++ b/eval-framework/scenarios/cgs-gate-check/mode-boundary/scenario.json @@ -9,7 +9,6 @@ "mustRead": [ ".agents/skills/cgs-gate-check/SKILL.md", "eval-framework/rubrics/skill-behavior.json", - "eval-framework/rubrics/skill-behavior.json", "eval-framework/scenarios/cgs-gate-check/mode-boundary/prompt.md" ], "mustChange": [ diff --git a/eval-framework/scenarios/cgs-skill-test/behavioral-spec/scenario.json b/eval-framework/scenarios/cgs-skill-test/behavioral-spec/scenario.json index 6864a16..9210eee 100644 --- a/eval-framework/scenarios/cgs-skill-test/behavioral-spec/scenario.json +++ b/eval-framework/scenarios/cgs-skill-test/behavioral-spec/scenario.json @@ -9,7 +9,6 @@ "mustRead": [ ".agents/skills/cgs-skill-test/SKILL.md", "eval-framework/rubrics/skill-behavior.json", - "eval-framework/rubrics/skill-behavior.json", "eval-framework/scenarios/cgs-skill-test/behavioral-spec/prompt.md" ], "mustChange": [ diff --git a/eval-framework/scenarios/cgs-story-done/done-verdict/scenario.json b/eval-framework/scenarios/cgs-story-done/done-verdict/scenario.json index 7b3faff..09f3f2c 100644 --- a/eval-framework/scenarios/cgs-story-done/done-verdict/scenario.json +++ b/eval-framework/scenarios/cgs-story-done/done-verdict/scenario.json @@ -9,7 +9,6 @@ "mustRead": [ ".agents/skills/cgs-story-done/SKILL.md", "eval-framework/rubrics/skill-behavior.json", - "eval-framework/rubrics/skill-behavior.json", "eval-framework/scenarios/cgs-story-done/done-verdict/prompt.md" ], "mustChange": [ diff --git a/eval-framework/scenarios/cgs-story-readiness/readiness-verdict/scenario.json b/eval-framework/scenarios/cgs-story-readiness/readiness-verdict/scenario.json index a28d173..d8a0629 100644 --- a/eval-framework/scenarios/cgs-story-readiness/readiness-verdict/scenario.json +++ b/eval-framework/scenarios/cgs-story-readiness/readiness-verdict/scenario.json @@ -9,7 +9,6 @@ "mustRead": [ ".agents/skills/cgs-story-readiness/SKILL.md", "eval-framework/rubrics/skill-behavior.json", - "eval-framework/rubrics/skill-behavior.json", "eval-framework/scenarios/cgs-story-readiness/readiness-verdict/prompt.md" ], "mustChange": [ diff --git a/eval-framework/scenarios/cgs-test-flakiness/flakiness-diagnosis/scenario.json b/eval-framework/scenarios/cgs-test-flakiness/flakiness-diagnosis/scenario.json index 58a818d..49a7dfe 100644 --- a/eval-framework/scenarios/cgs-test-flakiness/flakiness-diagnosis/scenario.json +++ b/eval-framework/scenarios/cgs-test-flakiness/flakiness-diagnosis/scenario.json @@ -9,7 +9,6 @@ "mustRead": [ ".agents/skills/cgs-test-flakiness/SKILL.md", "eval-framework/rubrics/skill-behavior.json", - "eval-framework/rubrics/skill-behavior.json", "eval-framework/scenarios/cgs-test-flakiness/flakiness-diagnosis/prompt.md" ], "mustChange": [ diff --git a/src/performance-evaluation.ts b/src/performance-evaluation.ts index 4ddb4e9..cf9eff2 100644 --- a/src/performance-evaluation.ts +++ b/src/performance-evaluation.ts @@ -178,6 +178,16 @@ function asArray(value: unknown): string[] { return Array.isArray(value) ? value.filter((item): item is string => typeof item === "string") : []; } +function duplicateValues(values: string[]): string[] { + const seen = new Set(); + const duplicates = new Set(); + for (const value of values) { + if (seen.has(value)) duplicates.add(value); + seen.add(value); + } + return [...duplicates]; +} + function hasExplicitReadDenial(evidence: string, required: string): boolean { const escaped = escapeRegExp(required); const denialBeforePath = new RegExp( @@ -273,7 +283,7 @@ export function summarizePerformanceEvaluationCoverage(framework: PerformanceEva } return { targets: framework.catalog.targets.length, - scenarios: framework.catalog.targets.reduce((total, target) => total + target.scenarios.length, 0), + scenarios: new Set(framework.catalog.targets.flatMap((target) => target.scenarios)).size, surfacePaths: framework.catalog.targets.reduce((total, target) => total + target.surfacePaths.length, 0), byKind }; @@ -336,6 +346,23 @@ export function validatePerformanceEvaluationFramework(root: string): Performanc : fail("performance_eval.scenarios.behavioral_expectations", scenarioProblems.join("; ")) ); + const duplicateExpectationProblems = framework.scenarios.flatMap((scenario) => { + const expectationArrays: Array<[string, string[]]> = [ + ["mustRead", scenario.expected.mustRead], + ["mustChange", scenario.expected.mustChange], + ["mustNotChange", scenario.expected.mustNotChange], + ["mustRunOrExplain", scenario.expected.mustRunOrExplain] + ]; + return expectationArrays.flatMap(([field, values]) => + duplicateValues(values).map((duplicate) => `${scenario.id} duplicates expected.${field} entry ${duplicate}`) + ); + }); + checks.push( + duplicateExpectationProblems.length === 0 + ? pass("performance_eval.scenarios.unique_expectations", "scenario expectation arrays do not duplicate required reads, changes, forbidden changes, or verification commands") + : fail("performance_eval.scenarios.unique_expectations", duplicateExpectationProblems.join("; ")) + ); + const rubricProblems = framework.rubrics.flatMap((rubric) => { const problems: string[] = []; if (rubric.manualOnly !== true) problems.push(`${rubric.id} is not manual-only`); diff --git a/tests/performance-evaluation-framework.test.ts b/tests/performance-evaluation-framework.test.ts index 7f21fe7..1006654 100644 --- a/tests/performance-evaluation-framework.test.ts +++ b/tests/performance-evaluation-framework.test.ts @@ -5,6 +5,7 @@ import { loadPerformanceEvaluationFramework, summarizePerformanceEvaluationCoverage, validatePerformanceEvaluationFramework, + type PerformanceEvaluationFramework, type PerformanceEvaluationScenario } from "../src/performance-evaluation.js"; @@ -70,6 +71,31 @@ describe("performance evaluation framework", () => { expect(result.failures.map((failure) => failure.id)).toContain("required-read-explicitly-denied"); }); + test("counts unique scenario references instead of duplicated target references", () => { + const duplicatePath = "eval-framework/scenarios/cgs-skill-test/behavioral-spec/scenario.json"; + const duplicateFramework: PerformanceEvaluationFramework = { + catalog: { + version: 1, + manualOnly: true, + lastReviewed: "2026-07-01", + targets: [{ + id: "duplicated-target", + kind: "skill", + priority: "critical", + manualOnly: true, + surfacePaths: [".agents/skills/cgs-skill-test/SKILL.md"], + rubric: "eval-framework/rubrics/skill-behavior.json", + scenarios: [duplicatePath, duplicatePath] + }], + runners: { harnessHosts: [], manualAgentHosts: [] } + }, + scenarios: [scenario], + rubrics: [] + }; + + expect(summarizePerformanceEvaluationCoverage(duplicateFramework).scenarios).toBe(1); + }); + test("loads first-pass coverage across workflow prompts, skills, and role prompts", () => { const framework = loadPerformanceEvaluationFramework(process.cwd()); const summary = summarizePerformanceEvaluationCoverage(framework); @@ -117,6 +143,7 @@ describe("performance evaluation framework", () => { expect(ids).toEqual(expect.arrayContaining([ "performance_eval.catalog.manual_only", "performance_eval.scenarios.behavioral_expectations", + "performance_eval.scenarios.unique_expectations", "performance_eval.rubrics.semantic_dimensions", "performance_eval.strategy.no_existence_only_checks", "performance_eval.token_estimation",