feat: add prompt discovery metadata

This commit is contained in:
MerlinH
2026-06-30 08:58:10 +10:00
parent 275d48cdd6
commit 841f906dc0
170 changed files with 1579 additions and 735 deletions
+7 -1
View File
@@ -9,7 +9,7 @@ const repoRoot = process.cwd();
describe("prompt surface audit", () => {
test("audits local and CCGS prompt surfaces with metadata, decisions, and tests", () => {
const output = execFileSync(process.execPath, ["--import", "tsx", "scripts/audit-prompt-surfaces.ts", "--json"], { cwd: repoRoot, encoding: "utf8" });
const report = JSON.parse(output) as { local: { agents: number; workflows: number; skills: number }; upstream: { agents: number; skills: number }; decisionLegend: string[]; rows: Array<{ localPath: string; sourcePath?: string; sourceHash?: string; decision: string; status: string; metadata: Record<string, boolean>; requiredTests: string[] }> };
const report = JSON.parse(output) as { local: { agents: number; workflows: number; skills: number }; upstream: { agents: number; skills: number }; decisionLegend: string[]; rows: Array<{ localPath: string; sourcePath?: string; sourceHash?: string; decision: string; status: string; metadata: Record<string, boolean>; discoveryMetadata: { text: string; valid: boolean; diagnostics: Array<{ id: string; message: string }> }; requiredTests: string[] }> };
expect(report.local.agents).toBeGreaterThanOrEqual(38);
expect(report.local.workflows).toBeGreaterThanOrEqual(31);
expect(report.local.skills).toBeGreaterThanOrEqual(79);
@@ -22,10 +22,16 @@ describe("prompt surface audit", () => {
expect(["adapt", "adopt", "merge", "split", "defer", "out-of-scope"]).toContain(prototype?.decision);
expect(prototype?.status).toMatch(/complete|needs-uplift|deferred/);
expect(prototype?.metadata.model).toBe(true);
expect(prototype?.discoveryMetadata.text).toContain("prototype");
expect(prototype?.discoveryMetadata.valid).toBe(true);
expect(prototype?.requiredTests).toContain("tests/validation.test.ts");
const gameDesigner = report.rows.find((row) => row.localPath === ".codex/agents/game-designer.toml");
expect(gameDesigner?.sourcePath).toBe(".claude/agents/game-designer.md");
expect(gameDesigner?.metadata.model).toBe(true);
expect(gameDesigner?.discoveryMetadata.valid).toBe(true);
const verticalSlice = report.rows.find((row) => row.localPath === ".codex/workflows/vertical-slice.md");
expect(verticalSlice?.discoveryMetadata.text).toContain("slice");
expect(verticalSlice?.discoveryMetadata.valid).toBe(true);
expect(existsSync(path.join(repoRoot, "references", "prompt-surface-uplift-matrix.json"))).toBe(true);
expect(readFileSync(path.join(repoRoot, "references", "prompt-surface-uplift-matrix.md"), "utf8")).toContain("cgs-prototype");
});
+46 -1
View File
@@ -1,6 +1,15 @@
import { describe, test } from "node:test";
import { expect } from "expect";
import { codexModelForComplexity, isCodexModelName, parsePromptSurfaceFrontmatter, validateModelPolicy, type PromptSurfaceComplexity } from "../src/prompt-surface-metadata.js";
import {
codexModelForComplexity,
isCodexModelName,
parsePromptSurfaceFrontmatter,
validateAgentDescriptionQuality,
validateModelPolicy,
validateSkillDescriptionQuality,
validateWorkflowArgumentHintQuality,
type PromptSurfaceComplexity
} from "../src/prompt-surface-metadata.js";
describe("prompt surface metadata", () => {
test("maps complexity to exact Codex model names", () => {
@@ -22,4 +31,40 @@ describe("prompt surface metadata", () => {
expect(parsed.frontmatter.model_reasoning_effort).toBe("high");
expect(parsed.body).toContain("# Body");
});
test("flags generic agent descriptions and accepts selection-oriented ownership metadata", () => {
const generic = validateAgentDescriptionQuality("Game development Game Designer agent for game-designer tasks in this template repository.", "game-designer");
expect(generic.valid).toBe(false);
expect(generic.diagnostics.map((diagnostic) => diagnostic.id)).toContain("prompt.discovery.agent.generic_description");
const specific = validateAgentDescriptionQuality(
"Owns implementable gameplay mechanics, tuning values, edge cases, and player-facing acceptance criteria for bounded feature design; hand off production scope or QA evidence decisions.",
"game-designer"
);
expect(specific.valid).toBe(true);
});
test("flags non-actionable skill descriptions and accepts trigger/outcome metadata", () => {
const generic = validateSkillDescriptionQuality("Use for Codex Game Studio prototype work.", "cgs-prototype");
expect(generic.valid).toBe(false);
expect(generic.diagnostics.map((diagnostic) => diagnostic.id)).toContain("prompt.discovery.skill.vague_description");
const specific = validateSkillDescriptionQuality(
"Use for prototype experiments: define a falsifiable gameplay hypothesis, throwaway boundary, success signal, cleanup plan, and verification evidence.",
"cgs-prototype"
);
expect(specific.valid).toBe(true);
});
test("flags generic workflow argument hints and accepts workflow-specific input guidance", () => {
const generic = validateWorkflowArgumentHintQuality("Describe the vertical-slice goal, target milestone/files, constraints, and required evidence.", "vertical-slice");
expect(generic.valid).toBe(false);
expect(generic.diagnostics.map((diagnostic) => diagnostic.id)).toContain("prompt.discovery.workflow.generic_argument_hint");
const specific = validateWorkflowArgumentHintQuality(
"Provide the playable slice goal, milestone target, involved scenes/assets/code paths, scope constraints, acceptance gates, and required build or playtest evidence.",
"vertical-slice"
);
expect(specific.valid).toBe(true);
});
});