import { existsSync, mkdirSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; import { mkdtempSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; import { describe, expect, test } from "vitest"; import { projectRolePromptSourceInput } from "../src/agents.js"; import { packageAssetPath } from "../src/paths.js"; import { freezeProject, initProject } from "../src/projects.js"; import { workflowSourceInput } from "../src/projects.js"; import { runValidation, validateProject } from "../src/validation.js"; import { hashGeneratedBody, stableHash, stripGeneratedMetadata } from "../src/generated-surfaces.js"; import { loadEngineConfigs } from "../src/engines.js"; import { engineReferenceRegistry } from "../src/engine-reference.js"; import { rolePackages } from "../src/roles.js"; import { behavioralEvaluationScenarios } from "../src/behavioral-evaluation.js"; const validApprovalRecord = { id: "appr_test", stage: "approved", role: "gameplay-programmer", objectiveSha256: "0".repeat(64), approvedGlobs: ["src/**/*.ts"], source: "draft-workflow", approvedBy: "designer", approvedAt: "2026-06-13T00:00:00.000Z" }; describe("validation", () => { test("package exposes codex-game-studio as the canonical installed CLI", () => { const pkg = JSON.parse(readFileSync(path.join(process.cwd(), "package.json"), "utf8")) as { name?: string; bin?: Record }; expect(pkg.name).toBe("codex-game-studio"); expect(pkg.bin?.["codex-game-studio"]).toBe("./dist/cli.js"); expect(Object.keys(pkg.bin ?? {})).toEqual(["codex-game-studio"]); }); test("source checkout wrapper uses built dist output instead of a generated bundle", () => { const wrapper = readFileSync(path.join(process.cwd(), "codex-game-studio"), "utf8"); expect(wrapper).toContain("dist/cli.js"); expect(wrapper).toContain("npm install && npm run build"); expect(wrapper).not.toContain("bin/codex-game-studio.mjs"); }); test("fresh initialized projects validate and failures are explicit", () => { for (const [name, engine, mode] of [ ["Godot Val", "godot", "prototype"], ["Unity Val", "unity", "design"], ["Unreal Val", "ue5", "development"] ] as const) { const cwd = mkdtempSync(path.join(tmpdir(), "ogs-val-")); const { projectRoot } = initProject({ name, engine, mode, nonInteractive: true }, cwd); expect(validateProject(projectRoot).filter((c) => c.status === "fail")).toEqual([]); } }); test("missing required project artifacts fail", () => { const cwd = mkdtempSync(path.join(tmpdir(), "ogs-val-")); const { projectRoot } = initProject({ name: "Broken Game", engine: "godot", mode: "prototype", nonInteractive: true }, cwd); rmSync(path.join(projectRoot, "AGENTS.md")); rmSync(path.join(projectRoot, "src", "project.godot")); const failures = validateProject(projectRoot).filter((c) => c.status === "fail"); expect(failures.map((f) => f.id)).toContain("project.agents_md"); expect(failures.map((f) => f.id)).toContain("project.engine_file"); }); test("malformed AGENTS contract and prompt files fail validation", () => { const cwd = mkdtempSync(path.join(tmpdir(), "ogs-val-")); const { projectRoot } = initProject({ name: "Prompt Game", engine: "godot", mode: "prototype", nonInteractive: true }, cwd); writeFileSync(path.join(projectRoot, "AGENTS.md"), "# Broken\n"); writeFileSync(path.join(projectRoot, ".codex", "prompts", "producer.md"), ""); const failures = validateProject(projectRoot).filter((c) => c.status === "fail"); expect(failures.map((f) => f.id)).toContain("codex.project.AGENTS.md.## Project Goal"); expect(failures.map((f) => f.id)).toContain("codex.prompt.producer"); }); test("empty timeline sections fail validation", () => { const cwd = mkdtempSync(path.join(tmpdir(), "ogs-val-")); const { projectRoot } = initProject({ name: "Timeline Game", engine: "godot", mode: "prototype", nonInteractive: true }, cwd); writeFileSync( path.join(projectRoot, "production", "timeline.md"), "# Timeline\n\nTBD\n\n# Milestones\n\n# Risks\n\n- Scope risk.\n\n# Next Validation Gate\n\nRun validation.\n" ); const failures = validateProject(projectRoot).filter((c) => c.status === "fail"); expect(failures.map((f) => f.id)).toContain("project.timeline.# Milestones"); }); test("Unity validation fails when ProjectSettings marker is missing", () => { const cwd = mkdtempSync(path.join(tmpdir(), "ogs-val-")); const { projectRoot } = initProject({ name: "Broken Unity", engine: "unity", mode: "design", nonInteractive: true }, cwd); rmSync(path.join(projectRoot, "src", "ProjectSettings", "ProjectSettings.asset")); const failures = validateProject(projectRoot).filter((c) => c.status === "fail"); expect(failures.map((f) => f.id)).toContain("project.engine_settings"); }); test("invalid studio json fails", () => { const cwd = mkdtempSync(path.join(tmpdir(), "ogs-val-")); const { projectRoot } = initProject({ name: "Stale Game", engine: "godot", mode: "prototype", nonInteractive: true }, cwd); writeFileSync(path.join(projectRoot, ".codex", "studio.json"), "{ invalid json"); expect(validateProject(projectRoot)[0].status).toBe("fail"); }); test("invalid studio mode fails project validation", () => { const cwd = mkdtempSync(path.join(tmpdir(), "ogs-val-")); const { projectRoot } = initProject({ name: "Studio Mode Val", engine: "godot", mode: "prototype", studioMode: "strict-studio", nonInteractive: true }, cwd); const studioPath = path.join(projectRoot, ".codex", "studio.json"); const studio = JSON.parse(readFileSync(studioPath, "utf8")); studio.studioMode = "ceremony-platform"; writeFileSync(studioPath, `${JSON.stringify(studio, null, 2)}\n`); expect(validateProject(projectRoot)[0]).toEqual( expect.objectContaining({ id: "codex.project.studio", status: "fail", message: expect.stringMatching(/studioMode/i) }) ); }); test("malformed approval store fails project validation", () => { const cwd = mkdtempSync(path.join(tmpdir(), "ogs-val-")); const { projectRoot } = initProject({ name: "Approval Val", engine: "godot", mode: "prototype", nonInteractive: true }, cwd); writeFileSync( path.join(projectRoot, ".codex", "approvals.json"), `${JSON.stringify({ schemaVersion: 1, product: "codex-game-studio", records: [{ id: "bad", stage: "approved", approvedGlobs: ["../escape.ts"] }] }, null, 2)}\n` ); const failures = validateProject(projectRoot).filter((c) => c.status === "fail"); expect(failures).toContainEqual( expect.objectContaining({ id: "codex.project.approvals", message: expect.stringMatching(/approval store invalid/i) }) ); }); test("context manifest schema and stale freshness metadata fail project validation", () => { const cwd = mkdtempSync(path.join(tmpdir(), "ogs-val-")); const { projectRoot } = initProject({ name: "Manifest Val", engine: "godot", mode: "prototype", nonInteractive: true }, cwd); writeFileSync(path.join(projectRoot, ".codex", "context-manifest.json"), "{ invalid json"); expect(validateProject(projectRoot).filter((c) => c.status === "fail")).toContainEqual( expect.objectContaining({ id: "codex.project.context_manifest", message: expect.stringMatching(/invalid JSON/i) }) ); const { projectRoot: staleProject } = initProject({ name: "Stale Manifest Val", engine: "godot", mode: "prototype", studioMode: "strict-studio", nonInteractive: true }, mkdtempSync(path.join(tmpdir(), "ogs-val-"))); const metaPath = path.join(staleProject, ".codex", "context-manifest.meta.json"); const meta = JSON.parse(readFileSync(metaPath, "utf8")); meta.studioMode = "guided-studio"; meta.manifestSha256 = "0".repeat(64); writeFileSync(metaPath, `${JSON.stringify(meta, null, 2)}\n`); const failures = validateProject(staleProject).filter((c) => c.status === "fail"); expect(failures.map((f) => f.id)).toContain("codex.project.context_manifest.freshness"); }); test("approval store symlink escape fails project validation", () => { const cwd = mkdtempSync(path.join(tmpdir(), "ogs-val-")); const { projectRoot } = initProject({ name: "Approval Symlink Val", engine: "godot", mode: "prototype", nonInteractive: true }, cwd); const outside = mkdtempSync(path.join(tmpdir(), "ogs-outside-")); mkdirSync(path.join(projectRoot, "source"), { recursive: true }); symlinkSync(outside, path.join(projectRoot, "source", "outside-link")); writeFileSync( path.join(projectRoot, ".codex", "approvals.json"), `${JSON.stringify({ schemaVersion: 1, product: "codex-game-studio", records: [{ ...validApprovalRecord, approvedGlobs: ["source/outside-link/file.ts"] }] }, null, 2)}\n` ); const failures = validateProject(projectRoot).filter((c) => c.status === "fail"); expect(failures).toContainEqual( expect.objectContaining({ id: "codex.project.approvals", message: expect.stringMatching(/symlink outside project/i) }) ); }); test("freeze status-only changes keep project validation green", () => { const cwd = mkdtempSync(path.join(tmpdir(), "ogs-val-")); const { projectRoot } = initProject({ name: "Freeze Valid", engine: "godot", mode: "prototype", nonInteractive: true }, cwd); freezeProject(projectRoot, cwd); expect(validateProject(projectRoot).filter((c) => c.status === "fail")).toEqual([]); }); test("engine reference validation covers package metadata and project materialization", () => { const cwd = mkdtempSync(path.join(tmpdir(), "ogs-val-engine-reference-")); const { projectRoot } = initProject({ name: "Unity Reference Valid", engine: "unity", mode: "prototype", nonInteractive: true }, cwd); for (const file of [...engineReferenceRegistry.unity.requiredFiles, ...engineReferenceRegistry.unity.moduleFiles, ...engineReferenceRegistry.unity.pluginFiles]) { expect(existsSync(path.join(projectRoot, engineReferenceRegistry.unity.projectPath(file)))).toBe(true); } expect(existsSync(path.join(projectRoot, "docs", "engine-reference", "unreal", "VERSION.md"))).toBe(false); const checks = validateProject(projectRoot); expect(checks.filter((c) => c.id.startsWith("engine_reference.") && c.status === "fail")).toEqual([]); expect(checks).toContainEqual(expect.objectContaining({ id: "engine_reference.package.unity.VERSION.md.metadata", status: "pass" })); expect(checks).toContainEqual(expect.objectContaining({ id: "engine_reference.package.unity.modules/networking.md.metadata", status: "pass" })); expect(checks).toContainEqual(expect.objectContaining({ id: "engine_reference.project.unity.VERSION.md", status: "pass" })); expect(checks).toContainEqual(expect.objectContaining({ id: "engine_reference.project.unity.modules/networking.md", status: "pass" })); }); test("missing materialized engine reference fails project validation", () => { const cwd = mkdtempSync(path.join(tmpdir(), "ogs-val-engine-reference-")); const { projectRoot } = initProject({ name: "Broken Reference Valid", engine: "godot", mode: "prototype", nonInteractive: true }, cwd); rmSync(path.join(projectRoot, "docs", "engine-reference", "godot", "plugins.md")); expect(validateProject(projectRoot).filter((c) => c.status === "fail")).toContainEqual( expect.objectContaining({ id: "engine_reference.project.godot.plugins.md" }) ); }); test("wrong-engine specialist prompts are absent and active specialist prompt is validated", () => { const cwd = mkdtempSync(path.join(tmpdir(), "ogs-val-specialist-")); const { projectRoot } = initProject({ name: "Godot Specialist Valid", engine: "godot", mode: "prototype", nonInteractive: true }, cwd); expect(validateProject(projectRoot).filter((c) => c.status === "fail")).toEqual([]); rmSync(path.join(projectRoot, ".codex", "prompts", "godot-specialist.md")); const failures = validateProject(projectRoot).filter((c) => c.status === "fail"); expect(failures.map((f) => f.id)).toContain("codex.role.godot-specialist.prompt.exists"); expect(failures.map((f) => f.id)).not.toContain("codex.role.unity-specialist.prompt.exists"); }); test("validation fails when wrong-engine specialist prompts are materialized", () => { const cwd = mkdtempSync(path.join(tmpdir(), "ogs-val-specialist-")); const { projectRoot } = initProject({ name: "Wrong Specialist Valid", engine: "godot", mode: "prototype", nonInteractive: true }, cwd); writeFileSync(path.join(projectRoot, ".codex", "prompts", "unity-specialist.md"), "# Wrong specialist\n"); expect(validateProject(projectRoot).filter((c) => c.status === "fail")).toContainEqual( expect.objectContaining({ id: "codex.role.unity-specialist.prompt.absent" }) ); }); test("generated surface hashes are stable for object key order", () => { expect(stableHash({ b: 2, a: { d: 4, c: 3 } })).toBe(stableHash({ a: { c: 3, d: 4 }, b: 2 })); }); test("generated surface validation detects freshness, body tampering, and legacy metadata", () => { const cwd = mkdtempSync(path.join(tmpdir(), "ogs-val-")); const { projectRoot } = initProject({ name: "Freshness Game", engine: "godot", mode: "prototype", nonInteractive: true }, cwd); const rolePrompt = path.join(projectRoot, ".codex", "prompts", "market-analyst.md"); writeFileSync(rolePrompt, readFileSync(rolePrompt, "utf8").replace(/source-input-sha256: [a-f0-9]+/, "source-input-sha256: bad")); expect(validateProject(projectRoot).filter((c) => c.status === "fail").map((c) => c.id)).toContain("codex.role.market-analyst.prompt.freshness"); const { projectRoot: bodyProject } = initProject({ name: "Body Freshness Game", engine: "godot", mode: "prototype", nonInteractive: true }, mkdtempSync(path.join(tmpdir(), "ogs-val-"))); const bodyPrompt = path.join(bodyProject, ".codex", "prompts", "market-analyst.md"); writeFileSync(bodyPrompt, `${readFileSync(bodyPrompt, "utf8")}\nTampered body.\n`); expect(validateProject(bodyProject).filter((c) => c.status === "fail").map((c) => c.id)).toContain("codex.role.market-analyst.prompt.body"); const { projectRoot: metadataBodyProject } = initProject({ name: "Metadata Body Game", engine: "godot", mode: "prototype", nonInteractive: true }, mkdtempSync(path.join(tmpdir(), "ogs-val-"))); const metadataBodyPrompt = path.join(metadataBodyProject, ".codex", "prompts", "market-analyst.md"); writeFileSync(metadataBodyPrompt, `${readFileSync(metadataBodyPrompt, "utf8")}\n\n`); expect(validateProject(metadataBodyProject).filter((c) => c.status === "fail").map((c) => c.id)).toContain("codex.role.market-analyst.prompt.body"); const { projectRoot: rendererProject } = initProject({ name: "Renderer Drift Game", engine: "godot", mode: "prototype", nonInteractive: true }, mkdtempSync(path.join(tmpdir(), "ogs-val-"))); const rendererPrompt = path.join(rendererProject, ".codex", "prompts", "market-analyst.md"); const changedBody = readFileSync(rendererPrompt, "utf8").replace("# Market Analyst", "# Market Research Analyst"); const changedHash = hashGeneratedBody(stripGeneratedMetadata(changedBody)); writeFileSync(rendererPrompt, changedBody.replace(/rendered-body-sha256: [a-f0-9]+/, `rendered-body-sha256: ${changedHash}`)); expect(validateProject(rendererProject).filter((c) => c.status === "fail").map((c) => c.id)).toContain("codex.role.market-analyst.prompt.body"); const { projectRoot: workflowProject } = initProject({ name: "Workflow Freshness Game", engine: "godot", mode: "prototype", nonInteractive: true }, mkdtempSync(path.join(tmpdir(), "ogs-val-"))); const workflow = path.join(workflowProject, ".codex", "workflows", "ui-ux-review.md"); writeFileSync(workflow, readFileSync(workflow, "utf8").replace(/source-input-sha256: [a-f0-9]+/, "source-input-sha256: bad")); expect(validateProject(workflowProject).filter((c) => c.status === "fail").map((c) => c.id)).toContain("codex.workflow.ui-ux-review.freshness"); const { projectRoot: workflowBodyProject } = initProject({ name: "Workflow Body Game", engine: "godot", mode: "prototype", nonInteractive: true }, mkdtempSync(path.join(tmpdir(), "ogs-val-"))); const workflowBody = path.join(workflowBodyProject, ".codex", "workflows", "ui-ux-review.md"); writeFileSync(workflowBody, `${readFileSync(workflowBody, "utf8")}\nTampered workflow body.\n`); expect(validateProject(workflowBodyProject).filter((c) => c.status === "fail").map((c) => c.id)).toContain("codex.workflow.ui-ux-review.body"); const { projectRoot: legacyProject } = initProject({ name: "Legacy Surface Game", engine: "godot", mode: "prototype", nonInteractive: true }, mkdtempSync(path.join(tmpdir(), "ogs-val-"))); const legacyPrompt = path.join(legacyProject, ".codex", "prompts", "market-analyst.md"); writeFileSync(legacyPrompt, readFileSync(legacyPrompt, "utf8").replace(/^\n/gm, "")); const legacyChecks = validateProject(legacyProject); expect(legacyChecks).toContainEqual(expect.objectContaining({ id: "codex.role.market-analyst.prompt.freshness", status: "skip" })); expect(legacyChecks.filter((c) => c.status === "fail").map((c) => c.id)).not.toContain("codex.role.market-analyst.prompt.freshness"); }); test("generated surface validation fails malformed metadata without treating it as legacy", () => { const cwd = mkdtempSync(path.join(tmpdir(), "ogs-val-")); const { projectRoot: malformedProject } = initProject({ name: "Malformed Surface Game", engine: "godot", mode: "prototype", nonInteractive: true }, cwd); const malformedPrompt = path.join(malformedProject, ".codex", "prompts", "market-analyst.md"); writeFileSync(malformedPrompt, readFileSync(malformedPrompt, "utf8").replace(/^\n/m, "\n")); const malformedFailures = validateProject(malformedProject).filter((c) => c.status === "fail").map((c) => c.id); expect(malformedFailures).toContain("codex.role.market-analyst.prompt.freshness"); const { projectRoot: partialProject } = initProject({ name: "Partial Surface Game", engine: "godot", mode: "prototype", nonInteractive: true }, mkdtempSync(path.join(tmpdir(), "ogs-val-"))); const partialWorkflow = path.join(partialProject, ".codex", "workflows", "ui-ux-review.md"); writeFileSync(partialWorkflow, readFileSync(partialWorkflow, "utf8").replace(/^\n/m, "")); const partialFailures = validateProject(partialProject).filter((c) => c.status === "fail").map((c) => c.id); expect(partialFailures).toContain("codex.workflow.ui-ux-review.body"); const { projectRoot: legacyProject } = initProject({ name: "Commentless Surface Game", engine: "godot", mode: "prototype", nonInteractive: true }, mkdtempSync(path.join(tmpdir(), "ogs-val-"))); const legacyPrompt = path.join(legacyProject, ".codex", "prompts", "market-analyst.md"); writeFileSync(legacyPrompt, readFileSync(legacyPrompt, "utf8").replace(/^\n/gm, "")); expect(validateProject(legacyProject)).toContainEqual(expect.objectContaining({ id: "codex.role.market-analyst.prompt.freshness", status: "skip" })); }); test("generated surface source input covers rendered engine and role display fields", () => { const engines = loadEngineConfigs(packageAssetPath("engine_configs")); const config = initProject({ name: "Hash Coverage Game", engine: "godot", mode: "prototype", nonInteractive: true }, mkdtempSync(path.join(tmpdir(), "ogs-val-"))).config; const baseRoleHash = stableHash(projectRolePromptSourceInput("market-analyst", config, engines)); const renamedEngines = { ...engines, godot: { ...engines.godot, display_name: "Renamed Godot" } }; expect(stableHash(projectRolePromptSourceInput("market-analyst", config, renamedEngines))).not.toBe(baseRoleHash); const baseWorkflowHash = stableHash(workflowSourceInput("ui-ux-review")); const originalName = rolePackages["ui-ux-designer"].displayName; rolePackages["ui-ux-designer"].displayName = "Renamed UI UX Designer"; try { expect(stableHash(workflowSourceInput("ui-ux-review"))).not.toBe(baseWorkflowHash); } finally { rolePackages["ui-ux-designer"].displayName = originalName; } }); test("repo validation includes deterministic behavioral evaluation subchecks", async () => { const result = await runValidation(); const checks = result.checks.filter((check) => check.id.startsWith("behavioral.scenario.")); expect(checks.map((check) => check.id)).toEqual(behavioralEvaluationScenarios.map((scenario) => `behavioral.scenario.${scenario.id}`)); expect(checks.every((check) => check.status === "pass")).toBe(true); }); test("repo validation reports Codex readiness hard failure when unavailable", async () => { const old = process.env.CODEX_BIN; process.env.CODEX_BIN = "/missing/codex"; try { const result = await runValidation(); expect(result.checks.some((c) => c.id === "codex.cli" && c.status === "fail")).toBe(true); expect(result.failed).toBe(true); } finally { if (old === undefined) delete process.env.CODEX_BIN; else process.env.CODEX_BIN = old; } }); });