SnapOtter/tests/unit/helpers/generated-harness-contract.test.ts

import { readFileSync } from "node:fs";
import { join } from "node:path";
import { describe, expect, it } from "vitest";

const GENERATED = join(process.cwd(), "tests/integration/generated");
const source = (filename: string): string => readFileSync(join(GENERATED, filename), "utf8");

describe("generated QA harness contract", () => {
  it.each(["fuzz-settings.test.ts", "settings-pairwise.test.ts"])(
    "%s accounts for execution through the production v2 contract and uses explicit runtime skips",
    (filename) => {
      const text = source(filename);
      expect(text).toContain("GeneratedCaseAccounting");
      expect(text).toContain("runGeneratedTool");
      expect(text).toContain("context.skip(");
      expect(text).toContain("isExpectedGeneratedRejection");
      expect(text).toContain("findMissingGeneratedPrerequisite");
      expect(text).toContain("buildGeneratedProcessInputs(fixtures, config, tool.modality)");
      expect(text).not.toMatch(/if \(!config\) return;/);
      expect(text).not.toContain("config.process(");
      expect(text).not.toContain("CRASH_PATTERN");
      expect(text).not.toContain("Clean operational failure");
    },
  );

  it.each([
    "format-matrix-generated.shared.ts",
    "format-matrix-multimodal.test.ts",
    "format-matrix-ai.test.ts",
  ])("%s makes required-AI mode strict", (filename) => {
    const text = source(filename);
    expect(text).toContain("REQUIRE_AI_FEATURES");
    expect(text).toContain("featureUnavailableDisposition");
  });

  it("multimodal matrix consumes the shared image-inclusive fixture index", () => {
    const text = source("format-matrix-multimodal.test.ts");
    expect(text).toContain("generatedFixtureDirectories");
    expect(text).toContain("selectFixturesForTool");
  });

  it.each([
    "format-matrix-generated.shared.ts",
    "format-matrix-multimodal.test.ts",
    "settings-matrix.test.ts",
  ])("%s resolves every 202 and validates its completed artifact", (filename) => {
    const text = source(filename);
    expect(text).toContain("waitForGeneratedJobArtifact");
    expect(text).not.toContain("cancelAcceptedJobAndWait");
    expect(text).not.toMatch(
      /statusCode === 200 \|\| res\.statusCode === 202\) accounting\.accept/,
    );
  });

  it("fuzz executes the exact requested run count instead of interrupting successfully", () => {
    const text = source("fuzz-settings.test.ts");
    expect(text).not.toContain("interruptAfterTimeLimit");
    expect(text).toContain("expectedAttempts: FUZZ_CONFIG.runs + 1");
  });

  it("pairwise does not truncate the covering array after it is generated", () => {
    const text = source("settings-pairwise.test.ts");
    expect(text).not.toContain("MAX_CASES_PER_TOOL");
    expect(text).not.toMatch(/\.slice\(0,/);
  });

  it("settings capability gates preserve every collected variation", () => {
    const text = source("settings-matrix.test.ts");
    expect(text).toContain("allPythonVariationsUnavailable");
    expect(text).not.toContain("describe.skip(");
  });

  it("the nightly extended lane does not require AI features it never installs", () => {
    // Strict required-AI mode (absence of a bundle is a failure) is only valid
    // on a host with bundles installed, which is the release-QA fleet. The
    // extended-matrix lane has no bundle-install step, so requiring them there
    // 501s every installed-contract case on a bundle-less runner.
    const workflow = readFileSync(join(process.cwd(), ".github/workflows/nightly.yml"), "utf8");
    const extendedLane = workflow.slice(
      workflow.indexOf("  extended-matrix:"),
      workflow.indexOf("  api-fuzz:"),
    );
    expect(extendedLane).not.toContain("REQUIRE_AI_FEATURES");
  });
});