SnapOtter/tests/e2e-docker/full-tool-audit.mjs

/**
 * FULL TOOL AUDIT — Tests every tool in SnapOtter on Windows/amd64.
 * Verifies: (1) tool works, (2) GPU-capable tools do not unexpectedly fall
 * back, and (3) execution provenance matches the declared contract. OCR is
 * deliberately CPU-only on every host, including NVIDIA systems.
 */
import { readFileSync } from "node:fs";

function requiredEnv(name) {
  const value = process.env[name]?.trim();
  if (!value) throw new Error(`${name} is required`);
  return value;
}

const BASE = process.env.QA_BASE_URL?.trim() || "http://localhost:13499";
const USERNAME = process.env.QA_USERNAME?.trim() || "admin";
const PASSWORD = requiredEnv("QA_PASSWORD");
const IMG =
  process.env.QA_IMAGE_FIXTURE?.trim() ||
  new URL("../fixtures/image/valid/multi-face.webp", import.meta.url);
const CONTAINER_NAME = requiredEnv("QA_CONTAINER_NAME");

const results = [];
let token = "";

function log(tool, status, detail = "") {
  const icon = status === "PASS" ? "\u2713" : status === "FAIL" ? "\u2717" : "-";
  const line = `${icon} [${status}] ${tool}${detail ? ` -- ${detail}` : ""}`;
  console.log(line);
  results.push({ tool, status, detail });
}

async function callTool(path, settings, filename = "test.webp") {
  const imageBuffer = readFileSync(IMG);
  const imageBlob = new Blob([imageBuffer], { type: "image/webp" });
  const formData = new FormData();
  formData.append("file", new File([imageBlob], filename, { type: "image/webp" }));
  formData.append("settings", JSON.stringify(settings));
  // All audited tools are image-modality; section prefix hardcoded
  // (cannot import apiToolPath from @snapotter/shared in plain .mjs).
  const res = await fetch(`${BASE}/api/v1/tools/image/${path}`, {
    method: "POST",
    headers: { Authorization: `Bearer ${token}` },
    body: formData,
  });
  const body = await res.json().catch(() => ({ error: `HTTP ${res.status} (non-JSON)` }));
  return { status: res.status, ok: res.ok, body };
}

async function test(name, path, settings, checks = {}) {
  try {
    const { status, ok, body } = await callTool(path, settings);
    if (!ok || body.error) {
      log(name, "FAIL", body.details || body.error || `HTTP ${status}`);
      return;
    }
    // Check expected model/method/provenance
    if (checks.expectKey && checks.expectValue) {
      const actual = body[checks.expectKey];
      if (actual !== checks.expectValue) {
        log(
          name,
          "FAIL",
          `Expected ${checks.expectKey}=${checks.expectValue} but got ${actual} (FALLBACK DETECTED)`,
        );
        return;
      }
    }
    for (const [key, expected] of Object.entries(checks.expectFields ?? {})) {
      if (body[key] !== expected) {
        log(name, "FAIL", `Expected ${key}=${expected} but got ${body[key]}`);
        return;
      }
    }
    // Build detail string
    const parts = [];
    for (const k of [
      "method",
      "model",
      "engine",
      "provider",
      "device",
      "requestedQuality",
      "actualQuality",
      "runtimeVersion",
      "modelVersion",
      "format",
      "width",
      "height",
      "facesDetected",
      "steps",
    ]) {
      if (body[k] !== undefined) {
        const v = Array.isArray(body[k]) ? JSON.stringify(body[k]) : body[k];
        parts.push(`${k}=${v}`);
      }
    }
    log(name, "PASS", parts.join(", "));
  } catch (err) {
    log(name, "FAIL", err.message.slice(0, 200));
  }
}

async function main() {
  console.log("=============================================================");
  console.log("  SNAPOTTER FULL TOOL AUDIT — Windows amd64 + RTX 4070");
  console.log("=============================================================\n");

  // Login
  const loginRes = await fetch(`${BASE}/api/auth/login`, {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    body: JSON.stringify({ username: USERNAME, password: PASSWORD }),
  });
  const loginBody = await loginRes.json();
  token = loginBody.token;
  console.log("Authenticated.\n");

  // Check GPU status
  const healthRes = await fetch(`${BASE}/api/v1/admin/health`, {
    headers: { Authorization: `Bearer ${token}` },
  });
  const health = await healthRes.json();
  console.log(`GPU detected: ${health.ai?.gpu}`);
  console.log(`Version: ${health.version}\n`);

  const featuresRes = await fetch(`${BASE}/api/v1/features`, {
    headers: { Authorization: `Bearer ${token}` },
  });
  if (!featuresRes.ok) {
    throw new Error(`Could not read feature capabilities: HTTP ${featuresRes.status}`);
  }
  const features = await featuresRes.json();
  const ocrFeature = features.bundles?.find((bundle) => bundle.id === "ocr");
  if (!ocrFeature?.availableQualities?.includes("fast")) {
    throw new Error("OCR feature state does not advertise the built-in Fast tier");
  }

  // ════════════════════════════════════════════════════════════════
  // SECTION 1: GPU/AI TOOLS — verify correct model and declared provider
  // ════════════════════════════════════════════════════════════════
  console.log("--- GPU/AI TOOLS (GPU where supported; OCR is CPU-only) ---\n");

  // Background removal — all models
  await test(
    "Remove BG (birefnet-general-lite)",
    "remove-background",
    { model: "birefnet-general-lite" },
    { expectKey: "model", expectValue: "birefnet-general-lite" },
  );
  await test(
    "Remove BG (birefnet-portrait)",
    "remove-background",
    { model: "birefnet-portrait" },
    { expectKey: "model", expectValue: "birefnet-portrait" },
  );
  await test(
    "Remove BG (birefnet-general)",
    "remove-background",
    { model: "birefnet-general" },
    { expectKey: "model", expectValue: "birefnet-general" },
  );
  await test(
    "Remove BG (u2net)",
    "remove-background",
    { model: "u2net" },
    { expectKey: "model", expectValue: "u2net" },
  );
  await test(
    "Remove BG (bria-rmbg)",
    "remove-background",
    { model: "bria-rmbg" },
    { expectKey: "model", expectValue: "bria-rmbg" },
  );
  await test(
    "Remove BG (isnet-general-use)",
    "remove-background",
    { model: "isnet-general-use" },
    { expectKey: "model", expectValue: "isnet-general-use" },
  );
  await test(
    "Remove BG (birefnet-matting/Ultra)",
    "remove-background",
    { model: "birefnet-matting" },
    { expectKey: "model", expectValue: "birefnet-matting" },
  );

  // Upscale
  await test(
    "Upscale (realesrgan 2x)",
    "upscale",
    { scale: 2, model: "realesrgan" },
    { expectKey: "method", expectValue: "realesrgan" },
  );
  await test(
    "Upscale (realesrgan 4x)",
    "upscale",
    { scale: 4, model: "realesrgan" },
    { expectKey: "method", expectValue: "realesrgan" },
  );
  await test("Upscale (lanczos 2x)", "upscale", { scale: 2, model: "lanczos" });
  await test("Upscale (auto)", "upscale", { scale: 2, model: "auto" });

  // Face enhancement
  await test(
    "Face Enhance (gfpgan)",
    "enhance-faces",
    { model: "gfpgan" },
    { expectKey: "model", expectValue: "gfpgan" },
  );
  await test("Face Enhance (codeformer)", "enhance-faces", { model: "codeformer" });
  await test("Face Enhance (auto)", "enhance-faces", { model: "auto" });

  // Colorize
  await test(
    "Colorize (ddcolor)",
    "colorize",
    { model: "ddcolor" },
    { expectKey: "method", expectValue: "ddcolor" },
  );
  await test("Colorize (auto)", "colorize", { model: "auto" });

  // Noise removal — all tiers
  await test("Noise Removal (quick)", "noise-removal", { tier: "quick" });
  await test("Noise Removal (balanced)", "noise-removal", { tier: "balanced" });
  await test("Noise Removal (quality/SCUNet)", "noise-removal", { tier: "quality" });
  await test("Noise Removal (maximum/NAFNet)", "noise-removal", { tier: "maximum" });

  // Photo restoration
  await test("Photo Restoration", "restore-photo", {});

  // OCR
  await test(
    "OCR (Fast built-in)",
    "ocr",
    { quality: "fast" },
    {
      expectFields: {
        engine: "tesseract",
        provider: "native",
        device: "cpu",
        requestedQuality: "fast",
        actualQuality: "fast",
        degraded: false,
      },
    },
  );
  for (const quality of ["balanced", "best"]) {
    if (!ocrFeature.availableQualities.includes(quality)) {
      log(`OCR (${quality})`, "SKIP", `signed accurate runtime unavailable (${ocrFeature.status})`);
      continue;
    }
    await test(
      `OCR (${quality})`,
      "ocr",
      { quality },
      {
        expectFields: {
          engine: "rapidocr-onnx",
          provider: "CPUExecutionProvider",
          device: "cpu",
          requestedQuality: quality,
          actualQuality: quality,
          degraded: false,
        },
      },
    );
  }

  // Face operations (MediaPipe)
  await test("Face Blur", "blur-faces", { intensity: 50 });
  await test("Red-Eye Removal", "red-eye-removal", {});

  // Content-aware resize (caire binary)
  await test("Content-Aware Resize", "content-aware-resize", { width: 800, height: 800 });

  // Erase object (LaMa inpainting — needs mask, likely fails without one)
  // Skipping as it needs a mask input

  // Smart crop
  await test("Smart Crop", "smart-crop", { width: 400, height: 400 });

  // ════════════════════════════════════════════════════════════════
  // SECTION 2: IMAGE PROCESSING TOOLS (Sharp-based, CPU)
  // ════════════════════════════════════════════════════════════════
  console.log("\n--- IMAGE PROCESSING TOOLS (Sharp-based) ---\n");

  await test("Resize", "resize", { width: 512, height: 512, fit: "cover" });
  await test("Crop", "crop", { left: 100, top: 100, width: 500, height: 500 });
  await test("Rotate (90)", "rotate", { angle: 90 });
  await test("Rotate (45 + fill)", "rotate", { angle: 45, background: "#ffffff" });
  await test("Compress (webp q50)", "compress", { quality: 50 });
  await test("Convert (to PNG)", "convert", { format: "png" });
  await test("Convert (to JPEG)", "convert", { format: "jpg", quality: 85 });
  await test("Image Enhancement (auto)", "image-enhancement", { preset: "auto" });
  // color-adjustments is part of image-enhancement, not a separate tool
  await test("Image Enhancement (vivid)", "image-enhancement", { preset: "vivid" });
  await test("Sharpening", "sharpening", { sigma: 1.5, amount: 1.0 });
  await test("Border", "border", { size: 20, color: "#ff0000" });
  await test("Replace Color", "replace-color", {
    targetColor: "#ffffff",
    replacementColor: "#000000",
    tolerance: 30,
  });

  // ════════════════════════════════════════════════════════════════
  // SECTION 3: UTILITY TOOLS
  // ════════════════════════════════════════════════════════════════
  console.log("\n--- UTILITY TOOLS ---\n");

  await test("Info (metadata)", "info", {});
  await test("Strip Metadata", "strip-metadata", {});
  await test("Image to Base64", "image-to-base64", {});
  await test("Optimize for Web", "optimize-for-web", { maxWidth: 1920, quality: 80 });
  // Favicon returns binary ICO, not JSON — test via status code only
  try {
    const imageBuffer = readFileSync(IMG);
    const imageBlob = new Blob([imageBuffer], { type: "image/webp" });
    const formData = new FormData();
    formData.append("file", new File([imageBlob], "test.webp", { type: "image/webp" }));
    formData.append("settings", JSON.stringify({}));
    const res = await fetch(`${BASE}/api/v1/tools/image/favicon`, {
      method: "POST",
      headers: { Authorization: `Bearer ${token}` },
      body: formData,
    });
    log(
      "Favicon",
      res.ok ? "PASS" : "FAIL",
      `HTTP ${res.status}, ${res.headers.get("content-type")}`,
    );
  } catch (e) {
    log("Favicon", "FAIL", e.message.slice(0, 100));
  }

  // ════════════════════════════════════════════════════════════════
  // SECTION 4: MULTI-IMAGE / SPECIAL TOOLS (may need special input)
  // ════════════════════════════════════════════════════════════════
  console.log("\n--- SPECIAL TOOLS (may need specific inputs) ---\n");

  await test("QR Generate", "qr-generate", {
    text: "https://snapotter.app",
    size: 512,
    format: "png",
  });
  await test("Text Overlay", "text-overlay", {
    text: "TEST",
    fontSize: 48,
    color: "#ff0000",
    position: "center",
  });
  await test("Vectorize", "vectorize", {});
  await test("SVG to Raster", "svg-to-raster", {}); // Will fail - needs SVG input

  // ════════════════════════════════════════════════════════════════
  // SUMMARY
  // ════════════════════════════════════════════════════════════════
  console.log("\n=============================================================");
  console.log("  SUMMARY");
  console.log("=============================================================\n");

  const passed = results.filter((r) => r.status === "PASS");
  const failed = results.filter((r) => r.status === "FAIL");

  console.log(`PASSED: ${passed.length}`);
  console.log(`FAILED: ${failed.length}`);
  console.log(`TOTAL:  ${results.length}\n`);

  if (failed.length > 0) {
    console.log("FAILURES:");
    for (const r of failed) {
      // Truncate long error messages
      const detail = r.detail.length > 150 ? `${r.detail.slice(0, 150)}...` : r.detail;
      console.log(`  \u2717 ${r.tool}: ${detail}`);
    }
  }

  // Check GPU usage in docker logs
  console.log("\n--- GPU USAGE CHECK ---\n");
  const { spawnSync } = await import("node:child_process");
  const logResult = spawnSync("docker", ["logs", CONTAINER_NAME], {
    encoding: "utf-8",
    maxBuffer: 1024 * 1024,
  });
  if (logResult.error || logResult.status !== 0) {
    throw logResult.error ?? new Error(`docker logs failed with exit ${logResult.status}`);
  }
  const logs = `${logResult.stdout}${logResult.stderr}`;
  const gpuLines = logs
    .split("\n")
    .filter(
      (l) =>
        l.includes("[gpu]") ||
        l.includes("[bridge]") ||
        l.includes("[dispatcher]") ||
        l.includes("GPU") ||
        l.includes("CUDA") ||
        l.includes("CUDAExecution"),
    );
  for (const line of gpuLines.slice(0, 15)) {
    console.log(`  ${line.trim().slice(0, 120)}`);
  }

  // Check for any fallback warnings
  console.log("\n--- FALLBACK/MISMATCH WARNINGS ---\n");
  const warnLines = logs
    .split("\n")
    .filter(
      (l) =>
        l.includes("mismatch") ||
        l.includes("fallback") ||
        l.includes("Falling back") ||
        l.includes("degraded") ||
        (l.includes("lanczos") && l.includes("warn")),
    );
  if (warnLines.length === 0) {
    console.log("  None detected - no silent fallbacks occurred.");
  } else {
    for (const line of warnLines) {
      console.log(`  WARNING: ${line.trim().slice(0, 150)}`);
    }
  }

  process.exit(failed.length > 0 ? 1 : 0);
}

main().catch((err) => {
  console.error("Fatal:", err);
  process.exit(1);
});