feat: add finding confidence and aggregate audits

- classify findings as confirmed/probable/informational/likely_false_positive
- expose confidence in text/image/container JSON and human reports
- add audit_dir.py and audit_website.py for aggregate reports
- document the confidence taxonomy and audit commands in SKILL.md

Closes #13

skills/remove-ai-marks/SKILL.md

@@ -33,6 +33,8 @@ python3 "$SCRIPTS/clean_text.py" ...
 python3 "$SCRIPTS/inspect_image.py" ...
 python3 "$SCRIPTS/clean_image.py" ...
 python3 "$SCRIPTS/rewrite_text.py" ...
+python3 "$SCRIPTS/audit_dir.py" ...
+python3 "$SCRIPTS/audit_website.py" ...
 ```
 
 ## Ethics
@@ -50,7 +52,8 @@ Intended for **your own** content (privacy, hygiene, research). Do not market re
 | `.md` / `.html` | container clean (frontmatter/meta) + Layer A |
 | `.png` / `.jpg` / `.jpeg` | image metadata strip |
 | `.svg` / `.pdf` / `.docx` / `.odt` | container metadata strip |
-| Directory | batch each matching file |
+| Directory | aggregate report with `audit_dir.py` |
+| Website / sitemap | aggregate report with `audit_website.py` |
 | Mixed | run unified `inspect_file` / `clean_file` |
 
 ### 2. Inspect first
@@ -70,6 +73,27 @@ external reverse-SynthID scorer. That is **detection only**, not removal.
 Bootstrap the external checkout with `scripts/setup_synthid.sh`, or build a
 local image with `make docker-synthid-build`.
 
+### Aggregate audits and confidence
+
+Findings are classified as **confirmed**, **probable**, **informational**, or
+**likely_false_positive**. Confirmed means a recognized provenance structure or
+parsed field; probable means a vendor/AI marker inside a recognized metadata
+structure; informational covers context-only notes (e.g. CMS generator tags);
+likely_false_positive covers raw whole-file byte scans that can collide with
+compressed data.
+
+Audit a whole tree or a live sitemap for an aggregate report:
+
+```bash
+python3 "$SCRIPTS/audit_dir.py" DIR --json
+python3 "$SCRIPTS/audit_website.py" --sitemap https://example.com/sitemap.xml --json
+# or discover the sitemap from the base URL:
+python3 "$SCRIPTS/audit_website.py" --base https://example.com --json
+```
+
+`audit_website.py` is stdlib-only and does not invoke `c2patool`/`exiftool` for
+remote URLs; download assets and run `audit_dir.py` locally for those.
+
 ### 3. Deterministic clean (always for matching inputs)
 
 **Text — Layer A:**
@@ -225,4 +249,8 @@ python3 scripts/rewrite_text.py notes.md --backend print-prompt --strength parap
 # Images only
 python3 scripts/inspect_image.py shot.png
 python3 scripts/clean_image.py shot.png -o shot.cleaned.png
+
+# Aggregate audits
+python3 scripts/audit_dir.py ./src --json
+python3 scripts/audit_website.py --sitemap https://example.com/sitemap.xml --json
 ```

skills/remove-ai-marks/scripts/audit_dir.py

@@ -0,0 +1,108 @@
+#!/usr/bin/env python3
+"""Aggregate AI-provenance audit over a directory tree.
+
+Recursively inspects supported text/image/container files and emits one
+summary plus a per-file finding list with confidence classifications.
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+
+from audit_lib import aggregate, print_human_report, scan_file  # noqa: E402
+from common import MAX_INPUT_BYTES, emit_json, eprint  # noqa: E402
+
+DEFAULT_SKIP_DIRS = {
+    ".git",
+    ".hg",
+    ".svn",
+    "node_modules",
+    "__pycache__",
+    ".venv",
+    "venv",
+    ".tox",
+    ".mypy_cache",
+    ".pytest_cache",
+    "dist",
+    "build",
+    ".next",
+    "target",
+    ".cache",
+}
+
+
+def walk_files(root: Path, skip_dirs: set[str]):
+    for dirpath, dirnames, filenames in os.walk(root):
+        dirnames[:] = sorted(
+            d for d in dirnames if d not in skip_dirs and not d.startswith(".")
+        )
+        for fn in sorted(filenames):
+            path = Path(dirpath) / fn
+            if path.is_file():
+                yield path
+
+
+def main() -> int:
+    p = argparse.ArgumentParser(description=__doc__)
+    p.add_argument("path", type=Path, help="Directory to audit recursively")
+    p.add_argument("--json", action="store_true", help="Emit a JSON report")
+    p.add_argument(
+        "--skip",
+        default="",
+        help="Comma-separated extra directory names to skip",
+    )
+    args = p.parse_args()
+
+    root = args.path
+    if not root.is_dir():
+        eprint(f"not a directory: {root}")
+        return 2
+
+    skip_dirs = set(DEFAULT_SKIP_DIRS)
+    for name in args.skip.split(","):
+        name = name.strip()
+        if name:
+            skip_dirs.add(name)
+
+    files = []
+    skipped = []
+    for path in walk_files(root, skip_dirs):
+        try:
+            if path.stat().st_size > MAX_INPUT_BYTES:
+                skipped.append({"path": str(path), "reason": "too large"})
+                continue
+            files.append(scan_file(path))
+        except Exception as e:  # keep the audit going on one bad file
+            skipped.append({"path": str(path), "reason": str(e)})
+
+    summary = aggregate(files)
+    report = {
+        "root": str(root),
+        "files_scanned": len(files),
+        "files_skipped": skipped,
+        "summary": summary,
+        "files": files,
+    }
+
+    if args.json:
+        emit_json(report)
+    else:
+        print_human_report(
+            files,
+            summary,
+            extra_header={
+                "Root": report["root"],
+                "Files skipped": str(len(skipped)),
+            },
+        )
+
+    return 1 if summary["actionable_files"] else 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

skills/remove-ai-marks/scripts/audit_lib.py

@@ -0,0 +1,150 @@
+"""Shared helpers for aggregate directory and website audits.
+
+Both audits normalize every file/URL into the same per-item dict so a single
+aggregate summary can be computed and rendered consistently.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from common import CONFIDENCE_LEVELS, classify_finding_confidence
+from container_meta import inspect_container
+from image_meta import inspect_image
+from inspect_file import classify
+from text_unicode import inspect_text
+
+
+def text_hit_confidence(kind: str) -> str:
+    """Layer A space homoglyphs are weaker context than invisible carriers."""
+    return "informational" if kind == "space" else "probable"
+
+
+def text_findings(report: Any) -> tuple[list[str], list[str], int]:
+    """Flatten a TextInspectReport into finding strings + confidence lists."""
+    findings: list[str] = []
+    confidences: list[str] = []
+    for h in report.hits:
+        conf = text_hit_confidence(h.kind)
+        findings.append(f"layer-a [{h.kind}] {h.label} x{h.count}")
+        confidences.append(conf)
+    return findings, confidences, report.suspicious_total
+
+
+def scan_file(path: Path, display_name: str | None = None) -> dict[str, Any]:
+    """Inspect one local file and return a normalized audit item."""
+    name = display_name or str(path)
+    kind = classify(path)
+
+    if kind == "text":
+        try:
+            text = path.read_text(encoding="utf-8", errors="surrogateescape")
+        except OSError as e:
+            return {"path": name, "kind": "text", "error": str(e)}
+        report = inspect_text(text)
+        findings, confidences, suspicious = text_findings(report)
+        return {
+            "path": name,
+            "kind": "text",
+            "has_c2pa": False,
+            "has_ai_metadata": False,
+            "suspicious_total": suspicious,
+            "findings": findings,
+            "confidence": confidences,
+            "notes": report.notes,
+        }
+
+    if kind == "image":
+        report = inspect_image(path)
+        return {
+            "path": name,
+            "kind": report.format,
+            "has_c2pa": report.has_c2pa,
+            "has_ai_metadata": report.has_ai_metadata,
+            "suspicious_total": 0,
+            "findings": report.findings,
+            "confidence": [classify_finding_confidence(f) for f in report.findings],
+            "notes": report.notes,
+        }
+
+    report = inspect_container(path)
+    findings = list(report.findings)
+    confidences = [classify_finding_confidence(f) for f in report.findings]
+    suspicious = 0
+
+    # Text-bearing containers also get a Layer A scan of their visible text,
+    # mirroring the skill's "container + Layer A" workflow.
+    if report.format in ("html", "markdown"):
+        try:
+            text = path.read_text(encoding="utf-8", errors="surrogateescape")
+        except OSError:
+            text = ""
+        if text:
+            t_report = inspect_text(text)
+            t_findings, t_confidences, t_suspicious = text_findings(t_report)
+            findings.extend(t_findings)
+            confidences.extend(t_confidences)
+            suspicious = t_suspicious
+
+    return {
+        "path": name,
+        "kind": report.format,
+        "has_c2pa": report.has_c2pa,
+        "has_ai_metadata": report.has_ai_metadata,
+        "suspicious_total": suspicious,
+        "findings": findings,
+        "confidence": confidences,
+        "notes": report.notes,
+    }
+
+
+def is_actionable(item: dict[str, Any]) -> bool:
+    """A file is actionable when it has a confirmed/probable finding or C2PA."""
+    if item.get("has_c2pa"):
+        return True
+    return any(c in ("confirmed", "probable") for c in item.get("confidence", []))
+
+
+def aggregate(files: list[dict[str, Any]]) -> dict[str, Any]:
+    """Build the summary block shared by directory and website audits."""
+    summary = {
+        "total": len(files),
+        "by_kind": {},
+        "with_c2pa": 0,
+        "with_ai_metadata": 0,
+        "with_suspicious_text": 0,
+        "actionable_files": 0,
+        "findings_by_confidence": {c: 0 for c in CONFIDENCE_LEVELS},
+    }
+    for item in files:
+        kind = str(item.get("kind") or "error")
+        summary["by_kind"][kind] = summary["by_kind"].get(kind, 0) + 1
+        if item.get("has_c2pa"):
+            summary["with_c2pa"] += 1
+        if item.get("has_ai_metadata"):
+            summary["with_ai_metadata"] += 1
+        if item.get("suspicious_total", 0) > 0:
+            summary["with_suspicious_text"] += 1
+        for c in item.get("confidence", []):
+            if c in summary["findings_by_confidence"]:
+                summary["findings_by_confidence"][c] += 1
+        if is_actionable(item):
+            summary["actionable_files"] += 1
+    return summary
+
+
+def print_human_report(files: list[dict[str, Any]], summary: dict[str, Any], extra_header: dict[str, Any] | None = None) -> None:
+    """Shared plain-text rendering for audit scripts."""
+    for key, value in (extra_header or {}).items():
+        print(f"{key}: {value}")
+    print(f"Files scanned: {summary['total']}")
+    print(f"By kind: {summary['by_kind']}")
+    print(f"With C2PA: {summary['with_c2pa']}")
+    print(f"With AI metadata: {summary['with_ai_metadata']}")
+    print(f"With suspicious text: {summary['with_suspicious_text']}")
+    print(f"Actionable files: {summary['actionable_files']}")
+    print(f"Findings by confidence: {summary['findings_by_confidence']}")
+    for item in files:
+        for msg, conf in zip(item.get("findings", []), item.get("confidence", [])):
+            print(f"  [{conf}] {item['path']}: {msg}")

skills/remove-ai-marks/scripts/audit_website.py

@@ -0,0 +1,265 @@
+#!/usr/bin/env python3
+"""Aggregate AI-provenance audit over the URLs listed in a sitemap.
+
+Stdlib-only: downloads each URL, classifies it by content type/suffix/magic,
+and runs the same deterministic text/image/container inspections used by the
+local audit. Optional external tools (c2patool/exiftool) are not invoked for
+remote URLs; download the assets and run audit_dir.py locally for those.
+"""
+
+from __future__ import annotations
+
+import argparse
+import gzip
+import sys
+import tempfile
+import urllib.error
+import urllib.parse
+import urllib.request
+import xml.etree.ElementTree as ET
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+
+from audit_lib import aggregate, print_human_report, scan_file  # noqa: E402
+from common import emit_json, eprint  # noqa: E402
+
+DEFAULT_MAX_BYTES = 4 << 20
+DEFAULT_TIMEOUT = 15
+DEFAULT_MAX_PAGES = 200
+USER_AGENT = "remove-ai-marks-audit/1.0"
+
+_EXT_FOR_KIND = {
+    "png": ".png",
+    "jpeg": ".jpg",
+    "svg": ".svg",
+    "pdf": ".pdf",
+    "docx": ".docx",
+    "odt": ".odt",
+    "html": ".html",
+    "markdown": ".md",
+    "text": ".txt",
+}
+
+
+def _local(tag: str) -> str:
+    return tag.rsplit("}", 1)[-1]
+
+
+def parse_sitemap(data: bytes) -> tuple[str, list[str]]:
+    """Parse a (possibly gzip-compressed) sitemap into (kind, urls)."""
+    if data[:2] == b"\x1f\x8b":
+        data = gzip.decompress(data)
+    root = ET.fromstring(data)
+    kind = _local(root.tag)
+    urls = []
+    for el in root.iter():
+        if _local(el.tag) == "loc" and el.text:
+            urls.append(el.text.strip())
+    return kind, urls
+
+
+def guess_kind(url: str, data: bytes, content_type: str | None = None) -> str:
+    """Classify a downloaded URL from headers, suffix, then magic bytes."""
+    ct = (content_type or "").lower().split(";")[0].strip()
+    if "html" in ct:
+        return "html"
+    if ct == "image/png":
+        return "png"
+    if ct == "image/jpeg":
+        return "jpeg"
+    if "svg" in ct:
+        return "svg"
+    if ct == "application/pdf":
+        return "pdf"
+    if "wordprocessingml" in ct:
+        return "docx"
+    if "opendocument.text" in ct:
+        return "odt"
+    if "markdown" in ct:
+        return "markdown"
+    if ct == "text/plain":
+        return "text"
+
+    path = urllib.parse.urlparse(url).path.lower()
+    for ext, kind in (
+        (".png", "png"),
+        (".jpg", "jpeg"),
+        (".jpeg", "jpeg"),
+        (".svg", "svg"),
+        (".pdf", "pdf"),
+        (".docx", "docx"),
+        (".odt", "odt"),
+        (".html", "html"),
+        (".htm", "html"),
+        (".md", "markdown"),
+        (".markdown", "markdown"),
+        (".txt", "text"),
+    ):
+        if path.endswith(ext):
+            return kind
+
+    if data.startswith(b"\x89PNG"):
+        return "png"
+    if data.startswith(b"\xff\xd8"):
+        return "jpeg"
+    if data.startswith(b"%PDF"):
+        return "pdf"
+    if data[:100].lstrip().startswith(b"<") and b"svg" in data[:500].lower():
+        return "svg"
+    if b"<html" in data[:2000].lower() or data[:100].lstrip().lower().startswith(b"<"):
+        return "html"
+    return "text"
+
+
+def fetch(url: str, timeout: int, max_bytes: int) -> tuple[bytes, str | None]:
+    """Fetch *url* with a byte cap; returns (body, content_type)."""
+    req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
+    with urllib.request.urlopen(req, timeout=timeout) as resp:
+        content_type = resp.headers.get("Content-Type")
+        chunks = []
+        total = 0
+        while True:
+            chunk = resp.read(1 << 16)
+            if not chunk:
+                break
+            total += len(chunk)
+            if total > max_bytes:
+                raise ValueError(f"exceeds {max_bytes} bytes")
+            chunks.append(chunk)
+        return b"".join(chunks), content_type
+
+
+def inspect_remote(url: str, data: bytes, content_type: str | None = None) -> dict:
+    """Inspect downloaded bytes using the local scan_file pipeline."""
+    kind = guess_kind(url, data, content_type)
+    ext = _EXT_FOR_KIND.get(kind, ".bin")
+    with tempfile.TemporaryDirectory() as td:
+        tmp = Path(td) / f"asset{ext}"
+        tmp.write_bytes(data)
+        result = scan_file(tmp, display_name=url)
+    result["kind"] = kind
+    return result
+
+
+def discover_sitemap(base_url: str, timeout: int) -> str | None:
+    """Find a sitemap for *base_url* via /sitemap.xml then /robots.txt."""
+    base = base_url.rstrip("/")
+    for candidate in (f"{base}/sitemap.xml", f"{base}/sitemap_index.xml"):
+        try:
+            data, _ = fetch(candidate, timeout, DEFAULT_MAX_BYTES)
+            parse_sitemap(data)
+            return candidate
+        except Exception:
+            continue
+    try:
+        data, _ = fetch(f"{base}/robots.txt", timeout, 1 << 20)
+        text = data.decode("utf-8", errors="replace")
+        for line in text.splitlines():
+            if line.lower().startswith("sitemap:"):
+                return line.split(":", 1)[1].strip()
+    except Exception:
+        pass
+    return None
+
+
+def collect_urls(sitemap_url: str, timeout: int, max_pages: int) -> list[str]:
+    """Collect page/asset URLs from a sitemap, following nested indexes."""
+    urls: list[str] = []
+    seen: set[str] = set()
+
+    def _recurse(url: str, depth: int = 0) -> None:
+        if len(urls) >= max_pages or depth > 3:
+            return
+        data, _ = fetch(url, timeout, DEFAULT_MAX_BYTES)
+        kind, locs = parse_sitemap(data)
+        if kind == "sitemapindex":
+            for loc in locs:
+                if loc not in seen:
+                    seen.add(loc)
+                    _recurse(loc, depth + 1)
+        else:
+            for loc in locs:
+                if loc not in seen:
+                    seen.add(loc)
+                    urls.append(loc)
+
+    _recurse(sitemap_url)
+    return urls
+
+
+def main() -> int:
+    p = argparse.ArgumentParser(description=__doc__)
+    p.add_argument("--sitemap", help="Sitemap URL to audit")
+    p.add_argument("--base", help="Base URL; discover the sitemap automatically")
+    p.add_argument("--max-pages", type=int, default=DEFAULT_MAX_PAGES)
+    p.add_argument("--timeout", type=int, default=DEFAULT_TIMEOUT)
+    p.add_argument("--max-bytes", type=int, default=DEFAULT_MAX_BYTES)
+    p.add_argument("--json", action="store_true")
+    args = p.parse_args()
+
+    if not args.sitemap and not args.base:
+        eprint("provide --sitemap URL or --base URL")
+        return 2
+
+    sitemap_url = args.sitemap
+    if not sitemap_url:
+        sitemap_url = discover_sitemap(args.base, args.timeout)
+        if not sitemap_url:
+            eprint(f"no sitemap found for {args.base}")
+            return 2
+
+    try:
+        urls = collect_urls(sitemap_url, args.timeout, args.max_pages)
+    except Exception as e:
+        eprint(f"could not collect URLs from {sitemap_url}: {e}")
+        return 2
+    if not urls:
+        eprint("no URLs collected from sitemap")
+        return 2
+
+    files = []
+    failures = []
+    for url in urls[: args.max_pages]:
+        try:
+            data, content_type = fetch(url, args.timeout, args.max_bytes)
+        except Exception as e:
+            failures.append({"url": url, "error": str(e)})
+            continue
+        try:
+            files.append(inspect_remote(url, data, content_type))
+        except Exception as e:
+            failures.append({"url": url, "error": f"inspect failed: {e}"})
+
+    summary = aggregate(files)
+    report = {
+        "sitemap": sitemap_url,
+        "base": args.base,
+        "urls_collected": len(urls),
+        "urls_scanned": len(files),
+        "urls_failed": failures,
+        "summary": summary,
+        "files": files,
+    }
+
+    if args.json:
+        emit_json(report)
+    else:
+        print_human_report(
+            files,
+            summary,
+            extra_header={
+                "Sitemap": sitemap_url,
+                "URLs collected": str(len(urls)),
+                "URLs scanned": str(len(files)),
+                "URLs failed": str(len(failures)),
+            },
+        )
+        for failure in failures:
+            print(f"  [error] {failure['url']}: {failure['error']}")
+
+    return 1 if summary["actionable_files"] else 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

skills/remove-ai-marks/scripts/common.py

@@ -177,6 +177,96 @@ def emit_json(data: Any) -> None:
     sys.stdout.write("\n")
 
 
+CONFIDENCE_LEVELS = (
+    "confirmed",
+    "probable",
+    "informational",
+    "likely_false_positive",
+)
+
+
+def classify_finding_confidence(finding: str) -> str:
+    """Classify a scanner finding by confidence.
+
+    The four buckets are a heuristic mapping of *how strong* a finding is:
+
+    - confirmed: a recognized provenance structure (C2PA/JUMBF manifest, or a
+      parsed field such as digitalSourceType / trainedAlgorithmicMedia).
+    - probable: an AI/vendor marker found inside a recognized metadata
+      structure, but not a fully parsed provenance claim.
+    - informational: context-only notes (CMS generators, presence of an XMP
+      packet or customXml parts, unsupported/partial inspection).
+    - likely_false_positive: raw whole-file byte scans that can collide with
+      compressed image/stream data.
+
+    The mapping is intentionally conservative; a scanner finding is a signal,
+    not a verdict.
+    """
+    t = finding.lower()
+
+    if any(
+        s in t
+        for s in (
+            "c2patool reports",
+            "c2pa-related manifest",
+            "png chunk c2",
+            "png chunk cabx",
+            "png chunk jumb",
+            "png chunk jumd",
+            "jpeg app11 segment",
+            "digital_source_type",
+            "digitalsourcetype",
+            "trainedalgorithmicmedia",
+            "compositewithtrainedalgorithmicmedia",
+            "softwareagent",
+        )
+    ):
+        return "confirmed"
+
+    if t.startswith("info:") or any(
+        s in t
+        for s in (
+            "cms generator",
+            "customxml parts",
+            "xmp packet present",
+            "unsupported",
+            "not fully inspected",
+            "format not",
+            "svg <metadata> present",
+            "not a valid",
+            "truncated chunk",
+            "bad segment length",
+            "svg decode note",
+        )
+    ):
+        return "informational"
+
+    if "byte-scan" in t:
+        return "likely_false_positive"
+
+    if any(
+        s in t
+        for s in (
+            "ai:",
+            "marker:",
+            "meta:",
+            "frontmatter",
+            "json-ld",
+            "attr:",
+            "png ",
+            "jpeg app",
+            "exif",
+            "xmp",
+            "interesting",
+            "pdf-structured",
+            "layer-a",
+        )
+    ):
+        return "probable"
+
+    return "informational"
+
+
 def cleaned_path(src: Path, suffix: str = ".cleaned") -> Path:
     """path/to/file.ext -> path/to/file.cleaned.ext"""
     return src.with_name(f"{src.stem}{suffix}{src.suffix}")

skills/remove-ai-marks/scripts/container_meta.py

@@ -13,7 +13,7 @@ import zipfile
 from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Any
-from common import safe_arg, safe_write_bytes, safe_write_text, subprocess_preexec_fn, which
+from common import classify_finding_confidence, safe_arg, safe_write_bytes, safe_write_text, subprocess_preexec_fn, which
 from image_meta import AI_META_HINTS, C2PA_MARKERS, run_optional_tools
 
 # Frontmatter / meta keys that often carry AI provenance
@@ -75,6 +75,9 @@ class ContainerInspectReport:
             "has_c2pa": self.has_c2pa,
             "has_ai_metadata": self.has_ai_metadata,
             "findings": self.findings,
+            "findings_confidence": [
+                classify_finding_confidence(f) for f in self.findings
+            ],
             "tools": self.tools,
             "details": self.details,
             "notes": self.notes,

skills/remove-ai-marks/scripts/image_meta.py

@@ -13,7 +13,7 @@ from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Any
 
-from common import safe_arg, safe_write_bytes, subprocess_preexec_fn, which
+from common import classify_finding_confidence, safe_arg, safe_write_bytes, subprocess_preexec_fn, which
 
 SCRIPTS_DIR = Path(__file__).resolve().parent
 
@@ -72,6 +72,9 @@ class ImageInspectReport:
             "has_c2pa": self.has_c2pa,
             "has_ai_metadata": self.has_ai_metadata,
             "findings": self.findings,
+            "findings_confidence": [
+                classify_finding_confidence(f) for f in self.findings
+            ],
             "tools": self.tools,
             "synthid": self.synthid,
             "notes": self.notes,

skills/remove-ai-marks/scripts/inspect_file.py

@@ -9,7 +9,7 @@ from pathlib import Path
 
 sys.path.insert(0, str(Path(__file__).resolve().parent))
 
-from common import MAX_INPUT_BYTES, emit_json, eprint, read_text_input  # noqa: E402
+from common import MAX_INPUT_BYTES, classify_finding_confidence, emit_json, eprint, read_text_input  # noqa: E402
 from container_meta import detect_container_format, inspect_container  # noqa: E402
 from image_meta import detect_format as detect_image_format  # noqa: E402
 from image_meta import inspect_image  # noqa: E402
@@ -82,7 +82,7 @@ def main() -> int:
             print(f"C2PA: {report.has_c2pa}")
             print(f"AI metadata: {report.has_ai_metadata}")
             for f in report.findings:
-                print(f"  - {f}")
+                print(f"  - [{classify_finding_confidence(f)}] {f}")
         return 0 if not (report.has_c2pa or report.has_ai_metadata) else 1
 
     report = inspect_container(args.path)
@@ -95,7 +95,7 @@ def main() -> int:
         print(f"C2PA: {report.has_c2pa}")
         print(f"AI metadata: {report.has_ai_metadata}")
         for f in report.findings:
-            print(f"  - {f}")
+            print(f"  - [{classify_finding_confidence(f)}] {f}")
     return 0 if not (report.has_c2pa or report.has_ai_metadata) else 1
 
 

skills/remove-ai-marks/scripts/inspect_image.py

@@ -9,7 +9,7 @@ from pathlib import Path
 
 sys.path.insert(0, str(Path(__file__).resolve().parent))
 
-from common import emit_json  # noqa: E402
+from common import classify_finding_confidence, emit_json  # noqa: E402
 from image_meta import inspect_image  # noqa: E402
 
 
@@ -40,7 +40,7 @@ def main() -> int:
         if report.findings:
             print("Findings:")
             for f in report.findings:
-                print(f"  - {f}")
+                print(f"  - [{classify_finding_confidence(f)}] {f}")
         ct = report.tools.get("c2patool") or {}
         print(f"c2patool: {'yes' if ct.get('available') else 'no'}")
         et = report.tools.get("exiftool") or {}

skills/remove-ai-marks/scripts/text_unicode.py

@@ -277,6 +277,11 @@ def _char_label(ch: str) -> str:
     return f"U+{cp:04X} {name} ({cat})"
 
 
+def _hit_confidence(kind: str) -> str:
+    """Layer A hits are edit-based carriers; space homoglyphs are weaker context."""
+    return "informational" if kind == "space" else "probable"
+
+
 @dataclass
 class CharHit:
     codepoint: int
@@ -304,6 +309,7 @@ class TextInspectReport:
                     "label": h.label,
                     "count": h.count,
                     "kind": h.kind,
+                    "confidence": _hit_confidence(h.kind),
                     "sample_offsets": h.samples[:10],
                 }
                 for h in self.hits
@@ -434,7 +440,10 @@ def human_report(report: TextInspectReport) -> str:
     if report.hits:
         lines.append("Hits:")
         for h in report.hits:
-            lines.append(f"  [{h.kind}] {h.label} x{h.count} @ {h.samples[:5]}")
+            lines.append(
+                f"  [{h.kind}/{_hit_confidence(h.kind)}] "
+                f"{h.label} x{h.count} @ {h.samples[:5]}"
+            )
     for n in report.notes:
         lines.append(f"Note: {n}")
     return "\n".join(lines)

tests/test_audit.py

@@ -0,0 +1,169 @@
+"""Tests for finding confidence and aggregate audit scripts."""
+
+from __future__ import annotations
+
+import gzip
+import struct
+import sys
+import zlib
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+SCRIPTS = ROOT / "skills" / "remove-ai-marks" / "scripts"
+sys.path.insert(0, str(SCRIPTS))
+
+from audit_lib import aggregate, is_actionable, scan_file  # noqa: E402
+from audit_website import guess_kind, inspect_remote, parse_sitemap  # noqa: E402
+from common import classify_finding_confidence  # noqa: E402
+from container_meta import inspect_container  # noqa: E402
+from text_unicode import inspect_text  # noqa: E402
+
+
+def test_classify_finding_confidence_buckets():
+    cases = {
+        "c2patool reports a C2PA-related manifest": "confirmed",
+        "PNG chunk caBX (possible C2PA container)": "confirmed",
+        "JPEG APP11 segment (JUMBF/C2PA common)": "confirmed",
+        "pdf-structured:ai:digitalSourceType": "confirmed",
+        "pdf-structured:ai:AIGC": "probable",
+        "PNG tEXt: c2pa, contentcredentials": "probable",
+        "frontmatter key: generator": "probable",
+        "info: cms generator: <meta name=\"generator\" content=\"WordPress\">": "informational",
+        "customXml parts: 1": "informational",
+        "unsupported container: woff2": "informational",
+        "svg <metadata> present": "informational",
+        "byte-scan C2PA markers: c2pa": "likely_false_positive",
+    }
+    for finding, expected in cases.items():
+        assert classify_finding_confidence(finding) == expected, finding
+
+
+def test_text_report_hit_confidence():
+    report = inspect_text("a\u200bb")
+    assert report.to_dict()["hits"][0]["confidence"] == "probable"
+
+    # Exotic space homoglyphs are weaker context.
+    report = inspect_text("a\u2003b")
+    assert report.to_dict()["hits"][0]["confidence"] == "informational"
+
+
+def test_container_report_findings_confidence(tmp_path: Path):
+    src = tmp_path / "cms.html"
+    src.write_text(
+        '<html><head><meta name="generator" content="WordPress 6.0"></head></html>',
+        encoding="utf-8",
+    )
+    report = inspect_container(src)
+    assert report.to_dict()["findings_confidence"] == ["informational"]
+
+
+def _png_chunk(ctype: bytes, payload: bytes) -> bytes:
+    crc = zlib.crc32(ctype)
+    crc = zlib.crc32(payload, crc) & 0xFFFFFFFF
+    return struct.pack(">I", len(payload)) + ctype + payload + struct.pack(">I", crc)
+
+
+def _minimal_png_with_text() -> bytes:
+    sig = b"\x89PNG\r\n\x1a\n"
+    ihdr = struct.pack(">IIBBBBB", 1, 1, 8, 2, 0, 0, 0)
+    idat = zlib.compress(b"\x00\x00\x00")
+    text = b"Comment\x00c2pa contentcredentials"
+    return (
+        sig
+        + _png_chunk(b"IHDR", ihdr)
+        + _png_chunk(b"tEXt", text)
+        + _png_chunk(b"IDAT", idat)
+        + _png_chunk(b"IEND", b"")
+    )
+
+
+def test_image_report_findings_confidence(tmp_path: Path):
+    from image_meta import inspect_image
+
+    src = tmp_path / "t.png"
+    src.write_bytes(_minimal_png_with_text())
+    report = inspect_image(src)
+    d = report.to_dict()
+    assert "findings_confidence" in d
+    assert any(c in ("probable", "confirmed") for c in d["findings_confidence"])
+
+
+def test_scan_file_text_and_html(tmp_path: Path):
+    text = tmp_path / "a.txt"
+    text.write_text("Hello\u200bWorld\n", encoding="utf-8")
+    item = scan_file(text)
+    assert item["kind"] == "text"
+    assert is_actionable(item)
+
+    html = tmp_path / "b.html"
+    html.write_text(
+        '<html><head><meta name="generator" content="WordPress"></head><body>ok</body></html>',
+        encoding="utf-8",
+    )
+    item = scan_file(html)
+    assert item["kind"] == "html"
+    assert not is_actionable(item)
+
+
+def test_aggregate_summary():
+    files = [
+        {
+            "path": "a.txt",
+            "kind": "text",
+            "has_c2pa": False,
+            "has_ai_metadata": False,
+            "suspicious_total": 1,
+            "findings": ["layer-a [zwj_family] U+200B ZERO WIDTH SPACE (Cf) x1"],
+            "confidence": ["probable"],
+        },
+        {
+            "path": "b.html",
+            "kind": "html",
+            "has_c2pa": False,
+            "has_ai_metadata": False,
+            "suspicious_total": 0,
+            "findings": ["info: cms generator: <meta>"],
+            "confidence": ["informational"],
+        },
+    ]
+    summary = aggregate(files)
+    assert summary["total"] == 2
+    assert summary["actionable_files"] == 1
+    assert summary["findings_by_confidence"]["probable"] == 1
+    assert summary["findings_by_confidence"]["informational"] == 1
+    assert summary["with_suspicious_text"] == 1
+
+
+def test_parse_sitemap_urlset_and_index():
+    data = (
+        b'<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">'
+        b'<url><loc>https://x.test/a</loc></url></urlset>'
+    )
+    kind, urls = parse_sitemap(data)
+    assert kind == "urlset"
+    assert urls == ["https://x.test/a"]
+
+    idx = (
+        b'<sitemapindex><sitemap><loc>https://x.test/s1.xml</loc></sitemap>'
+        b"</sitemapindex>"
+    )
+    assert parse_sitemap(idx) == ("sitemapindex", ["https://x.test/s1.xml"])
+
+    assert parse_sitemap(gzip.compress(data))[0] == "urlset"
+
+
+def test_guess_kind():
+    assert guess_kind("https://x.test/a", b"<html>x</html>", "text/html") == "html"
+    assert guess_kind("https://x.test/a.png", b"\x89PNG\r\n\x1a\n", None) == "png"
+    assert guess_kind("https://x.test/a", b"%PDF-1.4", None) == "pdf"
+
+
+def test_inspect_remote_html_cms_informational():
+    result = inspect_remote(
+        "https://x.test/page",
+        b'<html><head><meta name="generator" content="WordPress"></head><body>hi</body></html>',
+        "text/html",
+    )
+    assert result["kind"] == "html"
+    assert result["confidence"] == ["informational"]
+    assert not is_actionable(result)