From 396c83dbae360cf852ef8a2b5190c6ef4cd3b99e Mon Sep 17 00:00:00 2001 From: "Guillaume Meyer (The Opinionated Man)" <1385518+guillaumemeyer@users.noreply.github.com> Date: Thu, 13 Aug 2026 08:25:57 -0700 Subject: [PATCH] feat: add finding confidence and aggregate audits - classify findings as confirmed/probable/informational/likely_false_positive - expose confidence in text/image/container JSON and human reports - add audit_dir.py and audit_website.py for aggregate reports - document the confidence taxonomy and audit commands in SKILL.md Closes #13 --- skills/remove-ai-marks/SKILL.md | 30 +- skills/remove-ai-marks/scripts/audit_dir.py | 108 +++++++ skills/remove-ai-marks/scripts/audit_lib.py | 150 ++++++++++ .../remove-ai-marks/scripts/audit_website.py | 265 ++++++++++++++++++ skills/remove-ai-marks/scripts/common.py | 90 ++++++ .../remove-ai-marks/scripts/container_meta.py | 5 +- skills/remove-ai-marks/scripts/image_meta.py | 5 +- .../remove-ai-marks/scripts/inspect_file.py | 6 +- .../remove-ai-marks/scripts/inspect_image.py | 4 +- .../remove-ai-marks/scripts/text_unicode.py | 11 +- tests/test_audit.py | 169 +++++++++++ 11 files changed, 834 insertions(+), 9 deletions(-) create mode 100644 skills/remove-ai-marks/scripts/audit_dir.py create mode 100644 skills/remove-ai-marks/scripts/audit_lib.py create mode 100644 skills/remove-ai-marks/scripts/audit_website.py create mode 100644 tests/test_audit.py diff --git a/skills/remove-ai-marks/SKILL.md b/skills/remove-ai-marks/SKILL.md index ca6abcd..bb34cc7 100644 --- a/skills/remove-ai-marks/SKILL.md +++ b/skills/remove-ai-marks/SKILL.md @@ -33,6 +33,8 @@ python3 "$SCRIPTS/clean_text.py" ... python3 "$SCRIPTS/inspect_image.py" ... python3 "$SCRIPTS/clean_image.py" ... python3 "$SCRIPTS/rewrite_text.py" ... +python3 "$SCRIPTS/audit_dir.py" ... +python3 "$SCRIPTS/audit_website.py" ... ``` ## Ethics @@ -50,7 +52,8 @@ Intended for **your own** content (privacy, hygiene, research). Do not market re | `.md` / `.html` | container clean (frontmatter/meta) + Layer A | | `.png` / `.jpg` / `.jpeg` | image metadata strip | | `.svg` / `.pdf` / `.docx` / `.odt` | container metadata strip | -| Directory | batch each matching file | +| Directory | aggregate report with `audit_dir.py` | +| Website / sitemap | aggregate report with `audit_website.py` | | Mixed | run unified `inspect_file` / `clean_file` | ### 2. Inspect first @@ -70,6 +73,27 @@ external reverse-SynthID scorer. That is **detection only**, not removal. Bootstrap the external checkout with `scripts/setup_synthid.sh`, or build a local image with `make docker-synthid-build`. +### Aggregate audits and confidence + +Findings are classified as **confirmed**, **probable**, **informational**, or +**likely_false_positive**. Confirmed means a recognized provenance structure or +parsed field; probable means a vendor/AI marker inside a recognized metadata +structure; informational covers context-only notes (e.g. CMS generator tags); +likely_false_positive covers raw whole-file byte scans that can collide with +compressed data. + +Audit a whole tree or a live sitemap for an aggregate report: + +```bash +python3 "$SCRIPTS/audit_dir.py" DIR --json +python3 "$SCRIPTS/audit_website.py" --sitemap https://example.com/sitemap.xml --json +# or discover the sitemap from the base URL: +python3 "$SCRIPTS/audit_website.py" --base https://example.com --json +``` + +`audit_website.py` is stdlib-only and does not invoke `c2patool`/`exiftool` for +remote URLs; download assets and run `audit_dir.py` locally for those. + ### 3. Deterministic clean (always for matching inputs) **Text — Layer A:** @@ -225,4 +249,8 @@ python3 scripts/rewrite_text.py notes.md --backend print-prompt --strength parap # Images only python3 scripts/inspect_image.py shot.png python3 scripts/clean_image.py shot.png -o shot.cleaned.png + +# Aggregate audits +python3 scripts/audit_dir.py ./src --json +python3 scripts/audit_website.py --sitemap https://example.com/sitemap.xml --json ``` diff --git a/skills/remove-ai-marks/scripts/audit_dir.py b/skills/remove-ai-marks/scripts/audit_dir.py new file mode 100644 index 0000000..eb760e4 --- /dev/null +++ b/skills/remove-ai-marks/scripts/audit_dir.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +"""Aggregate AI-provenance audit over a directory tree. + +Recursively inspects supported text/image/container files and emits one +summary plus a per-file finding list with confidence classifications. +""" + +from __future__ import annotations + +import argparse +import os +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from audit_lib import aggregate, print_human_report, scan_file # noqa: E402 +from common import MAX_INPUT_BYTES, emit_json, eprint # noqa: E402 + +DEFAULT_SKIP_DIRS = { + ".git", + ".hg", + ".svn", + "node_modules", + "__pycache__", + ".venv", + "venv", + ".tox", + ".mypy_cache", + ".pytest_cache", + "dist", + "build", + ".next", + "target", + ".cache", +} + + +def walk_files(root: Path, skip_dirs: set[str]): + for dirpath, dirnames, filenames in os.walk(root): + dirnames[:] = sorted( + d for d in dirnames if d not in skip_dirs and not d.startswith(".") + ) + for fn in sorted(filenames): + path = Path(dirpath) / fn + if path.is_file(): + yield path + + +def main() -> int: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("path", type=Path, help="Directory to audit recursively") + p.add_argument("--json", action="store_true", help="Emit a JSON report") + p.add_argument( + "--skip", + default="", + help="Comma-separated extra directory names to skip", + ) + args = p.parse_args() + + root = args.path + if not root.is_dir(): + eprint(f"not a directory: {root}") + return 2 + + skip_dirs = set(DEFAULT_SKIP_DIRS) + for name in args.skip.split(","): + name = name.strip() + if name: + skip_dirs.add(name) + + files = [] + skipped = [] + for path in walk_files(root, skip_dirs): + try: + if path.stat().st_size > MAX_INPUT_BYTES: + skipped.append({"path": str(path), "reason": "too large"}) + continue + files.append(scan_file(path)) + except Exception as e: # keep the audit going on one bad file + skipped.append({"path": str(path), "reason": str(e)}) + + summary = aggregate(files) + report = { + "root": str(root), + "files_scanned": len(files), + "files_skipped": skipped, + "summary": summary, + "files": files, + } + + if args.json: + emit_json(report) + else: + print_human_report( + files, + summary, + extra_header={ + "Root": report["root"], + "Files skipped": str(len(skipped)), + }, + ) + + return 1 if summary["actionable_files"] else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/remove-ai-marks/scripts/audit_lib.py b/skills/remove-ai-marks/scripts/audit_lib.py new file mode 100644 index 0000000..5e75cc9 --- /dev/null +++ b/skills/remove-ai-marks/scripts/audit_lib.py @@ -0,0 +1,150 @@ +"""Shared helpers for aggregate directory and website audits. + +Both audits normalize every file/URL into the same per-item dict so a single +aggregate summary can be computed and rendered consistently. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from common import CONFIDENCE_LEVELS, classify_finding_confidence +from container_meta import inspect_container +from image_meta import inspect_image +from inspect_file import classify +from text_unicode import inspect_text + + +def text_hit_confidence(kind: str) -> str: + """Layer A space homoglyphs are weaker context than invisible carriers.""" + return "informational" if kind == "space" else "probable" + + +def text_findings(report: Any) -> tuple[list[str], list[str], int]: + """Flatten a TextInspectReport into finding strings + confidence lists.""" + findings: list[str] = [] + confidences: list[str] = [] + for h in report.hits: + conf = text_hit_confidence(h.kind) + findings.append(f"layer-a [{h.kind}] {h.label} x{h.count}") + confidences.append(conf) + return findings, confidences, report.suspicious_total + + +def scan_file(path: Path, display_name: str | None = None) -> dict[str, Any]: + """Inspect one local file and return a normalized audit item.""" + name = display_name or str(path) + kind = classify(path) + + if kind == "text": + try: + text = path.read_text(encoding="utf-8", errors="surrogateescape") + except OSError as e: + return {"path": name, "kind": "text", "error": str(e)} + report = inspect_text(text) + findings, confidences, suspicious = text_findings(report) + return { + "path": name, + "kind": "text", + "has_c2pa": False, + "has_ai_metadata": False, + "suspicious_total": suspicious, + "findings": findings, + "confidence": confidences, + "notes": report.notes, + } + + if kind == "image": + report = inspect_image(path) + return { + "path": name, + "kind": report.format, + "has_c2pa": report.has_c2pa, + "has_ai_metadata": report.has_ai_metadata, + "suspicious_total": 0, + "findings": report.findings, + "confidence": [classify_finding_confidence(f) for f in report.findings], + "notes": report.notes, + } + + report = inspect_container(path) + findings = list(report.findings) + confidences = [classify_finding_confidence(f) for f in report.findings] + suspicious = 0 + + # Text-bearing containers also get a Layer A scan of their visible text, + # mirroring the skill's "container + Layer A" workflow. + if report.format in ("html", "markdown"): + try: + text = path.read_text(encoding="utf-8", errors="surrogateescape") + except OSError: + text = "" + if text: + t_report = inspect_text(text) + t_findings, t_confidences, t_suspicious = text_findings(t_report) + findings.extend(t_findings) + confidences.extend(t_confidences) + suspicious = t_suspicious + + return { + "path": name, + "kind": report.format, + "has_c2pa": report.has_c2pa, + "has_ai_metadata": report.has_ai_metadata, + "suspicious_total": suspicious, + "findings": findings, + "confidence": confidences, + "notes": report.notes, + } + + +def is_actionable(item: dict[str, Any]) -> bool: + """A file is actionable when it has a confirmed/probable finding or C2PA.""" + if item.get("has_c2pa"): + return True + return any(c in ("confirmed", "probable") for c in item.get("confidence", [])) + + +def aggregate(files: list[dict[str, Any]]) -> dict[str, Any]: + """Build the summary block shared by directory and website audits.""" + summary = { + "total": len(files), + "by_kind": {}, + "with_c2pa": 0, + "with_ai_metadata": 0, + "with_suspicious_text": 0, + "actionable_files": 0, + "findings_by_confidence": {c: 0 for c in CONFIDENCE_LEVELS}, + } + for item in files: + kind = str(item.get("kind") or "error") + summary["by_kind"][kind] = summary["by_kind"].get(kind, 0) + 1 + if item.get("has_c2pa"): + summary["with_c2pa"] += 1 + if item.get("has_ai_metadata"): + summary["with_ai_metadata"] += 1 + if item.get("suspicious_total", 0) > 0: + summary["with_suspicious_text"] += 1 + for c in item.get("confidence", []): + if c in summary["findings_by_confidence"]: + summary["findings_by_confidence"][c] += 1 + if is_actionable(item): + summary["actionable_files"] += 1 + return summary + + +def print_human_report(files: list[dict[str, Any]], summary: dict[str, Any], extra_header: dict[str, Any] | None = None) -> None: + """Shared plain-text rendering for audit scripts.""" + for key, value in (extra_header or {}).items(): + print(f"{key}: {value}") + print(f"Files scanned: {summary['total']}") + print(f"By kind: {summary['by_kind']}") + print(f"With C2PA: {summary['with_c2pa']}") + print(f"With AI metadata: {summary['with_ai_metadata']}") + print(f"With suspicious text: {summary['with_suspicious_text']}") + print(f"Actionable files: {summary['actionable_files']}") + print(f"Findings by confidence: {summary['findings_by_confidence']}") + for item in files: + for msg, conf in zip(item.get("findings", []), item.get("confidence", [])): + print(f" [{conf}] {item['path']}: {msg}") diff --git a/skills/remove-ai-marks/scripts/audit_website.py b/skills/remove-ai-marks/scripts/audit_website.py new file mode 100644 index 0000000..c601669 --- /dev/null +++ b/skills/remove-ai-marks/scripts/audit_website.py @@ -0,0 +1,265 @@ +#!/usr/bin/env python3 +"""Aggregate AI-provenance audit over the URLs listed in a sitemap. + +Stdlib-only: downloads each URL, classifies it by content type/suffix/magic, +and runs the same deterministic text/image/container inspections used by the +local audit. Optional external tools (c2patool/exiftool) are not invoked for +remote URLs; download the assets and run audit_dir.py locally for those. +""" + +from __future__ import annotations + +import argparse +import gzip +import sys +import tempfile +import urllib.error +import urllib.parse +import urllib.request +import xml.etree.ElementTree as ET +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from audit_lib import aggregate, print_human_report, scan_file # noqa: E402 +from common import emit_json, eprint # noqa: E402 + +DEFAULT_MAX_BYTES = 4 << 20 +DEFAULT_TIMEOUT = 15 +DEFAULT_MAX_PAGES = 200 +USER_AGENT = "remove-ai-marks-audit/1.0" + +_EXT_FOR_KIND = { + "png": ".png", + "jpeg": ".jpg", + "svg": ".svg", + "pdf": ".pdf", + "docx": ".docx", + "odt": ".odt", + "html": ".html", + "markdown": ".md", + "text": ".txt", +} + + +def _local(tag: str) -> str: + return tag.rsplit("}", 1)[-1] + + +def parse_sitemap(data: bytes) -> tuple[str, list[str]]: + """Parse a (possibly gzip-compressed) sitemap into (kind, urls).""" + if data[:2] == b"\x1f\x8b": + data = gzip.decompress(data) + root = ET.fromstring(data) + kind = _local(root.tag) + urls = [] + for el in root.iter(): + if _local(el.tag) == "loc" and el.text: + urls.append(el.text.strip()) + return kind, urls + + +def guess_kind(url: str, data: bytes, content_type: str | None = None) -> str: + """Classify a downloaded URL from headers, suffix, then magic bytes.""" + ct = (content_type or "").lower().split(";")[0].strip() + if "html" in ct: + return "html" + if ct == "image/png": + return "png" + if ct == "image/jpeg": + return "jpeg" + if "svg" in ct: + return "svg" + if ct == "application/pdf": + return "pdf" + if "wordprocessingml" in ct: + return "docx" + if "opendocument.text" in ct: + return "odt" + if "markdown" in ct: + return "markdown" + if ct == "text/plain": + return "text" + + path = urllib.parse.urlparse(url).path.lower() + for ext, kind in ( + (".png", "png"), + (".jpg", "jpeg"), + (".jpeg", "jpeg"), + (".svg", "svg"), + (".pdf", "pdf"), + (".docx", "docx"), + (".odt", "odt"), + (".html", "html"), + (".htm", "html"), + (".md", "markdown"), + (".markdown", "markdown"), + (".txt", "text"), + ): + if path.endswith(ext): + return kind + + if data.startswith(b"\x89PNG"): + return "png" + if data.startswith(b"\xff\xd8"): + return "jpeg" + if data.startswith(b"%PDF"): + return "pdf" + if data[:100].lstrip().startswith(b"<") and b"svg" in data[:500].lower(): + return "svg" + if b" tuple[bytes, str | None]: + """Fetch *url* with a byte cap; returns (body, content_type).""" + req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) + with urllib.request.urlopen(req, timeout=timeout) as resp: + content_type = resp.headers.get("Content-Type") + chunks = [] + total = 0 + while True: + chunk = resp.read(1 << 16) + if not chunk: + break + total += len(chunk) + if total > max_bytes: + raise ValueError(f"exceeds {max_bytes} bytes") + chunks.append(chunk) + return b"".join(chunks), content_type + + +def inspect_remote(url: str, data: bytes, content_type: str | None = None) -> dict: + """Inspect downloaded bytes using the local scan_file pipeline.""" + kind = guess_kind(url, data, content_type) + ext = _EXT_FOR_KIND.get(kind, ".bin") + with tempfile.TemporaryDirectory() as td: + tmp = Path(td) / f"asset{ext}" + tmp.write_bytes(data) + result = scan_file(tmp, display_name=url) + result["kind"] = kind + return result + + +def discover_sitemap(base_url: str, timeout: int) -> str | None: + """Find a sitemap for *base_url* via /sitemap.xml then /robots.txt.""" + base = base_url.rstrip("/") + for candidate in (f"{base}/sitemap.xml", f"{base}/sitemap_index.xml"): + try: + data, _ = fetch(candidate, timeout, DEFAULT_MAX_BYTES) + parse_sitemap(data) + return candidate + except Exception: + continue + try: + data, _ = fetch(f"{base}/robots.txt", timeout, 1 << 20) + text = data.decode("utf-8", errors="replace") + for line in text.splitlines(): + if line.lower().startswith("sitemap:"): + return line.split(":", 1)[1].strip() + except Exception: + pass + return None + + +def collect_urls(sitemap_url: str, timeout: int, max_pages: int) -> list[str]: + """Collect page/asset URLs from a sitemap, following nested indexes.""" + urls: list[str] = [] + seen: set[str] = set() + + def _recurse(url: str, depth: int = 0) -> None: + if len(urls) >= max_pages or depth > 3: + return + data, _ = fetch(url, timeout, DEFAULT_MAX_BYTES) + kind, locs = parse_sitemap(data) + if kind == "sitemapindex": + for loc in locs: + if loc not in seen: + seen.add(loc) + _recurse(loc, depth + 1) + else: + for loc in locs: + if loc not in seen: + seen.add(loc) + urls.append(loc) + + _recurse(sitemap_url) + return urls + + +def main() -> int: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--sitemap", help="Sitemap URL to audit") + p.add_argument("--base", help="Base URL; discover the sitemap automatically") + p.add_argument("--max-pages", type=int, default=DEFAULT_MAX_PAGES) + p.add_argument("--timeout", type=int, default=DEFAULT_TIMEOUT) + p.add_argument("--max-bytes", type=int, default=DEFAULT_MAX_BYTES) + p.add_argument("--json", action="store_true") + args = p.parse_args() + + if not args.sitemap and not args.base: + eprint("provide --sitemap URL or --base URL") + return 2 + + sitemap_url = args.sitemap + if not sitemap_url: + sitemap_url = discover_sitemap(args.base, args.timeout) + if not sitemap_url: + eprint(f"no sitemap found for {args.base}") + return 2 + + try: + urls = collect_urls(sitemap_url, args.timeout, args.max_pages) + except Exception as e: + eprint(f"could not collect URLs from {sitemap_url}: {e}") + return 2 + if not urls: + eprint("no URLs collected from sitemap") + return 2 + + files = [] + failures = [] + for url in urls[: args.max_pages]: + try: + data, content_type = fetch(url, args.timeout, args.max_bytes) + except Exception as e: + failures.append({"url": url, "error": str(e)}) + continue + try: + files.append(inspect_remote(url, data, content_type)) + except Exception as e: + failures.append({"url": url, "error": f"inspect failed: {e}"}) + + summary = aggregate(files) + report = { + "sitemap": sitemap_url, + "base": args.base, + "urls_collected": len(urls), + "urls_scanned": len(files), + "urls_failed": failures, + "summary": summary, + "files": files, + } + + if args.json: + emit_json(report) + else: + print_human_report( + files, + summary, + extra_header={ + "Sitemap": sitemap_url, + "URLs collected": str(len(urls)), + "URLs scanned": str(len(files)), + "URLs failed": str(len(failures)), + }, + ) + for failure in failures: + print(f" [error] {failure['url']}: {failure['error']}") + + return 1 if summary["actionable_files"] else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/remove-ai-marks/scripts/common.py b/skills/remove-ai-marks/scripts/common.py index 27f1784..94b0693 100755 --- a/skills/remove-ai-marks/scripts/common.py +++ b/skills/remove-ai-marks/scripts/common.py @@ -177,6 +177,96 @@ def emit_json(data: Any) -> None: sys.stdout.write("\n") +CONFIDENCE_LEVELS = ( + "confirmed", + "probable", + "informational", + "likely_false_positive", +) + + +def classify_finding_confidence(finding: str) -> str: + """Classify a scanner finding by confidence. + + The four buckets are a heuristic mapping of *how strong* a finding is: + + - confirmed: a recognized provenance structure (C2PA/JUMBF manifest, or a + parsed field such as digitalSourceType / trainedAlgorithmicMedia). + - probable: an AI/vendor marker found inside a recognized metadata + structure, but not a fully parsed provenance claim. + - informational: context-only notes (CMS generators, presence of an XMP + packet or customXml parts, unsupported/partial inspection). + - likely_false_positive: raw whole-file byte scans that can collide with + compressed image/stream data. + + The mapping is intentionally conservative; a scanner finding is a signal, + not a verdict. + """ + t = finding.lower() + + if any( + s in t + for s in ( + "c2patool reports", + "c2pa-related manifest", + "png chunk c2", + "png chunk cabx", + "png chunk jumb", + "png chunk jumd", + "jpeg app11 segment", + "digital_source_type", + "digitalsourcetype", + "trainedalgorithmicmedia", + "compositewithtrainedalgorithmicmedia", + "softwareagent", + ) + ): + return "confirmed" + + if t.startswith("info:") or any( + s in t + for s in ( + "cms generator", + "customxml parts", + "xmp packet present", + "unsupported", + "not fully inspected", + "format not", + "svg present", + "not a valid", + "truncated chunk", + "bad segment length", + "svg decode note", + ) + ): + return "informational" + + if "byte-scan" in t: + return "likely_false_positive" + + if any( + s in t + for s in ( + "ai:", + "marker:", + "meta:", + "frontmatter", + "json-ld", + "attr:", + "png ", + "jpeg app", + "exif", + "xmp", + "interesting", + "pdf-structured", + "layer-a", + ) + ): + return "probable" + + return "informational" + + def cleaned_path(src: Path, suffix: str = ".cleaned") -> Path: """path/to/file.ext -> path/to/file.cleaned.ext""" return src.with_name(f"{src.stem}{suffix}{src.suffix}") diff --git a/skills/remove-ai-marks/scripts/container_meta.py b/skills/remove-ai-marks/scripts/container_meta.py index 19d2724..e9644c7 100644 --- a/skills/remove-ai-marks/scripts/container_meta.py +++ b/skills/remove-ai-marks/scripts/container_meta.py @@ -13,7 +13,7 @@ import zipfile from dataclasses import dataclass, field from pathlib import Path from typing import Any -from common import safe_arg, safe_write_bytes, safe_write_text, subprocess_preexec_fn, which +from common import classify_finding_confidence, safe_arg, safe_write_bytes, safe_write_text, subprocess_preexec_fn, which from image_meta import AI_META_HINTS, C2PA_MARKERS, run_optional_tools # Frontmatter / meta keys that often carry AI provenance @@ -75,6 +75,9 @@ class ContainerInspectReport: "has_c2pa": self.has_c2pa, "has_ai_metadata": self.has_ai_metadata, "findings": self.findings, + "findings_confidence": [ + classify_finding_confidence(f) for f in self.findings + ], "tools": self.tools, "details": self.details, "notes": self.notes, diff --git a/skills/remove-ai-marks/scripts/image_meta.py b/skills/remove-ai-marks/scripts/image_meta.py index 9941876..f6a550f 100755 --- a/skills/remove-ai-marks/scripts/image_meta.py +++ b/skills/remove-ai-marks/scripts/image_meta.py @@ -13,7 +13,7 @@ from dataclasses import dataclass, field from pathlib import Path from typing import Any -from common import safe_arg, safe_write_bytes, subprocess_preexec_fn, which +from common import classify_finding_confidence, safe_arg, safe_write_bytes, subprocess_preexec_fn, which SCRIPTS_DIR = Path(__file__).resolve().parent @@ -72,6 +72,9 @@ class ImageInspectReport: "has_c2pa": self.has_c2pa, "has_ai_metadata": self.has_ai_metadata, "findings": self.findings, + "findings_confidence": [ + classify_finding_confidence(f) for f in self.findings + ], "tools": self.tools, "synthid": self.synthid, "notes": self.notes, diff --git a/skills/remove-ai-marks/scripts/inspect_file.py b/skills/remove-ai-marks/scripts/inspect_file.py index a6428f5..3b66107 100644 --- a/skills/remove-ai-marks/scripts/inspect_file.py +++ b/skills/remove-ai-marks/scripts/inspect_file.py @@ -9,7 +9,7 @@ from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) -from common import MAX_INPUT_BYTES, emit_json, eprint, read_text_input # noqa: E402 +from common import MAX_INPUT_BYTES, classify_finding_confidence, emit_json, eprint, read_text_input # noqa: E402 from container_meta import detect_container_format, inspect_container # noqa: E402 from image_meta import detect_format as detect_image_format # noqa: E402 from image_meta import inspect_image # noqa: E402 @@ -82,7 +82,7 @@ def main() -> int: print(f"C2PA: {report.has_c2pa}") print(f"AI metadata: {report.has_ai_metadata}") for f in report.findings: - print(f" - {f}") + print(f" - [{classify_finding_confidence(f)}] {f}") return 0 if not (report.has_c2pa or report.has_ai_metadata) else 1 report = inspect_container(args.path) @@ -95,7 +95,7 @@ def main() -> int: print(f"C2PA: {report.has_c2pa}") print(f"AI metadata: {report.has_ai_metadata}") for f in report.findings: - print(f" - {f}") + print(f" - [{classify_finding_confidence(f)}] {f}") return 0 if not (report.has_c2pa or report.has_ai_metadata) else 1 diff --git a/skills/remove-ai-marks/scripts/inspect_image.py b/skills/remove-ai-marks/scripts/inspect_image.py index fd4b7a9..4360412 100755 --- a/skills/remove-ai-marks/scripts/inspect_image.py +++ b/skills/remove-ai-marks/scripts/inspect_image.py @@ -9,7 +9,7 @@ from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) -from common import emit_json # noqa: E402 +from common import classify_finding_confidence, emit_json # noqa: E402 from image_meta import inspect_image # noqa: E402 @@ -40,7 +40,7 @@ def main() -> int: if report.findings: print("Findings:") for f in report.findings: - print(f" - {f}") + print(f" - [{classify_finding_confidence(f)}] {f}") ct = report.tools.get("c2patool") or {} print(f"c2patool: {'yes' if ct.get('available') else 'no'}") et = report.tools.get("exiftool") or {} diff --git a/skills/remove-ai-marks/scripts/text_unicode.py b/skills/remove-ai-marks/scripts/text_unicode.py index c568fae..f530e61 100755 --- a/skills/remove-ai-marks/scripts/text_unicode.py +++ b/skills/remove-ai-marks/scripts/text_unicode.py @@ -277,6 +277,11 @@ def _char_label(ch: str) -> str: return f"U+{cp:04X} {name} ({cat})" +def _hit_confidence(kind: str) -> str: + """Layer A hits are edit-based carriers; space homoglyphs are weaker context.""" + return "informational" if kind == "space" else "probable" + + @dataclass class CharHit: codepoint: int @@ -304,6 +309,7 @@ class TextInspectReport: "label": h.label, "count": h.count, "kind": h.kind, + "confidence": _hit_confidence(h.kind), "sample_offsets": h.samples[:10], } for h in self.hits @@ -434,7 +440,10 @@ def human_report(report: TextInspectReport) -> str: if report.hits: lines.append("Hits:") for h in report.hits: - lines.append(f" [{h.kind}] {h.label} x{h.count} @ {h.samples[:5]}") + lines.append( + f" [{h.kind}/{_hit_confidence(h.kind)}] " + f"{h.label} x{h.count} @ {h.samples[:5]}" + ) for n in report.notes: lines.append(f"Note: {n}") return "\n".join(lines) diff --git a/tests/test_audit.py b/tests/test_audit.py new file mode 100644 index 0000000..80e7cb6 --- /dev/null +++ b/tests/test_audit.py @@ -0,0 +1,169 @@ +"""Tests for finding confidence and aggregate audit scripts.""" + +from __future__ import annotations + +import gzip +import struct +import sys +import zlib +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SCRIPTS = ROOT / "skills" / "remove-ai-marks" / "scripts" +sys.path.insert(0, str(SCRIPTS)) + +from audit_lib import aggregate, is_actionable, scan_file # noqa: E402 +from audit_website import guess_kind, inspect_remote, parse_sitemap # noqa: E402 +from common import classify_finding_confidence # noqa: E402 +from container_meta import inspect_container # noqa: E402 +from text_unicode import inspect_text # noqa: E402 + + +def test_classify_finding_confidence_buckets(): + cases = { + "c2patool reports a C2PA-related manifest": "confirmed", + "PNG chunk caBX (possible C2PA container)": "confirmed", + "JPEG APP11 segment (JUMBF/C2PA common)": "confirmed", + "pdf-structured:ai:digitalSourceType": "confirmed", + "pdf-structured:ai:AIGC": "probable", + "PNG tEXt: c2pa, contentcredentials": "probable", + "frontmatter key: generator": "probable", + "info: cms generator: ": "informational", + "customXml parts: 1": "informational", + "unsupported container: woff2": "informational", + "svg present": "informational", + "byte-scan C2PA markers: c2pa": "likely_false_positive", + } + for finding, expected in cases.items(): + assert classify_finding_confidence(finding) == expected, finding + + +def test_text_report_hit_confidence(): + report = inspect_text("a\u200bb") + assert report.to_dict()["hits"][0]["confidence"] == "probable" + + # Exotic space homoglyphs are weaker context. + report = inspect_text("a\u2003b") + assert report.to_dict()["hits"][0]["confidence"] == "informational" + + +def test_container_report_findings_confidence(tmp_path: Path): + src = tmp_path / "cms.html" + src.write_text( + '', + encoding="utf-8", + ) + report = inspect_container(src) + assert report.to_dict()["findings_confidence"] == ["informational"] + + +def _png_chunk(ctype: bytes, payload: bytes) -> bytes: + crc = zlib.crc32(ctype) + crc = zlib.crc32(payload, crc) & 0xFFFFFFFF + return struct.pack(">I", len(payload)) + ctype + payload + struct.pack(">I", crc) + + +def _minimal_png_with_text() -> bytes: + sig = b"\x89PNG\r\n\x1a\n" + ihdr = struct.pack(">IIBBBBB", 1, 1, 8, 2, 0, 0, 0) + idat = zlib.compress(b"\x00\x00\x00") + text = b"Comment\x00c2pa contentcredentials" + return ( + sig + + _png_chunk(b"IHDR", ihdr) + + _png_chunk(b"tEXt", text) + + _png_chunk(b"IDAT", idat) + + _png_chunk(b"IEND", b"") + ) + + +def test_image_report_findings_confidence(tmp_path: Path): + from image_meta import inspect_image + + src = tmp_path / "t.png" + src.write_bytes(_minimal_png_with_text()) + report = inspect_image(src) + d = report.to_dict() + assert "findings_confidence" in d + assert any(c in ("probable", "confirmed") for c in d["findings_confidence"]) + + +def test_scan_file_text_and_html(tmp_path: Path): + text = tmp_path / "a.txt" + text.write_text("Hello\u200bWorld\n", encoding="utf-8") + item = scan_file(text) + assert item["kind"] == "text" + assert is_actionable(item) + + html = tmp_path / "b.html" + html.write_text( + 'ok', + encoding="utf-8", + ) + item = scan_file(html) + assert item["kind"] == "html" + assert not is_actionable(item) + + +def test_aggregate_summary(): + files = [ + { + "path": "a.txt", + "kind": "text", + "has_c2pa": False, + "has_ai_metadata": False, + "suspicious_total": 1, + "findings": ["layer-a [zwj_family] U+200B ZERO WIDTH SPACE (Cf) x1"], + "confidence": ["probable"], + }, + { + "path": "b.html", + "kind": "html", + "has_c2pa": False, + "has_ai_metadata": False, + "suspicious_total": 0, + "findings": ["info: cms generator: "], + "confidence": ["informational"], + }, + ] + summary = aggregate(files) + assert summary["total"] == 2 + assert summary["actionable_files"] == 1 + assert summary["findings_by_confidence"]["probable"] == 1 + assert summary["findings_by_confidence"]["informational"] == 1 + assert summary["with_suspicious_text"] == 1 + + +def test_parse_sitemap_urlset_and_index(): + data = ( + b'' + b'https://x.test/a' + ) + kind, urls = parse_sitemap(data) + assert kind == "urlset" + assert urls == ["https://x.test/a"] + + idx = ( + b'https://x.test/s1.xml' + b"" + ) + assert parse_sitemap(idx) == ("sitemapindex", ["https://x.test/s1.xml"]) + + assert parse_sitemap(gzip.compress(data))[0] == "urlset" + + +def test_guess_kind(): + assert guess_kind("https://x.test/a", b"x", "text/html") == "html" + assert guess_kind("https://x.test/a.png", b"\x89PNG\r\n\x1a\n", None) == "png" + assert guess_kind("https://x.test/a", b"%PDF-1.4", None) == "pdf" + + +def test_inspect_remote_html_cms_informational(): + result = inspect_remote( + "https://x.test/page", + b'hi', + "text/html", + ) + assert result["kind"] == "html" + assert result["confidence"] == ["informational"] + assert not is_actionable(result)