mirror of
https://github.com/guillaumemeyer/watermarks-remover.git
synced 2026-08-22 13:11:57 +02:00
chore: add Ruff linting and formatting with CI enforcement
Introduce Ruff (pinned at 0.16.3) as the project linter + formatter and enforce it in CI: - requirements-dev.txt: pin ruff==0.16.3 (exact pins, no drift) - ruff.toml: line-length 100, target py312; rule set E/F/W/I/UP/B/SIM/RUF/PLW/S with deliberate ignores (E501 for content strings, S603 for safe_arg subprocess calls, S101 asserts in tests) and per-file test ignores - Makefile: add lint / format / lint-fix targets - .github/workflows/ci.yml: add lint job (ruff check + format --check) - .gitignore: whitelist ruff.toml Also fix every finding the new gate surfaced so CI is green: - 109+ auto-fixes from ruff --fix (import sorting, simplifications, unused vars, re.I aliases, etc.) - explicit check=False on all subprocess.run calls (PLW1510) - harden sitemap XML parsing: reject DTD/entity declarations (S314) - replace hardcoded /tmp paths in tests with tmp_path (S108) - narrow/annotate intentional bare excepts (S110/S112), bind loop vars in closures (B023), raise ... from None (B904), strict= for zip (B905) - ruff format applied across service/ and tests/ Verified: ruff check + ruff format --check pass; 287 tests pass, 1 skip.
This commit is contained in:
parent
47a44e6ca8
commit
65f65eb179
@@ -46,3 +46,20 @@ jobs:
|
||||
run: |
|
||||
python -m pip_audit -r requirements-dev.txt \
|
||||
-r service/scripts/requirements-synthid-scorer.txt
|
||||
|
||||
lint:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Install test deps
|
||||
run: python -m pip install -r requirements-dev.txt
|
||||
|
||||
- name: Lint
|
||||
run: python -m ruff check service tests
|
||||
|
||||
- name: Format check
|
||||
run: python -m ruff format --check service tests
|
||||
|
||||
@@ -24,6 +24,7 @@
|
||||
!/pytest.ini
|
||||
!/README.md
|
||||
!/requirements-dev.txt
|
||||
!/ruff.toml
|
||||
!/SECURITY.md
|
||||
|
||||
# Exceptions even inside allowed trees
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
.PHONY: test smoke smoke-synthid bootstrap-synthid docker-synthid-build docker-synthid-help \
|
||||
.PHONY: test lint format lint-fix smoke smoke-synthid bootstrap-synthid docker-synthid-build docker-synthid-help \
|
||||
smoke-ctrlregen bootstrap-ctrlregen docker-ctrlregen-build docker-ctrlregen-help \
|
||||
smoke-markllm bootstrap-markllm docker-markllm-build docker-markllm-help \
|
||||
smoke-markdiffusion bootstrap-markdiffusion docker-markdiffusion-build docker-markdiffusion-help \
|
||||
@@ -11,6 +11,15 @@ PYTHON ?= $(shell if [ -x .venv/bin/python ]; then echo .venv/bin/python; else e
|
||||
test:
|
||||
$(PYTHON) -m pytest
|
||||
|
||||
lint:
|
||||
$(PYTHON) -m ruff check service tests
|
||||
|
||||
format:
|
||||
$(PYTHON) -m ruff format --check service tests
|
||||
|
||||
lint-fix:
|
||||
$(PYTHON) -m ruff check --fix service tests
|
||||
|
||||
smoke:
|
||||
-python3 $(SCRIPTS)/inspect_text.py tests/fixtures/sample_watermarked.txt
|
||||
python3 $(SCRIPTS)/clean_text.py tests/fixtures/sample_watermarked.txt -o /tmp/wm.cleaned.txt --stats
|
||||
|
||||
@@ -3,3 +3,4 @@
|
||||
pytest==9.1.1
|
||||
pip-audit==2.10.1
|
||||
openapi-spec-validator==0.9.0
|
||||
ruff==0.16.3
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
# Ruff configuration — lint + format.
|
||||
# Version pinned in requirements-dev.txt (exact pins, no drift).
|
||||
line-length = 100
|
||||
target-version = "py312"
|
||||
|
||||
[lint]
|
||||
# E/F/W: pycodestyle + pyflakes core; I: import sorting; UP: pyupgrade;
|
||||
# B: bugbear; SIM: simplify; RUF: ruff-specific; PLW: pylint warnings;
|
||||
# S: bandit security checks.
|
||||
select = ["E", "F", "W", "I", "UP", "B", "SIM", "RUF", "PLW", "S"]
|
||||
ignore = [
|
||||
"S101", # asserts are idiomatic in this test suite
|
||||
# E501 would flag long content strings (XML fixtures, help text) that the
|
||||
# formatter already keeps readable; ruff format enforces line length for code.
|
||||
"E501",
|
||||
# S603 fires on any subprocess call whose args aren't all literals, but this
|
||||
# codebase already guards args with safe_arg() and never uses shell=True.
|
||||
"S603",
|
||||
]
|
||||
|
||||
[lint.per-file-ignores]
|
||||
# Tests import scripts via sys.path manipulation; the delayed imports are
|
||||
# intentional, so don't demand E402 there (ruff already tolerates them, but
|
||||
# be explicit).
|
||||
"tests/**" = [
|
||||
"E402",
|
||||
# RUF001/RUF003: tests intentionally embed confusables / ambiguous Unicode
|
||||
# (NBSP, fullwidth, Cyrillic lookalikes) to exercise the scrubbing logic.
|
||||
"RUF001",
|
||||
"RUF003",
|
||||
]
|
||||
@@ -14,8 +14,8 @@ from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
|
||||
from audit_lib import aggregate, print_human_report, scan_file # noqa: E402
|
||||
from common import MAX_INPUT_BYTES, emit_json, eprint # noqa: E402
|
||||
from audit_lib import aggregate, print_human_report, scan_file
|
||||
from common import MAX_INPUT_BYTES, emit_json, eprint
|
||||
|
||||
DEFAULT_SKIP_DIRS = {
|
||||
".git",
|
||||
@@ -38,9 +38,7 @@ DEFAULT_SKIP_DIRS = {
|
||||
|
||||
def walk_files(root: Path, skip_dirs: set[str]):
|
||||
for dirpath, dirnames, filenames in os.walk(root):
|
||||
dirnames[:] = sorted(
|
||||
d for d in dirnames if d not in skip_dirs and not d.startswith(".")
|
||||
)
|
||||
dirnames[:] = sorted(d for d in dirnames if d not in skip_dirs and not d.startswith("."))
|
||||
for fn in sorted(filenames):
|
||||
path = Path(dirpath) / fn
|
||||
if path.is_file():
|
||||
@@ -69,10 +67,10 @@ def main() -> int:
|
||||
return 2
|
||||
|
||||
skip_dirs = set(DEFAULT_SKIP_DIRS)
|
||||
for name in args.skip.split(","):
|
||||
name = name.strip()
|
||||
if name:
|
||||
skip_dirs.add(name)
|
||||
for raw_part in args.skip.split(","):
|
||||
part = raw_part.strip()
|
||||
if part:
|
||||
skip_dirs.add(part)
|
||||
|
||||
files = []
|
||||
skipped = []
|
||||
|
||||
@@ -63,7 +63,9 @@ def scan_file(
|
||||
s_rep = score_text_stylometry(text, path=name)
|
||||
item["stylometry"] = s_rep.to_dict()
|
||||
if s_rep.score >= 0.65:
|
||||
item["findings"].append(f"stylometry [high_probability] score {s_rep.score:.2f} ({s_rep.confidence_level})")
|
||||
item["findings"].append(
|
||||
f"stylometry [high_probability] score {s_rep.score:.2f} ({s_rep.confidence_level})"
|
||||
)
|
||||
item["confidence"].append("probable")
|
||||
item["suspicious_total"] += 1
|
||||
return item
|
||||
@@ -98,7 +100,9 @@ def scan_file(
|
||||
s_rep = score_text_stylometry(text, path=name)
|
||||
stylometry_dict = s_rep.to_dict()
|
||||
if s_rep.score >= 0.65:
|
||||
findings.append(f"stylometry [high_probability] score {s_rep.score:.2f} ({s_rep.confidence_level})")
|
||||
findings.append(
|
||||
f"stylometry [high_probability] score {s_rep.score:.2f} ({s_rep.confidence_level})"
|
||||
)
|
||||
confidences.append("probable")
|
||||
suspicious += 1
|
||||
|
||||
@@ -152,7 +156,9 @@ def aggregate(files: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
return summary
|
||||
|
||||
|
||||
def print_human_report(files: list[dict[str, Any]], summary: dict[str, Any], extra_header: dict[str, Any] | None = None) -> None:
|
||||
def print_human_report(
|
||||
files: list[dict[str, Any]], summary: dict[str, Any], extra_header: dict[str, Any] | None = None
|
||||
) -> None:
|
||||
"""Shared plain-text rendering for audit scripts."""
|
||||
for key, value in (extra_header or {}).items():
|
||||
print(f"{key}: {value}")
|
||||
@@ -164,5 +170,5 @@ def print_human_report(files: list[dict[str, Any]], summary: dict[str, Any], ext
|
||||
print(f"Actionable files: {summary['actionable_files']}")
|
||||
print(f"Findings by confidence: {summary['findings_by_confidence']}")
|
||||
for item in files:
|
||||
for msg, conf in zip(item.get("findings", []), item.get("confidence", [])):
|
||||
for msg, conf in zip(item.get("findings", []), item.get("confidence", []), strict=False):
|
||||
print(f" [{conf}] {item['path']}: {msg}")
|
||||
|
||||
@@ -26,8 +26,10 @@ from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
|
||||
from audit_lib import aggregate, print_human_report, scan_file # noqa: E402
|
||||
from common import emit_json, eprint # noqa: E402
|
||||
import contextlib
|
||||
|
||||
from audit_lib import aggregate, print_human_report, scan_file
|
||||
from common import emit_json, eprint
|
||||
|
||||
DEFAULT_MAX_BYTES = 4 << 20
|
||||
DEFAULT_TIMEOUT = 15
|
||||
@@ -60,16 +62,17 @@ def parse_sitemap(data: bytes) -> tuple[str, list[str]]:
|
||||
data = stream.read(MAX_SITEMAP_DECOMPRESSED_BYTES + 1)
|
||||
if len(data) > MAX_SITEMAP_DECOMPRESSED_BYTES:
|
||||
raise ValueError(
|
||||
"sitemap decompressed size exceeds cap "
|
||||
f"({MAX_SITEMAP_DECOMPRESSED_BYTES} bytes)"
|
||||
f"sitemap decompressed size exceeds cap ({MAX_SITEMAP_DECOMPRESSED_BYTES} bytes)"
|
||||
)
|
||||
elif len(data) > MAX_SITEMAP_DECOMPRESSED_BYTES:
|
||||
raise ValueError(
|
||||
"sitemap size exceeds cap "
|
||||
f"({MAX_SITEMAP_DECOMPRESSED_BYTES} bytes)"
|
||||
)
|
||||
raise ValueError(f"sitemap size exceeds cap ({MAX_SITEMAP_DECOMPRESSED_BYTES} bytes)")
|
||||
|
||||
root = ET.fromstring(data)
|
||||
# stdlib ElementTree does not expand external entities, but internal entity
|
||||
# expansion (billion-laughs) is still possible; reject DTDs outright.
|
||||
if b"<!DOCTYPE" in data or b"<!ENTITY" in data:
|
||||
raise ValueError("sitemap declares a DTD / entities; refusing to parse")
|
||||
|
||||
root = ET.fromstring(data) # noqa: S314 - DTD/entity declarations rejected above
|
||||
kind = _local(root.tag)
|
||||
urls = []
|
||||
for el in root.iter():
|
||||
@@ -77,6 +80,7 @@ def parse_sitemap(data: bytes) -> tuple[str, list[str]]:
|
||||
urls.append(el.text.strip())
|
||||
return kind, urls
|
||||
|
||||
|
||||
def guess_kind(url: str, data: bytes, content_type: str | None = None) -> str:
|
||||
"""Classify a downloaded URL from headers, suffix, then magic bytes."""
|
||||
ct = (content_type or "").lower().split(";")[0].strip()
|
||||
@@ -222,10 +226,7 @@ def _validated_target(
|
||||
origin = _url_origin(url)
|
||||
|
||||
if expected_origin is not None and not _origin_allowed(origin, expected_origin):
|
||||
raise ValueError(
|
||||
f"cross-origin URL is not allowed: "
|
||||
f"{origin[0]}://{origin[1]}:{origin[2]}"
|
||||
)
|
||||
raise ValueError(f"cross-origin URL is not allowed: {origin[0]}://{origin[1]}:{origin[2]}")
|
||||
|
||||
return origin, _resolve_public_addresses(origin)
|
||||
|
||||
@@ -257,10 +258,8 @@ def _open_pinned_connection(
|
||||
if scheme == "https":
|
||||
context = ssl.create_default_context()
|
||||
context.minimum_version = ssl.TLSVersion.TLSv1_2
|
||||
try:
|
||||
with contextlib.suppress(NotImplementedError):
|
||||
context.set_alpn_protocols(["http/1.1"])
|
||||
except NotImplementedError:
|
||||
pass
|
||||
|
||||
wrapped = context.wrap_socket(raw, server_hostname=host)
|
||||
raw = None
|
||||
@@ -282,10 +281,8 @@ def _open_pinned_connection(
|
||||
except OSError as exc:
|
||||
last_error = exc
|
||||
if raw is not None:
|
||||
try:
|
||||
with contextlib.suppress(OSError):
|
||||
raw.close()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
if last_error is not None:
|
||||
raise last_error
|
||||
@@ -395,7 +392,7 @@ def discover_sitemap(base_url: str, timeout: int) -> str | None:
|
||||
)
|
||||
parse_sitemap(data)
|
||||
return candidate
|
||||
except Exception:
|
||||
except Exception: # noqa: S112 - try next candidate sitemap on any failure
|
||||
continue
|
||||
|
||||
try:
|
||||
@@ -414,17 +411,16 @@ def discover_sitemap(base_url: str, timeout: int) -> str | None:
|
||||
candidate_origin = _url_origin(candidate)
|
||||
|
||||
if not _origin_allowed(candidate_origin, origin):
|
||||
raise ValueError(
|
||||
f"cross-origin sitemap is not allowed: {candidate}"
|
||||
)
|
||||
raise ValueError(f"cross-origin sitemap is not allowed: {candidate}")
|
||||
|
||||
return candidate
|
||||
|
||||
except Exception:
|
||||
except Exception: # noqa: S110 - robots.txt is optional; fall through to None
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def collect_urls(
|
||||
sitemap_url: str,
|
||||
timeout: int,
|
||||
@@ -439,9 +435,7 @@ def collect_urls(
|
||||
candidate_origin = _url_origin(loc)
|
||||
|
||||
if not _origin_allowed(candidate_origin, origin):
|
||||
raise ValueError(
|
||||
f"cross-origin sitemap URL is not allowed: {loc}"
|
||||
)
|
||||
raise ValueError(f"cross-origin sitemap URL is not allowed: {loc}")
|
||||
|
||||
def _recurse(url: str, depth: int = 0) -> None:
|
||||
if len(urls) >= max_pages or depth > 3:
|
||||
@@ -478,6 +472,7 @@ def collect_urls(
|
||||
_recurse(sitemap_url)
|
||||
return urls
|
||||
|
||||
|
||||
def main() -> int:
|
||||
p = argparse.ArgumentParser(description=__doc__)
|
||||
p.add_argument("--sitemap", help="Sitemap URL to audit")
|
||||
|
||||
@@ -52,7 +52,7 @@ def resolve_device(raw: str | None) -> str:
|
||||
mps = getattr(torch.backends, "mps", None)
|
||||
if mps is not None and mps.is_available():
|
||||
return "mps"
|
||||
except Exception:
|
||||
except Exception: # noqa: S110 - optional torch device detection
|
||||
pass
|
||||
return "cpu"
|
||||
|
||||
@@ -131,8 +131,8 @@ def main() -> int:
|
||||
|
||||
sys.path.insert(0, str(src_dir))
|
||||
try:
|
||||
from PIL import Image # noqa: E402
|
||||
from ctrlregen.engine import CtrlRegenEngine, is_ctrlregen_available # noqa: E402
|
||||
from ctrlregen.engine import CtrlRegenEngine, is_ctrlregen_available
|
||||
from PIL import Image
|
||||
except ImportError as e:
|
||||
print(f"CtrlRegen dependencies missing: {e}", file=sys.stderr)
|
||||
print("run setup_ctrlregen.sh first", file=sys.stderr)
|
||||
|
||||
@@ -10,19 +10,19 @@ from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
|
||||
from common import ( # noqa: E402
|
||||
from common import (
|
||||
MAX_INPUT_BYTES,
|
||||
ROUTER_ADVICE,
|
||||
backup_path,
|
||||
cleaned_path,
|
||||
eprint,
|
||||
ROUTER_ADVICE,
|
||||
guard_binary,
|
||||
safe_write_text,
|
||||
)
|
||||
from container_meta import clean_container, detect_container_format # noqa: E402
|
||||
from format_dispatch import classify # noqa: E402
|
||||
from image_meta import clean_image # noqa: E402
|
||||
from text_unicode import clean_text # noqa: E402
|
||||
from container_meta import clean_container, detect_container_format
|
||||
from format_dispatch import classify
|
||||
from image_meta import clean_image
|
||||
from text_unicode import clean_text
|
||||
|
||||
|
||||
def main() -> int:
|
||||
|
||||
@@ -10,8 +10,8 @@ from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
|
||||
from common import backup_path, cleaned_path, eprint # noqa: E402
|
||||
from image_meta import clean_image # noqa: E402
|
||||
from common import backup_path, cleaned_path, eprint
|
||||
from image_meta import clean_image
|
||||
|
||||
|
||||
def main() -> int:
|
||||
@@ -186,10 +186,14 @@ def main() -> int:
|
||||
)
|
||||
if pr is not None:
|
||||
if pr.get("available"):
|
||||
engine = "CtrlRegen" if args.remove_pixel == "ctrlregen" else "DiffusionPurification"
|
||||
engine = (
|
||||
"CtrlRegen" if args.remove_pixel == "ctrlregen" else "DiffusionPurification"
|
||||
)
|
||||
eprint(f"{engine}: removed on {pr.get('device', 'unknown device')}")
|
||||
else:
|
||||
engine = "CtrlRegen" if args.remove_pixel == "ctrlregen" else "DiffusionPurification"
|
||||
engine = (
|
||||
"CtrlRegen" if args.remove_pixel == "ctrlregen" else "DiffusionPurification"
|
||||
)
|
||||
eprint(f"{engine}: unavailable: {pr.get('error', 'unknown error')}")
|
||||
if residual:
|
||||
eprint("warning: residual C2PA/AI signals may remain")
|
||||
|
||||
@@ -10,8 +10,14 @@ from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
|
||||
from common import backup_path, cleaned_path, eprint, read_text_input, write_text_output # noqa: E402
|
||||
from text_unicode import clean_text # noqa: E402
|
||||
from common import (
|
||||
backup_path,
|
||||
cleaned_path,
|
||||
eprint,
|
||||
read_text_input,
|
||||
write_text_output,
|
||||
)
|
||||
from text_unicode import clean_text
|
||||
|
||||
|
||||
def main() -> int:
|
||||
@@ -69,7 +75,7 @@ def main() -> int:
|
||||
eprint("--in-place requires a file path")
|
||||
return 2
|
||||
src = Path(args.path)
|
||||
bak = backup_path(src)
|
||||
backup_path(src) # side effect: keep a .bak of the original before overwriting
|
||||
out = str(src)
|
||||
elif out is None and args.path not in (None, "-"):
|
||||
out = str(cleaned_path(Path(args.path)))
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
@@ -36,10 +37,8 @@ def _reconfigure_stream(stream: Any, errors: str) -> None:
|
||||
"""
|
||||
reconfigure = getattr(stream, "reconfigure", None)
|
||||
if reconfigure is not None:
|
||||
try:
|
||||
with contextlib.suppress(OSError, ValueError):
|
||||
reconfigure(encoding="utf-8", errors=errors)
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def _configure_stdio() -> None:
|
||||
@@ -261,10 +260,8 @@ def safe_write_bytes(path: str | Path, data: bytes) -> None:
|
||||
os.fsync(f.fileno())
|
||||
os.replace(tmp_name, dest)
|
||||
except BaseException:
|
||||
try:
|
||||
with contextlib.suppress(OSError):
|
||||
os.unlink(tmp_name)
|
||||
except OSError:
|
||||
pass
|
||||
raise
|
||||
|
||||
|
||||
@@ -284,7 +281,7 @@ def backup_path(src: Path) -> Path:
|
||||
safe_write_bytes(bak, src.read_bytes())
|
||||
except OSError as e:
|
||||
eprint(f"cannot create backup {bak}: {e}")
|
||||
raise SystemExit(2)
|
||||
raise SystemExit(2) from None
|
||||
return bak
|
||||
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@ import zipfile
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from common import (
|
||||
classify_finding_confidence,
|
||||
safe_arg,
|
||||
@@ -25,7 +26,6 @@ from common import (
|
||||
from image_meta import (
|
||||
AI_META_HINTS,
|
||||
C2PA_MARKERS,
|
||||
detect_format as detect_image_format,
|
||||
inspect_isobmff,
|
||||
inspect_jpeg,
|
||||
inspect_png,
|
||||
@@ -36,6 +36,9 @@ from image_meta import (
|
||||
strip_png,
|
||||
strip_webp,
|
||||
)
|
||||
from image_meta import (
|
||||
detect_format as detect_image_format,
|
||||
)
|
||||
|
||||
# Frontmatter / meta keys that often carry AI provenance
|
||||
AI_FRONTMATTER_KEYS = frozenset(
|
||||
@@ -102,9 +105,7 @@ class ContainerInspectReport:
|
||||
"has_c2pa": self.has_c2pa,
|
||||
"has_ai_metadata": self.has_ai_metadata,
|
||||
"findings": self.findings,
|
||||
"findings_confidence": [
|
||||
classify_finding_confidence(f) for f in self.findings
|
||||
],
|
||||
"findings_confidence": [classify_finding_confidence(f) for f in self.findings],
|
||||
"tools": self.tools,
|
||||
"details": self.details,
|
||||
"notes": self.notes,
|
||||
@@ -192,7 +193,7 @@ def _inspect_embedded_data_uris(text: str) -> tuple[bool, bool, list[str]]:
|
||||
data = base64.b64decode(raw_b64)
|
||||
else:
|
||||
data = urllib.parse.unquote_to_bytes(payload)
|
||||
except Exception:
|
||||
except Exception: # noqa: S112 - malformed data URI; skip to next match
|
||||
continue
|
||||
|
||||
if not data:
|
||||
@@ -255,17 +256,11 @@ def _clean_embedded_data_uris(
|
||||
|
||||
try:
|
||||
if fmt == "png":
|
||||
cleaned_bytes, sub_actions = strip_png(
|
||||
data, strip_all_text=strip_all_metadata
|
||||
)
|
||||
cleaned_bytes, sub_actions = strip_png(data, strip_all_text=strip_all_metadata)
|
||||
elif fmt == "jpeg":
|
||||
cleaned_bytes, sub_actions = strip_jpeg(
|
||||
data, strip_all_app=strip_all_metadata
|
||||
)
|
||||
cleaned_bytes, sub_actions = strip_jpeg(data, strip_all_app=strip_all_metadata)
|
||||
elif fmt == "webp":
|
||||
cleaned_bytes, sub_actions = strip_webp(
|
||||
data, strip_all_metadata=strip_all_metadata
|
||||
)
|
||||
cleaned_bytes, sub_actions = strip_webp(data, strip_all_metadata=strip_all_metadata)
|
||||
elif fmt in ("avif", "heic"):
|
||||
cleaned_bytes, sub_actions = strip_isobmff(
|
||||
data, fmt, strip_all_metadata=strip_all_metadata
|
||||
@@ -278,9 +273,7 @@ def _clean_embedded_data_uris(
|
||||
if not any("drop" in a.lower() for a in sub_actions) or cleaned_bytes == data:
|
||||
return full_match
|
||||
|
||||
actions.append(
|
||||
f"cleaned embedded data:image/{mime} ({', '.join(sub_actions[:2])})"
|
||||
)
|
||||
actions.append(f"cleaned embedded data:image/{mime} ({', '.join(sub_actions[:2])})")
|
||||
|
||||
if is_b64:
|
||||
new_b64 = base64.b64encode(cleaned_bytes).decode("ascii")
|
||||
@@ -438,9 +431,9 @@ def _is_cms_generator_meta(tag: str) -> bool:
|
||||
).lower()
|
||||
if name_or_prop != "generator":
|
||||
return False
|
||||
if _GENERATOR_AI_RE.search(attrs.get("content", "")) or _GENERATOR_AI_RE.search(tag):
|
||||
return False
|
||||
return True
|
||||
return not (_GENERATOR_AI_RE.search(attrs.get("content", "")) or _GENERATOR_AI_RE.search(tag))
|
||||
|
||||
|
||||
_JSONLD_RE = re.compile(
|
||||
r"<script\b[^>]*type\s*=\s*[\"']application/ld\+json[\"'][^>]*>.*?</script>",
|
||||
re.I | re.DOTALL,
|
||||
@@ -530,6 +523,7 @@ def clean_html(text: str) -> tuple[str, list[str]]:
|
||||
# SVG
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def inspect_svg(data: bytes) -> tuple[bool, bool, list[str], dict]:
|
||||
findings: list[str] = []
|
||||
has_c2pa, has_ai, hits = _blob_hits(data)
|
||||
@@ -579,6 +573,7 @@ def clean_svg(data: bytes) -> tuple[bytes, list[str]]:
|
||||
if n:
|
||||
actions.append(f"drop xmpmeta x{n}")
|
||||
text = new
|
||||
|
||||
# Drop comments that look like provenance
|
||||
def _cmt(m: re.Match[str]) -> str:
|
||||
body = m.group(0)
|
||||
@@ -823,9 +818,9 @@ def clean_docx(data: bytes, *, also_layer_a_text: bool = True) -> tuple[bytes, l
|
||||
for tag, label in DOCX_SCRUB_FIELDS:
|
||||
pat = rf"(<{tag}\b[^>]*>)(.*?)(</{tag}>)"
|
||||
|
||||
def _empty(m: re.Match[str], _label=label) -> str:
|
||||
def _empty(m: re.Match[str], _label=label, _name=name) -> str:
|
||||
if m.group(2):
|
||||
actions.append(f"scrub {name} field {_label}")
|
||||
actions.append(f"scrub {_name} field {_label}")
|
||||
return m.group(1) + m.group(3)
|
||||
|
||||
new = re.sub(pat, _empty, new, flags=re.I | re.DOTALL)
|
||||
@@ -864,12 +859,12 @@ def clean_docx(data: bytes, *, also_layer_a_text: bool = True) -> tuple[bytes, l
|
||||
kept_names = {info.filename for info, _ in kept}
|
||||
final: list[tuple[zipfile.ZipInfo, bytes]] = []
|
||||
for info, raw in kept:
|
||||
part_raw = raw
|
||||
if info.filename.endswith(".rels"):
|
||||
new_raw, n = _prune_dangling_relationships(info.filename, raw, kept_names)
|
||||
part_raw, n = _prune_dangling_relationships(info.filename, raw, kept_names)
|
||||
if n:
|
||||
actions.append(f"prune dangling relationships x{n} in {info.filename}")
|
||||
raw = new_raw
|
||||
final.append((info, raw))
|
||||
final.append((info, part_raw))
|
||||
|
||||
out_buf = io.BytesIO()
|
||||
with zipfile.ZipFile(out_buf, "w", compression=zipfile.ZIP_DEFLATED) as zout:
|
||||
@@ -915,9 +910,10 @@ def clean_odt(data: bytes, *, also_layer_a_text: bool = True) -> tuple[bytes, li
|
||||
budget = [0]
|
||||
layer_removed = 0
|
||||
layer_replaced = 0
|
||||
with zipfile.ZipFile(io.BytesIO(data)) as zin, zipfile.ZipFile(
|
||||
out_buf, "w", compression=zipfile.ZIP_DEFLATED
|
||||
) as zout:
|
||||
with (
|
||||
zipfile.ZipFile(io.BytesIO(data)) as zin,
|
||||
zipfile.ZipFile(out_buf, "w", compression=zipfile.ZIP_DEFLATED) as zout,
|
||||
):
|
||||
for info in zin.infolist():
|
||||
name = info.filename
|
||||
_check_zip_budget(info, budget)
|
||||
@@ -933,6 +929,7 @@ def clean_odt(data: bytes, *, also_layer_a_text: bool = True) -> tuple[bytes, li
|
||||
if n:
|
||||
actions.append("drop meta:generator")
|
||||
text = new
|
||||
|
||||
# scrub creator-like if AI
|
||||
def _creator(m: re.Match[str]) -> str:
|
||||
if AI_META_NAME_RE.search(m.group(0)):
|
||||
@@ -1131,6 +1128,7 @@ def clean_pdf(path: Path, dest: Path) -> tuple[list[str], dict]:
|
||||
# Unified API
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def inspect_container(path: Path) -> ContainerInspectReport:
|
||||
data = path.read_bytes()
|
||||
fmt = detect_container_format(path, data)
|
||||
@@ -1169,15 +1167,17 @@ def inspect_container(path: Path) -> ContainerInspectReport:
|
||||
layer_a_total = ta["suspicious_total"]
|
||||
layer_a_hits = ta["hits"]
|
||||
for h in layer_a_hits:
|
||||
findings.append(
|
||||
f"layer-a: {h['codepoint']} {h['label']} x{h['count']} ({h['kind']})"
|
||||
)
|
||||
findings.append(f"layer-a: {h['codepoint']} {h['label']} x{h['count']} ({h['kind']})")
|
||||
|
||||
notes: list[str] = []
|
||||
if fmt == "pdf":
|
||||
notes.append("PDF inspection is best-effort; exiftool/c2patool give more reliable metadata detection")
|
||||
notes.append(
|
||||
"PDF inspection is best-effort; exiftool/c2patool give more reliable metadata detection"
|
||||
)
|
||||
elif fmt == "docx":
|
||||
notes.append("DOCX: only metadata/provenance parts are scanned; visible body text is ignored")
|
||||
notes.append(
|
||||
"DOCX: only metadata/provenance parts are scanned; visible body text is ignored"
|
||||
)
|
||||
if "unsupported" in details:
|
||||
notes.append(f"format not fully inspected: {fmt}")
|
||||
if layer_a_total:
|
||||
|
||||
@@ -32,7 +32,7 @@ from pathlib import Path
|
||||
SCRIPTS_DIR = Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(SCRIPTS_DIR))
|
||||
|
||||
from common import eprint, emit_json, read_text_input, safe_write_text # noqa: E402
|
||||
from common import emit_json, eprint, read_text_input, safe_write_text # noqa: E402
|
||||
|
||||
# Scheme name as the user types it -> MarkLLM algorithm name (config/{ALG}.json).
|
||||
SCHEMES = {
|
||||
@@ -74,7 +74,7 @@ def resolve_device(raw: str | None) -> str:
|
||||
mps = getattr(torch.backends, "mps", None)
|
||||
if mps is not None and mps.is_available():
|
||||
return "mps"
|
||||
except Exception:
|
||||
except Exception: # noqa: S110 - optional torch device detection
|
||||
pass
|
||||
return "cpu"
|
||||
|
||||
@@ -85,9 +85,9 @@ def _load_algorithm(
|
||||
"""Import the checkout and build an ``AutoWatermark`` instance."""
|
||||
sys.path.insert(0, str(upstream))
|
||||
try:
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer # noqa: E402
|
||||
from utils.transformers_config import TransformersConfig # noqa: E402
|
||||
from watermark.auto_watermark import AutoWatermark # noqa: E402
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||
from utils.transformers_config import TransformersConfig
|
||||
from watermark.auto_watermark import AutoWatermark
|
||||
except ImportError as e:
|
||||
raise _Unavailable(f"MarkLLM dependencies missing: {e}") from e
|
||||
|
||||
@@ -131,10 +131,7 @@ def _threshold_from_config(config: Path) -> float | None:
|
||||
|
||||
|
||||
def _resolve_config(upstream: Path, alg: str, config: str | None) -> Path:
|
||||
if config:
|
||||
path = Path(config).expanduser().resolve()
|
||||
else:
|
||||
path = upstream / "config" / f"{alg}.json"
|
||||
path = Path(config).expanduser().resolve() if config else upstream / "config" / f"{alg}.json"
|
||||
if not path.is_file():
|
||||
raise _Unavailable(f"MarkLLM config not found: {path}")
|
||||
try:
|
||||
@@ -142,9 +139,7 @@ def _resolve_config(upstream: Path, alg: str, config: str | None) -> Path:
|
||||
except OSError as e:
|
||||
raise _Unavailable(f"cannot stat MarkLLM config {path}: {e}") from e
|
||||
if size > MAX_CONFIG_BYTES:
|
||||
raise _Unavailable(
|
||||
f"MarkLLM config too large ({size} bytes > {MAX_CONFIG_BYTES}): {path}"
|
||||
)
|
||||
raise _Unavailable(f"MarkLLM config too large ({size} bytes > {MAX_CONFIG_BYTES}): {path}")
|
||||
return path
|
||||
|
||||
|
||||
@@ -159,9 +154,7 @@ def _cmd_detect(args: argparse.Namespace, upstream: Path, alg: str) -> int:
|
||||
try:
|
||||
config = _resolve_config(upstream, alg, args.config)
|
||||
threshold = _threshold_from_config(config)
|
||||
wm = _load_algorithm(
|
||||
upstream, alg, config, args.model, device, offline=args.offline
|
||||
)
|
||||
wm = _load_algorithm(upstream, alg, config, args.model, device, offline=args.offline)
|
||||
result = wm.detect_watermark(text, return_dict=True)
|
||||
except _Unavailable as e:
|
||||
eprint(str(e))
|
||||
@@ -207,9 +200,7 @@ def _cmd_watermark(args: argparse.Namespace, upstream: Path, alg: str) -> int:
|
||||
|
||||
try:
|
||||
config = _resolve_config(upstream, alg, args.config)
|
||||
wm = _load_algorithm(
|
||||
upstream, alg, config, args.model, device, offline=args.offline
|
||||
)
|
||||
wm = _load_algorithm(upstream, alg, config, args.model, device, offline=args.offline)
|
||||
if args.seed is not None:
|
||||
import torch
|
||||
|
||||
@@ -250,7 +241,9 @@ def _cmd_watermark(args: argparse.Namespace, upstream: Path, alg: str) -> int:
|
||||
else:
|
||||
print(f"{alg}: watermarked sample ({payload['watermarked_chars']} chars) -> {wm_out}")
|
||||
if unwatermarked is not None:
|
||||
print(f" unwatermarked sample ({payload['unwatermarked_chars']} chars) -> {args.unwatermarked_output}")
|
||||
print(
|
||||
f" unwatermarked sample ({payload['unwatermarked_chars']} chars) -> {args.unwatermarked_output}"
|
||||
)
|
||||
|
||||
return 0
|
||||
|
||||
@@ -308,10 +301,18 @@ def main() -> int:
|
||||
|
||||
wm = sub.add_parser("watermark", help="Generate watermarked sample text")
|
||||
wm.add_argument("prompt", help="Prompt file, or - for stdin")
|
||||
wm.add_argument("-o", "--watermarked-output", default=None,
|
||||
help="Output path for the watermarked sample (default: stdout)")
|
||||
wm.add_argument("-o2", "--unwatermarked-output", default=None,
|
||||
help="Also write an unwatermarked sample to this path")
|
||||
wm.add_argument(
|
||||
"-o",
|
||||
"--watermarked-output",
|
||||
default=None,
|
||||
help="Output path for the watermarked sample (default: stdout)",
|
||||
)
|
||||
wm.add_argument(
|
||||
"-o2",
|
||||
"--unwatermarked-output",
|
||||
default=None,
|
||||
help="Also write an unwatermarked sample to this path",
|
||||
)
|
||||
wm.add_argument("--max-new-tokens", type=int, default=200)
|
||||
wm.add_argument("--min-length", type=int, default=0)
|
||||
wm.add_argument("--seed", type=int, default=None, help="Optional RNG seed")
|
||||
|
||||
@@ -13,7 +13,13 @@ from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from common import classify_finding_confidence, safe_arg, safe_write_bytes, subprocess_preexec_fn, which
|
||||
from common import (
|
||||
classify_finding_confidence,
|
||||
safe_arg,
|
||||
safe_write_bytes,
|
||||
subprocess_preexec_fn,
|
||||
which,
|
||||
)
|
||||
|
||||
SCRIPTS_DIR = Path(__file__).resolve().parent
|
||||
|
||||
@@ -30,7 +36,9 @@ def ctrlregen_preexec_fn() -> None:
|
||||
import resource
|
||||
|
||||
resource.setrlimit(resource.RLIMIT_AS, (_CTRLREGEN_RLIMIT_AS, _CTRLREGEN_RLIMIT_AS))
|
||||
resource.setrlimit(resource.RLIMIT_FSIZE, (_CTRLREGEN_RLIMIT_FSIZE, _CTRLREGEN_RLIMIT_FSIZE))
|
||||
resource.setrlimit(
|
||||
resource.RLIMIT_FSIZE, (_CTRLREGEN_RLIMIT_FSIZE, _CTRLREGEN_RLIMIT_FSIZE)
|
||||
)
|
||||
except (ImportError, OSError, ValueError):
|
||||
pass
|
||||
|
||||
@@ -94,9 +102,7 @@ class ImageInspectReport:
|
||||
"has_c2pa": self.has_c2pa,
|
||||
"has_ai_metadata": self.has_ai_metadata,
|
||||
"findings": self.findings,
|
||||
"findings_confidence": [
|
||||
classify_finding_confidence(f) for f in self.findings
|
||||
],
|
||||
"findings_confidence": [classify_finding_confidence(f) for f in self.findings],
|
||||
"tools": self.tools,
|
||||
"synthid": self.synthid,
|
||||
"notes": self.notes,
|
||||
@@ -113,9 +119,9 @@ def detect_format(data: bytes) -> str:
|
||||
if len(data) >= 12 and data[4:8] == b"ftyp":
|
||||
box_size = struct.unpack(">I", data[0:4])[0]
|
||||
header_chunk = (
|
||||
data[8:min(box_size, len(data), 64)]
|
||||
data[8 : min(box_size, len(data), 64)]
|
||||
if box_size >= 8
|
||||
else data[8:min(len(data), 64)]
|
||||
else data[8 : min(len(data), 64)]
|
||||
)
|
||||
if any(b in header_chunk for b in (b"avif", b"avis", b"avio")):
|
||||
return "avif"
|
||||
@@ -221,8 +227,7 @@ def inspect_jpeg(data: bytes) -> tuple[bool, bool, list[str]]:
|
||||
if hits:
|
||||
has_ai = True
|
||||
if any(
|
||||
h.lower() in ("c2pa", "contentcredentials", "jumb", "contentauth")
|
||||
for h in hits
|
||||
h.lower() in ("c2pa", "contentcredentials", "jumb", "contentauth") for h in hits
|
||||
):
|
||||
has_c2pa = True
|
||||
findings.append(f"JPEG APP{marker - 0xE0}: {', '.join(hits[:8])}")
|
||||
@@ -241,9 +246,7 @@ def _webp_chunks(data: bytes) -> tuple[list[tuple[bytes, bytes, bytes]], list[st
|
||||
notes: list[str] = []
|
||||
declared_size = struct.unpack("<I", data[4:8])[0]
|
||||
if declared_size + 8 != len(data):
|
||||
notes.append(
|
||||
f"RIFF size mismatch: header={declared_size + 8} actual={len(data)}"
|
||||
)
|
||||
notes.append(f"RIFF size mismatch: header={declared_size + 8} actual={len(data)}")
|
||||
|
||||
chunks: list[tuple[bytes, bytes, bytes]] = []
|
||||
pos = 12
|
||||
@@ -257,9 +260,7 @@ def _webp_chunks(data: bytes) -> tuple[list[tuple[bytes, bytes, bytes]], list[st
|
||||
name = fourcc.decode("latin-1", errors="replace")
|
||||
notes.append(f"truncated WebP chunk {name}")
|
||||
break
|
||||
chunks.append(
|
||||
(fourcc, data[payload_start:payload_end], data[payload_end:padded_end])
|
||||
)
|
||||
chunks.append((fourcc, data[payload_start:payload_end], data[payload_end:padded_end]))
|
||||
pos = padded_end
|
||||
if pos != len(data) and not any("truncated" in note for note in notes):
|
||||
notes.append(f"trailing WebP bytes: {len(data) - pos}")
|
||||
@@ -351,8 +352,7 @@ def inspect_isobmff(data: bytes, fmt: str = "avif") -> tuple[bool, bool, list[st
|
||||
else:
|
||||
findings.append(f"{fmt.upper()} XMP uuid box")
|
||||
if any(
|
||||
h.lower() in ("c2pa", "contentcredentials", "jumb", "contentauth")
|
||||
for h in hits
|
||||
h.lower() in ("c2pa", "contentcredentials", "jumb", "contentauth") for h in hits
|
||||
):
|
||||
has_c2pa = True
|
||||
else:
|
||||
@@ -360,10 +360,7 @@ def inspect_isobmff(data: bytes, fmt: str = "avif") -> tuple[bool, bool, list[st
|
||||
if hits:
|
||||
has_ai = True
|
||||
findings.append(f"{fmt.upper()} uuid box: {', '.join(hits[:8])}")
|
||||
if any(
|
||||
h.lower() in ("c2pa", "contentcredentials", "jumb")
|
||||
for h in hits
|
||||
):
|
||||
if any(h.lower() in ("c2pa", "contentcredentials", "jumb") for h in hits):
|
||||
has_c2pa = True
|
||||
elif fourcc == b"meta":
|
||||
meta_sub = _parse_isobmff_boxes(payload, start=4)
|
||||
@@ -371,25 +368,16 @@ def inspect_isobmff(data: bytes, fmt: str = "avif") -> tuple[bool, bool, list[st
|
||||
s_name = s_fourcc.decode("latin-1", errors="replace")
|
||||
if s_fourcc in (b"jumb", b"c2pa") or s_name.lower().startswith("c2"):
|
||||
has_c2pa = True
|
||||
findings.append(
|
||||
f"{fmt.upper()} meta sub-box {s_name} (C2PA/JUMBF container)"
|
||||
)
|
||||
findings.append(f"{fmt.upper()} meta sub-box {s_name} (C2PA/JUMBF container)")
|
||||
elif s_fourcc == b"uuid":
|
||||
if s_payload.startswith(XMP_UUID):
|
||||
has_ai = True
|
||||
hits = _contains_any(
|
||||
s_payload[16:], AI_META_HINTS + C2PA_MARKERS
|
||||
)
|
||||
hits = _contains_any(s_payload[16:], AI_META_HINTS + C2PA_MARKERS)
|
||||
if hits:
|
||||
findings.append(
|
||||
f"{fmt.upper()} meta XMP uuid: {', '.join(hits[:8])}"
|
||||
)
|
||||
findings.append(f"{fmt.upper()} meta XMP uuid: {', '.join(hits[:8])}")
|
||||
else:
|
||||
findings.append(f"{fmt.upper()} meta XMP uuid box")
|
||||
if any(
|
||||
h.lower() in ("c2pa", "contentcredentials", "jumb")
|
||||
for h in hits
|
||||
):
|
||||
if any(h.lower() in ("c2pa", "contentcredentials", "jumb") for h in hits):
|
||||
has_c2pa = True
|
||||
else:
|
||||
hits = _contains_any(s_payload, AI_META_HINTS + C2PA_MARKERS)
|
||||
@@ -400,14 +388,9 @@ def inspect_isobmff(data: bytes, fmt: str = "avif") -> tuple[bool, bool, list[st
|
||||
hits = _contains_any(s_payload, AI_META_HINTS + C2PA_MARKERS)
|
||||
if hits:
|
||||
has_ai = True
|
||||
if any(
|
||||
h.lower() in ("c2pa", "contentcredentials", "jumb")
|
||||
for h in hits
|
||||
):
|
||||
if any(h.lower() in ("c2pa", "contentcredentials", "jumb") for h in hits):
|
||||
has_c2pa = True
|
||||
findings.append(
|
||||
f"{fmt.upper()} meta/{s_name}: {', '.join(hits[:8])}"
|
||||
)
|
||||
findings.append(f"{fmt.upper()} meta/{s_name}: {', '.join(hits[:8])}")
|
||||
|
||||
whole = _contains_any(data, C2PA_MARKERS)
|
||||
if whole and not has_c2pa:
|
||||
@@ -428,6 +411,7 @@ def run_optional_tools(path: Path) -> dict[str, Any]:
|
||||
text=True,
|
||||
timeout=30,
|
||||
preexec_fn=subprocess_preexec_fn,
|
||||
check=False,
|
||||
)
|
||||
out = (r.stdout or "") + (r.stderr or "")
|
||||
low = out.lower()
|
||||
@@ -440,9 +424,7 @@ def run_optional_tools(path: Path) -> dict[str, Any]:
|
||||
"available": True,
|
||||
"returncode": r.returncode,
|
||||
"snippet": out[:2000],
|
||||
"has_manifest": (
|
||||
"claim" in low or "c2pa" in low or "manifest" in low
|
||||
)
|
||||
"has_manifest": ("claim" in low or "c2pa" in low or "manifest" in low)
|
||||
and not no_manifest,
|
||||
}
|
||||
except Exception as e:
|
||||
@@ -459,6 +441,7 @@ def run_optional_tools(path: Path) -> dict[str, Any]:
|
||||
text=True,
|
||||
timeout=30,
|
||||
preexec_fn=subprocess_preexec_fn,
|
||||
check=False,
|
||||
)
|
||||
out = r.stdout or ""
|
||||
interesting = [
|
||||
@@ -511,6 +494,7 @@ def run_synthid_score(
|
||||
text=True,
|
||||
timeout=180,
|
||||
preexec_fn=subprocess_preexec_fn,
|
||||
check=False,
|
||||
)
|
||||
except Exception as e:
|
||||
return {"available": False, "error": str(e)}
|
||||
@@ -569,9 +553,7 @@ def run_markdiffusion_purify(
|
||||
if upstream_dir is None:
|
||||
upstream_dir = os.environ.get("MARKDIFFUSION_DIR")
|
||||
|
||||
upstream = (
|
||||
Path(upstream_dir).expanduser().resolve() if upstream_dir else None
|
||||
)
|
||||
upstream = Path(upstream_dir).expanduser().resolve() if upstream_dir else None
|
||||
if upstream is not None and not upstream.is_dir():
|
||||
return {
|
||||
"available": False,
|
||||
@@ -608,6 +590,7 @@ def run_markdiffusion_purify(
|
||||
text=True,
|
||||
timeout=timeout,
|
||||
preexec_fn=ctrlregen_subprocess_preexec_fn,
|
||||
check=False,
|
||||
)
|
||||
except subprocess.TimeoutExpired:
|
||||
return {
|
||||
@@ -686,6 +669,7 @@ def run_ctrlregen_clean(
|
||||
text=True,
|
||||
timeout=timeout,
|
||||
preexec_fn=ctrlregen_subprocess_preexec_fn,
|
||||
check=False,
|
||||
)
|
||||
except subprocess.TimeoutExpired:
|
||||
return {"available": False, "error": f"CtrlRegen timed out after {timeout}s"}
|
||||
@@ -717,9 +701,7 @@ def inspect_image(
|
||||
elif fmt in ("avif", "heic"):
|
||||
has_c2pa, has_ai, findings = inspect_isobmff(data, fmt)
|
||||
else:
|
||||
has_c2pa, has_ai, findings = False, False, [
|
||||
"unsupported format (PNG/JPEG/WebP/AVIF/HEIC)"
|
||||
]
|
||||
has_c2pa, has_ai, findings = False, False, ["unsupported format (PNG/JPEG/WebP/AVIF/HEIC)"]
|
||||
|
||||
notes: list[str] = []
|
||||
if fmt == "unknown":
|
||||
@@ -835,7 +817,6 @@ def strip_jpeg(data: bytes, *, strip_all_app: bool = True) -> tuple[bytes, list[
|
||||
seglen = struct.unpack(">H", data[i : i + 2])[0]
|
||||
# Find EOI from here carefully: after SOS segment header, scan for FF D9
|
||||
# not preceded by stuffed FF 00 issues — simple approach: copy from FF DA to end
|
||||
sos_start = i - 2 # points at 0xFF before marker... actually marker already consumed
|
||||
# Reconstruct: FF DA + rest of file
|
||||
out.extend(b"\xff\xda")
|
||||
out.extend(data[i:])
|
||||
@@ -899,9 +880,7 @@ def strip_webp(data: bytes, *, strip_all_metadata: bool = True) -> tuple[bytes,
|
||||
for fourcc, payload, padding in chunks:
|
||||
drop = fourcc.upper() == b"C2PA"
|
||||
if fourcc in metadata_flags:
|
||||
drop = strip_all_metadata or bool(
|
||||
_contains_any(payload, AI_META_HINTS + C2PA_MARKERS)
|
||||
)
|
||||
drop = strip_all_metadata or bool(_contains_any(payload, AI_META_HINTS + C2PA_MARKERS))
|
||||
if drop:
|
||||
name = fourcc.decode("latin-1", errors="replace")
|
||||
actions.append(f"drop WebP chunk {name}")
|
||||
@@ -911,12 +890,13 @@ def strip_webp(data: bytes, *, strip_all_metadata: bool = True) -> tuple[bytes,
|
||||
|
||||
body = bytearray(WEBP_SIG)
|
||||
for fourcc, payload, padding in kept:
|
||||
if fourcc == b"VP8X" and len(payload) >= 1 and removed_flags:
|
||||
payload = bytes([payload[0] & ~removed_flags]) + payload[1:]
|
||||
chunk = payload
|
||||
if fourcc == b"VP8X" and len(chunk) >= 1 and removed_flags:
|
||||
chunk = bytes([chunk[0] & ~removed_flags]) + chunk[1:]
|
||||
body.extend(fourcc)
|
||||
body.extend(struct.pack("<I", len(payload)))
|
||||
body.extend(payload)
|
||||
body.extend(padding if len(payload) & 1 else b"")
|
||||
body.extend(struct.pack("<I", len(chunk)))
|
||||
body.extend(chunk)
|
||||
body.extend(padding if len(chunk) & 1 else b"")
|
||||
|
||||
if not actions:
|
||||
actions.append("no WebP metadata chunks removed (already clean or none matched)")
|
||||
@@ -933,7 +913,7 @@ def strip_isobmff(
|
||||
actions: list[str] = []
|
||||
out = bytearray()
|
||||
|
||||
for fourcc, payload, size, header_size in boxes:
|
||||
for fourcc, payload, _size, _header_size in boxes:
|
||||
name = fourcc.decode("latin-1", errors="replace")
|
||||
if fourcc in (b"jumb", b"c2pa") or name.lower().startswith("c2"):
|
||||
actions.append(f"drop top-level {name} box (C2PA/JUMBF)")
|
||||
@@ -951,7 +931,7 @@ def strip_isobmff(
|
||||
meta_verflags = payload[:4] if len(payload) >= 4 else b"\x00\x00\x00\x00"
|
||||
sub_boxes = _parse_isobmff_boxes(payload, start=4)
|
||||
clean_sub = bytearray()
|
||||
for s_fourcc, s_payload, s_size, s_hdr in sub_boxes:
|
||||
for s_fourcc, s_payload, _s_size, _s_hdr in sub_boxes:
|
||||
s_name = s_fourcc.decode("latin-1", errors="replace")
|
||||
if s_fourcc in (b"jumb", b"c2pa") or s_name.lower().startswith("c2"):
|
||||
actions.append(f"drop meta sub-box {s_name} (C2PA/JUMBF)")
|
||||
@@ -960,33 +940,24 @@ def strip_isobmff(
|
||||
if s_payload.startswith(XMP_UUID):
|
||||
actions.append(f"drop meta sub-box {s_name} (XMP metadata)")
|
||||
continue
|
||||
if strip_all_metadata or _contains_any(
|
||||
s_payload, AI_META_HINTS + C2PA_MARKERS
|
||||
):
|
||||
if strip_all_metadata or _contains_any(s_payload, AI_META_HINTS + C2PA_MARKERS):
|
||||
actions.append(f"drop meta sub-box {s_name} (UUID metadata)")
|
||||
continue
|
||||
if s_fourcc in (b"xml ", b"bxml"):
|
||||
if strip_all_metadata or _contains_any(
|
||||
s_payload, AI_META_HINTS + C2PA_MARKERS
|
||||
):
|
||||
actions.append(f"drop meta sub-box {s_name} (XML metadata)")
|
||||
continue
|
||||
clean_sub.extend(
|
||||
struct.pack(">I", len(s_payload) + 8) + s_fourcc + s_payload
|
||||
)
|
||||
if s_fourcc in (b"xml ", b"bxml") and (
|
||||
strip_all_metadata or _contains_any(s_payload, AI_META_HINTS + C2PA_MARKERS)
|
||||
):
|
||||
actions.append(f"drop meta sub-box {s_name} (XML metadata)")
|
||||
continue
|
||||
clean_sub.extend(struct.pack(">I", len(s_payload) + 8) + s_fourcc + s_payload)
|
||||
|
||||
new_meta_payload = meta_verflags + clean_sub
|
||||
out.extend(
|
||||
struct.pack(">I", len(new_meta_payload) + 8) + b"meta" + new_meta_payload
|
||||
)
|
||||
out.extend(struct.pack(">I", len(new_meta_payload) + 8) + b"meta" + new_meta_payload)
|
||||
continue
|
||||
|
||||
out.extend(struct.pack(">I", len(payload) + 8) + fourcc + payload)
|
||||
|
||||
if not actions:
|
||||
actions.append(
|
||||
f"no {fmt.upper()} metadata boxes removed (already clean or none matched)"
|
||||
)
|
||||
actions.append(f"no {fmt.upper()} metadata boxes removed (already clean or none matched)")
|
||||
|
||||
return bytes(out), actions
|
||||
|
||||
@@ -1082,8 +1053,7 @@ def clean_image(
|
||||
)
|
||||
if pixel_removal.get("available"):
|
||||
actions.append(
|
||||
f"DiffusionPurification pixel removal "
|
||||
f"(strength {markdiffusion_strength})"
|
||||
f"DiffusionPurification pixel removal (strength {markdiffusion_strength})"
|
||||
)
|
||||
else:
|
||||
actions.append(
|
||||
|
||||
@@ -9,7 +9,7 @@ from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
|
||||
from common import ( # noqa: E402
|
||||
from common import (
|
||||
MAX_INPUT_BYTES,
|
||||
ROUTER_ADVICE,
|
||||
classify_finding_confidence,
|
||||
@@ -17,10 +17,10 @@ from common import ( # noqa: E402
|
||||
eprint,
|
||||
read_text_input,
|
||||
)
|
||||
from container_meta import inspect_container # noqa: E402
|
||||
from format_dispatch import classify # noqa: E402
|
||||
from image_meta import inspect_image # noqa: E402
|
||||
from text_unicode import human_report, inspect_text # noqa: E402
|
||||
from container_meta import inspect_container
|
||||
from format_dispatch import classify
|
||||
from image_meta import inspect_image
|
||||
from text_unicode import human_report, inspect_text
|
||||
|
||||
|
||||
def main() -> int:
|
||||
|
||||
@@ -9,8 +9,8 @@ from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
|
||||
from common import classify_finding_confidence, emit_json # noqa: E402
|
||||
from image_meta import inspect_image # noqa: E402
|
||||
from common import classify_finding_confidence, emit_json
|
||||
from image_meta import inspect_image
|
||||
|
||||
|
||||
def main() -> int:
|
||||
|
||||
@@ -10,9 +10,9 @@ from pathlib import Path
|
||||
# Allow running as script from any cwd
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
|
||||
from common import emit_json, read_text_input # noqa: E402
|
||||
from score_stylometry import print_human_stylometry_report, score_text_stylometry # noqa: E402
|
||||
from text_unicode import human_report, inspect_text # noqa: E402
|
||||
from common import emit_json, read_text_input
|
||||
from score_stylometry import print_human_stylometry_report, score_text_stylometry
|
||||
from text_unicode import human_report, inspect_text
|
||||
|
||||
|
||||
def main() -> int:
|
||||
|
||||
@@ -32,7 +32,6 @@ from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
@@ -41,7 +40,7 @@ from typing import Any
|
||||
SCRIPTS_DIR = Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(SCRIPTS_DIR))
|
||||
|
||||
from common import eprint, emit_json, read_text_input, safe_write_bytes # noqa: E402
|
||||
from common import emit_json, eprint, read_text_input, safe_write_bytes # noqa: E402
|
||||
|
||||
# User-facing scheme names -> MarkDiffusion algorithm names (image-only; the
|
||||
# video algorithms VideoShield/VideoMark are out of scope here). Canonical
|
||||
@@ -93,7 +92,7 @@ def resolve_device(raw: str | None) -> str:
|
||||
mps = getattr(torch.backends, "mps", None)
|
||||
if mps is not None and mps.is_available():
|
||||
return "mps"
|
||||
except Exception:
|
||||
except Exception: # noqa: S110 - optional torch device detection
|
||||
pass
|
||||
return "cpu"
|
||||
|
||||
@@ -104,10 +103,7 @@ def normalize_scheme(raw: str) -> str:
|
||||
return up
|
||||
if raw.lower() in SCHEMES:
|
||||
return SCHEMES[raw.lower()]
|
||||
raise ValueError(
|
||||
f"unknown scheme {raw!r}; image schemes: "
|
||||
+ ", ".join(sorted(SCHEMES))
|
||||
)
|
||||
raise ValueError(f"unknown scheme {raw!r}; image schemes: " + ", ".join(sorted(SCHEMES)))
|
||||
|
||||
|
||||
def _import_markdiffusion(upstream: Path | None) -> Any:
|
||||
@@ -118,8 +114,9 @@ def _import_markdiffusion(upstream: Path | None) -> Any:
|
||||
import markdiffusion
|
||||
except ImportError as e:
|
||||
raise _Unavailable(
|
||||
"markdiffusion not importable: " + str(e) +
|
||||
" (set MARKDIFFUSION_DIR / --upstream-dir to a checkout, or "
|
||||
"markdiffusion not importable: "
|
||||
+ str(e)
|
||||
+ " (set MARKDIFFUSION_DIR / --upstream-dir to a checkout, or "
|
||||
"pip install markdiffusion[optional])"
|
||||
) from e
|
||||
return markdiffusion
|
||||
@@ -178,14 +175,14 @@ def _json_safe(obj: Any) -> Any:
|
||||
|
||||
if isinstance(obj, np.generic):
|
||||
return obj.item()
|
||||
except Exception:
|
||||
except Exception: # noqa: S110 - optional numpy import
|
||||
pass
|
||||
try:
|
||||
import torch
|
||||
|
||||
if isinstance(obj, torch.Tensor):
|
||||
return obj.item()
|
||||
except Exception:
|
||||
except Exception: # noqa: S110 - optional torch import
|
||||
pass
|
||||
try:
|
||||
return float(obj)
|
||||
@@ -237,8 +234,8 @@ def _cmd_watermark(args: argparse.Namespace, upstream: Path | None, scheme: str)
|
||||
|
||||
try:
|
||||
_import_markdiffusion(upstream)
|
||||
from markdiffusion.watermark import AutoWatermark
|
||||
from markdiffusion.utils import DiffusionConfig
|
||||
from markdiffusion.watermark import AutoWatermark
|
||||
|
||||
pipe, scheduler = _load_diffusion(args.model, device, args.offline, args.size)
|
||||
diffusion_config = DiffusionConfig(
|
||||
@@ -312,10 +309,9 @@ def _cmd_detect(args: argparse.Namespace, upstream: Path | None, scheme: str) ->
|
||||
|
||||
try:
|
||||
_import_markdiffusion(upstream)
|
||||
from PIL import Image
|
||||
|
||||
from markdiffusion.watermark import AutoWatermark
|
||||
from markdiffusion.utils import DiffusionConfig
|
||||
from markdiffusion.watermark import AutoWatermark
|
||||
from PIL import Image
|
||||
|
||||
pipe, scheduler = _load_diffusion(args.model, device, args.offline, args.size)
|
||||
diffusion_config = DiffusionConfig(
|
||||
@@ -337,9 +333,7 @@ def _cmd_detect(args: argparse.Namespace, upstream: Path | None, scheme: str) ->
|
||||
kwargs: dict[str, Any] = {}
|
||||
if args.detector_type:
|
||||
kwargs["detector_type"] = args.detector_type
|
||||
result = wm.detect_watermark_in_media(
|
||||
image, prompt=args.prompt or "", **kwargs
|
||||
)
|
||||
result = wm.detect_watermark_in_media(image, prompt=args.prompt or "", **kwargs)
|
||||
config_dict = getattr(getattr(wm, "config", None), "config_dict", None)
|
||||
except _Unavailable as e:
|
||||
eprint(str(e))
|
||||
@@ -377,10 +371,9 @@ def _cmd_purify(args: argparse.Namespace, upstream: Path | None) -> int:
|
||||
|
||||
try:
|
||||
_import_markdiffusion(upstream)
|
||||
from PIL import Image
|
||||
|
||||
from markdiffusion.evaluation.tools.image_editor import DiffusionPurification
|
||||
from markdiffusion.utils import DiffusionConfig
|
||||
from PIL import Image
|
||||
|
||||
pipe, scheduler = _load_diffusion(args.model, device, args.offline, args.size)
|
||||
diffusion_config = DiffusionConfig(
|
||||
@@ -459,12 +452,21 @@ def main() -> int:
|
||||
|
||||
wm = sub.add_parser("watermark", help="Generate a watermarked sample image")
|
||||
wm.add_argument("prompt", help="Prompt file, or - for stdin")
|
||||
wm.add_argument("-o", "--watermarked-output", required=True,
|
||||
help="Output PNG path (images cannot go to stdout)")
|
||||
wm.add_argument("-o2", "--unwatermarked-output", default=None,
|
||||
help="Also write an unwatermarked sample to this path")
|
||||
wm.add_argument("--scheme", default="tr",
|
||||
help="Scheme: " + ", ".join(sorted(SCHEMES)) + " (default: tr)")
|
||||
wm.add_argument(
|
||||
"-o",
|
||||
"--watermarked-output",
|
||||
required=True,
|
||||
help="Output PNG path (images cannot go to stdout)",
|
||||
)
|
||||
wm.add_argument(
|
||||
"-o2",
|
||||
"--unwatermarked-output",
|
||||
default=None,
|
||||
help="Also write an unwatermarked sample to this path",
|
||||
)
|
||||
wm.add_argument(
|
||||
"--scheme", default="tr", help="Scheme: " + ", ".join(sorted(SCHEMES)) + " (default: tr)"
|
||||
)
|
||||
wm.add_argument("--config", default=None, help="Algorithm config JSON (default: bundled)")
|
||||
wm.add_argument("--size", type=int, default=512, help="Image size in px (default: 512)")
|
||||
wm.add_argument("--steps", type=int, default=50, help="Diffusion steps (default: 50)")
|
||||
@@ -476,12 +478,16 @@ def main() -> int:
|
||||
|
||||
det = sub.add_parser("detect", help="Same-scheme detection on an image")
|
||||
det.add_argument("path", help="Image file to detect on")
|
||||
det.add_argument("--scheme", default="tr",
|
||||
help="Scheme: " + ", ".join(sorted(SCHEMES)) + " (default: tr)")
|
||||
det.add_argument(
|
||||
"--scheme", default="tr", help="Scheme: " + ", ".join(sorted(SCHEMES)) + " (default: tr)"
|
||||
)
|
||||
det.add_argument("--config", default=None, help="Algorithm config JSON (default: bundled)")
|
||||
det.add_argument("--prompt", default=None, help="Optional prompt used at generation")
|
||||
det.add_argument("--detector-type", default=None,
|
||||
help="Detector variant (e.g. l1_distance, p_value; scheme-dependent)")
|
||||
det.add_argument(
|
||||
"--detector-type",
|
||||
default=None,
|
||||
help="Detector variant (e.g. l1_distance, p_value; scheme-dependent)",
|
||||
)
|
||||
det.add_argument("--size", type=int, default=512, help="Image size in px (default: 512)")
|
||||
det.add_argument("--steps", type=int, default=50, help="Diffusion steps (default: 50)")
|
||||
det.add_argument("--guidance", type=float, default=7.5, help="Guidance scale (default: 7.5)")
|
||||
@@ -493,9 +499,12 @@ def main() -> int:
|
||||
pf = sub.add_parser("purify", help="Run the DiffusionPurification regeneration attack")
|
||||
pf.add_argument("path", help="Image file to purify")
|
||||
pf.add_argument("-o", "--output", required=True, help="Output PNG path")
|
||||
pf.add_argument("--purification-strength", type=float, default=0.3,
|
||||
help="Fraction of the diffusion schedule to regenerate in (0, 1] "
|
||||
"(default: 0.3)")
|
||||
pf.add_argument(
|
||||
"--purification-strength",
|
||||
type=float,
|
||||
default=0.3,
|
||||
help="Fraction of the diffusion schedule to regenerate in (0, 1] (default: 0.3)",
|
||||
)
|
||||
pf.add_argument("--prompt", default=None, help="Optional prompt for denoising (default: '')")
|
||||
pf.add_argument("--size", type=int, default=512, help="Image size in px (default: 512)")
|
||||
pf.add_argument("--steps", type=int, default=50, help="Diffusion steps (default: 50)")
|
||||
|
||||
@@ -23,6 +23,7 @@ Security notes:
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import itertools
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
@@ -36,8 +37,8 @@ from urllib.parse import urlparse
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
|
||||
from common import cleaned_path, eprint, read_text_input, write_text_output # noqa: E402
|
||||
from text_unicode import clean_text # noqa: E402
|
||||
from common import cleaned_path, eprint, read_text_input, write_text_output
|
||||
from text_unicode import clean_text
|
||||
|
||||
DEFAULT_MARKLLM_MODEL = "facebook/opt-1.3b"
|
||||
|
||||
@@ -87,7 +88,7 @@ def _tokens(text: str) -> list[str]:
|
||||
|
||||
|
||||
def _bigrams(tokens: list[str]) -> set[tuple[str, str]]:
|
||||
return set(zip(tokens, tokens[1:]))
|
||||
return set(itertools.pairwise(tokens))
|
||||
|
||||
|
||||
def _lexical_divergence(original: str, candidate: str) -> float:
|
||||
@@ -169,7 +170,7 @@ class _NoRedirect(urllib.request.HTTPRedirectHandler):
|
||||
host behind the localhost allowlist. Any 3xx now surfaces as HTTPError.
|
||||
"""
|
||||
|
||||
def redirect_request(self, req, fp, code, msg, headers, newurl): # noqa: N802
|
||||
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
||||
raise urllib.error.HTTPError(req.full_url, code, msg, headers, fp)
|
||||
|
||||
|
||||
@@ -253,6 +254,7 @@ def _markllm_detect(
|
||||
text=True,
|
||||
timeout=timeout,
|
||||
preexec_fn=_markllm_preexec(),
|
||||
check=False,
|
||||
)
|
||||
except (OSError, subprocess.SubprocessError, TimeoutError) as e:
|
||||
return {"available": False, "error": f"MarkLLM adapter error: {e}"}
|
||||
@@ -296,7 +298,8 @@ def _http_json(url: str, payload: dict, headers: dict[str, str], timeout: float)
|
||||
if urlparse(url).scheme not in ("http", "https"):
|
||||
raise ValueError(f"refusing non-http(s) rewrite endpoint: {url}")
|
||||
body = json.dumps(payload).encode("utf-8")
|
||||
req = urllib.request.Request(
|
||||
# S310: URL scheme is restricted to http/https just above.
|
||||
req = urllib.request.Request( # noqa: S310
|
||||
url,
|
||||
data=body,
|
||||
headers={"Content-Type": "application/json", **headers},
|
||||
@@ -307,9 +310,7 @@ def _http_json(url: str, payload: dict, headers: dict[str, str], timeout: float)
|
||||
return json.loads(resp.read().decode("utf-8"))
|
||||
|
||||
|
||||
def call_ollama(
|
||||
base_url: str, model: str, prompt: str, timeout: float, temperature: float
|
||||
) -> str:
|
||||
def call_ollama(base_url: str, model: str, prompt: str, timeout: float, temperature: float) -> str:
|
||||
url = base_url.rstrip("/") + "/api/chat"
|
||||
data = _http_json(
|
||||
url,
|
||||
@@ -589,9 +590,7 @@ def main() -> int:
|
||||
temperature=args.temperature,
|
||||
candidates=args.candidates,
|
||||
allow_remote=allow_remote,
|
||||
reasoning_effort=(
|
||||
None if args.reasoning_effort == "off" else args.reasoning_effort
|
||||
),
|
||||
reasoning_effort=(None if args.reasoning_effort == "off" else args.reasoning_effort),
|
||||
markllm_scheme=args.markllm_scheme,
|
||||
markllm_dir=args.markllm_dir,
|
||||
markllm_model=args.markllm_model,
|
||||
|
||||
@@ -15,10 +15,8 @@ from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import unicodedata
|
||||
from collections import Counter
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
@@ -28,11 +26,8 @@ SCRIPTS_DIR = Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(SCRIPTS_DIR))
|
||||
|
||||
from common import ( # noqa: E402
|
||||
MAX_INPUT_BYTES,
|
||||
classify_finding_confidence,
|
||||
emit_json,
|
||||
eprint,
|
||||
looks_binary,
|
||||
read_text_input,
|
||||
)
|
||||
|
||||
@@ -48,14 +43,46 @@ AI_PHRASE_PATTERNS: tuple[tuple[str, str, float], ...] = (
|
||||
(r"\ba\s+testament\s+to\b", "a testament to", 1.1),
|
||||
(r"\brich\s+tapestry(?:\s+of)?\b", "rich tapestry", 1.3),
|
||||
(r"\bplays?\s+a\s+(?:pivotal|crucial|vital|key)\s+role\b", "plays a pivotal/crucial role", 1.0),
|
||||
(r"\bin\s+(?:today'?s|the)\s+(?:(?:fast-paced|ever-evolving|digital|rapidly\s+changing)\s+)*(?:world|landscape|era|environment)\b", "in today's fast-paced world/landscape", 1.4),
|
||||
(r"\bit\s+is\s+(?:important|essential|crucial|worth\s+noting)\s+to\s+(?:note|remember|consider|highlight)\b", "it is important/crucial to note", 0.9),
|
||||
(r"\bnot\s+only\b[\w\s,]+\bbut\s+(?:also\s+)?(?:serves\s+to|acts\s+as|highlights)\b", "not only ... but also serves to", 0.8),
|
||||
(r"\bserve(?:s|d)?\s+as\s+a\s+(?:beacon|reminder|catalyst|cornerstone)\b", "serves as a beacon/catalyst/cornerstone", 1.1),
|
||||
(r"\bunderscore(?:s|d)?\s+the\s+(?:importance|need|significance)\b", "underscores the importance/need", 0.9),
|
||||
(r"\bfoster(?:s|ing|ed)?\s+a\s+(?:sense|culture|deeper\s+understanding)\b", "fosters a sense/culture", 0.9),
|
||||
(r"\bseamlessly\s+(?:integrates?|integrated|blends?|combine[sd]?)\b", "seamlessly integrates/blends", 1.0),
|
||||
(r"\bnavigat(?:e|ing|es|ed)\s+the\s+(?:complexities|intricacies|nuances)\b", "navigating the complexities/nuances", 1.0),
|
||||
(
|
||||
r"\bin\s+(?:today'?s|the)\s+(?:(?:fast-paced|ever-evolving|digital|rapidly\s+changing)\s+)*(?:world|landscape|era|environment)\b",
|
||||
"in today's fast-paced world/landscape",
|
||||
1.4,
|
||||
),
|
||||
(
|
||||
r"\bit\s+is\s+(?:important|essential|crucial|worth\s+noting)\s+to\s+(?:note|remember|consider|highlight)\b",
|
||||
"it is important/crucial to note",
|
||||
0.9,
|
||||
),
|
||||
(
|
||||
r"\bnot\s+only\b[\w\s,]+\bbut\s+(?:also\s+)?(?:serves\s+to|acts\s+as|highlights)\b",
|
||||
"not only ... but also serves to",
|
||||
0.8,
|
||||
),
|
||||
(
|
||||
r"\bserve(?:s|d)?\s+as\s+a\s+(?:beacon|reminder|catalyst|cornerstone)\b",
|
||||
"serves as a beacon/catalyst/cornerstone",
|
||||
1.1,
|
||||
),
|
||||
(
|
||||
r"\bunderscore(?:s|d)?\s+the\s+(?:importance|need|significance)\b",
|
||||
"underscores the importance/need",
|
||||
0.9,
|
||||
),
|
||||
(
|
||||
r"\bfoster(?:s|ing|ed)?\s+a\s+(?:sense|culture|deeper\s+understanding)\b",
|
||||
"fosters a sense/culture",
|
||||
0.9,
|
||||
),
|
||||
(
|
||||
r"\bseamlessly\s+(?:integrates?|integrated|blends?|combine[sd]?)\b",
|
||||
"seamlessly integrates/blends",
|
||||
1.0,
|
||||
),
|
||||
(
|
||||
r"\bnavigat(?:e|ing|es|ed)\s+the\s+(?:complexities|intricacies|nuances)\b",
|
||||
"navigating the complexities/nuances",
|
||||
1.0,
|
||||
),
|
||||
(r"\bmultifaceted\s+(?:nature|approach|landscape)\b", "multifaceted nature/approach", 1.0),
|
||||
(r"\bharness(?:ing|ed|es)?\s+the\s+power\s+of\b", "harnessing the power of", 1.0),
|
||||
(r"\ba\s+myriad\s+of\b", "a myriad of", 0.8),
|
||||
@@ -118,9 +145,7 @@ class StylometryReport:
|
||||
"confidence_level": self.confidence_level,
|
||||
"status": self.status,
|
||||
"findings": self.findings,
|
||||
"findings_confidence": [
|
||||
classify_finding_confidence(f) for f in self.findings
|
||||
],
|
||||
"findings_confidence": [classify_finding_confidence(f) for f in self.findings],
|
||||
"notes": self.notes,
|
||||
}
|
||||
|
||||
@@ -285,19 +310,18 @@ def score_text_stylometry(text: str, path: str = "<text>") -> StylometryReport:
|
||||
ngram_score = 0.0
|
||||
|
||||
# Lexical uniformity subscore: LLMs cluster tightly around MATTR 0.65-0.78 for 50-word windows
|
||||
if 0.68 <= mattr <= 0.76:
|
||||
diversity_score = 0.40
|
||||
else:
|
||||
diversity_score = 0.10
|
||||
diversity_score = 0.4 if 0.68 <= mattr <= 0.76 else 0.1
|
||||
|
||||
# 4. Composite Scoring & Small-Sample Dampening
|
||||
raw_composite = (burstiness_score * 0.45) + (ngram_score * 0.45) + (diversity_score * 0.10)
|
||||
|
||||
# Dampening factor: scales smoothly from 0.4 at MIN_SAMPLE_WORDS up to 1.0 at FULL_WEIGHT_WORDS
|
||||
if word_count < FULL_WEIGHT_WORDS:
|
||||
dampener = 0.4 + 0.6 * ((word_count - MIN_SAMPLE_WORDS) / (FULL_WEIGHT_WORDS - MIN_SAMPLE_WORDS))
|
||||
dampener = 0.4 + 0.6 * (
|
||||
(word_count - MIN_SAMPLE_WORDS) / (FULL_WEIGHT_WORDS - MIN_SAMPLE_WORDS)
|
||||
)
|
||||
notes.append(
|
||||
f"Sample word count ({word_count}) is in calibration range ({MIN_SAMPLE_WORDS}–{FULL_WEIGHT_WORDS}); score dampened by factor {dampener:.2f}"
|
||||
f"Sample word count ({word_count}) is in calibration range ({MIN_SAMPLE_WORDS}-{FULL_WEIGHT_WORDS}); score dampened by factor {dampener:.2f}"
|
||||
)
|
||||
else:
|
||||
dampener = 1.0
|
||||
@@ -362,7 +386,7 @@ def print_human_stylometry_report(report: StylometryReport, explain: bool = Fals
|
||||
for m in report.matched_markers:
|
||||
print(f" * {m['phrase']} (occurrences: {m['count']}, weight: {m['weight']})")
|
||||
if m.get("samples"):
|
||||
print(f" sample: \"{m['samples'][0]}\"")
|
||||
print(f' sample: "{m["samples"][0]}"')
|
||||
|
||||
if report.notes:
|
||||
print("\nNotes:")
|
||||
@@ -376,9 +400,16 @@ def main() -> int:
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
p.add_argument("path", nargs="?", default="-", help="Text file to score ('-' for stdin)")
|
||||
p.add_argument("--threshold", type=float, default=DEFAULT_THRESHOLD, help=f"Score threshold to trigger exit code 1 (default: {DEFAULT_THRESHOLD})")
|
||||
p.add_argument(
|
||||
"--threshold",
|
||||
type=float,
|
||||
default=DEFAULT_THRESHOLD,
|
||||
help=f"Score threshold to trigger exit code 1 (default: {DEFAULT_THRESHOLD})",
|
||||
)
|
||||
p.add_argument("--json", action="store_true", help="Emit JSON output")
|
||||
p.add_argument("--explain", action="store_true", help="Include detailed matched phrase breakdown")
|
||||
p.add_argument(
|
||||
"--explain", action="store_true", help="Include detailed matched phrase breakdown"
|
||||
)
|
||||
|
||||
args = p.parse_args()
|
||||
|
||||
|
||||
@@ -77,9 +77,9 @@ def main() -> int:
|
||||
|
||||
sys.path.insert(0, str(extraction))
|
||||
try:
|
||||
import cv2 # noqa: E402
|
||||
from robust_extractor import RobustSynthIDExtractor # noqa: E402
|
||||
from synthid_bypass_v4 import SpectralCodebookV4 # noqa: E402
|
||||
import cv2
|
||||
from robust_extractor import RobustSynthIDExtractor
|
||||
from synthid_bypass_v4 import SpectralCodebookV4
|
||||
except ImportError as e:
|
||||
print(f"optional scorer dependencies missing: {e}", file=sys.stderr)
|
||||
return 3
|
||||
@@ -99,9 +99,7 @@ def main() -> int:
|
||||
codebook_v4.load(str(codebook))
|
||||
|
||||
extractor = RobustSynthIDExtractor()
|
||||
result = extractor.detect_from_v4_codebook(
|
||||
rgb, codebook_v4, model=args.model
|
||||
)
|
||||
result = extractor.detect_from_v4_codebook(rgb, codebook_v4, model=args.model)
|
||||
except Exception as e:
|
||||
print(f"scorer error: {e}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
+68
-31
@@ -35,17 +35,17 @@ from urllib.parse import urlparse
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
|
||||
from common import ( # noqa: E402
|
||||
from common import (
|
||||
MAX_INPUT_BYTES,
|
||||
looks_binary,
|
||||
eprint,
|
||||
looks_binary,
|
||||
which,
|
||||
)
|
||||
from container_meta import clean_container, inspect_container # noqa: E402
|
||||
from format_dispatch import classify_bytes # noqa: E402
|
||||
from image_meta import clean_image, inspect_image # noqa: E402
|
||||
from score_stylometry import score_text_stylometry # noqa: E402
|
||||
from text_unicode import clean_text, inspect_text # noqa: E402
|
||||
from container_meta import clean_container, inspect_container
|
||||
from format_dispatch import classify_bytes
|
||||
from image_meta import clean_image, inspect_image
|
||||
from score_stylometry import score_text_stylometry
|
||||
from text_unicode import clean_text, inspect_text
|
||||
|
||||
VERSION = os.environ.get("WATERMARKS_SERVER_VERSION", "dev")
|
||||
|
||||
@@ -97,6 +97,7 @@ def capabilities() -> dict[str, Any]:
|
||||
# plus live runtime values (version, auth, allowed options), so it can never
|
||||
# drift from the endpoints the handler actually serves. Served at /openapi.json.
|
||||
|
||||
|
||||
def _schema(**props: Any) -> dict[str, Any]:
|
||||
return props
|
||||
|
||||
@@ -133,7 +134,9 @@ def _clean_request_schema() -> dict[str, Any]:
|
||||
options[key] = _schema(type="string")
|
||||
return _file_request(
|
||||
{
|
||||
"properties": {"options": _schema(type="object", properties=options, additionalProperties=False)},
|
||||
"properties": {
|
||||
"options": _schema(type="object", properties=options, additionalProperties=False)
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
@@ -161,11 +164,15 @@ _OPENAPI_PATHS: dict[str, dict[str, Any]] = {
|
||||
"version": _schema(type="string"),
|
||||
"tools": _schema(
|
||||
type="object",
|
||||
properties={k: _schema(type="boolean") for k in ("c2patool", "exiftool", "qpdf")},
|
||||
properties={
|
||||
k: _schema(type="boolean") for k in ("c2patool", "exiftool", "qpdf")
|
||||
},
|
||||
),
|
||||
"pixel_backends": _schema(
|
||||
type="object",
|
||||
properties={k: _schema(type="boolean") for k in ("ctrlregen", "diffusion")},
|
||||
properties={
|
||||
k: _schema(type="boolean") for k in ("ctrlregen", "diffusion")
|
||||
},
|
||||
),
|
||||
"scorers": _schema(
|
||||
type="object",
|
||||
@@ -174,7 +181,9 @@ _OPENAPI_PATHS: dict[str, dict[str, Any]] = {
|
||||
"stylometry": _schema(type="boolean"),
|
||||
},
|
||||
),
|
||||
"harnesses": _schema(type="object", properties={"markllm": _schema(type="boolean")}),
|
||||
"harnesses": _schema(
|
||||
type="object", properties={"markllm": _schema(type="boolean")}
|
||||
),
|
||||
},
|
||||
)
|
||||
},
|
||||
@@ -221,7 +230,9 @@ _OPENAPI_PATHS: dict[str, dict[str, Any]] = {
|
||||
properties={
|
||||
"ok": _schema(type="boolean"),
|
||||
"kind": _schema(type="string", enum=["text", "image", "container"]),
|
||||
"cleaned": _schema(type="string", description="Base64-encoded cleaned file bytes"),
|
||||
"cleaned": _schema(
|
||||
type="string", description="Base64-encoded cleaned file bytes"
|
||||
),
|
||||
"report": _schema(type="object"),
|
||||
},
|
||||
)
|
||||
@@ -235,11 +246,23 @@ _ERROR_SCHEMA = _schema(
|
||||
properties={"ok": _schema(type="boolean", enum=[False]), "error": _schema(type="string")},
|
||||
)
|
||||
_COMMON_ERRORS = {
|
||||
"400": {"description": "Bad request", "content": {"application/json": {"schema": _ERROR_SCHEMA}}},
|
||||
"401": {"description": "Missing/invalid bearer token", "content": {"application/json": {"schema": _ERROR_SCHEMA}}},
|
||||
"400": {
|
||||
"description": "Bad request",
|
||||
"content": {"application/json": {"schema": _ERROR_SCHEMA}},
|
||||
},
|
||||
"401": {
|
||||
"description": "Missing/invalid bearer token",
|
||||
"content": {"application/json": {"schema": _ERROR_SCHEMA}},
|
||||
},
|
||||
"404": {"description": "Not found", "content": {"application/json": {"schema": _ERROR_SCHEMA}}},
|
||||
"413": {"description": "Request body too large", "content": {"application/json": {"schema": _ERROR_SCHEMA}}},
|
||||
"500": {"description": "Internal error", "content": {"application/json": {"schema": _ERROR_SCHEMA}}},
|
||||
"413": {
|
||||
"description": "Request body too large",
|
||||
"content": {"application/json": {"schema": _ERROR_SCHEMA}},
|
||||
},
|
||||
"500": {
|
||||
"description": "Internal error",
|
||||
"content": {"application/json": {"schema": _ERROR_SCHEMA}},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -256,7 +279,7 @@ def openapi_spec() -> dict[str, Any]:
|
||||
paths.setdefault(path, {})[method] = {
|
||||
"summary": op["summary"],
|
||||
"responses": responses,
|
||||
**(op.get("requestBody") and {"requestBody": op["requestBody"]} or {}),
|
||||
**((op.get("requestBody") and {"requestBody": op["requestBody"]}) or {}),
|
||||
}
|
||||
|
||||
spec: dict[str, Any] = {
|
||||
@@ -316,7 +339,7 @@ def _decode_input(body: dict[str, Any]) -> tuple[bytes, str]:
|
||||
try:
|
||||
data = base64.b64decode(raw, validate=True)
|
||||
except (binascii.Error, ValueError):
|
||||
raise ValueError("'file' is not valid base64")
|
||||
raise ValueError("'file' is not valid base64") from None
|
||||
return data, _safe_name(name or "")
|
||||
|
||||
|
||||
@@ -324,7 +347,7 @@ class Handler(BaseHTTPRequestHandler):
|
||||
server_version = f"watermarks-remover/{VERSION}"
|
||||
|
||||
def log_message(self, fmt: str, *args: object) -> None:
|
||||
eprint("%s - %s" % (self.address_string(), fmt % args))
|
||||
eprint(f"{self.address_string()} - {fmt % args}")
|
||||
|
||||
def _authorized(self) -> bool:
|
||||
if not API_KEY:
|
||||
@@ -356,7 +379,7 @@ class Handler(BaseHTTPRequestHandler):
|
||||
self.end_headers()
|
||||
self.wfile.write(data)
|
||||
|
||||
def do_GET(self) -> None: # noqa: N802 (http.server API)
|
||||
def do_GET(self) -> None:
|
||||
path = urlparse(self.path).path
|
||||
if not self._authorized():
|
||||
self._respond(HTTPStatus.UNAUTHORIZED, {"ok": False, "error": "unauthorized"})
|
||||
@@ -370,7 +393,7 @@ class Handler(BaseHTTPRequestHandler):
|
||||
else:
|
||||
self._respond(HTTPStatus.NOT_FOUND, {"ok": False, "error": "not found"})
|
||||
|
||||
def do_POST(self) -> None: # noqa: N802 (http.server API)
|
||||
def do_POST(self) -> None:
|
||||
path = urlparse(self.path).path
|
||||
if not self._authorized():
|
||||
self._respond(HTTPStatus.UNAUTHORIZED, {"ok": False, "error": "unauthorized"})
|
||||
@@ -399,9 +422,11 @@ class Handler(BaseHTTPRequestHandler):
|
||||
self._handle_clean(data, name, body)
|
||||
except ValueError as e:
|
||||
self._respond(HTTPStatus.BAD_REQUEST, {"ok": False, "error": str(e)})
|
||||
except Exception as e: # noqa: BLE001 — report, do not leak internals
|
||||
except Exception as e:
|
||||
eprint(f"error handling {path}: {e!r}")
|
||||
self._respond(HTTPStatus.INTERNAL_SERVER_ERROR, {"ok": False, "error": "internal error"})
|
||||
self._respond(
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR, {"ok": False, "error": "internal error"}
|
||||
)
|
||||
|
||||
def _handle_inspect(self, data: bytes, name: str) -> None:
|
||||
kind = classify_bytes(data, Path(name).suffix)
|
||||
@@ -410,7 +435,9 @@ class Handler(BaseHTTPRequestHandler):
|
||||
path.write_bytes(data)
|
||||
if kind == "text":
|
||||
if looks_binary(data):
|
||||
raise ValueError("refusing to inspect bytes that look like a binary container as text")
|
||||
raise ValueError(
|
||||
"refusing to inspect bytes that look like a binary container as text"
|
||||
)
|
||||
raw_text = data.decode("utf-8", errors="surrogateescape")
|
||||
report = inspect_text(raw_text).to_dict()
|
||||
s_rep = score_text_stylometry(raw_text, path=name or "<text>")
|
||||
@@ -419,10 +446,14 @@ class Handler(BaseHTTPRequestHandler):
|
||||
report = inspect_image(path).to_dict()
|
||||
else:
|
||||
report = inspect_container(path).to_dict()
|
||||
suspicious = bool(report.get("suspicious_total")) or bool(
|
||||
report.get("has_c2pa") or report.get("has_ai_metadata")
|
||||
) or bool(report.get("stylometry", {}).get("score", 0.0) >= 0.65)
|
||||
self._respond(HTTPStatus.OK, {"ok": True, "kind": kind, "report": report, "suspicious": suspicious})
|
||||
suspicious = (
|
||||
bool(report.get("suspicious_total"))
|
||||
or bool(report.get("has_c2pa") or report.get("has_ai_metadata"))
|
||||
or bool(report.get("stylometry", {}).get("score", 0.0) >= 0.65)
|
||||
)
|
||||
self._respond(
|
||||
HTTPStatus.OK, {"ok": True, "kind": kind, "report": report, "suspicious": suspicious}
|
||||
)
|
||||
|
||||
def _handle_clean(self, data: bytes, name: str, body: dict[str, Any]) -> None:
|
||||
kind = classify_bytes(data, Path(name).suffix)
|
||||
@@ -441,7 +472,9 @@ class Handler(BaseHTTPRequestHandler):
|
||||
src.write_bytes(data)
|
||||
if kind == "text":
|
||||
if looks_binary(data):
|
||||
raise ValueError("refusing to clean bytes that look like a binary container as text")
|
||||
raise ValueError(
|
||||
"refusing to clean bytes that look like a binary container as text"
|
||||
)
|
||||
text = data.decode("utf-8", errors="surrogateescape")
|
||||
cleaned, stats = clean_text(
|
||||
text,
|
||||
@@ -468,7 +501,9 @@ class Handler(BaseHTTPRequestHandler):
|
||||
report = {"kind": "image", **result}
|
||||
else:
|
||||
dest = _tmp_path(tmpdir, f"out{Path(name).suffix}")
|
||||
result = clean_container(src, dest, also_layer_a_text=bool(options.get("also_layer_a_text", True)))
|
||||
result = clean_container(
|
||||
src, dest, also_layer_a_text=bool(options.get("also_layer_a_text", True))
|
||||
)
|
||||
cleaned_bytes = dest.read_bytes()
|
||||
report = {"kind": "container", **result}
|
||||
report.pop("input", None)
|
||||
@@ -488,7 +523,9 @@ def main() -> int:
|
||||
global API_KEY # noqa: PLW0603 — CLI overrides env
|
||||
p = argparse.ArgumentParser(description=__doc__)
|
||||
p.add_argument("--host", default=os.environ.get("WATERMARKS_SERVER_HOST", "127.0.0.1"))
|
||||
p.add_argument("--port", type=int, default=int(os.environ.get("WATERMARKS_SERVER_PORT", "8765")))
|
||||
p.add_argument(
|
||||
"--port", type=int, default=int(os.environ.get("WATERMARKS_SERVER_PORT", "8765"))
|
||||
)
|
||||
p.add_argument("--api-key", default=API_KEY, help="require this bearer token (default: none)")
|
||||
p.add_argument("-V", "--version", action="store_true", help="print version and exit")
|
||||
args = p.parse_args()
|
||||
|
||||
@@ -164,7 +164,7 @@ LATIN_CONFUSABLES: dict[int, str] = {
|
||||
0xFF5A: "z",
|
||||
}
|
||||
|
||||
# Variation selectors beyond FE0x (VS17–VS256 in Supplementary Special-purpose)
|
||||
# Variation selectors beyond FE0x (VS17-VS256 in Supplementary Special-purpose)
|
||||
_VS_SUPPLEMENT = range(0xE0100, 0xE01F0)
|
||||
|
||||
|
||||
@@ -202,9 +202,7 @@ _PRESERVABLE_BIDI_CPS: frozenset[int] = frozenset(
|
||||
)
|
||||
|
||||
# Zero-width family (common edit-based carriers)
|
||||
_ZW_FAMILY: frozenset[int] = frozenset(
|
||||
{0x200B, 0x200C, 0x200D, 0x2060, 0xFEFF, 0x180E}
|
||||
)
|
||||
_ZW_FAMILY: frozenset[int] = frozenset({0x200B, 0x200C, 0x200D, 0x2060, 0xFEFF, 0x180E})
|
||||
|
||||
|
||||
def _is_private_use(cp: int) -> bool:
|
||||
@@ -217,12 +215,10 @@ def _is_strip_cp(cp: int) -> bool:
|
||||
return True
|
||||
if cp in _VS_SUPPLEMENT:
|
||||
return True
|
||||
# Tag characters used in some stego schemes (U+E0001–U+E007F)
|
||||
# Tag characters used in some stego schemes (U+E0001-U+E007F)
|
||||
if 0xE0001 <= cp <= 0xE007F:
|
||||
return True
|
||||
if _is_private_use(cp):
|
||||
return True
|
||||
return False
|
||||
return bool(_is_private_use(cp))
|
||||
|
||||
|
||||
def _strip_kind(cp: int) -> str:
|
||||
@@ -263,9 +259,8 @@ def _is_emoji_base(cp: int) -> bool:
|
||||
return True
|
||||
if cp in (0x00A9, 0x00AE, 0x2122, 0x3030, 0x303D, 0x3297, 0x3299):
|
||||
return True
|
||||
if cp in (0x0023, 0x002A) or 0x0030 <= cp <= 0x0039: # keycap bases
|
||||
return True
|
||||
return False
|
||||
# keycap bases
|
||||
return cp in (0x0023, 0x002A) or 0x0030 <= cp <= 0x0039
|
||||
|
||||
|
||||
# ZWNJ/ZWJ are orthographic inside complex scripts (Persian میروم, Devanagari
|
||||
@@ -647,8 +642,7 @@ def human_report(report: TextInspectReport) -> str:
|
||||
lines.append("Hits:")
|
||||
for h in report.hits:
|
||||
lines.append(
|
||||
f" [{h.kind}/{_hit_confidence(h.kind)}] "
|
||||
f"{h.label} x{h.count} @ {h.samples[:5]}"
|
||||
f" [{h.kind}/{_hit_confidence(h.kind)}] {h.label} x{h.count} @ {h.samples[:5]}"
|
||||
)
|
||||
for n in report.notes:
|
||||
lines.append(f"Note: {n}")
|
||||
|
||||
+13
-25
@@ -14,12 +14,12 @@ ROOT = Path(__file__).resolve().parents[1]
|
||||
SCRIPTS = ROOT / "service" / "scripts"
|
||||
sys.path.insert(0, str(SCRIPTS))
|
||||
|
||||
import audit_website # noqa: E402
|
||||
from audit_lib import aggregate, is_actionable, scan_file # noqa: E402
|
||||
from audit_website import guess_kind, inspect_remote, parse_sitemap # noqa: E402
|
||||
from common import classify_finding_confidence # noqa: E402
|
||||
from container_meta import inspect_container # noqa: E402
|
||||
from text_unicode import inspect_text # noqa: E402
|
||||
import audit_website
|
||||
from audit_lib import aggregate, is_actionable, scan_file
|
||||
from audit_website import guess_kind, inspect_remote, parse_sitemap
|
||||
from common import classify_finding_confidence
|
||||
from container_meta import inspect_container
|
||||
from text_unicode import inspect_text
|
||||
|
||||
|
||||
def test_classify_finding_confidence_buckets():
|
||||
@@ -31,7 +31,7 @@ def test_classify_finding_confidence_buckets():
|
||||
"pdf-structured:ai:AIGC": "probable",
|
||||
"PNG tEXt: c2pa, contentcredentials": "probable",
|
||||
"frontmatter key: generator": "probable",
|
||||
"info: cms generator: <meta name=\"generator\" content=\"WordPress\">": "informational",
|
||||
'info: cms generator: <meta name="generator" content="WordPress">': "informational",
|
||||
"customXml parts: 1": "informational",
|
||||
"unsupported container: woff2": "informational",
|
||||
"svg <metadata> present": "informational",
|
||||
@@ -151,16 +151,13 @@ def test_aggregate_summary():
|
||||
def test_parse_sitemap_urlset_and_index():
|
||||
data = (
|
||||
b'<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">'
|
||||
b'<url><loc>https://x.test/a</loc></url></urlset>'
|
||||
b"<url><loc>https://x.test/a</loc></url></urlset>"
|
||||
)
|
||||
kind, urls = parse_sitemap(data)
|
||||
assert kind == "urlset"
|
||||
assert urls == ["https://x.test/a"]
|
||||
|
||||
idx = (
|
||||
b'<sitemapindex><sitemap><loc>https://x.test/s1.xml</loc></sitemap>'
|
||||
b"</sitemapindex>"
|
||||
)
|
||||
idx = b"<sitemapindex><sitemap><loc>https://x.test/s1.xml</loc></sitemap></sitemapindex>"
|
||||
assert parse_sitemap(idx) == ("sitemapindex", ["https://x.test/s1.xml"])
|
||||
|
||||
assert parse_sitemap(gzip.compress(data))[0] == "urlset"
|
||||
@@ -183,7 +180,6 @@ def test_inspect_remote_html_cms_informational():
|
||||
assert not is_actionable(result)
|
||||
|
||||
|
||||
|
||||
def test_parse_sitemap_rejects_oversized_gzip(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
audit_website,
|
||||
@@ -202,9 +198,7 @@ def test_parse_sitemap_rejects_oversized_gzip(monkeypatch):
|
||||
|
||||
def test_validate_public_url_rejects_private_literal():
|
||||
with pytest.raises(ValueError, match="non-public address"):
|
||||
audit_website._validate_public_http_url(
|
||||
"http://127.0.0.1/secret"
|
||||
)
|
||||
audit_website._validate_public_http_url("http://127.0.0.1/secret")
|
||||
|
||||
|
||||
def test_validate_public_url_rejects_private_dns(monkeypatch):
|
||||
@@ -226,9 +220,7 @@ def test_validate_public_url_rejects_private_dns(monkeypatch):
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="non-public address"):
|
||||
audit_website._validate_public_http_url(
|
||||
"https://example.com/page"
|
||||
)
|
||||
audit_website._validate_public_http_url("https://example.com/page")
|
||||
|
||||
|
||||
def test_validate_public_url_accepts_public_dns(monkeypatch):
|
||||
@@ -249,9 +241,7 @@ def test_validate_public_url_accepts_public_dns(monkeypatch):
|
||||
fake_getaddrinfo,
|
||||
)
|
||||
|
||||
assert audit_website._validate_public_http_url(
|
||||
"https://example.com/page"
|
||||
) == (
|
||||
assert audit_website._validate_public_http_url("https://example.com/page") == (
|
||||
"https",
|
||||
"example.com",
|
||||
443,
|
||||
@@ -278,9 +268,7 @@ def test_collect_urls_rejects_cross_origin_entry(monkeypatch):
|
||||
allowed_origin=None,
|
||||
):
|
||||
return (
|
||||
b"<urlset><url>"
|
||||
b"<loc>http://127.0.0.1/secret</loc>"
|
||||
b"</url></urlset>",
|
||||
b"<urlset><url><loc>http://127.0.0.1/secret</loc></url></urlset>",
|
||||
"application/xml",
|
||||
)
|
||||
|
||||
|
||||
@@ -10,8 +10,8 @@ ROOT = Path(__file__).resolve().parents[1]
|
||||
SCRIPTS = ROOT / "service" / "scripts"
|
||||
sys.path.insert(0, str(SCRIPTS))
|
||||
|
||||
from format_dispatch import classify, classify_bytes # noqa: E402
|
||||
from image_meta import ( # noqa: E402
|
||||
from format_dispatch import classify_bytes
|
||||
from image_meta import (
|
||||
XMP_UUID,
|
||||
clean_image,
|
||||
detect_format,
|
||||
@@ -95,7 +95,7 @@ def test_inspect_isobmff_detects_c2pa_and_xmp():
|
||||
|
||||
def test_inspect_isobmff_heic_ai_metadata():
|
||||
heic_bytes = _minimal_heic_with_xmp()
|
||||
has_c2pa, has_ai, findings = inspect_isobmff(heic_bytes, "heic")
|
||||
_has_c2pa, has_ai, findings = inspect_isobmff(heic_bytes, "heic")
|
||||
assert has_ai is True
|
||||
assert any("trainedAlgorithmicMedia" in f or "XMP" in f for f in findings)
|
||||
|
||||
@@ -108,7 +108,7 @@ def test_strip_isobmff_removes_c2pa_and_xmp():
|
||||
assert any("xmp" in a.lower() or "uuid" in a.lower() for a in actions)
|
||||
|
||||
# Re-inspect cleaned bytes
|
||||
has_c2pa, has_ai, findings = inspect_isobmff(cleaned, "avif")
|
||||
has_c2pa, has_ai, _findings = inspect_isobmff(cleaned, "avif")
|
||||
assert has_c2pa is False
|
||||
assert has_ai is False
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@ ROOT = Path(__file__).resolve().parents[1]
|
||||
SCRIPTS = ROOT / "service" / "scripts"
|
||||
sys.path.insert(0, str(SCRIPTS))
|
||||
|
||||
from common import guard_binary, looks_binary # noqa: E402
|
||||
from common import guard_binary, looks_binary
|
||||
|
||||
DOCX_XML = (
|
||||
'<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'
|
||||
@@ -38,11 +38,13 @@ def run(script: str, *args: str) -> subprocess.CompletedProcess:
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=60,
|
||||
check=False,
|
||||
)
|
||||
|
||||
|
||||
# --- looks_binary ----------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"data,expected_fragment",
|
||||
[
|
||||
@@ -67,8 +69,8 @@ def test_flags_binary(data, expected_fragment):
|
||||
b"",
|
||||
b"Just some prose.\n",
|
||||
b"# Markdown\n\n- bullet\n",
|
||||
"Accented prose: naïve café résumé\n".encode("utf-8"),
|
||||
"Zero width and nbsp here\n".encode("utf-8"),
|
||||
"Accented prose: naïve café résumé\n".encode(),
|
||||
"Zero width and nbsp here\n".encode(),
|
||||
b"Latin-1 bytes: caf\xe9 na\xefve\n", # not UTF-8, still text
|
||||
b"a\tb\r\nc\x0cd\x1b[0m\n", # tabs, CRLF, form feed, ANSI escape
|
||||
],
|
||||
@@ -91,6 +93,7 @@ def test_guard_binary_can_be_overridden():
|
||||
|
||||
# --- CLI behaviour ---------------------------------------------------------
|
||||
|
||||
|
||||
def test_inspect_text_refuses_docx(tmp_path):
|
||||
docx = make_docx(tmp_path / "doc.docx")
|
||||
r = run("inspect_text.py", str(docx))
|
||||
@@ -209,6 +212,7 @@ def test_stdin_binary_is_refused():
|
||||
input=docx.getvalue(),
|
||||
capture_output=True,
|
||||
timeout=60,
|
||||
check=False,
|
||||
)
|
||||
assert r.returncode == 2
|
||||
assert b"looks like" in r.stderr
|
||||
@@ -237,6 +241,7 @@ def test_stdin_non_ascii_magic_is_refused_whatever_the_codec(data, label, io_enc
|
||||
capture_output=True,
|
||||
timeout=60,
|
||||
env=env,
|
||||
check=False,
|
||||
)
|
||||
assert r.returncode == 2, (label, io_encoding, r.stderr)
|
||||
assert label.encode() in r.stderr, (label, io_encoding, r.stderr)
|
||||
@@ -245,9 +250,10 @@ def test_stdin_non_ascii_magic_is_refused_whatever_the_codec(data, label, io_enc
|
||||
def test_stdin_text_still_flows_through():
|
||||
r = subprocess.run(
|
||||
[sys.executable, str(SCRIPTS / "clean_text.py")],
|
||||
input="plaintext\n".encode("utf-8"),
|
||||
input="plaintext\n".encode(),
|
||||
capture_output=True,
|
||||
timeout=60,
|
||||
check=False,
|
||||
)
|
||||
assert r.returncode == 0, r.stderr
|
||||
assert r.stdout.replace(b"\r\n", b"\n") == b"plaintext\n"
|
||||
|
||||
@@ -9,7 +9,7 @@ ROOT = Path(__file__).resolve().parents[1]
|
||||
SCRIPTS = ROOT / "service" / "scripts"
|
||||
sys.path.insert(0, str(SCRIPTS))
|
||||
|
||||
import image_meta # noqa: E402
|
||||
import image_meta
|
||||
|
||||
MANIFEST_OUTPUT = """{
|
||||
"active_manifest": "urn:c2pa:0000",
|
||||
|
||||
@@ -11,10 +11,9 @@ ROOT = Path(__file__).resolve().parents[1]
|
||||
SCRIPTS = ROOT / "service" / "scripts"
|
||||
sys.path.insert(0, str(SCRIPTS))
|
||||
|
||||
from image_meta import ( # noqa: E402
|
||||
from image_meta import (
|
||||
clean_image,
|
||||
detect_format,
|
||||
inspect_image,
|
||||
inspect_webp,
|
||||
strip_jpeg,
|
||||
strip_png,
|
||||
@@ -63,9 +62,7 @@ def _webp_chunk(fourcc: bytes, payload: bytes) -> bytes:
|
||||
|
||||
|
||||
def _minimal_webp(*chunks: tuple[bytes, bytes]) -> bytes:
|
||||
body = b"WEBP" + b"".join(
|
||||
_webp_chunk(fourcc, payload) for fourcc, payload in chunks
|
||||
)
|
||||
body = b"WEBP" + b"".join(_webp_chunk(fourcc, payload) for fourcc, payload in chunks)
|
||||
return b"RIFF" + struct.pack("<I", len(body)) + body
|
||||
|
||||
|
||||
|
||||
+13
-11
@@ -9,7 +9,7 @@ ROOT = Path(__file__).resolve().parents[1]
|
||||
SCRIPTS = ROOT / "service" / "scripts"
|
||||
sys.path.insert(0, str(SCRIPTS))
|
||||
|
||||
from text_unicode import clean_text, inspect_text # noqa: E402
|
||||
from text_unicode import clean_text, inspect_text
|
||||
|
||||
|
||||
def test_strips_zero_width_and_soft_hyphen():
|
||||
@@ -107,16 +107,16 @@ def test_clean_preserves_arrow_emoji_vs16():
|
||||
|
||||
|
||||
def test_clean_preserves_cjk_ideographic_variation_selector():
|
||||
raw = "\u845b\U000E0100" # CJK ideograph + VS17
|
||||
raw = "\u845b\U000e0100" # CJK ideograph + VS17
|
||||
cleaned, stats = clean_text(raw)
|
||||
assert cleaned == raw
|
||||
assert stats["removed_count"] == 0
|
||||
|
||||
|
||||
def test_clean_strips_repeated_cjk_variation_selector():
|
||||
raw = "\u845b\U000E0100\U000E0101"
|
||||
raw = "\u845b\U000e0100\U000e0101"
|
||||
cleaned, stats = clean_text(raw)
|
||||
assert cleaned == "\u845b\U000E0100"
|
||||
assert cleaned == "\u845b\U000e0100"
|
||||
assert stats["removed_count"] == 1
|
||||
|
||||
|
||||
@@ -128,14 +128,14 @@ def test_clean_preserves_mongolian_variation_selector():
|
||||
|
||||
|
||||
def test_clean_preserves_zwj_family():
|
||||
raw = "Family time: \U0001F468\u200D\U0001F469\u200D\U0001F467" # 👨👩👧
|
||||
raw = "Family time: \U0001f468\u200d\U0001f469\u200d\U0001f467" # 👨👩👧
|
||||
cleaned, stats = clean_text(raw)
|
||||
assert cleaned == raw
|
||||
assert stats["removed_count"] == 0
|
||||
|
||||
|
||||
def test_clean_preserves_zwj_chain():
|
||||
raw = "\u2764\ufe0f\u200d\U0001F525" # ❤️🔥
|
||||
raw = "\u2764\ufe0f\u200d\U0001f525" # ❤️🔥
|
||||
cleaned, stats = clean_text(raw)
|
||||
assert cleaned == raw
|
||||
assert stats["removed_count"] == 0
|
||||
@@ -149,7 +149,7 @@ def test_clean_strips_floating_emoji_glue():
|
||||
|
||||
|
||||
def test_inspect_emoji_glue_not_suspicious_by_default():
|
||||
raw = "Balance returns. \u2696\ufe0f Family time: \U0001F468\u200D\U0001F469\u200D\U0001F467"
|
||||
raw = "Balance returns. \u2696\ufe0f Family time: \U0001f468\u200d\U0001f469\u200d\U0001f467"
|
||||
report = inspect_text(raw)
|
||||
assert report.suspicious_total == 0
|
||||
|
||||
@@ -182,15 +182,15 @@ def test_clean_preserves_script_joiners():
|
||||
|
||||
def test_clean_preserves_flag_tag_sequence():
|
||||
# Scotland flag: emoji base U+1F3F4 + tag chars ending in U+E007F.
|
||||
raw = "\U0001F3F4\U000E0067\U000E0062\U000E0073\U000E0063\U000E0074\U000E007F"
|
||||
raw = "\U0001f3f4\U000e0067\U000e0062\U000e0073\U000e0063\U000e0074\U000e007f"
|
||||
cleaned, _ = clean_text(raw)
|
||||
assert cleaned == raw
|
||||
|
||||
|
||||
def test_clean_strips_incomplete_flag_tag_sequence():
|
||||
raw = "\U0001F3F4\U000E0067\U000E0062"
|
||||
raw = "\U0001f3f4\U000e0067\U000e0062"
|
||||
cleaned, stats = clean_text(raw)
|
||||
assert cleaned == "\U0001F3F4"
|
||||
assert cleaned == "\U0001f3f4"
|
||||
assert stats["removed_count"] == 2
|
||||
|
||||
|
||||
@@ -271,7 +271,9 @@ def test_clean_still_strips_floating_script_glue():
|
||||
# Isolated between Latin these are contraband, not orthography.
|
||||
for raw in ("a\u180bb", "a\u17b4b", "a\u115fb", "\u180b", "\u1160"):
|
||||
cleaned, _ = clean_text(raw)
|
||||
assert cleaned == raw.replace("\u180b", "").replace("\u17b4", "").replace("\u115f", "").replace("\u1160", "")
|
||||
assert cleaned == raw.replace("\u180b", "").replace("\u17b4", "").replace(
|
||||
"\u115f", ""
|
||||
).replace("\u1160", "")
|
||||
|
||||
|
||||
def test_clean_strip_emoji_glue_flag_strips_script_glue():
|
||||
|
||||
@@ -17,7 +17,7 @@ ROOT = Path(__file__).resolve().parents[1]
|
||||
SCRIPTS = ROOT / "service" / "scripts"
|
||||
sys.path.insert(0, str(SCRIPTS))
|
||||
|
||||
from container_meta import ( # noqa: E402
|
||||
from container_meta import (
|
||||
clean_container,
|
||||
clean_docx,
|
||||
clean_html,
|
||||
@@ -37,6 +37,7 @@ def _run(script: str, *args: str) -> subprocess.CompletedProcess:
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=60,
|
||||
check=False,
|
||||
)
|
||||
|
||||
|
||||
@@ -69,19 +70,11 @@ def test_markdown_frontmatter_with_blank_line_does_not_crash():
|
||||
|
||||
def test_markdown_drops_nested_children_of_dropped_key():
|
||||
"""Regression: nested values under a dropped AI key survived the clean."""
|
||||
text = (
|
||||
"---\n"
|
||||
"title: Demo\n"
|
||||
"model:\n"
|
||||
" name: claude-opus\n"
|
||||
" version: 4\n"
|
||||
"author: you\n"
|
||||
"---\nBody\n"
|
||||
)
|
||||
text = "---\ntitle: Demo\nmodel:\n name: claude-opus\n version: 4\nauthor: you\n---\nBody\n"
|
||||
cleaned, actions = clean_markdown(text)
|
||||
assert "claude-opus" not in cleaned # the leak
|
||||
assert "claude-opus" not in cleaned # the leak
|
||||
assert "version: 4" not in cleaned
|
||||
assert "title: Demo" in cleaned # siblings untouched
|
||||
assert "title: Demo" in cleaned # siblings untouched
|
||||
assert "author: you" in cleaned
|
||||
assert any("drop frontmatter key: model" in a for a in actions)
|
||||
|
||||
@@ -107,7 +100,7 @@ def test_html_meta_strip():
|
||||
<meta name="viewport" content="width=device-width">
|
||||
<meta name="description" content="ok">
|
||||
</head><body data-ai-model="gpt">Hi</body></html>"""
|
||||
_c2, has_ai, findings, _ = inspect_html(html)
|
||||
_c2, has_ai, _findings, _ = inspect_html(html)
|
||||
assert has_ai
|
||||
cleaned, actions = clean_html(html)
|
||||
assert "ChatGPT" not in cleaned
|
||||
@@ -126,7 +119,7 @@ def test_html_cms_generator_not_ai():
|
||||
|
||||
def test_html_cms_generator_preserved_by_clean():
|
||||
html = '<html><head><meta name="generator" content="WordPress 6.0"><meta name="viewport" content="width=device-width"></head></html>'
|
||||
cleaned, actions = clean_html(html)
|
||||
cleaned, _actions = clean_html(html)
|
||||
assert "WordPress" in cleaned
|
||||
assert "viewport" in cleaned
|
||||
|
||||
@@ -160,7 +153,7 @@ def test_pdf_stream_byte_collision_not_ai(tmp_path: Path):
|
||||
pdf = b"%PDF-1.4\n1 0 obj<< /Length 4 >>stream\nAIGC\nendstream\nendobj\n%%EOF\n"
|
||||
src = tmp_path / "collision.pdf"
|
||||
src.write_bytes(pdf)
|
||||
has_c2pa, has_ai, findings, _ = inspect_pdf(src, pdf)
|
||||
has_c2pa, has_ai, _findings, _ = inspect_pdf(src, pdf)
|
||||
assert not has_c2pa
|
||||
assert not has_ai
|
||||
|
||||
@@ -171,7 +164,7 @@ def test_svg_metadata():
|
||||
<metadata>c2pa contentcredentials Anthropic</metadata>
|
||||
<circle cx="1" cy="1" r="1"/>
|
||||
</svg>"""
|
||||
has_c2pa, has_ai, findings, _ = inspect_svg(svg)
|
||||
has_c2pa, has_ai, _findings, _ = inspect_svg(svg)
|
||||
assert has_c2pa or has_ai
|
||||
cleaned, actions = clean_svg(svg)
|
||||
assert b"<metadata" not in cleaned.lower() or b"c2pa" not in cleaned.lower()
|
||||
@@ -227,9 +220,7 @@ def _dangling_rels(zip_bytes: bytes) -> list[str]:
|
||||
for rels in (n for n in names if n.endswith(".rels")):
|
||||
base = posixpath.dirname(posixpath.dirname(rels))
|
||||
text = zf.read(rels).decode()
|
||||
for m in re.finditer(
|
||||
r'<Relationship\b[^>]*Target="([^"]*)"[^>]*/>', text, re.I
|
||||
):
|
||||
for m in re.finditer(r'<Relationship\b[^>]*Target="([^"]*)"[^>]*/>', text, re.I):
|
||||
target, tag = m.group(1), m.group(0)
|
||||
if re.search(r"\bTargetMode\s*=", tag, re.I):
|
||||
continue # external
|
||||
@@ -329,7 +320,7 @@ def test_docx_metadata_vendor_word_is_still_flagged():
|
||||
from container_meta import inspect_docx
|
||||
|
||||
data = _make_docx_with_app("Claude AI Writer")
|
||||
has_c2pa, has_ai, findings, _ = inspect_docx(data)
|
||||
_has_c2pa, has_ai, findings, _ = inspect_docx(data)
|
||||
assert has_ai
|
||||
assert any("Claude" in f for f in findings)
|
||||
|
||||
@@ -431,7 +422,14 @@ def test_docx_scrubs_docprops_provenance_fields_unconditionally():
|
||||
app = zf.read("docProps/app.xml").decode()
|
||||
|
||||
# Every provenance field is emptied...
|
||||
for field in ("dc:creator", "cp:lastModifiedBy", "dc:description", "cp:keywords", "dc:subject", "cp:category"):
|
||||
for field in (
|
||||
"dc:creator",
|
||||
"cp:lastModifiedBy",
|
||||
"dc:description",
|
||||
"cp:keywords",
|
||||
"dc:subject",
|
||||
"cp:category",
|
||||
):
|
||||
assert f"<{field}></{field}>" in core or f"<{field}/>" in core
|
||||
for field in ("Application", "AppVersion", "Company", "Manager"):
|
||||
assert f"<{field}></{field}>" in app or f"<{field}/>" in app
|
||||
@@ -441,9 +439,9 @@ def test_docx_scrubs_docprops_provenance_fields_unconditionally():
|
||||
assert "Generated by AI" not in core
|
||||
assert "OpenAI" not in app
|
||||
|
||||
# The output docProps remain well-formed XML.
|
||||
ET.fromstring(core)
|
||||
ET.fromstring(app)
|
||||
# The output docProps remain well-formed XML (trusted in-memory test data).
|
||||
ET.fromstring(core) # noqa: S314
|
||||
ET.fromstring(app) # noqa: S314
|
||||
|
||||
assert any("scrub docProps/core.xml field dc:creator" in a for a in actions)
|
||||
assert any("drop part docProps/custom.xml" in a for a in actions)
|
||||
@@ -605,9 +603,7 @@ def test_inspect_container_clean_file_stays_clean(tmp_path: Path):
|
||||
|
||||
def test_inspect_container_svg(tmp_path: Path):
|
||||
src = tmp_path / "a.svg"
|
||||
src.write_bytes(
|
||||
b'<svg xmlns="http://www.w3.org/2000/svg"><metadata>c2pa</metadata></svg>'
|
||||
)
|
||||
src.write_bytes(b'<svg xmlns="http://www.w3.org/2000/svg"><metadata>c2pa</metadata></svg>')
|
||||
report = inspect_container(src)
|
||||
assert report.format == "svg"
|
||||
assert report.has_c2pa or report.has_ai_metadata
|
||||
@@ -641,13 +637,7 @@ def test_pdf_degraded_clean_without_crash(tmp_path: Path):
|
||||
b"<?xpacket end='w'?>"
|
||||
)
|
||||
# Minimal-ish PDF skeleton (not renderable; enough for byte-level tools)
|
||||
pdf = (
|
||||
b"%PDF-1.4\n"
|
||||
b"1 0 obj<<>>endobj\n"
|
||||
b"trailer<<>>\n"
|
||||
+ xmp
|
||||
+ b"\n%%EOF\n"
|
||||
)
|
||||
pdf = b"%PDF-1.4\n1 0 obj<<>>endobj\ntrailer<<>>\n" + xmp + b"\n%%EOF\n"
|
||||
src = tmp_path / "t.pdf"
|
||||
dest = tmp_path / "t.cleaned.pdf"
|
||||
src.write_bytes(pdf)
|
||||
@@ -674,9 +664,18 @@ def test_clean_container_accepts_explicit_fmt(tmp_path: Path):
|
||||
@pytest.mark.parametrize(
|
||||
"ext,make_bytes",
|
||||
[
|
||||
("md", lambda: (Path(__file__).resolve().parent / "fixtures" / "sample_ai.md").read_bytes()),
|
||||
("html", lambda: (Path(__file__).resolve().parent / "fixtures" / "sample_ai.html").read_bytes()),
|
||||
("svg", lambda: (Path(__file__).resolve().parent / "fixtures" / "sample_meta.svg").read_bytes()),
|
||||
(
|
||||
"md",
|
||||
(Path(__file__).resolve().parent / "fixtures" / "sample_ai.md").read_bytes,
|
||||
),
|
||||
(
|
||||
"html",
|
||||
(Path(__file__).resolve().parent / "fixtures" / "sample_ai.html").read_bytes,
|
||||
),
|
||||
(
|
||||
"svg",
|
||||
(Path(__file__).resolve().parent / "fixtures" / "sample_meta.svg").read_bytes,
|
||||
),
|
||||
("docx", _make_docx_with_app),
|
||||
("odt", _make_odt),
|
||||
],
|
||||
@@ -689,4 +688,7 @@ def test_clean_file_in_place_for_every_container_ext(tmp_path: Path, ext: str, m
|
||||
assert path.with_suffix(path.suffix + ".bak").is_file()
|
||||
data = json.loads(r.stdout)
|
||||
assert data["kind"] == "container"
|
||||
assert data["format"] == {"docx": "docx", "odt": "odt", "svg": "svg", "md": "markdown", "html": "html"}[ext]
|
||||
assert (
|
||||
data["format"]
|
||||
== {"docx": "docx", "odt": "odt", "svg": "svg", "md": "markdown", "html": "html"}[ext]
|
||||
)
|
||||
|
||||
@@ -17,12 +17,12 @@ ROOT = Path(__file__).resolve().parents[1]
|
||||
SCRIPTS = ROOT / "service" / "scripts"
|
||||
sys.path.insert(0, str(SCRIPTS))
|
||||
|
||||
import image_meta # noqa: E402
|
||||
from image_meta import run_ctrlregen_clean # noqa: E402
|
||||
import image_meta
|
||||
from image_meta import run_ctrlregen_clean
|
||||
|
||||
CLEAN_SCRIPT = SCRIPTS / "clean_ctrlregen.py"
|
||||
|
||||
FAKE_PIL = '''\
|
||||
FAKE_PIL = """\
|
||||
class Image:
|
||||
def __init__(self, size=(10, 20)):
|
||||
self.size = size
|
||||
@@ -35,7 +35,7 @@ class Image:
|
||||
return self
|
||||
def save(self, fp, format=None, **kwargs):
|
||||
fp.write(b"FAKEIMAGE")
|
||||
'''
|
||||
"""
|
||||
|
||||
|
||||
def _fake_engine(fail_run: bool) -> str:
|
||||
@@ -73,6 +73,7 @@ def _run_adapter(*args: str) -> subprocess.CompletedProcess[str]:
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=env,
|
||||
check=False,
|
||||
)
|
||||
|
||||
|
||||
@@ -123,7 +124,14 @@ def test_cli_json_success(tmp_path: Path):
|
||||
img.write_bytes(b"x")
|
||||
out = tmp_path / "out.png"
|
||||
r = _run_adapter(
|
||||
str(img), "-o", str(out), "--upstream-dir", str(upstream), "--device", "cpu", "--json",
|
||||
str(img),
|
||||
"-o",
|
||||
str(out),
|
||||
"--upstream-dir",
|
||||
str(upstream),
|
||||
"--device",
|
||||
"cpu",
|
||||
"--json",
|
||||
)
|
||||
assert r.returncode == 0, r.stderr
|
||||
payload = json.loads(r.stdout)
|
||||
@@ -137,8 +145,14 @@ def test_cli_runtime_error(tmp_path: Path):
|
||||
img = tmp_path / "img.png"
|
||||
img.write_bytes(b"x")
|
||||
r = _run_adapter(
|
||||
str(img), "-o", str(tmp_path / "out.png"), "--upstream-dir", str(upstream),
|
||||
"--device", "cpu", "--json",
|
||||
str(img),
|
||||
"-o",
|
||||
str(tmp_path / "out.png"),
|
||||
"--upstream-dir",
|
||||
str(upstream),
|
||||
"--device",
|
||||
"cpu",
|
||||
"--json",
|
||||
)
|
||||
assert r.returncode == 1
|
||||
assert "model missing" in (r.stderr or "")
|
||||
@@ -156,7 +170,14 @@ def test_cli_refuses_symlink_output(tmp_path: Path):
|
||||
except OSError:
|
||||
pytest.skip("symlinks unavailable")
|
||||
r = _run_adapter(
|
||||
str(img), "-o", str(out), "--upstream-dir", str(upstream), "--device", "cpu", "--json",
|
||||
str(img),
|
||||
"-o",
|
||||
str(out),
|
||||
"--upstream-dir",
|
||||
str(upstream),
|
||||
"--device",
|
||||
"cpu",
|
||||
"--json",
|
||||
)
|
||||
assert r.returncode == 1
|
||||
assert victim.read_bytes() == b"original"
|
||||
@@ -172,11 +193,12 @@ def test_run_ctrlregen_clean_unconfigured_returns_unavailable(
|
||||
|
||||
|
||||
def test_run_ctrlregen_clean_success_parses_json(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
upstream = tmp_path / "upstream"
|
||||
upstream.mkdir()
|
||||
payload = {"available": True, "output": "/tmp/y.png", "device": "cpu"}
|
||||
payload = {"available": True, "output": str(tmp_path / "y.png"), "device": "cpu"}
|
||||
captured: dict = {}
|
||||
|
||||
def fake_run(cmd, **kwargs):
|
||||
@@ -205,7 +227,8 @@ def test_run_ctrlregen_clean_success_parses_json(
|
||||
|
||||
|
||||
def test_run_ctrlregen_clean_unavailable_exit3(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
upstream = tmp_path / "upstream"
|
||||
upstream.mkdir()
|
||||
@@ -220,7 +243,8 @@ def test_run_ctrlregen_clean_unavailable_exit3(
|
||||
|
||||
|
||||
def test_run_ctrlregen_clean_prefers_venv_python(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
upstream = tmp_path / "upstream"
|
||||
if os.name == "nt":
|
||||
@@ -259,7 +283,7 @@ def test_clean_image_ctrlregen_flag(tmp_path: Path, monkeypatch: pytest.MonkeyPa
|
||||
src,
|
||||
dest,
|
||||
remove_pixel="ctrlregen",
|
||||
ctrlregen_dir="/tmp/upstream",
|
||||
ctrlregen_dir=str(tmp_path / "upstream"),
|
||||
ctrlregen_strength=0.3,
|
||||
)
|
||||
|
||||
|
||||
@@ -11,7 +11,7 @@ SCRIPTS = ROOT / "service" / "scripts"
|
||||
sys.path.insert(0, str(ROOT))
|
||||
sys.path.insert(0, str(SCRIPTS))
|
||||
|
||||
from container_meta import ( # noqa: E402
|
||||
from container_meta import (
|
||||
clean_html,
|
||||
clean_markdown,
|
||||
clean_svg,
|
||||
@@ -19,7 +19,8 @@ from container_meta import ( # noqa: E402
|
||||
inspect_markdown,
|
||||
inspect_svg,
|
||||
)
|
||||
from tests.test_clean_image import _minimal_jpeg_with_app11, _minimal_png_with_text # noqa: E402
|
||||
|
||||
from tests.test_clean_image import _minimal_jpeg_with_app11, _minimal_png_with_text
|
||||
|
||||
|
||||
def test_svg_embedded_png_c2pa_cleaned():
|
||||
@@ -28,7 +29,7 @@ def test_svg_embedded_png_c2pa_cleaned():
|
||||
|
||||
svg_data = f"""<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
|
||||
<image width="100" height="100" xlink:href="data:image/png;base64,{png_b64}" />
|
||||
</svg>""".encode("utf-8")
|
||||
</svg>""".encode()
|
||||
|
||||
# Inspect
|
||||
has_c2pa, has_ai, findings, _ = inspect_svg(svg_data)
|
||||
@@ -41,7 +42,7 @@ def test_svg_embedded_png_c2pa_cleaned():
|
||||
assert any("cleaned embedded data:image/png" in a for a in actions)
|
||||
|
||||
# Re-inspect
|
||||
has_c2pa_after, has_ai_after, findings_after, _ = inspect_svg(cleaned_bytes)
|
||||
has_c2pa_after, has_ai_after, _findings_after, _ = inspect_svg(cleaned_bytes)
|
||||
assert has_c2pa_after is False
|
||||
assert has_ai_after is False
|
||||
assert b"c2pa" not in cleaned_bytes.lower()
|
||||
@@ -55,7 +56,7 @@ def test_svg_embedded_base64_with_newlines():
|
||||
|
||||
svg_data = f"""<svg xmlns="http://www.w3.org/2000/svg">
|
||||
<image href="data:image/png;base64,{multiline_b64}" />
|
||||
</svg>""".encode("utf-8")
|
||||
</svg>""".encode()
|
||||
|
||||
cleaned_bytes, actions = clean_svg(svg_data)
|
||||
assert any("cleaned embedded data:image/png" in a for a in actions)
|
||||
@@ -106,7 +107,7 @@ Here is an image:
|
||||

|
||||
"""
|
||||
|
||||
has_c2pa, has_ai, findings, _ = inspect_markdown(md_text)
|
||||
has_c2pa, _has_ai, findings, _ = inspect_markdown(md_text)
|
||||
assert has_c2pa is True
|
||||
assert any("embedded data:image/png" in f for f in findings)
|
||||
|
||||
@@ -135,7 +136,7 @@ def test_already_clean_embedded_image_no_op():
|
||||
def test_corrupted_data_uri_graceful_fallback():
|
||||
# Corrupted base64 that shouldn't crash the file cleaner
|
||||
bad_html = '<img src="data:image/png;base64,!!!NOT_VALID_BASE64###">'
|
||||
cleaned_text, actions = clean_html(bad_html)
|
||||
cleaned_text, _actions = clean_html(bad_html)
|
||||
assert cleaned_text == bad_html
|
||||
|
||||
|
||||
@@ -145,9 +146,11 @@ def test_nested_svg_data_uri_cleaned():
|
||||
|
||||
parent_html = f'<img src="data:image/svg+xml;base64,{nested_b64}">'
|
||||
|
||||
has_c2pa, has_ai, findings, _ = inspect_html(parent_html)
|
||||
_has_c2pa, has_ai, _findings, _ = inspect_html(parent_html)
|
||||
assert has_ai is True
|
||||
|
||||
cleaned_html, actions = clean_html(parent_html)
|
||||
assert any("cleaned embedded data:image/svg+xml" in a for a in actions)
|
||||
assert "DALL-E" not in base64.b64decode(cleaned_html.split("base64,")[1].split('"')[0]).decode("utf-8")
|
||||
assert "DALL-E" not in base64.b64decode(cleaned_html.split("base64,")[1].split('"')[0]).decode(
|
||||
"utf-8"
|
||||
)
|
||||
|
||||
@@ -17,7 +17,7 @@ ROOT = Path(__file__).resolve().parents[1]
|
||||
SCRIPTS = ROOT / "service" / "scripts"
|
||||
sys.path.insert(0, str(SCRIPTS))
|
||||
|
||||
import server # noqa: E402
|
||||
import server
|
||||
|
||||
|
||||
def _png_chunk(ctype: bytes, payload: bytes) -> bytes:
|
||||
@@ -175,23 +175,25 @@ def test_clean_markdown_container(conn):
|
||||
|
||||
|
||||
def test_unknown_option_rejected(conn):
|
||||
status, body = _post(conn, "/clean", {"file": _b64(b"x"), "name": "x.txt", "options": {"nope": 1}})
|
||||
status, body = _post(
|
||||
conn, "/clean", {"file": _b64(b"x"), "name": "x.txt", "options": {"nope": 1}}
|
||||
)
|
||||
assert status == 400
|
||||
assert "unknown option" in body["error"]
|
||||
|
||||
|
||||
def test_bad_base64_rejected(conn):
|
||||
status, body = _post(conn, "/inspect", {"file": "!!!not-base64!!!"})
|
||||
status, _body = _post(conn, "/inspect", {"file": "!!!not-base64!!!"})
|
||||
assert status == 400
|
||||
|
||||
|
||||
def test_binary_named_as_text_rejected(conn):
|
||||
status, body = _post(conn, "/clean", {"file": _b64(_watermarked_png()), "name": "x.txt"})
|
||||
status, _body = _post(conn, "/clean", {"file": _b64(_watermarked_png()), "name": "x.txt"})
|
||||
assert status == 400
|
||||
|
||||
|
||||
def test_missing_file_field_rejected(conn):
|
||||
status, body = _post(conn, "/inspect", {"name": "x.txt"})
|
||||
status, _body = _post(conn, "/inspect", {"name": "x.txt"})
|
||||
assert status == 400
|
||||
|
||||
|
||||
@@ -218,7 +220,7 @@ def test_traversal_name_does_not_escape(conn, tmp_path):
|
||||
def test_oversized_body_413(conn, monkeypatch):
|
||||
monkeypatch.setattr(server, "MAX_BODY_BYTES", 64)
|
||||
data = "x" * 200
|
||||
status, body = _post(conn, "/inspect", {"file": _b64(data.encode())})
|
||||
status, _body = _post(conn, "/inspect", {"file": _b64(data.encode())})
|
||||
assert status == 413
|
||||
|
||||
|
||||
@@ -233,7 +235,7 @@ def test_auth_required(conn, monkeypatch):
|
||||
|
||||
|
||||
def test_404(conn):
|
||||
status, body = _get(conn, "/nope")
|
||||
status, _body = _get(conn, "/nope")
|
||||
assert status == 404
|
||||
status, body = _post(conn, "/nope", {"file": _b64(b"x")})
|
||||
status, _body = _post(conn, "/nope", {"file": _b64(b"x")})
|
||||
assert status == 404
|
||||
|
||||
@@ -9,8 +9,8 @@ ROOT = Path(__file__).resolve().parents[1]
|
||||
SCRIPTS = ROOT / "service" / "scripts"
|
||||
sys.path.insert(0, str(SCRIPTS))
|
||||
|
||||
import clean_file # noqa: E402
|
||||
import clean_image # noqa: E402
|
||||
import clean_file
|
||||
import clean_image
|
||||
|
||||
|
||||
def _container_result(dest: Path, residual: bool, degraded: bool = False) -> dict:
|
||||
@@ -54,7 +54,9 @@ def test_clean_file_json_and_human_agree_on_clean(monkeypatch, tmp_path):
|
||||
|
||||
|
||||
def test_clean_file_degraded_pdf_is_not_a_failure_in_either_mode(monkeypatch, tmp_path):
|
||||
assert _run_clean_file(monkeypatch, tmp_path, json_flag=False, residual=True, degraded=True) == 0
|
||||
assert (
|
||||
_run_clean_file(monkeypatch, tmp_path, json_flag=False, residual=True, degraded=True) == 0
|
||||
)
|
||||
assert _run_clean_file(monkeypatch, tmp_path, json_flag=True, residual=True, degraded=True) == 0
|
||||
|
||||
|
||||
|
||||
@@ -17,8 +17,8 @@ ROOT = Path(__file__).resolve().parents[1]
|
||||
SCRIPTS = ROOT / "service" / "scripts"
|
||||
sys.path.insert(0, str(SCRIPTS))
|
||||
|
||||
import image_meta # noqa: E402
|
||||
from image_meta import run_markdiffusion_purify # noqa: E402
|
||||
import image_meta
|
||||
from image_meta import run_markdiffusion_purify
|
||||
|
||||
HARNESS_SCRIPT = SCRIPTS / "markdiffusion_harness.py"
|
||||
|
||||
@@ -39,7 +39,7 @@ class Image:
|
||||
fp.write(b"FAKEPNG")
|
||||
"""
|
||||
|
||||
FAKE_MARKDIFFUSION = '''\
|
||||
FAKE_MARKDIFFUSION = """\
|
||||
class DiffusionConfig:
|
||||
def __init__(self, scheduler=None, pipe=None, device="cpu", **kwargs):
|
||||
self.device = device
|
||||
@@ -71,7 +71,7 @@ class DiffusionPurification:
|
||||
return Image.new("RGB", (16, 16))
|
||||
|
||||
from types import SimpleNamespace
|
||||
'''
|
||||
"""
|
||||
|
||||
|
||||
def _make_fake_upstream(tmp_path: Path) -> Path:
|
||||
@@ -139,6 +139,7 @@ def test_cli_unavailable_without_upstream(tmp_path: Path):
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=env,
|
||||
check=False,
|
||||
)
|
||||
assert r.returncode == 3
|
||||
assert "markdiffusion not importable" in (r.stderr or "")
|
||||
@@ -149,6 +150,7 @@ def test_cli_bad_input_missing_file(tmp_path: Path):
|
||||
[sys.executable, str(HARNESS_SCRIPT), "detect", str(tmp_path / "missing.png")],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
assert r.returncode == 2
|
||||
|
||||
@@ -160,6 +162,7 @@ def test_cli_bad_scheme(tmp_path: Path):
|
||||
[sys.executable, str(HARNESS_SCRIPT), "detect", str(img), "--scheme", "bogus"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
assert r.returncode == 2
|
||||
assert "unknown scheme" in (r.stderr or "")
|
||||
@@ -172,6 +175,7 @@ def test_cli_missing_purify_output(tmp_path: Path):
|
||||
[sys.executable, str(HARNESS_SCRIPT), "purify", str(img)],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
assert r.returncode == 2
|
||||
|
||||
@@ -243,9 +247,7 @@ def test_cmd_detect_p_value(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
|
||||
"_load_diffusion",
|
||||
lambda model, device, offline, size: (object(), object()),
|
||||
)
|
||||
args = _build_args(
|
||||
mod, "detect", path=str(img), detector_type="p_value"
|
||||
)
|
||||
args = _build_args(mod, "detect", path=str(img), detector_type="p_value")
|
||||
assert mod._cmd_detect(args, upstream, "tr") == 0
|
||||
|
||||
|
||||
@@ -300,9 +302,7 @@ def test_cmd_purify_runtime_error(tmp_path: Path, monkeypatch: pytest.MonkeyPatc
|
||||
raise RuntimeError("model missing")
|
||||
|
||||
monkeypatch.setattr(mod, "_load_diffusion", _boom)
|
||||
args = _build_args(
|
||||
mod, "purify", path=str(img), output=str(tmp_path / "out.png")
|
||||
)
|
||||
args = _build_args(mod, "purify", path=str(img), output=str(tmp_path / "out.png"))
|
||||
assert mod._cmd_purify(args, upstream) == 1
|
||||
|
||||
|
||||
@@ -315,12 +315,10 @@ def test_run_purify_unconfigured_returns_unavailable(monkeypatch: pytest.MonkeyP
|
||||
assert result["available"] is False
|
||||
|
||||
|
||||
def test_run_purify_success_parses_json(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
def test_run_purify_success_parses_json(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
|
||||
upstream = tmp_path / "upstream"
|
||||
upstream.mkdir()
|
||||
payload = {"available": True, "output": "/tmp/y.png", "device": "cpu"}
|
||||
payload = {"available": True, "output": str(tmp_path / "y.png"), "device": "cpu"}
|
||||
captured: dict = {}
|
||||
|
||||
def fake_run(cmd, **kwargs):
|
||||
@@ -348,15 +346,10 @@ def test_run_purify_success_parses_json(
|
||||
assert "--upstream-dir" in cmd and str(upstream) in cmd
|
||||
assert captured["kwargs"]["timeout"] == 99
|
||||
if os.name == "posix":
|
||||
assert (
|
||||
captured["kwargs"]["preexec_fn"]
|
||||
is image_meta.ctrlregen_subprocess_preexec_fn
|
||||
)
|
||||
assert captured["kwargs"]["preexec_fn"] is image_meta.ctrlregen_subprocess_preexec_fn
|
||||
|
||||
|
||||
def test_run_purify_runtime_error_is_reported(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
def test_run_purify_runtime_error_is_reported(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
|
||||
upstream = tmp_path / "upstream"
|
||||
upstream.mkdir()
|
||||
|
||||
@@ -364,16 +357,12 @@ def test_run_purify_runtime_error_is_reported(
|
||||
return SimpleNamespace(returncode=1, stdout="", stderr="boom")
|
||||
|
||||
monkeypatch.setattr(image_meta.subprocess, "run", fake_run)
|
||||
result = run_markdiffusion_purify(
|
||||
Path("x.png"), Path("y.png"), upstream_dir=str(upstream)
|
||||
)
|
||||
result = run_markdiffusion_purify(Path("x.png"), Path("y.png"), upstream_dir=str(upstream))
|
||||
assert result["available"] is False
|
||||
assert "boom" in result["error"]
|
||||
|
||||
|
||||
def test_run_purify_prefers_venv_python(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
def test_run_purify_prefers_venv_python(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
|
||||
upstream = tmp_path / "upstream"
|
||||
if os.name == "nt":
|
||||
venv_python = upstream / ".venv" / "Scripts" / "python.exe"
|
||||
@@ -388,15 +377,11 @@ def test_run_purify_prefers_venv_python(
|
||||
return SimpleNamespace(returncode=0, stdout="{}", stderr="")
|
||||
|
||||
monkeypatch.setattr(image_meta.subprocess, "run", fake_run)
|
||||
run_markdiffusion_purify(
|
||||
Path("x.png"), Path("y.png"), upstream_dir=str(upstream)
|
||||
)
|
||||
run_markdiffusion_purify(Path("x.png"), Path("y.png"), upstream_dir=str(upstream))
|
||||
assert captured["cmd"][0] == str(venv_python)
|
||||
|
||||
|
||||
def test_clean_image_diffusion_flag(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
def test_clean_image_diffusion_flag(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.delenv("MARKDIFFUSION_DIR", raising=False)
|
||||
monkeypatch.delenv("REVERSE_SYNTHID_DIR", raising=False)
|
||||
src = tmp_path / "t.png"
|
||||
@@ -415,7 +400,7 @@ def test_clean_image_diffusion_flag(
|
||||
src,
|
||||
dest,
|
||||
remove_pixel="diffusion",
|
||||
markdiffusion_dir="/tmp/upstream",
|
||||
markdiffusion_dir=str(tmp_path / "upstream"),
|
||||
markdiffusion_strength=0.3,
|
||||
)
|
||||
|
||||
|
||||
+100
-31
@@ -51,7 +51,11 @@ SYNTHID_CONFIG = '{"algorithm_name": "SynthID", "threshold": 0.52, "detector_typ
|
||||
|
||||
|
||||
def _fake_auto_watermark(*, fail_detect: bool = False, fail_generate: bool = False) -> str:
|
||||
detect_body = 'raise RuntimeError("boom")' if fail_detect else 'return {"is_watermarked": True, "score": 3.5}'
|
||||
detect_body = (
|
||||
'raise RuntimeError("boom")'
|
||||
if fail_detect
|
||||
else 'return {"is_watermarked": True, "score": 3.5}'
|
||||
)
|
||||
gen_body = 'raise RuntimeError("boom")' if fail_generate else "return 'WATERMARKED SAMPLE'"
|
||||
return (
|
||||
"class _WM:\n"
|
||||
@@ -90,9 +94,8 @@ def _make_fake_upstream(
|
||||
watermark.mkdir(parents=True)
|
||||
(watermark / "__init__.py").write_text("")
|
||||
(watermark / "auto_watermark.py").write_text(
|
||||
"from types import SimpleNamespace\n" + _fake_auto_watermark(
|
||||
fail_detect=fail_detect, fail_generate=fail_generate
|
||||
)
|
||||
"from types import SimpleNamespace\n"
|
||||
+ _fake_auto_watermark(fail_detect=fail_detect, fail_generate=fail_generate)
|
||||
)
|
||||
utils_dir = upstream / "utils"
|
||||
utils_dir.mkdir(parents=True)
|
||||
@@ -112,6 +115,7 @@ def _run_adapter(*args: str) -> subprocess.CompletedProcess[str]:
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=env,
|
||||
check=False,
|
||||
)
|
||||
|
||||
|
||||
@@ -193,8 +197,15 @@ def test_cli_detect_json_success(tmp_path: Path):
|
||||
f = tmp_path / "t.txt"
|
||||
f.write_text("hello world")
|
||||
r = _run_adapter(
|
||||
"detect", str(f), "--scheme", "kgw", "--upstream-dir", str(upstream),
|
||||
"--device", "cpu", "--json",
|
||||
"detect",
|
||||
str(f),
|
||||
"--scheme",
|
||||
"kgw",
|
||||
"--upstream-dir",
|
||||
str(upstream),
|
||||
"--device",
|
||||
"cpu",
|
||||
"--json",
|
||||
)
|
||||
assert r.returncode == 0, r.stderr
|
||||
payload = json.loads(r.stdout)
|
||||
@@ -211,8 +222,15 @@ def test_cli_detect_synthid_alias(tmp_path: Path):
|
||||
f = tmp_path / "t.txt"
|
||||
f.write_text("hello world")
|
||||
r = _run_adapter(
|
||||
"detect", str(f), "--scheme", "synthid-text", "--upstream-dir", str(upstream),
|
||||
"--device", "cpu", "--json",
|
||||
"detect",
|
||||
str(f),
|
||||
"--scheme",
|
||||
"synthid-text",
|
||||
"--upstream-dir",
|
||||
str(upstream),
|
||||
"--device",
|
||||
"cpu",
|
||||
"--json",
|
||||
)
|
||||
assert r.returncode == 0, r.stderr
|
||||
payload = json.loads(r.stdout)
|
||||
@@ -225,8 +243,15 @@ def test_cli_detect_runtime_error(tmp_path: Path):
|
||||
f = tmp_path / "t.txt"
|
||||
f.write_text("hello world")
|
||||
r = _run_adapter(
|
||||
"detect", str(f), "--scheme", "kgw", "--upstream-dir", str(upstream),
|
||||
"--device", "cpu", "--json",
|
||||
"detect",
|
||||
str(f),
|
||||
"--scheme",
|
||||
"kgw",
|
||||
"--upstream-dir",
|
||||
str(upstream),
|
||||
"--device",
|
||||
"cpu",
|
||||
"--json",
|
||||
)
|
||||
assert r.returncode == 1
|
||||
assert "boom" in (r.stderr or "")
|
||||
@@ -237,8 +262,16 @@ def test_cli_detect_offline_flag(tmp_path: Path):
|
||||
f = tmp_path / "t.txt"
|
||||
f.write_text("hello world")
|
||||
r = _run_adapter(
|
||||
"detect", str(f), "--scheme", "kgw", "--upstream-dir", str(upstream),
|
||||
"--device", "cpu", "--json", "--offline",
|
||||
"detect",
|
||||
str(f),
|
||||
"--scheme",
|
||||
"kgw",
|
||||
"--upstream-dir",
|
||||
str(upstream),
|
||||
"--device",
|
||||
"cpu",
|
||||
"--json",
|
||||
"--offline",
|
||||
)
|
||||
assert r.returncode == 0, r.stderr
|
||||
assert "local_files_only" in (r.stderr or "")
|
||||
@@ -252,8 +285,14 @@ def test_cli_config_too_large(tmp_path: Path):
|
||||
f = tmp_path / "t.txt"
|
||||
f.write_text("hello world")
|
||||
r = _run_adapter(
|
||||
"detect", str(f), "--scheme", "kgw", "--config", str(big),
|
||||
"--upstream-dir", str(upstream),
|
||||
"detect",
|
||||
str(f),
|
||||
"--scheme",
|
||||
"kgw",
|
||||
"--config",
|
||||
str(big),
|
||||
"--upstream-dir",
|
||||
str(upstream),
|
||||
)
|
||||
assert r.returncode == 3
|
||||
assert "too large" in (r.stderr or "")
|
||||
@@ -266,9 +305,19 @@ def test_cli_watermark_json_success(tmp_path: Path):
|
||||
wm_out = tmp_path / "wm.txt"
|
||||
uwm_out = tmp_path / "uwm.txt"
|
||||
r = _run_adapter(
|
||||
"watermark", str(prompt), "--scheme", "kgw",
|
||||
"-o", str(wm_out), "-o2", str(uwm_out),
|
||||
"--upstream-dir", str(upstream), "--device", "cpu", "--json",
|
||||
"watermark",
|
||||
str(prompt),
|
||||
"--scheme",
|
||||
"kgw",
|
||||
"-o",
|
||||
str(wm_out),
|
||||
"-o2",
|
||||
str(uwm_out),
|
||||
"--upstream-dir",
|
||||
str(upstream),
|
||||
"--device",
|
||||
"cpu",
|
||||
"--json",
|
||||
)
|
||||
assert r.returncode == 0, r.stderr
|
||||
payload = json.loads(r.stdout)
|
||||
@@ -282,8 +331,15 @@ def test_cli_watermark_runtime_error(tmp_path: Path):
|
||||
prompt = tmp_path / "prompt.txt"
|
||||
prompt.write_text("write about capybaras")
|
||||
r = _run_adapter(
|
||||
"watermark", str(prompt), "--scheme", "kgw",
|
||||
"--upstream-dir", str(upstream), "--device", "cpu", "--json",
|
||||
"watermark",
|
||||
str(prompt),
|
||||
"--scheme",
|
||||
"kgw",
|
||||
"--upstream-dir",
|
||||
str(upstream),
|
||||
"--device",
|
||||
"cpu",
|
||||
"--json",
|
||||
)
|
||||
assert r.returncode == 1
|
||||
assert "boom" in (r.stderr or "")
|
||||
@@ -295,7 +351,11 @@ def test_rewrite_markllm_detect_missing_venv(tmp_path: Path):
|
||||
upstream = tmp_path / "MarkLLM"
|
||||
upstream.mkdir()
|
||||
result = rewrite_text._markllm_detect(
|
||||
"hello", scheme="kgw", upstream_dir=str(upstream), model="x", timeout=5,
|
||||
"hello",
|
||||
scheme="kgw",
|
||||
upstream_dir=str(upstream),
|
||||
model="x",
|
||||
timeout=5,
|
||||
)
|
||||
assert result["available"] is False
|
||||
|
||||
@@ -322,7 +382,11 @@ def test_rewrite_markllm_detect_parses_json(tmp_path: Path, monkeypatch: pytest.
|
||||
|
||||
monkeypatch.setattr(rewrite_text.subprocess, "run", fake_run)
|
||||
result = rewrite_text._markllm_detect(
|
||||
"hello", scheme="kgw", upstream_dir=str(upstream), model="x", timeout=5,
|
||||
"hello",
|
||||
scheme="kgw",
|
||||
upstream_dir=str(upstream),
|
||||
model="x",
|
||||
timeout=5,
|
||||
)
|
||||
assert result["available"] is True
|
||||
assert result["score"] == 2.0
|
||||
@@ -347,7 +411,11 @@ def test_rewrite_markllm_detect_adapter_failure(tmp_path: Path, monkeypatch: pyt
|
||||
|
||||
monkeypatch.setattr(rewrite_text.subprocess, "run", fake_run)
|
||||
result = rewrite_text._markllm_detect(
|
||||
"hello", scheme="kgw", upstream_dir=str(upstream), model="x", timeout=5,
|
||||
"hello",
|
||||
scheme="kgw",
|
||||
upstream_dir=str(upstream),
|
||||
model="x",
|
||||
timeout=5,
|
||||
)
|
||||
assert result["available"] is False
|
||||
assert "deps missing" in result["error"]
|
||||
@@ -371,7 +439,8 @@ def test_markllm_preexec_env(monkeypatch: pytest.MonkeyPatch):
|
||||
|
||||
|
||||
def test_rewrite_markllm_detect_applies_rlimit(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
import rewrite_text
|
||||
|
||||
@@ -387,13 +456,15 @@ def test_rewrite_markllm_detect_applies_rlimit(
|
||||
|
||||
def fake_run(cmd, **kwargs):
|
||||
captured["preexec_fn"] = kwargs.get("preexec_fn")
|
||||
return SimpleNamespace(
|
||||
returncode=0, stdout='{"available": true}', stderr=""
|
||||
)
|
||||
return SimpleNamespace(returncode=0, stdout='{"available": true}', stderr="")
|
||||
|
||||
monkeypatch.setattr(rewrite_text.subprocess, "run", fake_run)
|
||||
result = rewrite_text._markllm_detect(
|
||||
"hello", scheme="kgw", upstream_dir=str(upstream), model="x", timeout=5,
|
||||
"hello",
|
||||
scheme="kgw",
|
||||
upstream_dir=str(upstream),
|
||||
model="x",
|
||||
timeout=5,
|
||||
)
|
||||
assert result["available"] is True
|
||||
assert callable(captured["preexec_fn"])
|
||||
@@ -406,9 +477,7 @@ def test_rewrite_markllm_hook_records_before_after(tmp_path: Path, monkeypatch:
|
||||
return {"available": True, "is_watermarked": text == "ORIG", "score": 3.0}
|
||||
|
||||
monkeypatch.setattr(rewrite_text, "_markllm_detect", fake_detect)
|
||||
monkeypatch.setattr(
|
||||
rewrite_text, "call_ollama", lambda *a, **k: "REWRITTEN OUTPUT"
|
||||
)
|
||||
monkeypatch.setattr(rewrite_text, "call_ollama", lambda *a, **k: "REWRITTEN OUTPUT")
|
||||
out, info = rewrite_text.rewrite(
|
||||
"ORIG",
|
||||
backend="ollama",
|
||||
@@ -423,7 +492,7 @@ def test_rewrite_markllm_hook_records_before_after(tmp_path: Path, monkeypatch:
|
||||
temperature=0.9,
|
||||
candidates=1,
|
||||
markllm_scheme="kgw",
|
||||
markllm_dir="/tmp/x",
|
||||
markllm_dir=str(tmp_path / "x"),
|
||||
markllm_model="opt-1.3b",
|
||||
markllm_timeout=5,
|
||||
)
|
||||
|
||||
@@ -20,8 +20,8 @@ ROOT = Path(__file__).resolve().parents[1]
|
||||
SCRIPTS = ROOT / "service" / "scripts"
|
||||
sys.path.insert(0, str(SCRIPTS))
|
||||
|
||||
import container_meta # noqa: E402
|
||||
from container_meta import clean_pdf # noqa: E402
|
||||
import container_meta
|
||||
from container_meta import clean_pdf
|
||||
|
||||
|
||||
class _Completed:
|
||||
@@ -104,7 +104,7 @@ def test_with_qpdf_the_document_is_rebuilt(monkeypatch, tmp_path: Path):
|
||||
src.write_bytes(_ai_pdf())
|
||||
seen = _fake_tools(monkeypatch, qpdf=True)
|
||||
|
||||
actions, meta = clean_pdf(src, dest)
|
||||
_actions, meta = clean_pdf(src, dest)
|
||||
|
||||
assert meta["structural_rewrite"] is True
|
||||
qpdf_cmd = [c for c in seen if c[0].endswith("qpdf")]
|
||||
|
||||
@@ -16,8 +16,7 @@ ROOT = Path(__file__).resolve().parents[1]
|
||||
SCRIPTS = ROOT / "service" / "scripts"
|
||||
sys.path.insert(0, str(SCRIPTS))
|
||||
|
||||
import rewrite_text # noqa: E402
|
||||
from rewrite_text import ( # noqa: E402
|
||||
from rewrite_text import (
|
||||
_check_remote,
|
||||
_flag_env,
|
||||
_lexical_divergence,
|
||||
@@ -178,7 +177,7 @@ def test_openai_compatible_sends_reasoning_effort_when_set():
|
||||
self.end_headers()
|
||||
self.wfile.write(b'{"choices": [{"message": {"content": "rewritten"}}]}')
|
||||
|
||||
def log_message(self, format, *args): # noqa: A002
|
||||
def log_message(self, format, *args):
|
||||
pass
|
||||
|
||||
server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Collector)
|
||||
@@ -227,7 +226,7 @@ def test_rewrite_blocks_redirect_and_never_sends_key():
|
||||
)
|
||||
self.end_headers()
|
||||
|
||||
def log_message(self, format, *args): # noqa: A002
|
||||
def log_message(self, format, *args):
|
||||
pass
|
||||
|
||||
class Collector(http.server.BaseHTTPRequestHandler):
|
||||
@@ -236,11 +235,9 @@ def test_rewrite_blocks_redirect_and_never_sends_key():
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.end_headers()
|
||||
self.wfile.write(
|
||||
b'{"choices": [{"message": {"content": "rewritten"}}]}'
|
||||
)
|
||||
self.wfile.write(b'{"choices": [{"message": {"content": "rewritten"}}]}')
|
||||
|
||||
def log_message(self, format, *args): # noqa: A002
|
||||
def log_message(self, format, *args):
|
||||
pass
|
||||
|
||||
collector = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Collector)
|
||||
@@ -252,9 +249,7 @@ def test_rewrite_blocks_redirect_and_never_sends_key():
|
||||
with pytest.raises(urllib.error.HTTPError):
|
||||
rewrite(
|
||||
"secret text",
|
||||
**_rewrite_http_kwargs(
|
||||
f"http://127.0.0.1:{redirector.server_address[1]}"
|
||||
),
|
||||
**_rewrite_http_kwargs(f"http://127.0.0.1:{redirector.server_address[1]}"),
|
||||
)
|
||||
time.sleep(0.2)
|
||||
assert captured == {}, "redirect target received a request (key leak?)"
|
||||
|
||||
@@ -14,15 +14,15 @@ ROOT = Path(__file__).resolve().parents[1]
|
||||
SCRIPTS = ROOT / "service" / "scripts"
|
||||
sys.path.insert(0, str(SCRIPTS))
|
||||
|
||||
import common # noqa: E402
|
||||
from common import ( # noqa: E402
|
||||
import common
|
||||
from common import (
|
||||
backup_path,
|
||||
read_text_input,
|
||||
safe_arg,
|
||||
safe_write_bytes,
|
||||
safe_write_text,
|
||||
)
|
||||
from container_meta import ( # noqa: E402
|
||||
from container_meta import (
|
||||
MAX_ZIP_DECOMPRESSED_BYTES,
|
||||
_check_zip_budget,
|
||||
inspect_docx,
|
||||
@@ -196,4 +196,4 @@ def test_reconfigure_stream_writes_utf8():
|
||||
common._reconfigure_stream(stream, "backslashreplace")
|
||||
stream.write("\u200b")
|
||||
stream.flush()
|
||||
assert buf.getvalue() == "\u200b".encode("utf-8")
|
||||
assert buf.getvalue() == "\u200b".encode("utf-8")
|
||||
|
||||
@@ -7,14 +7,12 @@ import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
SCRIPTS_DIR = Path(__file__).resolve().parent.parent / "service" / "scripts"
|
||||
FIXTURES_DIR = Path(__file__).resolve().parent / "fixtures"
|
||||
sys.path.insert(0, str(SCRIPTS_DIR))
|
||||
|
||||
from audit_lib import scan_file # noqa: E402
|
||||
from score_stylometry import ( # noqa: E402
|
||||
from audit_lib import scan_file
|
||||
from score_stylometry import (
|
||||
compute_burstiness,
|
||||
compute_mattr,
|
||||
extract_sentences,
|
||||
@@ -22,11 +20,13 @@ from score_stylometry import ( # noqa: E402
|
||||
scan_ai_phrases,
|
||||
score_text_stylometry,
|
||||
)
|
||||
from server import capabilities # noqa: E402
|
||||
from server import capabilities
|
||||
|
||||
|
||||
def test_sentence_and_word_extraction():
|
||||
sample = "Hello world! This is a test. Here is a code block:\n```python\nprint('hi')\n```\nDone."
|
||||
sample = (
|
||||
"Hello world! This is a test. Here is a code block:\n```python\nprint('hi')\n```\nDone."
|
||||
)
|
||||
sentences = extract_sentences(sample)
|
||||
assert len(sentences) == 4
|
||||
assert sentences[0] == "Hello world!"
|
||||
@@ -46,7 +46,11 @@ def test_burstiness_variance():
|
||||
assert compute_burstiness(["Only one sentence here."])[1] == 0.0
|
||||
|
||||
# Uniform sentences (every sentence is 5 words) -> CV should be 0.0
|
||||
uniform = ["One two three four five.", "Six seven eight nine ten.", "Alpha beta gamma delta epsilon."]
|
||||
uniform = [
|
||||
"One two three four five.",
|
||||
"Six seven eight nine ten.",
|
||||
"Alpha beta gamma delta epsilon.",
|
||||
]
|
||||
mean, std, cv = compute_burstiness(uniform)
|
||||
assert mean == 5.0
|
||||
assert std == 0.0
|
||||
@@ -124,6 +128,7 @@ def test_score_stylometry_cli():
|
||||
[sys.executable, str(script), str(ai_path), "--json"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
assert res_ai.returncode == 1
|
||||
data_ai = json.loads(res_ai.stdout)
|
||||
@@ -135,6 +140,7 @@ def test_score_stylometry_cli():
|
||||
[sys.executable, str(script), str(human_path), "--json"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
assert res_human.returncode == 0
|
||||
data_human = json.loads(res_human.stdout)
|
||||
@@ -145,6 +151,7 @@ def test_score_stylometry_cli():
|
||||
[sys.executable, str(script), str(ai_path), "--threshold", "0.99"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
assert res_thresh.returncode == 0
|
||||
|
||||
@@ -157,6 +164,7 @@ def test_inspect_text_stylometry_flag():
|
||||
[sys.executable, str(script), str(ai_path), "--stylometry", "--json"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
assert res.returncode == 1
|
||||
data = json.loads(res.stdout)
|
||||
|
||||
@@ -14,8 +14,8 @@ ROOT = Path(__file__).resolve().parents[1]
|
||||
SCRIPTS = ROOT / "service" / "scripts"
|
||||
sys.path.insert(0, str(SCRIPTS))
|
||||
|
||||
import image_meta # noqa: E402
|
||||
from image_meta import ImageInspectReport, run_synthid_score # noqa: E402
|
||||
import image_meta
|
||||
from image_meta import ImageInspectReport, run_synthid_score
|
||||
|
||||
SCORE_SCRIPT = SCRIPTS / "score_synthid.py"
|
||||
|
||||
@@ -31,6 +31,7 @@ def test_score_synthid_cli_unavailable_without_upstream(
|
||||
[sys.executable, str(SCORE_SCRIPT), str(dummy)],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
|
||||
assert r.returncode == 3
|
||||
@@ -45,16 +46,18 @@ def test_run_synthid_score_unconfigured_returns_none(
|
||||
|
||||
|
||||
def test_run_synthid_score_unavailable_returns_none(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
def fake_run(*args, **kwargs):
|
||||
return SimpleNamespace(returncode=3, stdout="", stderr="unavailable")
|
||||
|
||||
monkeypatch.setattr(image_meta.subprocess, "run", fake_run)
|
||||
assert run_synthid_score(Path("x.png"), upstream_dir="/tmp/upstream") is None
|
||||
assert run_synthid_score(Path("x.png"), upstream_dir=str(tmp_path / "upstream")) is None
|
||||
|
||||
|
||||
def test_run_synthid_score_parses_json(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
payload = {
|
||||
@@ -70,22 +73,23 @@ def test_run_synthid_score_parses_json(
|
||||
return SimpleNamespace(returncode=0, stdout=json.dumps(payload), stderr="")
|
||||
|
||||
monkeypatch.setattr(image_meta.subprocess, "run", fake_run)
|
||||
result = run_synthid_score(Path("img.png"), upstream_dir="/tmp/upstream")
|
||||
result = run_synthid_score(Path("img.png"), upstream_dir=str(tmp_path / "upstream"))
|
||||
|
||||
assert result == payload
|
||||
assert "--json" in captured["cmd"]
|
||||
assert "--upstream-dir" in captured["cmd"]
|
||||
assert "/tmp/upstream" in captured["cmd"]
|
||||
assert str(tmp_path / "upstream") in captured["cmd"]
|
||||
|
||||
|
||||
def test_run_synthid_score_runtime_error_is_reported(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
def fake_run(*args, **kwargs):
|
||||
return SimpleNamespace(returncode=1, stdout="", stderr="boom")
|
||||
|
||||
monkeypatch.setattr(image_meta.subprocess, "run", fake_run)
|
||||
result = run_synthid_score(Path("img.png"), upstream_dir="/tmp/upstream")
|
||||
result = run_synthid_score(Path("img.png"), upstream_dir=str(tmp_path / "upstream"))
|
||||
|
||||
assert result is not None
|
||||
assert result.get("available") is False
|
||||
|
||||
@@ -58,9 +58,11 @@ def test_json_stdout_survives_noisy_upstream(tmp_path):
|
||||
img.write_bytes(b"\x89PNG\r\n\x1a\nstub")
|
||||
|
||||
r = subprocess.run(
|
||||
[sys.executable, str(SCRIPT), str(img),
|
||||
"--upstream-dir", str(upstream), "--json"],
|
||||
capture_output=True, text=True, timeout=60,
|
||||
[sys.executable, str(SCRIPT), str(img), "--upstream-dir", str(upstream), "--json"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=60,
|
||||
check=False,
|
||||
)
|
||||
|
||||
assert r.returncode == 0, r.stderr
|
||||
|
||||
Reference in New Issue
Block a user