From a2e72ed019cfff46e4d31c5c1ee523b8d39d24ed Mon Sep 17 00:00:00 2001 From: "Guillaume Meyer (The Opinionated Man)" <1385518+guillaumemeyer@users.noreply.github.com> Date: Mon, 17 Aug 2026 18:31:22 -0700 Subject: [PATCH] feat: vendor text-watermark detection (Gemini SynthID, Claude seam, MarkLLM) + SynthID image scorer sidecar (#109) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add vendor text-watermark detection and SynthID image scorer sidecar Adds Layer B watermark detection as a first-class service capability: - text_detectors.py: a registry of text-watermark detectors behind one interface — Google's official SynthID-text detector via the Gemini API (taskType DETECT_TEXT_WATERMARK), a Claude placeholder ready for Anthropic's announced detection API, and the MarkLLM research harness (KGW / SynthID, same-config-only). Fail-soft: unconfigured or errored detectors never block cleaning. - server.py: new POST /detect endpoint, detect_before / detect_after options on /clean (before/after scoring for text and images), an opt-in /inspect "detect" flag, and /capabilities gains text_detectors and scorers.synthid_http. - synthid_score_server.py: a stdlib HTTP sidecar for the reverse-SynthID scorer, so the published core image never bundles the non-commercial upstream code; wired via WATERMARKS_SYNTHID_SCORER_URL. - score_synthid.py: extract score_file() so the CLI and the sidecar share one implementation. - compose.yaml / Dockerfile.synthid / .env.example: wr-synthid-score sidecar service and env wiring. - README + skill docs, plus tests for the detectors, the /detect endpoint, and the image sidecar. * feat: per-candidate watermark detection for Layer B rewrite candidates When --candidates N (N > 1) is combined with --markllm-scheme or WATERMARKS_GEMINI_API_KEY, run every configured text detector from the text_detectors.py registry on each candidate and report per-candidate measurements in --json-stats as candidate_scores entries carrying lexical_divergence, selection_score, selected, and per-detector reports (is_watermarked, score, threshold where the detector provides one). Candidate selection stays purely lexical; the detections are observability for correlating lexical divergence with watermark removal (issue #106). Converges rewrite_text.py onto the shared detector registry: - MarkLLMTextDetector gains constructor overrides (scheme, upstream_dir, model, timeout) plus the checkout-venv interpreter preference and the WATERMARKS_MARKLLM_RLIMIT_AS preexec guard ported from rewrite_text.py; the old _markllm_detect / _venv_python / _markllm_preexec helpers are gone. - run_all_text_detectors() accepts an injected MarkLLM instance and an include_markllm switch so CLI flag gating stays intact. - before/after/cleared semantics unchanged; detection remains fail-soft. * docs: pin Watermarks in the Sand reference to arXiv v5 * fix: mark only one rewrite candidate as selected (#110) --------- Co-authored-by: Zhenxin Ai <142008897+ai-kunkun@users.noreply.github.com> --- .env.example | 24 ++ README.md | 101 ++++- compose.yaml | 30 ++ service/Dockerfile.synthid | 1 + service/scripts/rewrite_text.py | 166 +++----- service/scripts/score_synthid.py | 173 +++++---- service/scripts/server.py | 124 +++++- service/scripts/synthid_score_server.py | 176 +++++++++ service/scripts/text_detectors.py | 487 ++++++++++++++++++++++++ skills/remove-ai-marks/SKILL.md | 37 +- tests/test_detect_endpoint.py | 255 +++++++++++++ tests/test_markllm_detect.py | 136 +------ tests/test_rewrite_text.py | 203 ++++++++++ tests/test_text_detectors.py | 378 ++++++++++++++++++ 14 files changed, 1963 insertions(+), 328 deletions(-) create mode 100644 service/scripts/synthid_score_server.py create mode 100644 service/scripts/text_detectors.py create mode 100644 tests/test_detect_endpoint.py create mode 100644 tests/test_text_detectors.py diff --git a/.env.example b/.env.example index ad0abf0..270f2be 100644 --- a/.env.example +++ b/.env.example @@ -9,6 +9,23 @@ # `Authorization: Bearer `. WATERMARKS_SERVER_API_KEY= +# --------------------------------------------------------------------------- +# Vendor text-watermark detection (wr-core) +# --------------------------------------------------------------------------- +# Optional Google Gemini API key. When set, the service can run Google's +# official SynthID-text watermark detector via POST /detect and the +# detect_before / detect_after clean options. Env only — never on argv. +# Privacy: text is sent to Google only when this key is configured. +# WATERMARKS_GEMINI_API_KEY= +# WATERMARKS_GEMINI_MODEL=gemini-2.5-flash +# WATERMARKS_GEMINI_TIMEOUT=30 +# WATERMARKS_GEMINI_MAX_CHARS=1000000 + +# Optional MarkLLM research harness (host checkouts only; not in the core +# image). Same-config-only detection — not a vendor oracle. +# WATERMARKS_MARKLLM_DIR=~/MarkLLM +# WATERMARKS_MARKLLM_SCHEME=kgw # kgw | synthid + # --------------------------------------------------------------------------- # Harness / heavy backends (only used by the harness/heavy profiles) # --------------------------------------------------------------------------- @@ -16,6 +33,13 @@ WATERMARKS_SERVER_API_KEY= # MarkDiffusion score models). Env only — never on argv. HF_TOKEN= +# SynthID image scoring over HTTP (heavy profile): point wr-core at the +# wr-synthid-score sidecar and share the same bearer key on both sides. +# With the heavy profile up, uncomment: +# WATERMARKS_SYNTHID_SCORER_URL=http://wr-synthid-score:8766 +# WATERMARKS_SYNTHID_SCORER_API_KEY= +# WATERMARKS_SYNTHID_SCORER_TIMEOUT=60 + # --------------------------------------------------------------------------- # Client-side (used by the skill or curl, NOT by compose) # --------------------------------------------------------------------------- diff --git a/README.md b/README.md index 5589e76..926dd11 100644 --- a/README.md +++ b/README.md @@ -168,6 +168,7 @@ The same machinery runs as a stdlib HTTP service (`service/scripts/server.py`) | GET | `/capabilities` | — | optional tools / backends present | | GET | `/openapi.json` | — | dynamically generated OpenAPI 3.0.3 spec | | POST | `/inspect` | `{"file": "", "name": "notes.md"}` | `{"ok", "kind", "suspicious", "report"}` | +| POST | `/detect` | `{"file": "", "name": "notes.txt"}` | `{"ok", "kind", "detections": [...]}` | | POST | `/clean` | `{"file": "", "name": "notes.md", "options": {...}}` | `{"ok", "kind", "cleaned": "", "report"}` | ```bash @@ -180,6 +181,33 @@ curl -s -X POST "$WM/clean" -H 'Content-Type: application/json' \ The service routes by filename extension then magic bytes, so text / image / container are auto-detected. Set `WATERMARKS_SERVER_API_KEY` to require `Authorization: Bearer ` on every request. Loopback-only bind by default (`--host` to override); intended for a trusted network. +### Watermark detection (`/detect` and `detect_before` / `detect_after`) + +Detection is a separate step from cleaning — the service never calls vendor +APIs unless you ask it to: + +- **`POST /detect`** runs the configured watermark detectors on a file. + Text → vendor detectors + stylometry; image → SynthID pixel score. +- **`/inspect`** accepts an opt-in `"detect": true` flag that appends + detector results to the text report (and can flip `suspicious`). +- **`/clean`** accepts `"detect_before"` / `"detect_after"` options to + score the input and the cleaned output, so you can measure what a clean + actually changed. + +Text detectors (see `/capabilities` → `text_detectors`): + +| Detector | Activated by | Notes | +| --- | --- | --- | +| `gemini-synthid-text` | `WATERMARKS_GEMINI_API_KEY` | Google's official SynthID-text detector via the Gemini API (`taskType: DETECT_TEXT_WATERMARK`). Sends text to Google only when the operator sets the key. | +| `markllm` | `MARKLLM_DIR` (host checkout) | Research harness (KGW / SynthID schemes), same-config-only — not a vendor oracle. | +| `claude-text` | — (placeholder) | Anthropic has announced a watermark detection API; this seam activates when it ships. | + +Image scoring: when `WATERMARKS_SYNTHID_SCORER_URL` is set, the service +scores images through the `wr-synthid-score` sidecar (heavy profile); with a +local `REVERSE_SYNTHID_DIR` it uses the checkout directly. Detection is +fail-soft: unconfigured, timed-out, or errored detectors report +`{"available": false, "error": ...}` and never block cleaning. + ## Docker / compose Published images (GHCR): @@ -190,7 +218,7 @@ Published images (GHCR): | `…:markllm-` / `:markllm-latest` | MarkLLM text-watermark harness (Apache-2.0 upstream) | Yes | | `…:markdiffusion-` / `:markdiffusion-latest` | MarkDiffusion image harness (Apache-2.0 upstream) | Yes | | `watermarks-remover-ctrlregen:local` | CtrlRegen pixel removal — **never published** (`noai-watermark` ships no LICENSE) | Local build only | -| `watermarks-remover-synthid-scorer:local` | reverse-SynthID scorer — **never published** (non-commercial Research License) | Local build only | +| `watermarks-remover-synthid-scorer:local` | reverse-SynthID scorer — **never published** (non-commercial Research License) | Local build only (CLI scorer + optional `wr-synthid-score` HTTP sidecar under the `heavy` profile) | Build and run the core service: @@ -247,6 +275,11 @@ set -a; . ./.env; set +a; python3 service/scripts/rewrite_text.py /tmp/x.txt -o | Var | Reaches | Purpose | | --- | --- | --- | | `WATERMARKS_SERVER_API_KEY` | `wr-core` (via compose `environment`) | Require `Authorization: Bearer ` on the HTTP API | +| `WATERMARKS_GEMINI_API_KEY` | `wr-core` | Enable Google's SynthID-text detector (`/detect`, `detect_before/after`) — env only, never on argv | +| `WATERMARKS_GEMINI_MODEL` | `wr-core` | Gemini model for detection (default `gemini-2.5-flash`) | +| `WATERMARKS_SYNTHID_SCORER_URL` | `wr-core` | Point core at the `wr-synthid-score` sidecar for SynthID image scoring (e.g. `http://wr-synthid-score:8766` under the heavy profile) | +| `WATERMARKS_SYNTHID_SCORER_API_KEY` | `wr-core` + `wr-synthid-score` | Shared bearer key for the scorer sidecar (empty = no auth) | +| `WATERMARKS_MARKLLM_SCHEME` | `text_detectors.py` (host) | MarkLLM scheme for `/detect`: `kgw` (default) / `synthid` | | `HF_TOKEN` | harness/heavy services | Hugging Face token for gated models | | `WATERMARKS_SERVICE_URL` | client only (skill / curl) | Where to reach the service; default `http://127.0.0.1:8765` | | `WATERMARKS_REWRITE_BACKEND` | `rewrite_text.py` hook | `print-prompt` (default) / `ollama` / `openai-compatible` | @@ -310,8 +343,29 @@ docker run --rm \ The image is built locally from the upstream source at build time. It is not published, so it does not redistribute the upstream code. +### Option 3: HTTP scorer sidecar (docker compose) + +Under the `heavy` profile the compose stack also runs the scorer as an HTTP +sidecar (`wr-synthid-score`) so the **published core service** can score +images before/after cleaning without bundling the non-commercial upstream +code. Point `wr-core` at it and share a bearer key (see `.env.example`): + +```bash +# .env +WATERMARKS_SYNTHID_SCORER_URL=http://wr-synthid-score:8766 +WATERMARKS_SYNTHID_SCORER_API_KEY=change-me + +docker compose --profile heavy up -d +``` + +Then `POST /clean` with `{"options": {"detect_before": true, +"detect_after": true}}` returns `synthid_before` / `synthid_after` in the +report, and `POST /detect` on an image returns the SynthID score. Fail-soft: +if the sidecar is down or unconfigured, reports carry +`{"available": false, "error": ...}` and cleaning still succeeds. + V4 scoring uses `artifacts/spectral_codebook_v4.npz` from the upstream checkout -(~220 MB). This is **detection/scoring only** — it does not remove pixel +(`220 MB). This is **detection/scoring only** — it does not remove pixel watermarks. ## Optional CtrlRegen pixel removal @@ -455,6 +509,39 @@ MARKLLM_DIR=~/MarkLLM \ --markllm-scheme kgw --markllm-dir "$HOME/MarkLLM" --json-stats ``` +**Per-candidate detection:** when `--candidates N` (`N > 1`) is combined with +`--markllm-scheme` (or with `WATERMARKS_GEMINI_API_KEY` set), every generated +candidate is run through the configured text detectors and `--json-stats` +reports per-candidate measurements. Candidate selection stays purely lexical; +the detections exist so you can see whether divergence actually correlates with +watermark removal: + +```json +"candidate_scores": [ + { + "lexical_divergence": 0.91, + "selection_score": 0.91, + "selected": true, + "detections": [ + {"detector": "markllm", "available": true, "scheme": "kgw", + "is_watermarked": true, "score": 4.3, "threshold": 3.0} + ] + }, + { + "lexical_divergence": 0.84, + "selection_score": 0.84, + "selected": false, + "detections": [ + {"detector": "markllm", "available": true, "scheme": "kgw", + "is_watermarked": false, "score": 1.7, "threshold": 3.0} + ] + } +] +``` + +A detector that is unconfigured, times out, or errors yields an +`"available": false` entry with an `error` reason and never fails the rewrite. + If the backend is unconfigured or its deps are missing, the rewrite proceeds and the report notes verification was unavailable. A GPU is recommended; CPU runs work but are slow, and the model download is a few GB. @@ -466,8 +553,8 @@ Hardening knobs: Custom remote code is never executed (transformers `trust_remote_code` is never enabled). - `WATERMARKS_MARKLLM_RLIMIT_AS=` (env, POSIX) applies an address-space - limit to the MarkLLM subprocess spawned by `rewrite_text.py`. Off by default - because torch/CUDA usually needs large address spaces. + limit to the MarkLLM detector subprocess. Off by default because torch/CUDA + usually needs large address spaces. - Config files are capped at 1 MiB; the upstream checkout and the base image are pinned by SHA/digest. @@ -561,7 +648,7 @@ on the host instead. Model downloads still hit the HF hub on first run. | Channel | Claude | Gemini/SynthID | OpenAI | Open-LLM | | --- | --- | --- | --- | --- | | Unicode / edit-based text | Layer A | Layer A | Layer A | Layer A | -| Statistical sampling text | Layer B best-effort | Layer B best-effort | Layer B if present | Layer B best-effort | +| **Statistical sampling text** | Layer B best-effort + optional vendor detector (`gemini-synthid-text`; Claude seam when Anthropic's detection API ships) | Layer B best-effort + optional vendor detector (`gemini-synthid-text`) | Layer B if present | Layer B best-effort + optional MarkLLM harness | | C2PA / file metadata | Yes (listed formats) | Yes when present | Yes when present | Yes when present | | Pixel image marks | Out of scope | Optional SynthID score + CtrlRegen removal (external); optional MarkDiffusion same-scheme detect + DiffusionPurification removal (external) | Out of scope | Optional CtrlRegen / MarkDiffusion removal (external) | | Training backdoors | Out of scope | Out of scope | Out of scope | Out of scope | @@ -728,7 +815,7 @@ make smoke # quick CLI smoke on fixtures **MarkLLM text-watermark harness (optional)** - New optional harness (external `THU-BPM/MarkLLM` checkout, Apache-2.0): `detect_text_watermark.py` with `detect` / `watermark` subcommands for KGW and SynthID schemes -- `rewrite_text.py --markllm-scheme` runs before/after detection around a Layer B rewrite (env-gated; reports `cleared`) +- `rewrite_text.py --markllm-scheme` runs before/after detection around a Layer B rewrite and per-candidate detection when `--candidates N>1` (env-gated; reports `cleared`) - `setup_markllm.sh` bootstrap + `requirements-markllm.txt` (pinned deps) + `Dockerfile.markllm` and Makefile `bootstrap-markllm` / `smoke-markllm` / `docker-markllm-build` / `docker-markllm-help` - Hardening: `--offline` cache-only model loading (no HF egress, no remote code), 1 MiB config cap, optional `WATERMARKS_MARKLLM_RLIMIT_AS` on the rewrite subprocess, pinned torch in the Dockerfile, and clone-SHA verification in `Dockerfile.markllm` - Mock-based tests (`tests/test_markllm_detect.py`, 21 cases) — no torch in CI; verification-harness caveat (same-config-only, not a vendor-detector oracle) documented in README, SKILL.md, `removal-matrix.md`, `vendor-notes.md` @@ -855,7 +942,7 @@ MIT — see [LICENSE](LICENSE). - Kirchenbauer et al., [*A Watermark for Large Language Models*](https://arxiv.org/abs/2301.10226) - [THU-BPM/MarkLLM](https://github.com/THU-BPM/MarkLLM) (unified toolkit for evaluating LLM watermarking algorithms) - Pan et al., [*MarkDiffusion: An Open-Source Toolkit for Generative Watermarking of Latent Diffusion Models*](https://arxiv.org/abs/2509.10569) (JMLR) — the embedding toolkit this repo's optional image-watermark harness wraps — [code](https://github.com/THU-BPM/MarkDiffusion), [docs](https://markdiffusion.readthedocs.io) -- Zhang et al., [*Watermarks in the Sand: Impossibility of Strong Watermarking for Generative Models*](https://arxiv.org/abs/2311.04378) (ICML 2024) +- Zhang et al., [*Watermarks in the Sand: Impossibility of Strong Watermarking for Generative Models*](https://arxiv.org/abs/2311.04378v5) (ICML 2024) - [google-deepmind/synthid-text](https://github.com/google-deepmind/synthid-text) (research reference; not used for detection here) - [aloshdenny/reverse-SynthID](https://github.com/aloshdenny/reverse-SynthID) (research reference) - Liu et al., [*Image Watermarks are Removable Using Controllable Regeneration from Clean Noise*](https://arxiv.org/abs/2410.05470) (ICLR 2025) — the pixel-regeneration method the optional CtrlRegen backend implements — [code](https://github.com/yepengliu/CtrlRegen) diff --git a/compose.yaml b/compose.yaml index 7ec88ec..e5eabb9 100644 --- a/compose.yaml +++ b/compose.yaml @@ -27,6 +27,13 @@ services: environment: # Empty by default (no auth). Set to require `Authorization: Bearer `. WATERMARKS_SERVER_API_KEY: ${WATERMARKS_SERVER_API_KEY:-} + # Optional Gemini API key enabling vendor SynthID-text watermark + # detection via /detect and detect_before/after (see .env.example). + WATERMARKS_GEMINI_API_KEY: ${WATERMARKS_GEMINI_API_KEY:-} + WATERMARKS_GEMINI_MODEL: ${WATERMARKS_GEMINI_MODEL:-} + # Optional SynthID image scorer sidecar (heavy profile). + WATERMARKS_SYNTHID_SCORER_URL: ${WATERMARKS_SYNTHID_SCORER_URL:-} + WATERMARKS_SYNTHID_SCORER_API_KEY: ${WATERMARKS_SYNTHID_SCORER_API_KEY:-} read_only: true tmpfs: - /tmp @@ -109,6 +116,29 @@ services: volumes: - synthid-cache:/home/scorer/.cache/huggingface + # HTTP SynthID scorer sidecar: lets wr-core score images before/after + # cleaning without bundling the non-commercial reverse-SynthID code in the + # published core image. Reach it via WATERMARKS_SYNTHID_SCORER_URL on + # wr-core (see .env.example) and require the same bearer key on both sides. + wr-synthid-score: + profiles: [heavy] + build: + context: service + dockerfile: Dockerfile.synthid + image: watermarks-remover-synthid-scorer:local + entrypoint: ["python3"] + command: ["/app/synthid_score_server.py", "--host", "0.0.0.0", "--port", "8766"] + read_only: true + tmpfs: + - /tmp + init: true + user: "10001:10001" + environment: + WATERMARKS_SYNTHID_SCORER_API_KEY: ${WATERMARKS_SYNTHID_SCORER_API_KEY:-} + REVERSE_SYNTHID_DIR: /opt/reverse-synthid + volumes: + - synthid-cache:/home/scorer/.cache/huggingface + volumes: markllm-cache: markdiffusion-cache: diff --git a/service/Dockerfile.synthid b/service/Dockerfile.synthid index 7153e3f..6f9439e 100644 --- a/service/Dockerfile.synthid +++ b/service/Dockerfile.synthid @@ -45,6 +45,7 @@ RUN git clone --depth 1 --filter=blob:none --sparse \ COPY scripts/requirements-synthid-scorer.txt /app/requirements-synthid-scorer.txt COPY scripts/score_synthid.py /app/score_synthid.py +COPY scripts/synthid_score_server.py /app/synthid_score_server.py RUN python3 -m pip install --no-cache-dir "pip==26.2.1" \ && python3 -m pip install --no-cache-dir -r /app/requirements-synthid-scorer.txt diff --git a/service/scripts/rewrite_text.py b/service/scripts/rewrite_text.py index 2a89629..4b04ce1 100644 --- a/service/scripts/rewrite_text.py +++ b/service/scripts/rewrite_text.py @@ -27,17 +27,16 @@ import itertools import json import os import re -import subprocess import sys import urllib.error import urllib.request -from collections.abc import Callable from pathlib import Path from urllib.parse import urlparse sys.path.insert(0, str(Path(__file__).resolve().parent)) from common import cleaned_path, eprint, read_text_input, write_text_output +from text_detectors import MarkLLMTextDetector, run_all_text_detectors from text_unicode import clean_text DEFAULT_MARKLLM_MODEL = "facebook/opt-1.3b" @@ -174,100 +173,31 @@ class _NoRedirect(urllib.request.HTTPRedirectHandler): raise urllib.error.HTTPError(req.full_url, code, msg, headers, fp) -SCRIPTS_DIR = Path(__file__).resolve().parent +def _per_candidate_detections( + candidates: list[str], + markllm_detector: MarkLLMTextDetector | None, +) -> list[list[dict]]: + """Run every configured text detector on each rewrite candidate. - -def _venv_python(upstream: Path) -> Path | None: - """Locate the MarkLLM checkout's venv interpreter, if it exists.""" - if os.name == "nt": - candidate = upstream / ".venv" / "Scripts" / "python.exe" - else: - candidate = upstream / ".venv" / "bin" / "python" - return candidate if candidate.is_file() else None - - -def _markllm_preexec() -> Callable[[], None] | None: - """Optional RLIMIT_AS guard for the MarkLLM child; None means "no limit". - - torch/CUDA usually needs large address spaces, so unlike the - exiftool/c2patool/SynthID children (common.subprocess_rlimits) this is - opt-in via WATERMARKS_MARKLLM_RLIMIT_AS (byte count, hex/octal allowed). - POSIX only; on Windows preexec_fn must stay None. + Fail-soft: a detector that is unconfigured, times out, or errors yields + an ``available: False`` entry and never fails the rewrite. The MarkLLM + harness is only included when ``markllm_detector`` is given (i.e. the + caller passed --markllm-scheme); other detectors (e.g. + gemini-synthid-text) are key-gated by their own environment. """ - raw = os.environ.get("WATERMARKS_MARKLLM_RLIMIT_AS") - if not raw or os.name != "posix": - return None - try: - limit = int(raw, 0) - except ValueError: - return None - - def _apply() -> None: - import resource - - resource.setrlimit(resource.RLIMIT_AS, (limit, limit)) - - return _apply - - -def _markllm_detect( - text: str, - *, - scheme: str, - upstream_dir: str, - model: str, - timeout: float, -) -> dict: - """Run the MarkLLM adapter on *text*; never fails the rewrite. - - Returns the adapter's JSON payload, or an ``available: False`` dict with - an ``error`` string when the backend is unconfigured or broken. The Layer B - rewrite proceeds regardless; MarkLLM verification is best-effort. - """ - if not upstream_dir: - return {"available": False, "error": "no MARKLLM_DIR set"} - upstream = Path(upstream_dir).expanduser().resolve() - if not upstream.is_dir() or not (upstream / "watermark").is_dir(): - return {"available": False, "error": f"MarkLLM checkout missing: {upstream}"} - venv_python = _venv_python(upstream) - if venv_python is None: - return {"available": False, "error": f"MarkLLM venv missing: {upstream}"} - - cmd = [ - str(venv_python), - str(SCRIPTS_DIR / "detect_text_watermark.py"), - "detect", - "-", - "--scheme", - scheme, - "--upstream-dir", - str(upstream), - "--model", - model, - "--json", - ] - try: - r = subprocess.run( - cmd, - input=text, - capture_output=True, - text=True, - timeout=timeout, - preexec_fn=_markllm_preexec(), - check=False, - ) - except (OSError, subprocess.SubprocessError, TimeoutError) as e: - return {"available": False, "error": f"MarkLLM adapter error: {e}"} - - if r.returncode != 0: - return { - "available": False, - "error": (r.stderr or "").strip() or f"adapter exited {r.returncode}", - } - try: - return json.loads(r.stdout) - except ValueError as e: - return {"available": False, "error": f"adapter JSON parse error: {e}"} + detections: list[list[dict]] = [] + for cand in candidates: + try: + detections.append( + run_all_text_detectors( + cand, + markllm=markllm_detector, + include_markllm=markllm_detector is not None, + ) + ) + except Exception as e: # defensive: the registry contract is fail-soft + detections.append([{"available": False, "error": f"candidate detection failed: {e}"}]) + return detections def build_prompt(strength: str, text: str, *, lang: str, original_lang: str) -> str: @@ -400,16 +330,17 @@ def rewrite( info["reasoning_effort"] = reasoning_effort markllm: dict | None = None + markllm_detector: MarkLLMTextDetector | None = None if markllm_scheme: + markllm_detector = MarkLLMTextDetector( + scheme=markllm_scheme, + upstream_dir=markllm_dir, + model=markllm_model or DEFAULT_MARKLLM_MODEL, + timeout=markllm_timeout, + ) markllm = { "scheme": markllm_scheme, - "before": _markllm_detect( - text, - scheme=markllm_scheme, - upstream_dir=markllm_dir or "", - model=markllm_model or DEFAULT_MARKLLM_MODEL, - timeout=markllm_timeout, - ), + "before": markllm_detector.detect(text), } if not markllm["before"]["available"]: eprint(f"markllm verification unavailable: {markllm['before']['error']}") @@ -447,7 +378,29 @@ def rewrite( else: info["candidates"] = n out, scores = _select_candidate(text, outs) - info["candidate_scores"] = scores + selected_idx = max(range(len(outs)), key=lambda i: scores[i]) + trigger = markllm_scheme is not None or bool( + os.environ.get("WATERMARKS_GEMINI_API_KEY", "").strip() + ) + detections = _per_candidate_detections(outs, markllm_detector) if trigger else [] + info["candidate_scores"] = [] + for i, cand in enumerate(outs): + info["candidate_scores"].append( + { + "lexical_divergence": _lexical_divergence(text, cand), + "selection_score": scores[i], + "selected": i == selected_idx, + "detections": detections[i] if trigger else [], + } + ) + if trigger and detections: + names = sorted( + {d.get("detector", "?") for dets in detections for d in dets if d.get("available")} + ) + eprint( + f"note: running per-candidate watermark detection on {n} candidates" + + (f" ({', '.join(names)})" if names else "") + ) if layer_a_after: out, stats = clean_text(out) @@ -461,13 +414,8 @@ def rewrite( ) if markllm: - after = _markllm_detect( - out, - scheme=markllm["scheme"], - upstream_dir=markllm_dir or "", - model=markllm_model or DEFAULT_MARKLLM_MODEL, - timeout=markllm_timeout, - ) + assert markllm_detector is not None # set together with markllm above + after = markllm_detector.detect(out) markllm["after"] = after before = markllm["before"] if before.get("available") and after.get("available"): diff --git a/service/scripts/score_synthid.py b/service/scripts/score_synthid.py index 468d9ae..b5fced5 100755 --- a/service/scripts/score_synthid.py +++ b/service/scripts/score_synthid.py @@ -11,6 +11,9 @@ Exit codes: 1 scorer runtime error 2 bad input (missing/unreadable image, bad args) 3 scorer unavailable (not configured / missing deps / missing codebook) + +The scoring logic lives in :func:score_file so the CLI and the HTTP +sidecar (synthid_score_server.py) share one implementation. """ from __future__ import annotations @@ -21,6 +24,7 @@ import json import os import sys from pathlib import Path +from typing import Any def resolve_upstream(raw: str | None) -> Path | None: @@ -32,6 +36,90 @@ def resolve_upstream(raw: str | None) -> Path | None: return upstream +def score_file( + path: Path, + *, + upstream_dir: str | None = None, + codebook: Path | None = None, + model: str | None = None, +) -> tuple[int, dict[str, Any] | None]: + """Score *path* with the reverse-SynthID extractor. + + Returns (exit_code, payload) matching the CLI exit-code contract: + 0 = scored (payload present), 2 = bad input (payload None), + 3 = scorer unavailable (payload None). Errors are printed to stderr so + callers parsing stdout JSON are never corrupted. + """ + if not path.is_file(): + print(f"not a file: {path}", file=sys.stderr) + return 2, None + + raw_upstream = upstream_dir or os.environ.get("REVERSE_SYNTHID_DIR") + upstream = resolve_upstream(str(raw_upstream) if raw_upstream else None) + if upstream is None: + print( + "SynthID scorer not configured: set REVERSE_SYNTHID_DIR or pass --upstream-dir", + file=sys.stderr, + ) + return 3, None + + extraction = upstream / "src" / "extraction" + if not extraction.is_dir(): + print(f"upstream extraction dir not found: {extraction}", file=sys.stderr) + return 3, None + + codebook_path = codebook or upstream / "artifacts" / "spectral_codebook_v4.npz" + codebook_path = Path(codebook_path).expanduser().resolve() + if not codebook_path.is_file(): + print(f"codebook not found: {codebook_path}", file=sys.stderr) + return 3, None + + sys.path.insert(0, str(extraction)) + try: + import cv2 + from robust_extractor import RobustSynthIDExtractor + from synthid_bypass_v4 import SpectralCodebookV4 + except ImportError as e: + print(f"optional scorer dependencies missing: {e}", file=sys.stderr) + return 3, None + + try: + img = cv2.imread(str(path)) + if img is None: + print(f"could not load image: {path}", file=sys.stderr) + return 2, None + rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) + + # Upstream prints progress ("CodebookV4 loaded: ...") straight to + # stdout, which corrupts --json for any caller that parses us + # (image_meta.py json.loads our stdout). Keep stdout ours alone. + with contextlib.redirect_stdout(sys.stderr): + codebook_v4 = SpectralCodebookV4() + codebook_v4.load(str(codebook_path)) + + extractor = RobustSynthIDExtractor() + result = extractor.detect_from_v4_codebook(rgb, codebook_v4, model=model) + except Exception as e: + print(f"scorer error: {e}", file=sys.stderr) + return 1, None + + payload = { + "available": True, + "upstream_dir": str(upstream), + "codebook": str(codebook_path), + "model": model, + "profile_key": result.details.get("profile_key"), + "exact_match": result.details.get("exact_match"), + "is_watermarked": result.is_watermarked, + "confidence": result.confidence, + "phase_match": result.phase_match, + "per_channel_scores": result.details.get("per_channel_scores"), + "per_channel_n": result.details.get("per_channel_n"), + "multi_scale_consistency": result.multi_scale_consistency, + } + return 0, payload + + def main() -> int: p = argparse.ArgumentParser(description=__doc__) p.add_argument("path", type=Path, help="Image to score (PNG/JPEG/etc.)") @@ -51,83 +139,24 @@ def main() -> int: p.add_argument("--json", action="store_true", help="Emit JSON on stdout") args = p.parse_args() - if not args.path.is_file(): - print(f"not a file: {args.path}", file=sys.stderr) - return 2 - - raw_upstream = args.upstream_dir or os.environ.get("REVERSE_SYNTHID_DIR") - upstream = resolve_upstream(str(raw_upstream) if raw_upstream else None) - if upstream is None: - print( - "SynthID scorer not configured: set REVERSE_SYNTHID_DIR or pass --upstream-dir", - file=sys.stderr, - ) - return 3 - - extraction = upstream / "src" / "extraction" - if not extraction.is_dir(): - print(f"upstream extraction dir not found: {extraction}", file=sys.stderr) - return 3 - - codebook = args.codebook or upstream / "artifacts" / "spectral_codebook_v4.npz" - codebook = Path(codebook).expanduser().resolve() - if not codebook.is_file(): - print(f"codebook not found: {codebook}", file=sys.stderr) - return 3 - - sys.path.insert(0, str(extraction)) - try: - import cv2 - from robust_extractor import RobustSynthIDExtractor - from synthid_bypass_v4 import SpectralCodebookV4 - except ImportError as e: - print(f"optional scorer dependencies missing: {e}", file=sys.stderr) - return 3 - - try: - img = cv2.imread(str(args.path)) - if img is None: - print(f"could not load image: {args.path}", file=sys.stderr) - return 2 - rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) - - # Upstream prints progress ("CodebookV4 loaded: ...") straight to - # stdout, which corrupts --json for any caller that parses us - # (image_meta.py json.loads our stdout). Keep stdout ours alone. - with contextlib.redirect_stdout(sys.stderr): - codebook_v4 = SpectralCodebookV4() - codebook_v4.load(str(codebook)) - - extractor = RobustSynthIDExtractor() - result = extractor.detect_from_v4_codebook(rgb, codebook_v4, model=args.model) - except Exception as e: - print(f"scorer error: {e}", file=sys.stderr) - return 1 - - payload = { - "available": True, - "upstream_dir": str(upstream), - "codebook": str(codebook), - "model": args.model, - "profile_key": result.details.get("profile_key"), - "exact_match": result.details.get("exact_match"), - "is_watermarked": result.is_watermarked, - "confidence": result.confidence, - "phase_match": result.phase_match, - "per_channel_scores": result.details.get("per_channel_scores"), - "per_channel_n": result.details.get("per_channel_n"), - "multi_scale_consistency": result.multi_scale_consistency, - } + code, payload = score_file( + args.path, + upstream_dir=str(args.upstream_dir) if args.upstream_dir else None, + codebook=args.codebook, + model=args.model, + ) + if code != 0: + return code if args.json: json.dump(payload, sys.stdout, indent=2) sys.stdout.write("\n") else: - label = "yes" if result.is_watermarked else "no" - print(f"SynthID score: confidence {result.confidence:.3f} (watermarked: {label})") - print(f" phase_match: {result.phase_match:.3f}") - if result.details.get("profile_key"): - print(f" profile: {result.details['profile_key']}") + label = "yes" if payload["is_watermarked"] else "no" + print(f"SynthID score: confidence {payload['confidence']:.3f} (watermarked: {label})") + print(f" phase_match: {payload['phase_match']:.3f}") + if payload.get("profile_key"): + print(f" profile: {payload['profile_key']}") return 0 diff --git a/service/scripts/server.py b/service/scripts/server.py index dec0281..2c2093f 100644 --- a/service/scripts/server.py +++ b/service/scripts/server.py @@ -9,6 +9,7 @@ Endpoints: GET /capabilities -> which optional tools / pixel backends are present GET /openapi.json -> dynamically generated OpenAPI 3.0.3 spec POST /inspect -> {"file": , "name": "x.png"} -> findings JSON + POST /detect -> {"file": , "name": "x.txt"} -> watermark detector reports POST /clean -> {"file": , "name": "x.png", "options": {...}} -> {"cleaned": , "report": {...}} @@ -43,8 +44,9 @@ from common import ( ) from container_meta import clean_container, inspect_container from format_dispatch import classify_bytes -from image_meta import clean_image, inspect_image +from image_meta import clean_image, inspect_image, run_synthid_score from score_stylometry import score_text_stylometry +from text_detectors import detector_status, run_all_text_detectors, run_text_detectors from text_unicode import clean_text, inspect_text VERSION = os.environ.get("WATERMARKS_SERVER_VERSION", "dev") @@ -64,6 +66,8 @@ ALLOWED_CLEAN_OPTIONS = { "also_layer_a_text": bool, "remove_pixel": str, "strip_all_metadata": bool, + "detect_before": bool, + "detect_after": bool, } @@ -85,8 +89,10 @@ def capabilities() -> dict[str, Any]: }, "scorers": { "synthid": bool(os.environ.get("REVERSE_SYNTHID_DIR")), + "synthid_http": bool(os.environ.get("WATERMARKS_SYNTHID_SCORER_URL")), "stylometry": True, }, + "text_detectors": detector_status(), "harnesses": { "markllm": bool(os.environ.get("MARKLLM_DIR")), }, @@ -178,12 +184,17 @@ _OPENAPI_PATHS: dict[str, dict[str, Any]] = { type="object", properties={ "synthid": _schema(type="boolean"), + "synthid_http": _schema(type="boolean"), "stylometry": _schema(type="boolean"), }, ), "harnesses": _schema( type="object", properties={"markllm": _schema(type="boolean")} ), + "text_detectors": _schema( + type="object", + additionalProperties=_schema(type="boolean"), + ), }, ) }, @@ -202,7 +213,25 @@ _OPENAPI_PATHS: dict[str, dict[str, Any]] = { "summary": "Inspect a file for AI provenance marks (text / image / container auto-routed)", "requestBody": _schema( required=True, - content={"application/json": _schema(schema=_file_request())}, + content={ + "application/json": _schema( + schema=_file_request( + { + "properties": { + "detect": _schema( + type="boolean", + description=( + "Also run configured text watermark detectors " + "(opt-in; may call vendor APIs and send text " + "to them)" + ), + ) + }, + "required": [], + } + ) + ) + }, ), "responses": { "200": _schema( @@ -239,6 +268,25 @@ _OPENAPI_PATHS: dict[str, dict[str, Any]] = { }, } }, + "/detect": { + "post": { + "summary": "Run watermark detectors on a file (text: vendor/statistical; image: SynthID score)", + "requestBody": _schema( + required=True, + content={"application/json": _schema(schema=_file_request())}, + ), + "responses": { + "200": _schema( + type="object", + properties={ + "ok": _schema(type="boolean"), + "kind": _schema(type="string", enum=["text", "image", "container"]), + "detections": _schema(type="array", items=_schema(type="object")), + }, + ) + }, + } + }, } _ERROR_SCHEMA = _schema( @@ -398,7 +446,7 @@ class Handler(BaseHTTPRequestHandler): if not self._authorized(): self._respond(HTTPStatus.UNAUTHORIZED, {"ok": False, "error": "unauthorized"}) return - if path not in ("/inspect", "/clean"): + if path not in ("/inspect", "/clean", "/detect"): self._respond(HTTPStatus.NOT_FOUND, {"ok": False, "error": "not found"}) return body = self._read_json() @@ -417,7 +465,9 @@ class Handler(BaseHTTPRequestHandler): return try: if path == "/inspect": - self._handle_inspect(data, name) + self._handle_inspect(data, name, body) + elif path == "/detect": + self._handle_detect(data, name) else: self._handle_clean(data, name, body) except ValueError as e: @@ -428,7 +478,7 @@ class Handler(BaseHTTPRequestHandler): HTTPStatus.INTERNAL_SERVER_ERROR, {"ok": False, "error": "internal error"} ) - def _handle_inspect(self, data: bytes, name: str) -> None: + def _handle_inspect(self, data: bytes, name: str, body: dict[str, Any]) -> None: kind = classify_bytes(data, Path(name).suffix) if kind == "unknown": self._respond( @@ -443,6 +493,7 @@ class Handler(BaseHTTPRequestHandler): }, ) return + run_detect = body.get("detect") is True with tempfile.TemporaryDirectory(prefix="wm-inspect-") as tmp: path = _tmp_path(Path(tmp), name or "input") path.write_bytes(data) @@ -455,19 +506,69 @@ class Handler(BaseHTTPRequestHandler): report = inspect_text(raw_text).to_dict() s_rep = score_text_stylometry(raw_text, path=name or "") report["stylometry"] = s_rep.to_dict() + if run_detect: + report["text_detectors"] = run_all_text_detectors(raw_text) elif kind == "image": report = inspect_image(path).to_dict() else: report = inspect_container(path).to_dict() + detected_wm = any( + entry.get("available") and entry.get("is_watermarked") + for entry in report.get("text_detectors") or [] + ) suspicious = ( bool(report.get("suspicious_total")) or bool(report.get("has_c2pa") or report.get("has_ai_metadata")) or bool(report.get("stylometry", {}).get("score", 0.0) >= 0.65) + or detected_wm ) self._respond( HTTPStatus.OK, {"ok": True, "kind": kind, "report": report, "suspicious": suspicious} ) + def _handle_detect(self, data: bytes, name: str) -> None: + kind = classify_bytes(data, Path(name).suffix) + with tempfile.TemporaryDirectory(prefix="wm-detect-") as tmp: + path = _tmp_path(Path(tmp), name or "input") + path.write_bytes(data) + if kind == "text": + if looks_binary(data): + raise ValueError( + "refusing to detect bytes that look like a binary container as text" + ) + raw_text = data.decode("utf-8", errors="surrogateescape") + detections: list[dict[str, Any]] = run_all_text_detectors(raw_text) + s_rep = score_text_stylometry(raw_text, path=name or "") + detections.append({"detector": "stylometry", "available": True, **s_rep.to_dict()}) + elif kind == "image": + score = run_synthid_score(path) + if score is None: + score = { + "detector": "synthid", + "available": False, + "error": ( + "no SynthID scorer configured (set " + "WATERMARKS_SYNTHID_SCORER_URL or REVERSE_SYNTHID_DIR)" + ), + } + else: + score.setdefault("detector", "synthid") + detections = [score] + else: + detections = [] + report = inspect_container(path).to_dict() + self._respond( + HTTPStatus.OK, + { + "ok": True, + "kind": kind, + "detections": detections, + "report": report, + }, + ) + return + self._respond(HTTPStatus.OK, {"ok": True, "kind": kind, "detections": detections}) + def _handle_clean(self, data: bytes, name: str, body: dict[str, Any]) -> None: kind = classify_bytes(data, Path(name).suffix) if kind == "unknown": @@ -498,13 +599,22 @@ class Handler(BaseHTTPRequestHandler): "refusing to clean bytes that look like a binary container as text" ) text = data.decode("utf-8", errors="surrogateescape") + detect_before = bool(options.get("detect_before")) + detect_after = bool(options.get("detect_after")) + detector_reports: dict[str, Any] = {} + if detect_before: + detector_reports["before"] = run_text_detectors(text) cleaned, stats = clean_text( text, nfkc=bool(options.get("nfkc")), aggressive_homoglyphs=bool(options.get("aggressive_homoglyphs")), ) + if detect_after: + detector_reports["after"] = run_text_detectors(cleaned) cleaned_bytes = cleaned.encode("utf-8", errors="surrogateescape") report: dict[str, Any] = {"kind": "text", "stats": stats, "length": len(cleaned)} + if detector_reports: + report["text_detectors"] = detector_reports elif kind == "image": dest = tmpdir / "out.png" strip_all = not bool(options.get("keep_non_ai_metadata")) @@ -519,6 +629,10 @@ class Handler(BaseHTTPRequestHandler): strip_all_metadata=strip_all, remove_pixel=remove_pixel, ) + if bool(options.get("detect_before")) and result.get("synthid_before") is None: + result["synthid_before"] = run_synthid_score(src) + if bool(options.get("detect_after")) and result.get("synthid_after") is None: + result["synthid_after"] = run_synthid_score(dest) cleaned_bytes = dest.read_bytes() report = {"kind": "image", **result} else: diff --git a/service/scripts/synthid_score_server.py b/service/scripts/synthid_score_server.py new file mode 100644 index 0000000..345db0b --- /dev/null +++ b/service/scripts/synthid_score_server.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +"""Tiny stdlib HTTP sidecar exposing the reverse-SynthID pixel scorer. + +Runs inside the local-only wr-synthid heavy image so the published core +image never bundles the non-commercial reverse-SynthID code. The core +service calls this sidecar for SynthID image scoring when +WATERMARKS_SYNTHID_SCORER_URL is set (see compose.yaml / .env.example). + +Endpoints: + GET /health -> {"ok": true, "version": ...} + POST /score -> {"file": } -> score_synthid payload + +Hardening mirrors server.py: optional bearer key, input size caps, +unprivileged user, read-only rootfs with a /tmp tmpfs. Intended for the +compose network or a trusted network only. +""" + +from __future__ import annotations + +import argparse +import base64 +import binascii +import json +import os +import sys +import tempfile +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any +from urllib.parse import urlparse + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from score_synthid import score_file + +VERSION = os.environ.get("WATERMARKS_SYNTHID_SERVER_VERSION", "dev") + +# Mirror common.MAX_INPUT_BYTES (env-overridable) with the base64 envelope +# headroom. Read at import; the sidecar image does not copy common.py, so the +# default is repeated here. +MAX_INPUT_BYTES = int(os.environ.get("WATERMARKS_MAX_INPUT_BYTES", str(256 << 20))) +MAX_BODY_BYTES = MAX_INPUT_BYTES + (MAX_INPUT_BYTES >> 1) + +API_KEY = os.environ.get("WATERMARKS_SYNTHID_SCORER_API_KEY", "").strip() +MODEL = os.environ.get("WATERMARKS_SYNTHID_MODEL", "").strip() or None + + +def _json_ok(payload: dict[str, Any]) -> bytes: + return json.dumps(payload, ensure_ascii=False, indent=2).encode("utf-8") + + +class Handler(BaseHTTPRequestHandler): + server_version = f"watermarks-remover-synthid/{VERSION}" + + def log_message(self, fmt: str, *args: object) -> None: + print(f"{self.address_string()} - {fmt % args}", file=sys.stderr) + + def _authorized(self) -> bool: + if not API_KEY: + return True + return self.headers.get("Authorization", "") == f"Bearer {API_KEY}" + + def _read_json(self) -> dict[str, Any] | None: + raw = self.headers.get("Content-Length") + if raw is None or not raw.isdigit(): + return None + length = int(raw) + if length > MAX_BODY_BYTES: + return None + try: + body = json.loads(self.rfile.read(length).decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError, OSError): + return None + return body if isinstance(body, dict) else None + + def _respond(self, status: int, payload: dict[str, Any]) -> None: + data = _json_ok(payload) + self.send_response(status) + self.send_header("Content-Type", "application/json; charset=utf-8") + self.send_header("Content-Length", str(len(data))) + self.send_header("Cache-Control", "no-store") + self.end_headers() + self.wfile.write(data) + + def do_GET(self) -> None: + if not self._authorized(): + self._respond(HTTPStatus.UNAUTHORIZED, {"ok": False, "error": "unauthorized"}) + return + if urlparse(self.path).path == "/health": + self._respond(HTTPStatus.OK, {"ok": True, "version": VERSION}) + else: + self._respond(HTTPStatus.NOT_FOUND, {"ok": False, "error": "not found"}) + + def do_POST(self) -> None: + if not self._authorized(): + self._respond(HTTPStatus.UNAUTHORIZED, {"ok": False, "error": "unauthorized"}) + return + if urlparse(self.path).path != "/score": + self._respond(HTTPStatus.NOT_FOUND, {"ok": False, "error": "not found"}) + return + body = self._read_json() + if body is None: + raw_len = self.headers.get("Content-Length") + oversized = raw_len is not None and raw_len.isdigit() and int(raw_len) > MAX_BODY_BYTES + self._respond( + HTTPStatus.REQUEST_ENTITY_TOO_LARGE if oversized else HTTPStatus.BAD_REQUEST, + {"ok": False, "error": "invalid request body"}, + ) + return + + raw = body.get("file") + if not isinstance(raw, str): + self._respond(HTTPStatus.BAD_REQUEST, {"ok": False, "error": "missing 'file' field"}) + return + try: + data = base64.b64decode(raw, validate=True) + except (binascii.Error, ValueError): + self._respond( + HTTPStatus.BAD_REQUEST, {"ok": False, "error": "'file' is not valid base64"} + ) + return + if len(data) > MAX_INPUT_BYTES: + self._respond( + HTTPStatus.REQUEST_ENTITY_TOO_LARGE, {"ok": False, "error": "file too large"} + ) + return + + with tempfile.TemporaryDirectory(prefix="wm-synthid-") as tmp: + path = Path(tmp) / "input.png" + try: + path.write_bytes(data) + except OSError as e: + self._respond(HTTPStatus.INTERNAL_SERVER_ERROR, {"ok": False, "error": str(e)}) + return + code, payload = score_file(path, model=MODEL) + + if code == 0 and payload is not None: + self._respond(HTTPStatus.OK, payload) + elif code == 2: + self._respond(HTTPStatus.BAD_REQUEST, {"ok": False, "error": "could not load image"}) + else: + # exit 1 (runtime error) or 3 (unavailable) -> fail-soft payload, + # matching the shape image_meta.run_synthid_score expects. + self._respond( + HTTPStatus.OK, + {"available": False, "error": "scorer unavailable (see sidecar stderr)"}, + ) + + +def main() -> int: + global API_KEY # noqa: PLW0603 — CLI overrides env + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--host", default=os.environ.get("WATERMARKS_SYNTHID_SERVER_HOST", "127.0.0.1")) + p.add_argument( + "--port", type=int, default=int(os.environ.get("WATERMARKS_SYNTHID_SERVER_PORT", "8766")) + ) + p.add_argument("--api-key", default=API_KEY, help="require this bearer token (default: none)") + args = p.parse_args() + + if args.host not in ("127.0.0.1", "localhost", "::1"): + print( + f"warning: binding {args.host} — intended for a trusted network only", file=sys.stderr + ) + API_KEY = args.api_key + print(f"synthid scorer sidecar {VERSION} on http://{args.host}:{args.port}", file=sys.stderr) + server = ThreadingHTTPServer((args.host, args.port), Handler) + try: + server.serve_forever() + except KeyboardInterrupt: + server.shutdown() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/service/scripts/text_detectors.py b/service/scripts/text_detectors.py new file mode 100644 index 0000000..0e8865c --- /dev/null +++ b/service/scripts/text_detectors.py @@ -0,0 +1,487 @@ +#!/usr/bin/env python3 +"""Vendor and research text-watermark detectors behind one interface. + +Detects statistical (Layer B) text watermarks using vendor-provided or +research detectors. Every detector implements the same small protocol: + + name: str stable identifier (surfaced in /capabilities) + available() -> bool configured and usable right now + detect(text) -> dict JSON-safe report; never raises + +Reports follow the fail-soft contract: a detector that is unconfigured, +times out, or errors returns {"available": False, "error": ...} and can +never block cleaning. + +Detectors: + +- gemini-synthid-text — Google's official SynthID-text detector, called + through the Gemini API (taskType DETECT_TEXT_WATERMARK). Activated by + WATERMARKS_GEMINI_API_KEY. User text is sent to Google only when the + operator sets that key. +- markllm — optional research harness (KGW / SynthID schemes) via + detect_text_watermark.py, activated by MARKLLM_DIR. Same-config-only + detection; not a vendor oracle. +- claude-text — placeholder for Anthropic's announced text-watermark + detection API. Reports unavailable until a public endpoint exists; the + interface it must implement is already defined here. +""" + +from __future__ import annotations + +import contextlib +import json +import os +import subprocess +import sys +import tempfile +import time +import urllib.error +import urllib.request +from collections.abc import Callable +from pathlib import Path +from typing import Any, Protocol +from urllib.parse import urlparse + +GEMINI_DETECT_URL = ( + "https://generativelanguage.googleapis.com/v1beta/models/{model}:generateContent" +) +DEFAULT_GEMINI_MODEL = "gemini-2.5-flash" +DEFAULT_GEMINI_TIMEOUT = 30.0 +DEFAULT_GEMINI_MAX_CHARS = 1_000_000 +DEFAULT_MARKLLM_SCHEME = "kgw" +DEFAULT_MARKLLM_TIMEOUT = 600.0 + + +class DetectorError(RuntimeError): + """A detector call failed (network, HTTP error, timeout).""" + + +class TextDetector(Protocol): + name: str + + def available(self) -> bool: ... + + def detect(self, text: str) -> dict[str, Any]: ... + + +def _env_float(name: str, default: float) -> float: + try: + return float(os.environ.get(name, str(default))) + except ValueError: + return default + + +def _env_int(name: str, default: int) -> int: + try: + return int(os.environ.get(name, str(default))) + except ValueError: + return default + + +# --------------------------------------------------------------------------- +# Gemini (Google's official SynthID-text detector) +# --------------------------------------------------------------------------- + +_WATERMARKED_VERDICTS = ("watermarked", "ai-generated", "ai generated", "likely ai") + + +def _verdict_is_watermarked(verdict: str | None) -> bool | None: + """Map the detector model's free-text verdict to a boolean, or None.""" + if not verdict: + return None + low = verdict.strip().lower() + if low.startswith(("unlikely", "no", "not")): + return False + return any(marker in low for marker in _WATERMARKED_VERDICTS) + + +def _extract_numeric_score(candidate: dict[str, Any], top: dict[str, Any]) -> float | None: + """Pull a numeric watermark score from any of the known response shapes.""" + for container in (candidate, top): + for key in ( + "syntheticTextScore", + "synthetic_text_score", + "watermarkScore", + "watermark_score", + "score", + ): + value = container.get(key) + if isinstance(value, (int, float)): + return float(value) + attribution = candidate.get("attributionMetadata") or {} + if isinstance(attribution, dict): + for key in ("syntheticTextScore", "synthetic_text_score", "score"): + value = attribution.get(key) + if isinstance(value, (int, float)): + return float(value) + st = attribution.get("syntheticText") + if isinstance(st, dict): + for key in ("score", "confidence"): + value = st.get(key) + if isinstance(value, (int, float)): + return float(value) + return None + + +def parse_gemini_detect_response(data: dict[str, Any]) -> dict[str, Any]: + """Parse a generateContent response from a DETECT_TEXT_WATERMARK call. + + The endpoint can answer with either a free-text verdict + ("Likely AI-generated") or a structured score; both shapes are handled + defensively so upstream schema changes degrade to an error report + instead of a crash. + """ + candidates = data.get("candidates") or [] + candidate = candidates[0] if candidates else {} + if not isinstance(candidate, dict): + candidate = {} + + if not candidate: + feedback = data.get("promptFeedback") or {} + block = feedback.get("blockReason") + if block: + raise DetectorError(f"Gemini blocked the request: {block}") + raise DetectorError("Gemini returned no candidates") + + verdict: str | None = None + content = candidate.get("content") or {} + parts = content.get("parts") or [] + if parts and isinstance(parts[0], dict): + verdict = parts[0].get("text") + + score = _extract_numeric_score(candidate, data) + is_watermarked = _verdict_is_watermarked(verdict) + if is_watermarked is None and score is not None: + is_watermarked = score >= 0.5 + + if verdict is None and score is None: + raise DetectorError( + f"unexpected Gemini response (no verdict or score): {json.dumps(data)[:400]}" + ) + + raw = { + key: candidate[key] + for key in ("attributionMetadata", "finishReason", "index") + if candidate.get(key) is not None + } + return { + "is_watermarked": is_watermarked, + "score": score, + "verdict": verdict, + "raw": raw, + } + + +def _post_json(url: str, body: dict[str, Any], api_key: str, timeout: float) -> dict[str, Any]: + """POST *body* to *url*, retrying once on transient failures.""" + if urlparse(url).scheme not in ("http", "https"): + raise DetectorError(f"refusing non-http(s) Gemini endpoint: {url}") + # S310: URL scheme is restricted to http/https just above. + req = urllib.request.Request( # noqa: S310 + url, + data=json.dumps(body).encode("utf-8"), + headers={"Content-Type": "application/json", "x-goog-api-key": api_key}, + method="POST", + ) + last_err = "Gemini API call failed" + for attempt in range(2): + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: # noqa: S310 + payload = json.loads(resp.read().decode("utf-8")) + if not isinstance(payload, dict): + raise DetectorError("non-object Gemini response") + return payload + except urllib.error.HTTPError as e: + last_err = f"Gemini API HTTP {e.code}: {e.read().decode('utf-8', 'replace')[:300]}" + if e.code not in (429, 500, 502, 503, 504): + raise DetectorError(last_err) from e + except (urllib.error.URLError, TimeoutError, OSError) as e: + last_err = f"Gemini API unreachable: {e}" + if attempt == 0: + time.sleep(1.0) + raise DetectorError(last_err) + + +class GeminiSynthIDTextDetector: + """Google's official SynthID-text detector via the Gemini API.""" + + name = "gemini-synthid-text" + vendor = "google" + + def available(self) -> bool: + return bool(os.environ.get("WATERMARKS_GEMINI_API_KEY", "").strip()) + + def detect(self, text: str) -> dict[str, Any]: + api_key = os.environ.get("WATERMARKS_GEMINI_API_KEY", "").strip() + if not api_key: + return { + "detector": self.name, + "vendor": self.vendor, + "available": False, + "error": "WATERMARKS_GEMINI_API_KEY not set", + } + + max_chars = _env_int("WATERMARKS_GEMINI_MAX_CHARS", DEFAULT_GEMINI_MAX_CHARS) + if len(text) > max_chars: + return { + "detector": self.name, + "vendor": self.vendor, + "available": True, + "skipped": True, + "reason": f"text longer than {max_chars} chars", + "is_watermarked": None, + } + + model = ( + os.environ.get("WATERMARKS_GEMINI_MODEL", DEFAULT_GEMINI_MODEL) or DEFAULT_GEMINI_MODEL + ) + timeout = _env_float("WATERMARKS_GEMINI_TIMEOUT", DEFAULT_GEMINI_TIMEOUT) + url = GEMINI_DETECT_URL.format(model=model) + body = { + "contents": [{"role": "user", "parts": [{"text": text}]}], + "generationConfig": {"taskType": "DETECT_TEXT_WATERMARK"}, + } + report: dict[str, Any] = { + "detector": self.name, + "vendor": self.vendor, + "model": model, + "available": True, + } + try: + data = _post_json(url, body, api_key, timeout) + except DetectorError as e: + report["available"] = False + report["error"] = str(e) + return report + try: + parsed = parse_gemini_detect_response(data) + except DetectorError as e: + report["available"] = False + report["error"] = str(e) + return report + report.update(parsed) + return report + + +# --------------------------------------------------------------------------- +# MarkLLM (open-source research harness: KGW / SynthID schemes) +# --------------------------------------------------------------------------- + + +def _venv_python(upstream: Path) -> Path | None: + """Prefer the MarkLLM checkout's venv interpreter, if it exists.""" + if os.name == "nt": + candidate = upstream / ".venv" / "Scripts" / "python.exe" + else: + candidate = upstream / ".venv" / "bin" / "python" + return candidate if candidate.is_file() else None + + +def _markllm_preexec() -> Callable[[], None] | None: + """Optional RLIMIT_AS guard for the MarkLLM child; None means "no limit". + + torch/CUDA usually needs large address spaces, so this is opt-in via + WATERMARKS_MARKLLM_RLIMIT_AS (byte count, hex/octal allowed). POSIX only; + on Windows preexec_fn must stay None. + """ + raw = os.environ.get("WATERMARKS_MARKLLM_RLIMIT_AS") + if not raw or os.name != "posix": + return None + try: + limit = int(raw, 0) + except ValueError: + return None + + def _apply() -> None: + import resource + + resource.setrlimit(resource.RLIMIT_AS, (limit, limit)) + + return _apply + + +class MarkLLMTextDetector: + """Same-config-only research detection via detect_text_watermark.py. + + Constructor overrides (scheme, upstream_dir, model, timeout) take + precedence over the environment, so callers such as rewrite_text.py can + keep CLI flags driving the harness. When the MarkLLM checkout has a + venv, its interpreter runs the child process; otherwise the current + interpreter is used (the service image bundles the harness deps). + """ + + name = "markllm" + + def __init__( + self, + *, + scheme: str | None = None, + upstream_dir: str | None = None, + model: str | None = None, + timeout: float | None = None, + ) -> None: + self._scheme = scheme + self._upstream_dir = upstream_dir + self._model = model + self._timeout = timeout + + def available(self) -> bool: + upstream = self._upstream_dir or os.environ.get("MARKLLM_DIR", "").strip() + return bool(upstream) + + def detect(self, text: str) -> dict[str, Any]: + upstream = self._upstream_dir or os.environ.get("MARKLLM_DIR", "").strip() + scheme = ( + self._scheme + or os.environ.get("WATERMARKS_MARKLLM_SCHEME", "") + or DEFAULT_MARKLLM_SCHEME + ) + report: dict[str, Any] = { + "detector": self.name, + "scheme": scheme, + "vendor": "open-llm", + "available": False, + } + if not upstream: + report["error"] = "MARKLLM_DIR not set" + return report + + script = Path(__file__).resolve().parent / "detect_text_watermark.py" + timeout = ( + self._timeout + if self._timeout is not None + else _env_float("WATERMARKS_MARKLLM_TIMEOUT", DEFAULT_MARKLLM_TIMEOUT) + ) + venv_python = _venv_python(Path(upstream).expanduser().resolve()) + python = str(venv_python) if venv_python is not None else sys.executable + + with tempfile.NamedTemporaryFile("w", suffix=".txt", encoding="utf-8", delete=False) as f: + f.write(text) + tmp = f.name + + cmd = [python, str(script), "detect", tmp, "--scheme", scheme, "--json"] + if self._model: + cmd += ["--model", self._model] + if self._upstream_dir: + cmd += ["--upstream-dir", str(Path(upstream).expanduser().resolve())] + + try: + try: + r = subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=timeout, + preexec_fn=_markllm_preexec(), + check=False, + ) + except subprocess.TimeoutExpired: + report["error"] = "MarkLLM detection timed out" + return report + if r.returncode == 3: + report["error"] = (r.stderr or "").strip()[:400] or "MarkLLM unavailable" + return report + if r.returncode != 0: + report["error"] = (r.stderr or "").strip()[:400] or f"MarkLLM exit {r.returncode}" + return report + try: + payload = json.loads(r.stdout or "{}") + except json.JSONDecodeError as e: + report["error"] = f"bad MarkLLM JSON: {e}" + return report + finally: + with contextlib.suppress(OSError): + Path(tmp).unlink() + + if not isinstance(payload, dict): + report["error"] = "bad MarkLLM response" + return report + payload["available"] = True + payload["detector"] = self.name + payload["note"] = ( + "MarkLLM is a research harness: detection is only valid against the " + "same scheme config and keys used at generation; not a vendor detector." + ) + return payload + + +# --------------------------------------------------------------------------- +# Claude (Anthropic) — announced detector API, not yet public +# --------------------------------------------------------------------------- + + +class ClaudeTextDetector: + """Placeholder for Anthropic's announced text-watermark detection API. + + Anthropic has announced a watermark detection API for Claude-generated + text; no public endpoint exists yet. When it ships, set + WATERMARKS_CLAUDE_API_KEY, flip available() to check it, and fill in + detect() against the documented endpoint. + """ + + name = "claude-text" + vendor = "anthropic" + + def available(self) -> bool: + return False + + def detect(self, text: str) -> dict[str, Any]: + return { + "detector": self.name, + "vendor": self.vendor, + "available": False, + "error": ( + "Anthropic has announced a text-watermark detection API for " + "Claude; no public endpoint is available yet. When it ships, " + "set WATERMARKS_CLAUDE_API_KEY and implement ClaudeTextDetector." + ), + } + + +# --------------------------------------------------------------------------- +# Registry +# --------------------------------------------------------------------------- + + +def all_detectors( + markllm: MarkLLMTextDetector | None = None, *, include_markllm: bool = True +) -> list[TextDetector]: + detectors: list[TextDetector] = [GeminiSynthIDTextDetector()] + if include_markllm: + detectors.append(markllm or MarkLLMTextDetector()) + detectors.append(ClaudeTextDetector()) + return detectors + + +def detector_status() -> dict[str, bool]: + """Configured/usable status per detector (for /capabilities).""" + return {d.name: d.available() for d in all_detectors()} + + +def run_all_text_detectors( + text: str, + *, + markllm: MarkLLMTextDetector | None = None, + include_markllm: bool = True, +) -> list[dict[str, Any]]: + """Run every detector (including unavailable ones, with reasons). + + markllm injects a caller-parameterized MarkLLM detector (e.g. one + driven by rewrite_text.py CLI flags); pass include_markllm=False to + exclude the MarkLLM harness entirely. + """ + return [d.detect(text) for d in all_detectors(markllm, include_markllm=include_markllm)] + + +def run_text_detectors( + text: str, + *, + markllm: MarkLLMTextDetector | None = None, + include_markllm: bool = True, +) -> list[dict[str, Any]]: + """Run only the detectors that are configured and usable.""" + return [ + d.detect(text) + for d in all_detectors(markllm, include_markllm=include_markllm) + if d.available() + ] diff --git a/skills/remove-ai-marks/SKILL.md b/skills/remove-ai-marks/SKILL.md index 6476b43..9fae978 100644 --- a/skills/remove-ai-marks/SKILL.md +++ b/skills/remove-ai-marks/SKILL.md @@ -56,10 +56,13 @@ curl -s "$WM/capabilities" ``` Reports which optional tools are available server-side (`c2patool`, `exiftool`, -`qpdf`), scorers present (`scorers.stylometry`, `scorers.synthid`), and which heavy -backends are configured (`pixel_backends.ctrlregen`, `pixel_backends.diffusion`, -`harnesses.markllm`). **Drive your advice from this**: only recommend pixel -removal / SynthID scoring when the service reports the backend present. +`qpdf`), scorers present (`scorers.stylometry`, `scorers.synthid`, +`scorers.synthid_http`), vendor text-watermark detectors +(`text_detectors.gemini-synthid-text`, `text_detectors.markllm`, +`text_detectors.claude-text`), and which heavy backends are configured +(`pixel_backends.ctrlregen`, `pixel_backends.diffusion`, `harnesses.markllm`). +**Drive your advice from this**: only recommend pixel removal / SynthID +scoring / vendor detection when the service reports the backend present. ## HTTP API (curl) @@ -72,6 +75,7 @@ field and writes it to the output path itself. | GET | `/capabilities` | — | optional tools / backends present | | GET | `/openapi.json` | — | dynamically generated OpenAPI 3.0.3 spec | | POST | `/inspect` | `{"file": "", "name": "notes.md"}` | `{"ok", "kind", "suspicious", "report"}` | +| POST | `/detect` | `{"file": "", "name": "notes.txt"}` | `{"ok", "kind", "detections": [...]}` | | POST | `/clean` | `{"file": "", "name": "notes.md", "options": {...}}` | `{"ok", "kind", "cleaned": "", "report"}` | `/clean` and `/inspect` route by the uploaded `name` extension plus the bytes; @@ -85,7 +89,9 @@ clients. `options` accepted by `/clean`: `nfkc`, `aggressive_homoglyphs` (text), `keep_non_ai_metadata`, `strip_all_metadata`, `remove_pixel` (`ctrlregen` | -`diffusion`) (images), `also_layer_a_text` (containers). +`diffusion`) (images), `also_layer_a_text` (containers), `detect_before` / +`detect_after` (text and images: run watermark detection on the input and on +the cleaned output, included in the report). **Inspect first** (decide, don't guess): @@ -143,6 +149,25 @@ external heavy backends. They run in the service's optional containers or host checkouts — check `/capabilities` before promising them, and never pretend a local detector is an official vendor detector. +### 2b. Watermark detection before/after (when configured) + +When `/capabilities` reports a vendor detector (`text_detectors.gemini-synthid-text`) +or an image scorer (`scorers.synthid_http` / `scorers.synthid`), measure the +result by detecting before and after cleaning: + +```bash +curl -s -X POST "$WM/detect" -H 'Content-Type: application/json' \ + -d '{"file": "'"$(base64 -w0 notes.txt)"'", "name": "notes.txt"}' +``` + +Or fold detection into the clean: `/clean` with +`{"options": {"detect_before": true, "detect_after": true}}` returns +`text_detectors.before/after` (text) or `synthid_before/synthid_after` +(images) in the report. Note: vendor detection sends text to the configured +provider (Gemini) — only use it with user consent, and report the vendor's +verdict honestly (official SynthID-text detector for Gemini; MarkLLM is +same-config-only research; Claude's detector is not public yet). + ### 3. Deterministic clean (always for matching inputs) **Any supported file (unified):** @@ -286,7 +311,7 @@ Always state: - Layer B cannot be gold-verified without vendor detectors / keys. Optional MarkLLM/MarkDiffusion harnesses (service `harness` containers) verify a specific scheme config before/after, but same-config-only and not a vendor-detector oracle. - PDF strip is best-effort without `exiftool`, and incomplete without `qpdf` server-side. - Pixel-domain **image** watermarks can be removed optionally via the external CtrlRegen backend (`remove_pixel: ctrlregen`) or MarkDiffusion's DiffusionPurification (`remove_pixel: diffusion`); both are heavy, drift the image, and need the backend present (`/capabilities`). Audio/video watermarks remain out of scope. -- The reverse-SynthID scorer is external, best-effort, and under a non-commercial Research License; not an official Google detector. +- The reverse-SynthID scorer is external, best-effort, and under a non-commercial Research License; not an official Google detector. The Gemini text detector, by contrast, is Google's official SynthID-text detector when `WATERMARKS_GEMINI_API_KEY` is configured server-side. Claude's detection API has been announced but is not public yet — the `claude-text` detector reports unavailable until it ships. - **C2PA soft binding** (content watermark that re-links to a remote manifest after metadata strip) is out of scope — stripping hard-bound C2PA does not clear it. - Data-driven / backdoor model marks (trigger phrases) are out of scope. diff --git a/tests/test_detect_endpoint.py b/tests/test_detect_endpoint.py new file mode 100644 index 0000000..9c8a9ef --- /dev/null +++ b/tests/test_detect_endpoint.py @@ -0,0 +1,255 @@ +"""Tests for the HTTP detection surface: /detect, /inspect detect flag, +/clean detect_before/detect_after, capabilities, and the SynthID sidecar.""" + +from __future__ import annotations + +import base64 +import http.client +import json +import struct +import sys +import threading +import zlib +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +SCRIPTS = ROOT / "service" / "scripts" +sys.path.insert(0, str(SCRIPTS)) + +import image_meta +import server +import text_detectors + + +def _png_chunk(ctype: bytes, payload: bytes) -> bytes: + crc = zlib.crc32(ctype) + crc = zlib.crc32(payload, crc) & 0xFFFFFFFF + return struct.pack(">I", len(payload)) + ctype + payload + struct.pack(">I", crc) + + +def _watermarked_png() -> bytes: + sig = b"\x89PNG\r\n\x1a\n" + ihdr = struct.pack(">IIBBBBB", 1, 1, 8, 2, 0, 0, 0) + idat = zlib.compress(b"\x00\x00\x00") + text = b"Comment\x00c2pa test contentcredentials" + return ( + sig + + _png_chunk(b"IHDR", ihdr) + + _png_chunk(b"tEXt", text) + + _png_chunk(b"IDAT", idat) + + _png_chunk(b"IEND", b"") + ) + + +def _b64(data: bytes) -> str: + return base64.b64encode(data).decode("ascii") + + +def _post(conn, path: str, payload: dict) -> tuple[int, dict]: + conn.request( + "POST", + path, + body=json.dumps(payload).encode("utf-8"), + headers={"Content-Type": "application/json"}, + ) + resp = conn.getresponse() + data = resp.read() + return resp.status, json.loads(data) if data else {} + + +def _get(conn, path: str) -> tuple[int, dict]: + conn.request("GET", path) + resp = conn.getresponse() + data = resp.read() + return resp.status, json.loads(data) if data else {} + + +class _FakeResp: + def __init__(self, data: dict): + self._data = json.dumps(data).encode("utf-8") + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def read(self): + return self._data + + +@pytest.fixture(scope="module") +def conn() -> http.client.HTTPConnection: + srv = server.ThreadingHTTPServer(("127.0.0.1", 0), server.Handler) + thread = threading.Thread(target=srv.serve_forever, daemon=True) + thread.start() + c = http.client.HTTPConnection("127.0.0.1", srv.server_address[1]) + yield c + c.close() + srv.shutdown() + srv.server_close() + thread.join(timeout=5) + + +@pytest.fixture(autouse=True) +def _clean_env(monkeypatch): + for key in ( + "WATERMARKS_GEMINI_API_KEY", + "WATERMARKS_GEMINI_MODEL", + "WATERMARKS_SYNTHID_SCORER_URL", + "WATERMARKS_SYNTHID_SCORER_API_KEY", + "MARKLLM_DIR", + ): + monkeypatch.delenv(key, raising=False) + + +def test_capabilities_exposes_detectors(conn): + status, body = _get(conn, "/capabilities") + assert status == 200 + assert set(body["text_detectors"]) == {"gemini-synthid-text", "markllm", "claude-text"} + assert "synthid_http" in body["scorers"] + + +def test_openapi_includes_detect(conn): + status, body = _get(conn, "/openapi.json") + assert status == 200 + assert "/detect" in body["paths"] + assert ( + "detect_before" + in body["paths"]["/clean"]["post"]["requestBody"]["content"]["application/json"]["schema"][ + "properties" + ]["options"]["properties"] + ) + + +def test_detect_text_without_detectors(conn): + payload = {"file": _b64(b"some plain text"), "name": "notes.txt"} + status, body = _post(conn, "/detect", payload) + assert status == 200 + assert body["kind"] == "text" + names = {d["detector"] for d in body["detections"]} + assert "stylometry" in names + assert "claude-text" in names # placeholder always reports unavailable + gemini = next(d for d in body["detections"] if d["detector"] == "gemini-synthid-text") + assert gemini["available"] is False + + +def test_detect_text_with_gemini(conn, monkeypatch): + monkeypatch.setenv("WATERMARKS_GEMINI_API_KEY", "test-key") + monkeypatch.setattr( + text_detectors.urllib.request, + "urlopen", + lambda *a, **k: _FakeResp( + {"candidates": [{"content": {"parts": [{"text": "Likely AI-generated"}]}}]} + ), + ) + payload = {"file": _b64(b"watermarked prose here"), "name": "notes.txt"} + status, body = _post(conn, "/detect", payload) + assert status == 200 + gemini = next(d for d in body["detections"] if d["detector"] == "gemini-synthid-text") + assert gemini["available"] is True + assert gemini["is_watermarked"] is True + + +def test_inspect_detect_is_opt_in(conn, monkeypatch): + monkeypatch.setenv("WATERMARKS_GEMINI_API_KEY", "test-key") + monkeypatch.setattr( + text_detectors.urllib.request, + "urlopen", + lambda *a, **k: _FakeResp( + {"candidates": [{"content": {"parts": [{"text": "Likely AI-generated"}]}}]} + ), + ) + txt = b"watermarked prose here" + # without the flag: no vendor calls, no text_detectors key + status, body = _post(conn, "/inspect", {"file": _b64(txt), "name": "notes.txt"}) + assert status == 200 + assert "text_detectors" not in body["report"] + # with the flag: detector results appear and can flip suspicious + status, body = _post(conn, "/inspect", {"file": _b64(txt), "name": "notes.txt", "detect": True}) + assert status == 200 + assert "text_detectors" in body["report"] + assert body["suspicious"] is True + + +def test_clean_text_detect_before_after(conn, monkeypatch): + monkeypatch.setenv("WATERMARKS_GEMINI_API_KEY", "test-key") + monkeypatch.setattr( + text_detectors.urllib.request, + "urlopen", + lambda *a, **k: _FakeResp( + {"candidates": [{"content": {"parts": [{"text": "Likely AI-generated"}]}}]} + ), + ) + txt = ("watermarked prose here. " * 5).encode("utf-8") + status, body = _post( + conn, + "/clean", + { + "file": _b64(txt), + "name": "notes.txt", + "options": {"detect_before": True, "detect_after": True}, + }, + ) + assert status == 200 + det = body["report"]["text_detectors"] + assert set(det) == {"before", "after"} + assert det["before"][0]["is_watermarked"] is True + assert det["after"][0]["is_watermarked"] is True + + +def test_clean_image_detect_before_after_sidecar(conn, monkeypatch): + monkeypatch.setenv("WATERMARKS_SYNTHID_SCORER_URL", "http://scorer:8766") + monkeypatch.setattr( + image_meta.urllib.request, + "urlopen", + lambda *a, **k: _FakeResp( + { + "available": True, + "is_watermarked": True, + "confidence": 0.91, + "phase_match": 0.8, + } + ), + ) + status, body = _post( + conn, + "/clean", + { + "file": _b64(_watermarked_png()), + "name": "shot.png", + "options": {"detect_before": True, "detect_after": True}, + }, + ) + assert status == 200 + report = body["report"] + assert report["synthid_before"]["is_watermarked"] is True + assert report["synthid_after"]["is_watermarked"] is True + + +def test_run_synthid_score_http_mode(tmp_path, monkeypatch): + monkeypatch.setenv("WATERMARKS_SYNTHID_SCORER_URL", "http://scorer:8766") + seen = {} + + def fake_urlopen(req, timeout=None): + seen["url"] = req.full_url + seen["timeout"] = timeout + return _FakeResp({"available": True, "is_watermarked": False, "confidence": 0.1}) + + monkeypatch.setattr(image_meta.urllib.request, "urlopen", fake_urlopen) + img = tmp_path / "x.png" + img.write_bytes(_watermarked_png()) + payload = image_meta.run_synthid_score(img) + assert payload["available"] is True + assert payload["is_watermarked"] is False + assert seen["url"] == "http://scorer:8766/score" + assert seen["timeout"] == 60.0 + + +def test_detect_image_no_scorer(conn): + status, body = _post(conn, "/detect", {"file": _b64(_watermarked_png()), "name": "shot.png"}) + assert status == 200 + assert body["kind"] == "image" + assert body["detections"][0]["available"] is False diff --git a/tests/test_markllm_detect.py b/tests/test_markllm_detect.py index 2bde465..b17f827 100644 --- a/tests/test_markllm_detect.py +++ b/tests/test_markllm_detect.py @@ -7,7 +7,6 @@ import os import subprocess import sys from pathlib import Path -from types import SimpleNamespace import pytest @@ -345,138 +344,17 @@ def test_cli_watermark_runtime_error(tmp_path: Path): assert "boom" in (r.stderr or "") -def test_rewrite_markllm_detect_missing_venv(tmp_path: Path): - import rewrite_text - - upstream = tmp_path / "MarkLLM" - upstream.mkdir() - result = rewrite_text._markllm_detect( - "hello", - scheme="kgw", - upstream_dir=str(upstream), - model="x", - timeout=5, - ) - assert result["available"] is False - - -def test_rewrite_markllm_detect_parses_json(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): - import rewrite_text - - upstream = tmp_path / "MarkLLM" - if os.name == "nt": - venv_python = upstream / ".venv" / "Scripts" / "python.exe" - else: - venv_python = upstream / ".venv" / "bin" / "python" - venv_python.parent.mkdir(parents=True) - venv_python.write_text("") - (upstream / "watermark").mkdir() - - payload = {"available": True, "is_watermarked": True, "score": 2.0} - captured: dict = {} - - def fake_run(cmd, **kwargs): - captured["cmd"] = cmd - captured["input"] = kwargs.get("input") - return SimpleNamespace(returncode=0, stdout=json.dumps(payload), stderr="") - - monkeypatch.setattr(rewrite_text.subprocess, "run", fake_run) - result = rewrite_text._markllm_detect( - "hello", - scheme="kgw", - upstream_dir=str(upstream), - model="x", - timeout=5, - ) - assert result["available"] is True - assert result["score"] == 2.0 - assert captured["input"] == "hello" - assert captured["cmd"][0] == str(venv_python) - - -def test_rewrite_markllm_detect_adapter_failure(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): - import rewrite_text - - upstream = tmp_path / "MarkLLM" - if os.name == "nt": - venv_python = upstream / ".venv" / "Scripts" / "python.exe" - else: - venv_python = upstream / ".venv" / "bin" / "python" - venv_python.parent.mkdir(parents=True) - venv_python.write_text("") - (upstream / "watermark").mkdir() - - def fake_run(cmd, **kwargs): - return SimpleNamespace(returncode=3, stdout="", stderr="deps missing") - - monkeypatch.setattr(rewrite_text.subprocess, "run", fake_run) - result = rewrite_text._markllm_detect( - "hello", - scheme="kgw", - upstream_dir=str(upstream), - model="x", - timeout=5, - ) - assert result["available"] is False - assert "deps missing" in result["error"] - - -def test_markllm_preexec_default_off(monkeypatch: pytest.MonkeyPatch): - import rewrite_text - - monkeypatch.delenv("WATERMARKS_MARKLLM_RLIMIT_AS", raising=False) - assert rewrite_text._markllm_preexec() is None - - -def test_markllm_preexec_env(monkeypatch: pytest.MonkeyPatch): - import rewrite_text - - if os.name != "posix": - pytest.skip("preexec_fn is POSIX-only") - monkeypatch.setenv("WATERMARKS_MARKLLM_RLIMIT_AS", "0x40000000") - fn = rewrite_text._markllm_preexec() - assert callable(fn) - - -def test_rewrite_markllm_detect_applies_rlimit( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -): - import rewrite_text - - if os.name != "posix": - pytest.skip("preexec_fn is POSIX-only") - upstream = tmp_path / "MarkLLM" - venv_python = upstream / ".venv" / "bin" / "python" - venv_python.parent.mkdir(parents=True) - venv_python.write_text("") - (upstream / "watermark").mkdir() - monkeypatch.setenv("WATERMARKS_MARKLLM_RLIMIT_AS", "1073741824") - captured: dict = {} - - def fake_run(cmd, **kwargs): - captured["preexec_fn"] = kwargs.get("preexec_fn") - return SimpleNamespace(returncode=0, stdout='{"available": true}', stderr="") - - monkeypatch.setattr(rewrite_text.subprocess, "run", fake_run) - result = rewrite_text._markllm_detect( - "hello", - scheme="kgw", - upstream_dir=str(upstream), - model="x", - timeout=5, - ) - assert result["available"] is True - assert callable(captured["preexec_fn"]) - - def test_rewrite_markllm_hook_records_before_after(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): import rewrite_text - def fake_detect(text, **kwargs): - return {"available": True, "is_watermarked": text == "ORIG", "score": 3.0} + class _FakeDetector: + def __init__(self, **kwargs): + pass - monkeypatch.setattr(rewrite_text, "_markllm_detect", fake_detect) + def detect(self, text): + return {"available": True, "is_watermarked": text == "ORIG", "score": 3.0} + + monkeypatch.setattr(rewrite_text, "MarkLLMTextDetector", _FakeDetector) monkeypatch.setattr(rewrite_text, "call_ollama", lambda *a, **k: "REWRITTEN OUTPUT") out, info = rewrite_text.rewrite( "ORIG", diff --git a/tests/test_rewrite_text.py b/tests/test_rewrite_text.py index 92c6885..ea338f0 100644 --- a/tests/test_rewrite_text.py +++ b/tests/test_rewrite_text.py @@ -16,6 +16,7 @@ ROOT = Path(__file__).resolve().parents[1] SCRIPTS = ROOT / "service" / "scripts" sys.path.insert(0, str(SCRIPTS)) +import rewrite_text from rewrite_text import ( _check_remote, _flag_env, @@ -110,6 +111,208 @@ def test_select_candidate_prefers_more_divergent(): assert len(scores) == 3 +# --------------------------------------------------------------------------- +# Per-candidate watermark detection (issue #106) +# --------------------------------------------------------------------------- + + +class _FakeMarkLLM: + """Stand-in for text_detectors.MarkLLMTextDetector (no subprocess).""" + + name = "markllm" + + def __init__(self, **kwargs): + self._kwargs = kwargs + + def available(self) -> bool: + return True + + def detect(self, text: str) -> dict: + return { + "detector": "markllm", + "scheme": "kgw", + "vendor": "open-llm", + "available": True, + "is_watermarked": text == "the cat sat on the mat", + "score": 3.0, + "threshold": 3.0, + } + + +def _rewrite_candidates_kwargs(**overrides): + kwargs = dict( + backend="ollama", + model="m", + base_url="http://127.0.0.1:11434", + api_key=None, + strength="paraphrase", + lang="French", + original_lang="English", + timeout=10, + layer_a_after=False, + temperature=0.9, + candidates=2, + ) + kwargs.update(overrides) + return kwargs + + +def _two_candidates(monkeypatch): + """call_ollama yields an identical then a fully divergent candidate.""" + texts = iter(["the cat sat on the mat", "alpha beta gamma delta"]) + monkeypatch.setattr(rewrite_text, "call_ollama", lambda *a, **k: next(texts)) + + +def test_duplicate_candidates_mark_only_one_selected(monkeypatch): + monkeypatch.delenv("WATERMARKS_GEMINI_API_KEY", raising=False) + + texts = iter( + [ + "alpha beta gamma delta", + "alpha beta gamma delta", + ] + ) + monkeypatch.setattr( + rewrite_text, + "call_ollama", + lambda *a, **k: next(texts), + ) + + out, info = rewrite( + "the cat sat on the mat", + **_rewrite_candidates_kwargs(), + ) + + assert out == "alpha beta gamma delta" + + selected = [entry["selected"] for entry in info["candidate_scores"]] + assert selected == [True, False] + + +def test_candidate_scores_restructured_with_detections(monkeypatch): + monkeypatch.setattr(rewrite_text, "MarkLLMTextDetector", _FakeMarkLLM) + calls: list = [] + + def fake_run_all(text, *, markllm=None, include_markllm=True): + calls.append((text, markllm, include_markllm)) + return [ + { + "detector": "markllm", + "available": True, + "is_watermarked": False, + "score": 1.0, + "threshold": 3.0, + } + ] + + monkeypatch.setattr(rewrite_text, "run_all_text_detectors", fake_run_all) + _two_candidates(monkeypatch) + out, info = rewrite( + "the cat sat on the mat", + **_rewrite_candidates_kwargs(markllm_scheme="kgw", markllm_dir="/x"), + ) + # selection is still purely lexical (most divergent candidate wins) + assert out == "alpha beta gamma delta" + cs = info["candidate_scores"] + assert isinstance(cs, list) and len(cs) == 2 + assert set(cs[0]) == {"lexical_divergence", "selection_score", "selected", "detections"} + assert cs[0]["selected"] is False + assert cs[1]["selected"] is True + assert cs[0]["lexical_divergence"] == 0.0 + assert cs[1]["lexical_divergence"] == 1.0 + assert cs[1]["selection_score"] >= cs[0]["selection_score"] + assert cs[0]["detections"][0]["detector"] == "markllm" + # every candidate was detected, with the CLI-parameterized markllm injected + assert len(calls) == 2 + assert calls[0][1]._kwargs == { + "scheme": "kgw", + "upstream_dir": "/x", + "model": "facebook/opt-1.3b", + "timeout": 180.0, + } + assert calls[0][2] is True + # before/after detection on the original and the final output + mk = info["markllm"] + assert mk["before"]["is_watermarked"] is True + assert mk["after"]["is_watermarked"] is False + assert mk["cleared"] is True + + +def test_candidate_detections_gemini_trigger_excludes_markllm(monkeypatch): + monkeypatch.setenv("WATERMARKS_GEMINI_API_KEY", "k") + calls: list = [] + + def fake_run_all(text, *, markllm=None, include_markllm=True): + calls.append((text, markllm, include_markllm)) + return [ + { + "detector": "gemini-synthid-text", + "available": True, + "is_watermarked": False, + "score": 0.2, + } + ] + + monkeypatch.setattr(rewrite_text, "run_all_text_detectors", fake_run_all) + _two_candidates(monkeypatch) + out, info = rewrite("the cat sat on the mat", **_rewrite_candidates_kwargs()) + assert out == "alpha beta gamma delta" + assert "markllm" not in info + cs = info["candidate_scores"] + assert cs[0]["detections"][0]["detector"] == "gemini-synthid-text" + # no --markllm-scheme -> the MarkLLM harness is excluded from the loop + assert calls[0][1] is None + assert calls[0][2] is False + + +def test_candidate_detections_off_without_trigger(monkeypatch): + monkeypatch.setattr( + rewrite_text, + "run_all_text_detectors", + lambda *a, **k: pytest.fail("detection must not run without a trigger"), + ) + _two_candidates(monkeypatch) + out, info = rewrite("the cat sat on the mat", **_rewrite_candidates_kwargs()) + assert out == "alpha beta gamma delta" + assert all(cs["detections"] == [] for cs in info["candidate_scores"]) + + +def test_candidate_detection_fail_soft(monkeypatch): + monkeypatch.setattr(rewrite_text, "MarkLLMTextDetector", _FakeMarkLLM) + + def boom(*a, **k): + raise RuntimeError("detector exploded") + + monkeypatch.setattr(rewrite_text, "run_all_text_detectors", boom) + _two_candidates(monkeypatch) + out, info = rewrite( + "the cat sat on the mat", + **_rewrite_candidates_kwargs(markllm_scheme="kgw", markllm_dir="/x"), + ) + assert out == "alpha beta gamma delta" + entry = info["candidate_scores"][0]["detections"][0] + assert entry["available"] is False + assert "exploded" in entry["error"] + + +def test_single_candidate_keeps_no_candidate_scores(monkeypatch): + monkeypatch.setattr(rewrite_text, "MarkLLMTextDetector", _FakeMarkLLM) + monkeypatch.setattr( + rewrite_text, + "run_all_text_detectors", + lambda *a, **k: pytest.fail("single candidate: no per-candidate detection"), + ) + monkeypatch.setattr(rewrite_text, "call_ollama", lambda *a, **k: "REWRITTEN OUTPUT") + out, info = rewrite( + "the cat sat on the mat", + **_rewrite_candidates_kwargs(candidates=1, markllm_scheme="kgw", markllm_dir="/x"), + ) + assert out == "REWRITTEN OUTPUT" + assert "candidate_scores" not in info + assert "candidates" not in info + assert info["markllm"]["cleared"] is True + + # --------------------------------------------------------------------------- # HTTP client hardening: default-deny allowlist, scheme guard, no redirects # --------------------------------------------------------------------------- diff --git a/tests/test_text_detectors.py b/tests/test_text_detectors.py new file mode 100644 index 0000000..eaa98fe --- /dev/null +++ b/tests/test_text_detectors.py @@ -0,0 +1,378 @@ +"""Tests for text_detectors.py (vendor/research text-watermark detectors).""" + +from __future__ import annotations + +import http.client +import io +import json +import os +import subprocess +import sys +import urllib.error +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +SCRIPTS = ROOT / "service" / "scripts" +sys.path.insert(0, str(SCRIPTS)) + +import text_detectors + + +class _FakeResp: + def __init__(self, data: bytes | dict): + self._data = json.dumps(data).encode("utf-8") if isinstance(data, dict) else data + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def read(self): + return self._data + + +def _http_error(code: int) -> urllib.error.HTTPError: + hdrs = http.client.HTTPMessage() + return urllib.error.HTTPError( + "http://generativelanguage.invalid", code, "err", hdrs, io.BytesIO(b'{"error": "denied"}') + ) + + +def _gemini_success(verdict: str | None = None, score: float | None = None) -> dict: + candidate: dict = {} + if verdict is not None: + candidate["content"] = {"parts": [{"text": verdict}]} + if score is not None: + candidate["attributionMetadata"] = {"syntheticText": {"score": score}} + return {"candidates": [candidate]} + + +@pytest.fixture(autouse=True) +def _clean_env(monkeypatch): + for key in ( + "WATERMARKS_GEMINI_API_KEY", + "WATERMARKS_GEMINI_MODEL", + "WATERMARKS_GEMINI_MAX_CHARS", + "WATERMARKS_MARKLLM_SCHEME", + "MARKLLM_DIR", + "WATERMARKS_MARKLLM_TIMEOUT", + ): + monkeypatch.delenv(key, raising=False) + + +# --- Gemini ---------------------------------------------------------------- + + +def test_gemini_unconfigured(): + report = text_detectors.GeminiSynthIDTextDetector().detect("hello") + assert report["available"] is False + assert "WATERMARKS_GEMINI_API_KEY" in report["error"] + assert text_detectors.GeminiSynthIDTextDetector().available() is False + + +def test_gemini_verdict_watermarked(monkeypatch): + monkeypatch.setenv("WATERMARKS_GEMINI_API_KEY", "k") + monkeypatch.setattr( + text_detectors.urllib.request, + "urlopen", + lambda *a, **k: _FakeResp(_gemini_success(verdict="Likely AI-generated")), + ) + report = text_detectors.GeminiSynthIDTextDetector().detect("some text") + assert report["available"] is True + assert report["is_watermarked"] is True + assert report["verdict"] == "Likely AI-generated" + + +def test_gemini_verdict_unlikely(monkeypatch): + monkeypatch.setenv("WATERMARKS_GEMINI_API_KEY", "k") + monkeypatch.setattr( + text_detectors.urllib.request, + "urlopen", + lambda *a, **k: _FakeResp(_gemini_success(verdict="Unlikely AI-generated")), + ) + report = text_detectors.GeminiSynthIDTextDetector().detect("some text") + assert report["is_watermarked"] is False + + +def test_gemini_numeric_score(monkeypatch): + monkeypatch.setenv("WATERMARKS_GEMINI_API_KEY", "k") + monkeypatch.setattr( + text_detectors.urllib.request, + "urlopen", + lambda *a, **k: _FakeResp(_gemini_success(score=0.87)), + ) + report = text_detectors.GeminiSynthIDTextDetector().detect("some text") + assert report["is_watermarked"] is True + assert report["score"] == 0.87 + + +def test_gemini_http_error(monkeypatch): + monkeypatch.setenv("WATERMARKS_GEMINI_API_KEY", "k") + monkeypatch.setattr( + text_detectors.urllib.request, + "urlopen", + lambda *a, **k: (_ for _ in ()).throw(_http_error(401)), + ) + report = text_detectors.GeminiSynthIDTextDetector().detect("some text") + assert report["available"] is False + assert "HTTP 401" in report["error"] + + +def test_gemini_retries_once_on_429(monkeypatch): + monkeypatch.setenv("WATERMARKS_GEMINI_API_KEY", "k") + calls = {"n": 0} + + def flaky(*a, **k): + calls["n"] += 1 + if calls["n"] == 1: + raise _http_error(429) + return _FakeResp(_gemini_success(verdict="Likely AI-generated")) + + monkeypatch.setattr(text_detectors.urllib.request, "urlopen", flaky) + report = text_detectors.GeminiSynthIDTextDetector().detect("some text") + assert calls["n"] == 2 + assert report["is_watermarked"] is True + + +def test_gemini_oversize_skips(monkeypatch): + monkeypatch.setenv("WATERMARKS_GEMINI_API_KEY", "k") + monkeypatch.setenv("WATERMARKS_GEMINI_MAX_CHARS", "10") + report = text_detectors.GeminiSynthIDTextDetector().detect("x" * 100) + assert report["available"] is True + assert report["skipped"] is True + assert report["is_watermarked"] is None + + +def test_gemini_malformed_max_chars_env_does_not_crash(monkeypatch): + monkeypatch.setenv("WATERMARKS_GEMINI_API_KEY", "k") + monkeypatch.setenv("WATERMARKS_GEMINI_MAX_CHARS", "not-a-number") + monkeypatch.setattr( + text_detectors.urllib.request, + "urlopen", + lambda *a, **k: _FakeResp(_gemini_success(verdict="Likely AI-generated")), + ) + report = text_detectors.GeminiSynthIDTextDetector().detect("short text") + assert report["available"] is True + assert report["is_watermarked"] is True + + +def test_gemini_no_candidates(monkeypatch): + monkeypatch.setenv("WATERMARKS_GEMINI_API_KEY", "k") + monkeypatch.setattr( + text_detectors.urllib.request, "urlopen", lambda *a, **k: _FakeResp({"candidates": []}) + ) + report = text_detectors.GeminiSynthIDTextDetector().detect("some text") + assert report["available"] is False + assert "no candidates" in report["error"] + + +# --- MarkLLM --------------------------------------------------------------- + + +def test_markllm_unconfigured(): + assert text_detectors.MarkLLMTextDetector().available() is False + report = text_detectors.MarkLLMTextDetector().detect("hello") + assert report["available"] is False + assert "MARKLLM_DIR" in report["error"] + + +def test_markllm_success(monkeypatch): + monkeypatch.setenv("MARKLLM_DIR", "/fake/MarkLLM") + payload = {"is_watermarked": True, "score": 4.2, "threshold": 4.0} + monkeypatch.setattr( + text_detectors.subprocess, + "run", + lambda *a, **k: subprocess.CompletedProcess(a[0], 0, stdout=json.dumps(payload)), + ) + report = text_detectors.MarkLLMTextDetector().detect("hello") + assert report["available"] is True + assert report["is_watermarked"] is True + assert "research harness" in report["note"] + + +def test_markllm_unavailable_exit3(monkeypatch): + monkeypatch.setenv("MARKLLM_DIR", "/fake/MarkLLM") + monkeypatch.setattr( + text_detectors.subprocess, + "run", + lambda *a, **k: subprocess.CompletedProcess(a[0], 3, stdout="", stderr="missing deps"), + ) + report = text_detectors.MarkLLMTextDetector().detect("hello") + assert report["available"] is False + assert "missing deps" in report["error"] + + +def test_markllm_scheme_env(monkeypatch): + monkeypatch.setenv("MARKLLM_DIR", "/fake/MarkLLM") + monkeypatch.setenv("WATERMARKS_MARKLLM_SCHEME", "synthid") + seen = {} + + def fake_run(*args, **kwargs): + seen["argv"] = args[0] + return subprocess.CompletedProcess(args[0], 0, stdout="{}") + + monkeypatch.setattr(text_detectors.subprocess, "run", fake_run) + text_detectors.MarkLLMTextDetector().detect("hello") + assert "--scheme" in seen["argv"] + assert seen["argv"][seen["argv"].index("--scheme") + 1] == "synthid" + + +def test_markllm_prefers_checkout_venv(monkeypatch, tmp_path): + upstream = tmp_path / "MarkLLM" + if os.name == "nt": + venv_python = upstream / ".venv" / "Scripts" / "python.exe" + else: + venv_python = upstream / ".venv" / "bin" / "python" + venv_python.parent.mkdir(parents=True) + venv_python.write_text("") + (upstream / "watermark").mkdir() + seen: dict = {} + + def fake_run(cmd, **kwargs): + seen["argv"] = cmd + return subprocess.CompletedProcess(cmd, 0, stdout="{}") + + monkeypatch.setattr(text_detectors.subprocess, "run", fake_run) + det = text_detectors.MarkLLMTextDetector(upstream_dir=str(upstream)) + assert det.available() is True + det.detect("hello") + assert seen["argv"][0] == str(venv_python) + + +def test_markllm_falls_back_to_sys_executable(monkeypatch, tmp_path): + upstream = tmp_path / "MarkLLM" + upstream.mkdir() + (upstream / "watermark").mkdir() + seen: dict = {} + + def fake_run(cmd, **kwargs): + seen["argv"] = cmd + return subprocess.CompletedProcess(cmd, 0, stdout="{}") + + monkeypatch.setattr(text_detectors.subprocess, "run", fake_run) + det = text_detectors.MarkLLMTextDetector(upstream_dir=str(upstream)) + det.detect("hello") + assert seen["argv"][0] == sys.executable + + +def test_markllm_ctor_overrides_passed_to_adapter(monkeypatch, tmp_path): + upstream = tmp_path / "MarkLLM" + upstream.mkdir() + (upstream / "watermark").mkdir() + seen: dict = {} + + def fake_run(cmd, **kwargs): + seen["argv"] = cmd + return subprocess.CompletedProcess(cmd, 0, stdout="{}") + + monkeypatch.setattr(text_detectors.subprocess, "run", fake_run) + det = text_detectors.MarkLLMTextDetector( + scheme="synthid", + upstream_dir=str(upstream), + model="opt-1.3b", + timeout=5, + ) + det.detect("hello") + argv = seen["argv"] + assert argv[argv.index("--scheme") + 1] == "synthid" + assert argv[argv.index("--model") + 1] == "opt-1.3b" + assert argv[argv.index("--upstream-dir") + 1] == str(upstream.resolve()) + + +def test_markllm_ctor_available_with_override(monkeypatch, tmp_path): + upstream = tmp_path / "MarkLLM" + upstream.mkdir() + det = text_detectors.MarkLLMTextDetector(upstream_dir=str(upstream)) + assert det.available() is True + + +def test_markllm_preexec_default_off(monkeypatch): + monkeypatch.delenv("WATERMARKS_MARKLLM_RLIMIT_AS", raising=False) + assert text_detectors._markllm_preexec() is None + + +def test_markllm_preexec_env(monkeypatch): + if os.name != "posix": + pytest.skip("preexec_fn is POSIX-only") + monkeypatch.setenv("WATERMARKS_MARKLLM_RLIMIT_AS", "0x40000000") + assert callable(text_detectors._markllm_preexec()) + + +def test_markllm_detect_applies_rlimit(monkeypatch, tmp_path): + if os.name != "posix": + pytest.skip("preexec_fn is POSIX-only") + upstream = tmp_path / "MarkLLM" + upstream.mkdir() + (upstream / "watermark").mkdir() + monkeypatch.setenv("WATERMARKS_MARKLLM_RLIMIT_AS", "1073741824") + captured: dict = {} + + def fake_run(cmd, **kwargs): + captured["preexec_fn"] = kwargs.get("preexec_fn") + return subprocess.CompletedProcess(cmd, 0, stdout='{"available": true}') + + monkeypatch.setattr(text_detectors.subprocess, "run", fake_run) + det = text_detectors.MarkLLMTextDetector(upstream_dir=str(upstream)) + report = det.detect("hello") + assert report["available"] is True + assert callable(captured["preexec_fn"]) + + +def test_run_all_text_detectors_can_exclude_markllm(monkeypatch): + monkeypatch.setattr( + text_detectors, + "MarkLLMTextDetector", + lambda: pytest.fail("must not construct MarkLLM when excluded"), + ) + reports = text_detectors.run_all_text_detectors("hello", include_markllm=False) + assert len(reports) == 2 # gemini (unconfigured) + claude placeholder + assert {r["detector"] for r in reports} == {"gemini-synthid-text", "claude-text"} + + +def test_run_all_text_detectors_injects_markllm_instance(monkeypatch, tmp_path): + upstream = tmp_path / "MarkLLM" + upstream.mkdir() + det = text_detectors.MarkLLMTextDetector(upstream_dir=str(upstream), scheme="synthid") + seen: list = [] + + def fake_run(cmd, **kwargs): + seen.append(cmd) + return subprocess.CompletedProcess(cmd, 0, stdout="{}") + + monkeypatch.setattr(text_detectors.subprocess, "run", fake_run) + text_detectors.run_all_text_detectors("hello", markllm=det) + markllm_cmd = next(c for c in seen if "--scheme" in c) + assert markllm_cmd[markllm_cmd.index("--scheme") + 1] == "synthid" + + +# --- Claude placeholder ---------------------------------------------------- + + +def test_claude_placeholder(): + det = text_detectors.ClaudeTextDetector() + assert det.available() is False + report = det.detect("hello") + assert report["available"] is False + assert "WATERMARKS_CLAUDE_API_KEY" in report["error"] + + +# --- Registry -------------------------------------------------------------- + + +def test_detector_status_keys(): + status = text_detectors.detector_status() + assert set(status) == {"gemini-synthid-text", "markllm", "claude-text"} + + +def test_run_all_text_detectors_length(): + reports = text_detectors.run_all_text_detectors("hello") + assert len(reports) == 3 + assert all("detector" in r for r in reports) + + +def test_run_text_detectors_filters_unavailable(): + reports = text_detectors.run_text_detectors("hello") + assert reports == []