mirror of
https://github.com/guillaumemeyer/watermarks-remover.git
synced 2026-08-22 13:11:57 +02:00
* feat: add vendor text-watermark detection and SynthID image scorer sidecar Adds Layer B watermark detection as a first-class service capability: - text_detectors.py: a registry of text-watermark detectors behind one interface — Google's official SynthID-text detector via the Gemini API (taskType DETECT_TEXT_WATERMARK), a Claude placeholder ready for Anthropic's announced detection API, and the MarkLLM research harness (KGW / SynthID, same-config-only). Fail-soft: unconfigured or errored detectors never block cleaning. - server.py: new POST /detect endpoint, detect_before / detect_after options on /clean (before/after scoring for text and images), an opt-in /inspect "detect" flag, and /capabilities gains text_detectors and scorers.synthid_http. - synthid_score_server.py: a stdlib HTTP sidecar for the reverse-SynthID scorer, so the published core image never bundles the non-commercial upstream code; wired via WATERMARKS_SYNTHID_SCORER_URL. - score_synthid.py: extract score_file() so the CLI and the sidecar share one implementation. - compose.yaml / Dockerfile.synthid / .env.example: wr-synthid-score sidecar service and env wiring. - README + skill docs, plus tests for the detectors, the /detect endpoint, and the image sidecar. * feat: per-candidate watermark detection for Layer B rewrite candidates When --candidates N (N > 1) is combined with --markllm-scheme or WATERMARKS_GEMINI_API_KEY, run every configured text detector from the text_detectors.py registry on each candidate and report per-candidate measurements in --json-stats as candidate_scores entries carrying lexical_divergence, selection_score, selected, and per-detector reports (is_watermarked, score, threshold where the detector provides one). Candidate selection stays purely lexical; the detections are observability for correlating lexical divergence with watermark removal (issue #106). Converges rewrite_text.py onto the shared detector registry: - MarkLLMTextDetector gains constructor overrides (scheme, upstream_dir, model, timeout) plus the checkout-venv interpreter preference and the WATERMARKS_MARKLLM_RLIMIT_AS preexec guard ported from rewrite_text.py; the old _markllm_detect / _venv_python / _markllm_preexec helpers are gone. - run_all_text_detectors() accepts an injected MarkLLM instance and an include_markllm switch so CLI flag gating stays intact. - before/after/cleared semantics unchanged; detection remains fail-soft. * docs: pin Watermarks in the Sand reference to arXiv v5 * fix: mark only one rewrite candidate as selected (#110) --------- Co-authored-by: Zhenxin Ai <142008897+ai-kunkun@users.noreply.github.com>
177 lines
6.6 KiB
Python
177 lines
6.6 KiB
Python
#!/usr/bin/env python3
|
|
"""Tiny stdlib HTTP sidecar exposing the reverse-SynthID pixel scorer.
|
|
|
|
Runs inside the local-only wr-synthid heavy image so the published core
|
|
image never bundles the non-commercial reverse-SynthID code. The core
|
|
service calls this sidecar for SynthID image scoring when
|
|
WATERMARKS_SYNTHID_SCORER_URL is set (see compose.yaml / .env.example).
|
|
|
|
Endpoints:
|
|
GET /health -> {"ok": true, "version": ...}
|
|
POST /score -> {"file": <base64>} -> score_synthid payload
|
|
|
|
Hardening mirrors server.py: optional bearer key, input size caps,
|
|
unprivileged user, read-only rootfs with a /tmp tmpfs. Intended for the
|
|
compose network or a trusted network only.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import base64
|
|
import binascii
|
|
import json
|
|
import os
|
|
import sys
|
|
import tempfile
|
|
from http import HTTPStatus
|
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
from pathlib import Path
|
|
from typing import Any
|
|
from urllib.parse import urlparse
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
|
|
from score_synthid import score_file
|
|
|
|
VERSION = os.environ.get("WATERMARKS_SYNTHID_SERVER_VERSION", "dev")
|
|
|
|
# Mirror common.MAX_INPUT_BYTES (env-overridable) with the base64 envelope
|
|
# headroom. Read at import; the sidecar image does not copy common.py, so the
|
|
# default is repeated here.
|
|
MAX_INPUT_BYTES = int(os.environ.get("WATERMARKS_MAX_INPUT_BYTES", str(256 << 20)))
|
|
MAX_BODY_BYTES = MAX_INPUT_BYTES + (MAX_INPUT_BYTES >> 1)
|
|
|
|
API_KEY = os.environ.get("WATERMARKS_SYNTHID_SCORER_API_KEY", "").strip()
|
|
MODEL = os.environ.get("WATERMARKS_SYNTHID_MODEL", "").strip() or None
|
|
|
|
|
|
def _json_ok(payload: dict[str, Any]) -> bytes:
|
|
return json.dumps(payload, ensure_ascii=False, indent=2).encode("utf-8")
|
|
|
|
|
|
class Handler(BaseHTTPRequestHandler):
|
|
server_version = f"watermarks-remover-synthid/{VERSION}"
|
|
|
|
def log_message(self, fmt: str, *args: object) -> None:
|
|
print(f"{self.address_string()} - {fmt % args}", file=sys.stderr)
|
|
|
|
def _authorized(self) -> bool:
|
|
if not API_KEY:
|
|
return True
|
|
return self.headers.get("Authorization", "") == f"Bearer {API_KEY}"
|
|
|
|
def _read_json(self) -> dict[str, Any] | None:
|
|
raw = self.headers.get("Content-Length")
|
|
if raw is None or not raw.isdigit():
|
|
return None
|
|
length = int(raw)
|
|
if length > MAX_BODY_BYTES:
|
|
return None
|
|
try:
|
|
body = json.loads(self.rfile.read(length).decode("utf-8"))
|
|
except (UnicodeDecodeError, json.JSONDecodeError, OSError):
|
|
return None
|
|
return body if isinstance(body, dict) else None
|
|
|
|
def _respond(self, status: int, payload: dict[str, Any]) -> None:
|
|
data = _json_ok(payload)
|
|
self.send_response(status)
|
|
self.send_header("Content-Type", "application/json; charset=utf-8")
|
|
self.send_header("Content-Length", str(len(data)))
|
|
self.send_header("Cache-Control", "no-store")
|
|
self.end_headers()
|
|
self.wfile.write(data)
|
|
|
|
def do_GET(self) -> None:
|
|
if not self._authorized():
|
|
self._respond(HTTPStatus.UNAUTHORIZED, {"ok": False, "error": "unauthorized"})
|
|
return
|
|
if urlparse(self.path).path == "/health":
|
|
self._respond(HTTPStatus.OK, {"ok": True, "version": VERSION})
|
|
else:
|
|
self._respond(HTTPStatus.NOT_FOUND, {"ok": False, "error": "not found"})
|
|
|
|
def do_POST(self) -> None:
|
|
if not self._authorized():
|
|
self._respond(HTTPStatus.UNAUTHORIZED, {"ok": False, "error": "unauthorized"})
|
|
return
|
|
if urlparse(self.path).path != "/score":
|
|
self._respond(HTTPStatus.NOT_FOUND, {"ok": False, "error": "not found"})
|
|
return
|
|
body = self._read_json()
|
|
if body is None:
|
|
raw_len = self.headers.get("Content-Length")
|
|
oversized = raw_len is not None and raw_len.isdigit() and int(raw_len) > MAX_BODY_BYTES
|
|
self._respond(
|
|
HTTPStatus.REQUEST_ENTITY_TOO_LARGE if oversized else HTTPStatus.BAD_REQUEST,
|
|
{"ok": False, "error": "invalid request body"},
|
|
)
|
|
return
|
|
|
|
raw = body.get("file")
|
|
if not isinstance(raw, str):
|
|
self._respond(HTTPStatus.BAD_REQUEST, {"ok": False, "error": "missing 'file' field"})
|
|
return
|
|
try:
|
|
data = base64.b64decode(raw, validate=True)
|
|
except (binascii.Error, ValueError):
|
|
self._respond(
|
|
HTTPStatus.BAD_REQUEST, {"ok": False, "error": "'file' is not valid base64"}
|
|
)
|
|
return
|
|
if len(data) > MAX_INPUT_BYTES:
|
|
self._respond(
|
|
HTTPStatus.REQUEST_ENTITY_TOO_LARGE, {"ok": False, "error": "file too large"}
|
|
)
|
|
return
|
|
|
|
with tempfile.TemporaryDirectory(prefix="wm-synthid-") as tmp:
|
|
path = Path(tmp) / "input.png"
|
|
try:
|
|
path.write_bytes(data)
|
|
except OSError as e:
|
|
self._respond(HTTPStatus.INTERNAL_SERVER_ERROR, {"ok": False, "error": str(e)})
|
|
return
|
|
code, payload = score_file(path, model=MODEL)
|
|
|
|
if code == 0 and payload is not None:
|
|
self._respond(HTTPStatus.OK, payload)
|
|
elif code == 2:
|
|
self._respond(HTTPStatus.BAD_REQUEST, {"ok": False, "error": "could not load image"})
|
|
else:
|
|
# exit 1 (runtime error) or 3 (unavailable) -> fail-soft payload,
|
|
# matching the shape image_meta.run_synthid_score expects.
|
|
self._respond(
|
|
HTTPStatus.OK,
|
|
{"available": False, "error": "scorer unavailable (see sidecar stderr)"},
|
|
)
|
|
|
|
|
|
def main() -> int:
|
|
global API_KEY # noqa: PLW0603 — CLI overrides env
|
|
p = argparse.ArgumentParser(description=__doc__)
|
|
p.add_argument("--host", default=os.environ.get("WATERMARKS_SYNTHID_SERVER_HOST", "127.0.0.1"))
|
|
p.add_argument(
|
|
"--port", type=int, default=int(os.environ.get("WATERMARKS_SYNTHID_SERVER_PORT", "8766"))
|
|
)
|
|
p.add_argument("--api-key", default=API_KEY, help="require this bearer token (default: none)")
|
|
args = p.parse_args()
|
|
|
|
if args.host not in ("127.0.0.1", "localhost", "::1"):
|
|
print(
|
|
f"warning: binding {args.host} — intended for a trusted network only", file=sys.stderr
|
|
)
|
|
API_KEY = args.api_key
|
|
print(f"synthid scorer sidecar {VERSION} on http://{args.host}:{args.port}", file=sys.stderr)
|
|
server = ThreadingHTTPServer((args.host, args.port), Handler)
|
|
try:
|
|
server.serve_forever()
|
|
except KeyboardInterrupt:
|
|
server.shutdown()
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|