mirror of
https://github.com/guillaumemeyer/watermarks-remover.git
synced 2026-08-22 13:11:57 +02:00
* feat: add vendor text-watermark detection and SynthID image scorer sidecar Adds Layer B watermark detection as a first-class service capability: - text_detectors.py: a registry of text-watermark detectors behind one interface — Google's official SynthID-text detector via the Gemini API (taskType DETECT_TEXT_WATERMARK), a Claude placeholder ready for Anthropic's announced detection API, and the MarkLLM research harness (KGW / SynthID, same-config-only). Fail-soft: unconfigured or errored detectors never block cleaning. - server.py: new POST /detect endpoint, detect_before / detect_after options on /clean (before/after scoring for text and images), an opt-in /inspect "detect" flag, and /capabilities gains text_detectors and scorers.synthid_http. - synthid_score_server.py: a stdlib HTTP sidecar for the reverse-SynthID scorer, so the published core image never bundles the non-commercial upstream code; wired via WATERMARKS_SYNTHID_SCORER_URL. - score_synthid.py: extract score_file() so the CLI and the sidecar share one implementation. - compose.yaml / Dockerfile.synthid / .env.example: wr-synthid-score sidecar service and env wiring. - README + skill docs, plus tests for the detectors, the /detect endpoint, and the image sidecar. * feat: per-candidate watermark detection for Layer B rewrite candidates When --candidates N (N > 1) is combined with --markllm-scheme or WATERMARKS_GEMINI_API_KEY, run every configured text detector from the text_detectors.py registry on each candidate and report per-candidate measurements in --json-stats as candidate_scores entries carrying lexical_divergence, selection_score, selected, and per-detector reports (is_watermarked, score, threshold where the detector provides one). Candidate selection stays purely lexical; the detections are observability for correlating lexical divergence with watermark removal (issue #106). Converges rewrite_text.py onto the shared detector registry: - MarkLLMTextDetector gains constructor overrides (scheme, upstream_dir, model, timeout) plus the checkout-venv interpreter preference and the WATERMARKS_MARKLLM_RLIMIT_AS preexec guard ported from rewrite_text.py; the old _markllm_detect / _venv_python / _markllm_preexec helpers are gone. - run_all_text_detectors() accepts an injected MarkLLM instance and an include_markllm switch so CLI flag gating stays intact. - before/after/cleared semantics unchanged; detection remains fail-soft. * docs: pin Watermarks in the Sand reference to arXiv v5 * fix: mark only one rewrite candidate as selected (#110) --------- Co-authored-by: Zhenxin Ai <142008897+ai-kunkun@users.noreply.github.com>
147 lines
4.8 KiB
YAML
147 lines
4.8 KiB
YAML
# Whole-infra bring-up for watermarks-remover.
|
|
#
|
|
# docker compose up --build -d # core HTTP service only
|
|
# docker compose --profile harness up --build -d # + markllm / markdiffusion harnesses
|
|
# docker compose --profile heavy up --build -d # + ctrlregen / synthid (local builds)
|
|
#
|
|
# The skill and any web app talk to the wr-core service at http://127.0.0.1:8765
|
|
# (loopback-only host mapping). The heavy/harness images are one-shot CLIs:
|
|
# `up` starts them with `--help` to confirm the image, and real jobs run via
|
|
# `docker compose run`, e.g.:
|
|
# docker compose run --rm wr-ctrlregen /data/shot.png -o /data/out.png
|
|
#
|
|
# ctrlregen and synthid bake in upstream code that is not publicly
|
|
# redistributable (all-rights-reserved / non-commercial Research License), so
|
|
# they build from source locally and are never pushed to GHCR.
|
|
|
|
name: watermarks-remover
|
|
|
|
services:
|
|
wr-core:
|
|
build:
|
|
context: service
|
|
dockerfile: Dockerfile
|
|
image: ghcr.io/guillaumemeyer/watermarks-remover:latest
|
|
ports:
|
|
- "127.0.0.1:8765:8765"
|
|
environment:
|
|
# Empty by default (no auth). Set to require `Authorization: Bearer <key>`.
|
|
WATERMARKS_SERVER_API_KEY: ${WATERMARKS_SERVER_API_KEY:-}
|
|
# Optional Gemini API key enabling vendor SynthID-text watermark
|
|
# detection via /detect and detect_before/after (see .env.example).
|
|
WATERMARKS_GEMINI_API_KEY: ${WATERMARKS_GEMINI_API_KEY:-}
|
|
WATERMARKS_GEMINI_MODEL: ${WATERMARKS_GEMINI_MODEL:-}
|
|
# Optional SynthID image scorer sidecar (heavy profile).
|
|
WATERMARKS_SYNTHID_SCORER_URL: ${WATERMARKS_SYNTHID_SCORER_URL:-}
|
|
WATERMARKS_SYNTHID_SCORER_API_KEY: ${WATERMARKS_SYNTHID_SCORER_API_KEY:-}
|
|
read_only: true
|
|
tmpfs:
|
|
- /tmp
|
|
init: true
|
|
user: "10001:10001"
|
|
restart: unless-stopped
|
|
|
|
wr-markllm:
|
|
profiles: [harness]
|
|
build:
|
|
context: service
|
|
dockerfile: Dockerfile.markllm
|
|
image: ghcr.io/guillaumemeyer/watermarks-remover:markllm-latest
|
|
# One-shot CLI: `up` just confirms the image; run real jobs with
|
|
# `docker compose run --rm wr-markllm detect /data/wm.txt --scheme kgw`.
|
|
command: ["--help"]
|
|
read_only: true
|
|
tmpfs:
|
|
- /tmp
|
|
init: true
|
|
user: "10001:10001"
|
|
environment:
|
|
HF_TOKEN: ${HF_TOKEN:-}
|
|
HF_HOME: /home/markllm/.cache/huggingface
|
|
volumes:
|
|
- markllm-cache:/home/markllm/.cache/huggingface
|
|
|
|
wr-markdiffusion:
|
|
profiles: [harness]
|
|
build:
|
|
context: service
|
|
dockerfile: Dockerfile.markdiffusion
|
|
image: ghcr.io/guillaumemeyer/watermarks-remover:markdiffusion-latest
|
|
command: ["--help"]
|
|
read_only: true
|
|
tmpfs:
|
|
- /tmp
|
|
init: true
|
|
user: "10001:10001"
|
|
environment:
|
|
HF_TOKEN: ${HF_TOKEN:-}
|
|
HF_HOME: /home/markdiffusion/.cache/huggingface
|
|
volumes:
|
|
- markdiffusion-cache:/home/markdiffusion/.cache/huggingface
|
|
|
|
wr-ctrlregen:
|
|
profiles: [heavy]
|
|
build:
|
|
context: service
|
|
dockerfile: Dockerfile.ctrlregen
|
|
# Local-only tag: upstream `noai-watermark` ships no LICENSE (all-rights-
|
|
# reserved), so this image is never published.
|
|
image: watermarks-remover-ctrlregen:local
|
|
command: ["--help"]
|
|
read_only: true
|
|
tmpfs:
|
|
- /tmp
|
|
init: true
|
|
user: "10001:10001"
|
|
environment:
|
|
HF_TOKEN: ${HF_TOKEN:-}
|
|
NOAI_WATERMARK_DIR: /opt/noai-watermark
|
|
volumes:
|
|
- ctrlregen-cache:/home/remover/.cache/huggingface
|
|
|
|
wr-synthid:
|
|
profiles: [heavy]
|
|
build:
|
|
context: service
|
|
dockerfile: Dockerfile.synthid
|
|
# Local-only tag: upstream `reverse-SynthID` is under a non-commercial
|
|
# Research License, so this image is never published.
|
|
image: watermarks-remover-synthid-scorer:local
|
|
command: ["--help"]
|
|
read_only: true
|
|
tmpfs:
|
|
- /tmp
|
|
init: true
|
|
user: "10001:10001"
|
|
volumes:
|
|
- synthid-cache:/home/scorer/.cache/huggingface
|
|
|
|
# HTTP SynthID scorer sidecar: lets wr-core score images before/after
|
|
# cleaning without bundling the non-commercial reverse-SynthID code in the
|
|
# published core image. Reach it via WATERMARKS_SYNTHID_SCORER_URL on
|
|
# wr-core (see .env.example) and require the same bearer key on both sides.
|
|
wr-synthid-score:
|
|
profiles: [heavy]
|
|
build:
|
|
context: service
|
|
dockerfile: Dockerfile.synthid
|
|
image: watermarks-remover-synthid-scorer:local
|
|
entrypoint: ["python3"]
|
|
command: ["/app/synthid_score_server.py", "--host", "0.0.0.0", "--port", "8766"]
|
|
read_only: true
|
|
tmpfs:
|
|
- /tmp
|
|
init: true
|
|
user: "10001:10001"
|
|
environment:
|
|
WATERMARKS_SYNTHID_SCORER_API_KEY: ${WATERMARKS_SYNTHID_SCORER_API_KEY:-}
|
|
REVERSE_SYNTHID_DIR: /opt/reverse-synthid
|
|
volumes:
|
|
- synthid-cache:/home/scorer/.cache/huggingface
|
|
|
|
volumes:
|
|
markllm-cache:
|
|
markdiffusion-cache:
|
|
ctrlregen-cache:
|
|
synthid-cache:
|