From dfad1d55f169b7e921d3e3770e8a8e7423926f97 Mon Sep 17 00:00:00 2001 From: guillaumemeyer Date: Wed, 19 Aug 2026 15:17:24 -0700 Subject: [PATCH] =?UTF-8?q?feat:=20publishable=20benchmark=20harness=20?= =?UTF-8?q?=E2=80=94=20env-ready=20and=20runnable?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tracks the research/ arXiv-v1 paper kit (per PR #174's gitignore plan) and closes the environment gaps needed to actually run the 3,500-cell study: - detect_text_watermark.py / multilingual_gen.py: new --torch-dtype (auto|fp32|bf16). bf16 is ~8x faster for opt-1.3b CPU cached decode (0.5 vs 4 s/token on aarch64) and keeps same-config gen/detect consistent. - run_experiments.py: --quality-python (defaults to repo .venv-quality, else MarkLLM venv with a warning); --seeds/--prompts now limit the generate loop instead of only the plan; workers force --torch-dtype bf16; the 'none' control attack is a passthrough (was 'unknown attack'). - evaluate_quality.py: BERTScore on roberta-large (deberta-xlarge-mnli crashes under transformers>=5 in set_truncation_and_padding; WATERMARKS_BERTSCORE_MODEL override). - Makefile: research-check target (documented but missing). - .env.example: MARKLLM_DIR naming (code reads MARKLLM_DIR, not WATERMARKS_MARKLLM_DIR). - research/pins-quality.txt: pinned quality-venv freeze (protocol §7). - tests: quality-python plumbing, CLI subset, bf16 worker flag, none attack, multilingual fake_load signature. Validation: make research-check (78 tests) and make lint green; a restricted end-to-end run (generate -> attack -> detect -> evaluate -> report) passes with real metrics. --- .env.example | 2 +- .gitignore | 3 + Makefile | 5 +- research/01-experiment-protocol.md | 258 ++++ research/02-paper-outline.md | 188 +++ research/03-related-work.md | 80 ++ research/04-ethics-and-legal.md | 124 ++ research/05-arxiv-readiness.md | 85 ++ research/README.md | 84 ++ research/configs/EXP.json | 7 + research/configs/KGW-d1.json | 10 + research/configs/KGW-d2.json | 10 + research/configs/KGW-d4.json | 10 + research/configs/SIR.json | 11 + research/configs/SynthID.json | 16 + research/configs/Unigram.json | 7 + research/corpus/de/01.txt | 1 + research/corpus/de/02.txt | 1 + research/corpus/de/03.txt | 1 + research/corpus/de/04.txt | 1 + research/corpus/de/05.txt | 1 + research/corpus/de/06.txt | 1 + research/corpus/de/07.txt | 1 + research/corpus/de/08.txt | 1 + research/corpus/de/09.txt | 1 + research/corpus/de/10.txt | 1 + research/corpus/de/11.txt | 1 + research/corpus/de/12.txt | 1 + research/corpus/de/13.txt | 1 + research/corpus/de/14.txt | 1 + research/corpus/de/15.txt | 1 + research/corpus/de/16.txt | 1 + research/corpus/de/17.txt | 1 + research/corpus/de/18.txt | 1 + research/corpus/de/19.txt | 1 + research/corpus/de/20.txt | 1 + research/corpus/de/21.txt | 1 + research/corpus/de/22.txt | 1 + research/corpus/de/23.txt | 1 + research/corpus/de/24.txt | 1 + research/corpus/de/25.txt | 1 + research/corpus/en/01.txt | 1 + research/corpus/en/02.txt | 1 + research/corpus/en/03.txt | 1 + research/corpus/en/04.txt | 1 + research/corpus/en/05.txt | 1 + research/corpus/en/06.txt | 1 + research/corpus/en/07.txt | 1 + research/corpus/en/08.txt | 1 + research/corpus/en/09.txt | 1 + research/corpus/en/10.txt | 1 + research/corpus/en/11.txt | 1 + research/corpus/en/12.txt | 1 + research/corpus/en/13.txt | 1 + research/corpus/en/14.txt | 1 + research/corpus/en/15.txt | 1 + research/corpus/en/16.txt | 1 + research/corpus/en/17.txt | 1 + research/corpus/en/18.txt | 1 + research/corpus/en/19.txt | 1 + research/corpus/en/20.txt | 1 + research/corpus/en/21.txt | 1 + research/corpus/en/22.txt | 1 + research/corpus/en/23.txt | 1 + research/corpus/en/24.txt | 1 + research/corpus/en/25.txt | 1 + research/corpus/es/01.txt | 1 + research/corpus/es/02.txt | 1 + research/corpus/es/03.txt | 1 + research/corpus/es/04.txt | 1 + research/corpus/es/05.txt | 1 + research/corpus/es/06.txt | 1 + research/corpus/es/07.txt | 1 + research/corpus/es/08.txt | 1 + research/corpus/es/09.txt | 1 + research/corpus/es/10.txt | 1 + research/corpus/es/11.txt | 1 + research/corpus/es/12.txt | 1 + research/corpus/es/13.txt | 1 + research/corpus/es/14.txt | 1 + research/corpus/es/15.txt | 1 + research/corpus/es/16.txt | 1 + research/corpus/es/17.txt | 1 + research/corpus/es/18.txt | 1 + research/corpus/es/19.txt | 1 + research/corpus/es/20.txt | 1 + research/corpus/es/21.txt | 1 + research/corpus/es/22.txt | 1 + research/corpus/es/23.txt | 1 + research/corpus/es/24.txt | 1 + research/corpus/es/25.txt | 1 + research/corpus/fr/01.txt | 1 + research/corpus/fr/02.txt | 1 + research/corpus/fr/03.txt | 1 + research/corpus/fr/04.txt | 1 + research/corpus/fr/05.txt | 1 + research/corpus/fr/06.txt | 1 + research/corpus/fr/07.txt | 1 + research/corpus/fr/08.txt | 1 + research/corpus/fr/09.txt | 1 + research/corpus/fr/10.txt | 1 + research/corpus/fr/11.txt | 1 + research/corpus/fr/12.txt | 1 + research/corpus/fr/13.txt | 1 + research/corpus/fr/14.txt | 1 + research/corpus/fr/15.txt | 1 + research/corpus/fr/16.txt | 1 + research/corpus/fr/17.txt | 1 + research/corpus/fr/18.txt | 1 + research/corpus/fr/19.txt | 1 + research/corpus/fr/20.txt | 1 + research/corpus/fr/21.txt | 1 + research/corpus/fr/22.txt | 1 + research/corpus/fr/23.txt | 1 + research/corpus/fr/24.txt | 1 + research/corpus/fr/25.txt | 1 + research/paper/README.md | 56 + research/paper/abstract.tex | 25 + research/paper/acknowledgments.tex | 12 + research/paper/ethics.tex | 30 + research/paper/main.tex | 472 +++++++ research/paper/refs.bib | 388 ++++++ research/pins-quality.txt | 77 ++ research/requirements-quality.txt | 34 + research/scripts/analyze_roc.py | 371 ++++++ research/scripts/attacks/cheap.py | 338 +++++ research/scripts/evaluate_quality.py | 505 ++++++++ research/scripts/make_figures.py | 865 +++++++++++++ research/scripts/make_tables.py | 1091 +++++++++++++++++ research/scripts/multilingual_gen.py | 562 +++++++++ research/scripts/pins.py | 117 ++ research/scripts/run_experiments.py | 1430 ++++++++++++++++++++++ research/tests/test_analyze_roc.py | 232 ++++ research/tests/test_cheap.py | 184 +++ research/tests/test_corpus.py | 70 ++ research/tests/test_corpus_gap05a6.py | 123 ++ research/tests/test_evaluate_quality.py | 366 ++++++ research/tests/test_make_tables.py | 178 +++ research/tests/test_multilingual_gen.py | 567 +++++++++ research/tests/test_paper_skeleton.py | 293 +++++ research/tests/test_run_experiments.py | 272 ++++ service/scripts/detect_text_watermark.py | 23 +- 142 files changed, 9678 insertions(+), 3 deletions(-) create mode 100644 research/01-experiment-protocol.md create mode 100644 research/02-paper-outline.md create mode 100644 research/03-related-work.md create mode 100644 research/04-ethics-and-legal.md create mode 100644 research/05-arxiv-readiness.md create mode 100644 research/README.md create mode 100644 research/configs/EXP.json create mode 100644 research/configs/KGW-d1.json create mode 100644 research/configs/KGW-d2.json create mode 100644 research/configs/KGW-d4.json create mode 100644 research/configs/SIR.json create mode 100644 research/configs/SynthID.json create mode 100644 research/configs/Unigram.json create mode 100644 research/corpus/de/01.txt create mode 100644 research/corpus/de/02.txt create mode 100644 research/corpus/de/03.txt create mode 100644 research/corpus/de/04.txt create mode 100644 research/corpus/de/05.txt create mode 100644 research/corpus/de/06.txt create mode 100644 research/corpus/de/07.txt create mode 100644 research/corpus/de/08.txt create mode 100644 research/corpus/de/09.txt create mode 100644 research/corpus/de/10.txt create mode 100644 research/corpus/de/11.txt create mode 100644 research/corpus/de/12.txt create mode 100644 research/corpus/de/13.txt create mode 100644 research/corpus/de/14.txt create mode 100644 research/corpus/de/15.txt create mode 100644 research/corpus/de/16.txt create mode 100644 research/corpus/de/17.txt create mode 100644 research/corpus/de/18.txt create mode 100644 research/corpus/de/19.txt create mode 100644 research/corpus/de/20.txt create mode 100644 research/corpus/de/21.txt create mode 100644 research/corpus/de/22.txt create mode 100644 research/corpus/de/23.txt create mode 100644 research/corpus/de/24.txt create mode 100644 research/corpus/de/25.txt create mode 100644 research/corpus/en/01.txt create mode 100644 research/corpus/en/02.txt create mode 100644 research/corpus/en/03.txt create mode 100644 research/corpus/en/04.txt create mode 100644 research/corpus/en/05.txt create mode 100644 research/corpus/en/06.txt create mode 100644 research/corpus/en/07.txt create mode 100644 research/corpus/en/08.txt create mode 100644 research/corpus/en/09.txt create mode 100644 research/corpus/en/10.txt create mode 100644 research/corpus/en/11.txt create mode 100644 research/corpus/en/12.txt create mode 100644 research/corpus/en/13.txt create mode 100644 research/corpus/en/14.txt create mode 100644 research/corpus/en/15.txt create mode 100644 research/corpus/en/16.txt create mode 100644 research/corpus/en/17.txt create mode 100644 research/corpus/en/18.txt create mode 100644 research/corpus/en/19.txt create mode 100644 research/corpus/en/20.txt create mode 100644 research/corpus/en/21.txt create mode 100644 research/corpus/en/22.txt create mode 100644 research/corpus/en/23.txt create mode 100644 research/corpus/en/24.txt create mode 100644 research/corpus/en/25.txt create mode 100644 research/corpus/es/01.txt create mode 100644 research/corpus/es/02.txt create mode 100644 research/corpus/es/03.txt create mode 100644 research/corpus/es/04.txt create mode 100644 research/corpus/es/05.txt create mode 100644 research/corpus/es/06.txt create mode 100644 research/corpus/es/07.txt create mode 100644 research/corpus/es/08.txt create mode 100644 research/corpus/es/09.txt create mode 100644 research/corpus/es/10.txt create mode 100644 research/corpus/es/11.txt create mode 100644 research/corpus/es/12.txt create mode 100644 research/corpus/es/13.txt create mode 100644 research/corpus/es/14.txt create mode 100644 research/corpus/es/15.txt create mode 100644 research/corpus/es/16.txt create mode 100644 research/corpus/es/17.txt create mode 100644 research/corpus/es/18.txt create mode 100644 research/corpus/es/19.txt create mode 100644 research/corpus/es/20.txt create mode 100644 research/corpus/es/21.txt create mode 100644 research/corpus/es/22.txt create mode 100644 research/corpus/es/23.txt create mode 100644 research/corpus/es/24.txt create mode 100644 research/corpus/es/25.txt create mode 100644 research/corpus/fr/01.txt create mode 100644 research/corpus/fr/02.txt create mode 100644 research/corpus/fr/03.txt create mode 100644 research/corpus/fr/04.txt create mode 100644 research/corpus/fr/05.txt create mode 100644 research/corpus/fr/06.txt create mode 100644 research/corpus/fr/07.txt create mode 100644 research/corpus/fr/08.txt create mode 100644 research/corpus/fr/09.txt create mode 100644 research/corpus/fr/10.txt create mode 100644 research/corpus/fr/11.txt create mode 100644 research/corpus/fr/12.txt create mode 100644 research/corpus/fr/13.txt create mode 100644 research/corpus/fr/14.txt create mode 100644 research/corpus/fr/15.txt create mode 100644 research/corpus/fr/16.txt create mode 100644 research/corpus/fr/17.txt create mode 100644 research/corpus/fr/18.txt create mode 100644 research/corpus/fr/19.txt create mode 100644 research/corpus/fr/20.txt create mode 100644 research/corpus/fr/21.txt create mode 100644 research/corpus/fr/22.txt create mode 100644 research/corpus/fr/23.txt create mode 100644 research/corpus/fr/24.txt create mode 100644 research/corpus/fr/25.txt create mode 100644 research/paper/README.md create mode 100644 research/paper/abstract.tex create mode 100644 research/paper/acknowledgments.tex create mode 100644 research/paper/ethics.tex create mode 100644 research/paper/main.tex create mode 100644 research/paper/refs.bib create mode 100644 research/pins-quality.txt create mode 100644 research/requirements-quality.txt create mode 100644 research/scripts/analyze_roc.py create mode 100644 research/scripts/attacks/cheap.py create mode 100644 research/scripts/evaluate_quality.py create mode 100644 research/scripts/make_figures.py create mode 100644 research/scripts/make_tables.py create mode 100644 research/scripts/multilingual_gen.py create mode 100644 research/scripts/pins.py create mode 100644 research/scripts/run_experiments.py create mode 100644 research/tests/test_analyze_roc.py create mode 100644 research/tests/test_cheap.py create mode 100644 research/tests/test_corpus.py create mode 100644 research/tests/test_corpus_gap05a6.py create mode 100644 research/tests/test_evaluate_quality.py create mode 100644 research/tests/test_make_tables.py create mode 100644 research/tests/test_multilingual_gen.py create mode 100644 research/tests/test_paper_skeleton.py create mode 100644 research/tests/test_run_experiments.py diff --git a/.env.example b/.env.example index 347e994..c1cc544 100644 --- a/.env.example +++ b/.env.example @@ -18,7 +18,7 @@ WATERMARKS_SERVER_API_KEY= # Optional MarkLLM research harness (host checkouts only; not in the core # image). Same-config-only detection — not a vendor oracle. -# WATERMARKS_MARKLLM_DIR=~/MarkLLM +# MARKLLM_DIR=~/MarkLLM # read by run_experiments.py / multilingual_gen.py / detect_text_watermark.py # WATERMARKS_MARKLLM_SCHEME=kgw # kgw | synthid # --------------------------------------------------------------------------- diff --git a/.gitignore b/.gitignore index 247e8d1..89e8416 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,9 @@ !/requirements-dev.txt !/ruff.toml !/SECURITY.md +!/research/ +!/research/** +/research/results/ # Exceptions even inside allowed trees .env diff --git a/Makefile b/Makefile index 03329de..cee2de5 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: test lint format lint-fix smoke smoke-synthid bootstrap-synthid docker-synthid-build docker-synthid-help \ +.PHONY: test research-check lint format lint-fix smoke smoke-synthid bootstrap-synthid docker-synthid-build docker-synthid-help \ smoke-ctrlregen bootstrap-ctrlregen docker-ctrlregen-build docker-ctrlregen-help \ smoke-markllm bootstrap-markllm docker-markllm-build docker-markllm-help \ smoke-markdiffusion bootstrap-markdiffusion docker-markdiffusion-build docker-markdiffusion-help \ @@ -12,6 +12,9 @@ PYTHON ?= $(shell if [ -x .venv/bin/python ]; then echo .venv/bin/python; else e test: $(PYTHON) -m pytest +research-check: + $(PYTHON) -m pytest research/tests -q + lint: $(PYTHON) -m ruff check service tests diff --git a/research/01-experiment-protocol.md b/research/01-experiment-protocol.md new file mode 100644 index 0000000..3c807c6 --- /dev/null +++ b/research/01-experiment-protocol.md @@ -0,0 +1,258 @@ +# 01 — Experimental Protocol + +*Working title of the study: "How fragile are deployed text watermarks? +A measurement study of multi-layer watermark removal under realistic +user-side editing."* + +Version: v0.2 (2026-08-18). Scope locked for arXiv v1 (see +research/README.md decision log). Owner: Guillaume. + +--- + +## 1. Research questions + +- **RQ1 (robustness).** How robust are deployed-class text watermarking + schemes (KGW, SynthID-Text, EXP, Unigram, SIR) to realistic user-side + editing (paraphrase, translation round-trip, structural rewrite, + humanization, Unicode/formatting cleanup), measured with ROC-based + detection metrics? +- **RQ2 (layering).** Does a *layered* removal pipeline (formatting-layer + cleanup **+** statistical rewrite) outperform single-layer baselines at + equal text-quality cost? Is the gain additive, or does one layer dominate? +- **RQ3 (frontier).** What is the quality–detectability Pareto frontier? + I.e., what detection rate can a watermarker keep while text remains + usable (PPL/BERTScore within tolerance), under each attack? +- **RQ4 (policy, secondary).** The EU AI Act Art. 50 transparency regime + (in force 2026-08-02) leans on watermarking. Does the mechanism survive + contact with real users? (Feeds §7 of the paper and the ethics/position + angle; not required for the core experiment.) + +**Primary claim to defend:** *Under realistic editing, quality-preserving +removal collapses TPR@1%FPR of KGW-class watermarks to near chance; +SynthID-Text resists token-level substitution but not paraphrase / +back-translation; a layered pipeline dominates single layers at equal +quality cost.* + +**Anti-claim we must preempt:** "Your detector is misconfigured / your +attacks destroy the text / a stronger watermark config would survive." +Mitigations in §5.3, §6, §10. + +--- + +## 2. Design overview + +Factorial experiment, paired design: the same watermarked document is +attacked by every attack condition, and detection is run on every +(doc, attack) pair with the same watermark key/config. Unwatermarked +control texts run through the same attack+detect pipeline +(already supported via `--restamp-control`). + +### v1 factorial (locked 2026-08-18) + +| Grid | Schemes | Lengths | Temp | Langs | Prompts | Seeds | Cells | +| --- | --- | --- | --- | --- | --- | --- | --- | +| EN core | KGW (γ=.25, δ=1); KGW (γ=.5, δ=2); KGW (γ=.5, δ=4); SynthID-Text (default MarkLLM config); EXP (Gumbel); Unigram; SIR | 100, 300 | 0.7 | en | 25 | 5 | 1,750 | +| EN temp axis | 4 core schemes (KGW×3, SynthID) | 300 | 1.0 | en | 25 | 5 | 500 | +| EN length axis | 4 core schemes | 500 | 0.7 | en | 25 | 5 | 500 | +| Multilingual | KGW (γ=.5, δ=2); SynthID-Text | 300 | 0.7 | de, fr, es | 25 | 5 | 750 | +| **Total** | | | | | | | **3,500 cells** | + +→ 3,500 watermarked + 3,500 unwatermarked generations = **7,000 +texts**; each passed through 8 attack cells (§4) = **56,000 attack +outputs**; ~65,000 detection runs (originals + attacks + controls) +plus ~2,000 unwatermarked texts for the empirical null. + +Restrictions are deliberate (a naive full cartesian — 7 schemes × 3 +lengths × 2 temps × 4 langs × 25 × 5 — would be 21,000 cells and +infeasible on CPU): the temp axis, length axis, and multilingual grid +each vary exactly one factor against the 4 core schemes so the effects +are attributable. See §3 for the multilingual generator (model holdout). + +### Deferred (post-v1, not in arXiv v1) + +- Human eval (20 samples × 3 annotators) — journal/ACL requirement. +- File/metadata mini-study (C2PA/EXIF strip on 20 self-made files; + v0.1 §4.3) — separate mini-experiment, revisit for v2. + +--- + +## 3. Generation protocol + +- **EN core:** `facebook/opt-1.3b` via MarkLLM (matches existing + harness; CPU-feasible). +- **Multilingual (DE/FR/ES):** opt-1.3b is English-only, so these cells + use a CPU-feasible multilingual generator — **Qwen/Qwen2.5-1.5B- + Instruct** (fallback: 0.5B if too slow) — reported explicitly as a + model-holdout factor in the paper (scheme × language, not confounded + with the EN core). If the multilingual pilot (§9) shows unacceptable + output quality, shrink the grid to 10 prompts per language before + committing the full run. +- Prompts: extend `benchmarks/corpus/` to **25 factual, neutral EN + seeds** (50-90 words, varied domains; keep claims checkable) plus + **3×25 translated DE/FR/ES sets**; record all prompts in + `research/corpus/` (additive, not a fork). +- Decoding: **temperature 0.7, top-p 0.95** (realistic); record + `do_sample=True`, fixed seeds 1..5 per (prompt, seed) pair. +- Watermark configs: fixed per scheme; copy the MarkLLM config JSONs + (KGW: γ/δ/hash_key/f_scheme/window_scheme; SynthID; EXP; Unigram; + SIR) into `research/configs/` and **use the same JSON for + generation and detection**. Synthetic keys — fine to commit; note it. +- Control: same prompts, same seeds, no watermark → empirical null + distribution for FPR calibration (§5.2). + +## 4. Attack conditions + +### 4.1 Text-level attacks + +| # | Attack | Implementation | Prior-art anchor | +| --- | --- | --- | --- | +| A0 | None (control) | — | — | +| A1 | Layer A only: Unicode/invisible-char, bidi, tag cleanup | `clean_text.py` (deterministic) | formatting-layer marks (zero-width/steganography class) | +| A2 | Paraphrase, single pass | `rewrite_text.py --strength paraphrase --candidates 1 --max-loops 1` | paraphrase attacks in watermark literature | +| A3 | Paraphrase, adaptive (early stop on detection, up to 3 loops) | `rewrite_text.py --strength paraphrase --candidates 3 --max-loops 3 --markllm-scheme ` | same; our eval-loop is the "oracle" version | +| A4 | Back-translation round trip EN→DE→EN | `rewrite_text.py --strength backtranslate --lang German` | "Can Watermarks Survive Translation?" (X-SIR, ACL 2024) | +| A5 | Structural: outline → regenerate | `rewrite_text.py --strength structural` | summarization/outline attacks | +| A6 | Humanize | `rewrite_text.py --strength humanize` | style-transfer attacks | +| A7 | Cheap baselines: synonym substitution, random word deletion (5-10%), sentence reorder | `research/scripts/attacks/cheap.py` (to be written — gap 05-A3) | Random Walk / impossibility (ICML 2024) | +| A8 | **Full pipeline: A1 → A2/A3** | `clean_text.py` then `rewrite_text.py` | our layered contribution | + +Notes: +- CLI names above match the real `rewrite_text.py` flags: `--strength` + selects the rewrite type; `--backend` selects the transport + (`ollama` / `openai-compatible`); `--candidates` × `--max-loops` + control the detection-guided loop. +- The Layer B rewrite loop stops early when detection passes — that's an + *adaptive* attack (stronger than one-shot). Report both: with + early-stop (adaptive, "removal oracle") and forced single-pass. +- Record rewrite cost (tokens in/out, wall time, USD) per cell — this is + the "practical attack cost" contribution (Table 6). + +### 4.2 Detector-side conditions + +- Detectors: MarkLLM same-config (KGW z-score; SynthID mean/weighted/ + bayesian; EXP/Unigram/SIR per their configs). +- Also run **best-effort detection**: threshold tuned on held-out data + (favorable to the watermarker; preempts "weak threshold" criticism). +- Explicit **strawman preemption**: run the strongest config we have + (KGW γ=0.5 δ=4, long text) as the "best case for watermarking" column. + +## 5. Detection & evaluation protocol + +### 5.1 Metrics (primary) + +- **AUROC** over (watermarked, unwatermarked) score distributions. +- **TPR @ FPR ∈ {0.1%, 1%, 10%}** read off the ROC. +- Full ROC curves saved per cell (for Figure 2). +- Secondary: clear rate (existing bench metric, before-positive → + after-negative), mean/median score suppression. +- **95% bootstrap CIs (10k resamples) on AUROC and TPR@FPR**; report + effect sizes, not just p-values. + +### 5.2 Empirical null + +- ≥1,000 unwatermarked generations (same prompts/seeds/temps), scored by + the *same* detector. Do **not** assume z-scores are standard normal for + FPR calibration; SynthID tournament scores definitely aren't. Empirical + null is mandatory — use all unwatermarked controls. + +### 5.3 Quality metrics + +| Metric | Model/tool | Why | +| --- | --- | --- | +| PPL | `gpt2-large` (independent of generator opt-1.3b — never score with the generator) | fluency | +| BERTScore | `deberta-xlarge-mnli` | semantic preservation | +| ROUGE-L | standard | content overlap | +| SBERT cosine | `all-MiniLM-L6-v2` | cheap semantic similarity | +| Levenshtein distance % | stdlib | edit magnitude | +| Length drift %, number/URL survival | already in bench | practical usability | + +**Quality @ detection-collapse**: for each attack, report quality metrics +*at the operating point where detection collapses* (the Pareto framing). +This is the figure that makes the paper defensible: "removal without +destruction." To bound CPU cost, compute full quality on a stratified +subset (~2-4k texts covering every scheme × attack × collapse point). + +### 5.4 Statistics + +- 5 seeds/cell, paired design; 95% bootstrap CIs as above. +- Detection randomness: detection scores are deterministic given + (text, key) for KGW; SynthID scoring may have randomness — seed it. +- Multiple-comparison discipline: pre-register the headline cells (the + v1 matrix); treat nothing else as confirmatory. + +## 6. Budget & compute (estimates) + +CPU-only (opt-1.3b; Qwen2.5-1.5B for multilingual; MarkLLM). Rough +per-op figures from existing runs: gen ≈ 1-3 min/text (CPU), detect ≈ +10-40 s/text. + +| Stage | Volume | CPU-hours (1 core) | Parallel 8 cores | +| --- | --- | --- | --- | +| Generate (incl. multilingual) | 7,000 texts | ~250-400 | ~35-50 h | +| Detect (originals+attacks+controls) | ~65,000 | ~180-720 | ~25-90 h | +| Null corpus scoring | 2,000 | ~10-20 | ~2-3 h | +| Rewrites (API) | 56,000 outputs | — | ~1-2 days (rate-limited) | +| Quality metrics (stratified subset) | ~2-4k texts | ~60-150 | ~10-20 h | + +API rewrite budget: **~$50-120** (paraphrase ≈ 1.3× input tokens per +pass; back-translate ≈ 2×2 passes; structural ≈ 2 passes; cheap +instruct model, temperature 0.9 — record model + version). + +→ **v1 ≈ 2.5-4 weeks of part-time work on one 8-core box + API +budget.** That is the arXiv-v1 core; the extended scope adds ~1-2 weeks +over the original Tier-1-only estimate. + +## 7. Reproducibility + +- Pin: MarkLLM checkout commit (`git -C /home/guillaume/MarkLLM + rev-parse HEAD`), HF model revisions (opt-1.3b, Qwen2.5-1.5B, + gpt2-large, deberta-xlarge-mnli, all-MiniLM-L6-v2), all scheme + configs (copied into `research/configs/`). +- Seed policy: every random op seeded; seeds recorded per cell in + `results/manifest.json`. +- Environment: record `pip freeze` for the MarkLLM env (it is separate + from the service stdlib env) and for the quality-metric env. +- Release plan: `research/results/` (JSONL per cell), analysis + notebooks, corpus, and a `Makefile`-style runner + (`run_experiments.py --plan`). + +## 8. Artifacts & release + +1. `research/corpus/` — 25 EN + 75 multilingual prompts. +2. `research/configs/` — scheme configs + keys + model pins. +3. `research/results/` — scores, ROC data, quality metrics, manifest. +4. `research/scripts/` — attack additions (cheap.py, gap 05-A3), + analysis notebooks. +5. Paper artifacts: tables 1-7, figures 1-6 (spec in 02 §6). + +## 9. Smoke test (do this first, before the full run) + +Purpose: verify the pipeline end-to-end (generation → attack → detect +→ evaluate) on one scheme and a handful of prompts before committing +the 3,500-cell run. + +```bash +# Requires the multi-scheme bench refactor (gap 05-A1) to have landed. +python3 service/scripts/bench_synthid_text.py \ + --corpus research/corpus --docs 5 --seeds 2 --max-new-tokens 300 \ + --variants paraphrase:3 --restamp-control --tag smoke \ + --markllm-model facebook/opt-1.3b +``` + +Then, once the ROC/quality analysis modules (gaps 05-B1/B2) exist: +compute AUROC/TPR@FPR + quality on the smoke output, and sanity-check +the multilingual pilot (5 prompts × 1 seed per language, DE/FR/ES, via +Qwen2.5-1.5B) for output quality before the full multilingual grid. + +## 10. Risk register + +| Risk | Mitigation | +| --- | --- | +| "Detector is misconfigured" | Same-config detection is the field standard; add best-effort tuned detector + strongest-config column; identical config JSON for gen and detect (§3) | +| "Attacks destroy text" | Quality metrics at collapse point; show PPL/BERTScore within tolerance | +| "Not real vendors" | Honest limitation; vendor APIs are black-box/retired (Google retired SynthID API Aug 2026); optional Claude API probe study if ToS allows | +| "Already known" (novelty) | Novelty = layered attack + systematic ROC measurement across schemes + quality Pareto + policy measurement; check 03-related-work for overlap | +| Ethics rejection | 04-ethics-and-legal.md; frame as robustness evaluation of deployed mechanisms | +| Compute blowup | Locked restricted matrix (3,500 cells, not 21,000); `--plan` budget mode in run_experiments.py; stage checkpoints | +| **Multilingual generation quality** | opt-1.3b is English-only → Qwen2.5-1.5B holdout; 5-prompt pilot check before the grid; fallback: shrink to 10 prompts/lang or defer to v1.1 | +| Bench hardcodes SynthID | Add `--scheme kgw|synthid|exp|unigram|sir` to bench (gap 05-A1; small refactor) | diff --git a/research/02-paper-outline.md b/research/02-paper-outline.md new file mode 100644 index 0000000..a8eca1d --- /dev/null +++ b/research/02-paper-outline.md @@ -0,0 +1,188 @@ +# 02 — Paper Outline & Venue Strategy + +Version: v0.2 (2026-08-18). Status: skeleton; fill as results land. +Scope locked for arXiv v1 (see research/README.md decision log). + +--- + +## 1. Working title (locked) + +1. **"How Fragile Are Deployed Text Watermarks? An Empirical Study of + Layered Watermark Removal under Realistic User-Side Editing"** + (safe, descriptive — **locked for arXiv v1**; measurement-study + framing per the decision log) + +Dropped on 2026-08-18 (kept here only as history): *"Watermarks Are +Speed Bumps…"* — tone risk at archival venues; *"Removing Provenance +Marks…"* — neutral fallback if the title must change. + +## 2. Abstract draft (~150 words) + +> Text watermarking is the primary mechanism proposed for EU AI Act +> Art. 50 transparency obligations on machine-generated content. We +> measure its robustness against realistic user-side editing. Using +> same-config detection over 3,500 watermarked and 3,500 unwatermarked +> generations (KGW, SynthID-Text, EXP, Unigram, and SIR schemes, in +> English and German/French/Spanish), we evaluate a layered removal +> pipeline that combines formatting-layer cleanup (invisible Unicode, +> bidi) with statistical rewriting driven by detection feedback, and we +> report ROC-based metrics (AUROC, TPR@1%FPR) and quality metrics +> (perplexity, BERTScore, ROUGE-L) at the point of detection collapse. +> We find that quality-preserving paraphrase and translation round-trip +> collapse KGW-class detection to near chance, that SynthID-Text resists +> token substitution but not paraphrase, that layering dominates single +> attacks at equal quality cost, and that multilingual texts are +> systematically more fragile. We discuss implications for Art. 50 +> compliance and release our corpus, configs, and harness. + +(Recheck numbers against the final run during W2.) + +## 3. Venue plan + +| Step | Venue | When | Effort | +| --- | --- | --- | --- | +| 1 (**active**) | **arXiv preprint v1** | target ~2–4 weeks after core results | full paper: core + selected extensions | +| 2 (future) | NeurIPS 2026 workshop → ACL 2027 / TIFS | post-v1, per CFP | reuse v1; see 04 for ethics/dual-use per venue | + +Strategy: **arXiv v1 now**; do not split the same skeleton across two +main tracks — workshops + one archival venue is the clean post-v1 path. + +## 4. Contributions (4 bullets, in final form) + +1. **Measurement.** First ROC-based robustness study of deployed-class + text watermarking (KGW, SynthID-Text, EXP, Unigram, SIR) under + realistic, layered user-side editing, with empirical-null FPR + calibration. +2. **Method.** A layered removal pipeline (formatting + statistical) + with detection-feedback rewriting; we show it dominates single-layer + baselines at equal quality cost (Pareto). +3. **Resource.** Open corpus (25 EN + 75 multilingual prompts), + config-pinned harness (MarkLLM-based), and results (JSONL) for + reproducible attack/defense benchmarking. +4. **Policy measurement.** Evidence on whether Art. 50 transparency + obligations can rely on watermarking, with concrete recommendations + (provenance at platform level, metadata, robust-but-invisible + schemes, honest failure modes). + +## 5. Section skeleton + +### 1. Introduction (~1 p) +- Hook: Art. 50 in force 2026-08-02; vendor rollouts (Claude, Gemini); + Google retired SynthID-Text API Aug 2026 (context: the market is + consolidating on methods that don't survive editing). +- The viral deployment of our removal tool as motivation (1 short + paragraph, no metrics needed) → "users edit their own output; do + watermarks survive?" +- Contributions (4 bullets above). **Fig 1** here. + +### 2. Background & Related Work (~1.5 p) +- Watermarking families: sampling-based (KGW line), tournament + (SynthID-Text), semantic (SIR/X-SIR), provable (UPV, Christ-family). +- Attack literature: paraphrase, back-translation, random-walk + impossibility, watermark stealing. Metadata/C2PA: one short sentence + (the full metadata mini-study is deferred to v2). +- **Positioning paragraph:** what we add = layered attack + systematic + ROC measurement + quality Pareto + policy measurement. + (Full citation map in 03.) + +### 3. Threat Model & System (~1 p) +- Who: end users editing text they generated with their own account + (explicitly **not** third-party content; see ethics file). +- What: watermark-as-label (not access control); detector = same-config + MarkLLM detector (standard in literature; vendors black-box). +- System: 2 layers for v1 (A formatting / B statistical) with a + detection-feedback rewrite loop. **Fig 1** (pipeline diagram). + (Files/metadata layer deferred to v2.) +- Definitions box: TPR@FPR, AUROC, "detection collapse", "quality cost". + +### 4. Experimental Setup (~1.5 p) +- Design summary (→ 01 §2): locked v1 factorial — 3,500 cells, + 7 schemes, lengths 100/300/500, temps 0.7/1.0, languages en/de/fr/es + with the restricted subsets; generation protocol; attack cells + (A0–A8); detection protocol incl. empirical null and best-effort + detector; quality metrics table. +- Reproducibility: pins, seeds, configs, release URLs. + +### 5. Results (~3 p) +- **Table 2** (money table) → discussion of each scheme's failure mode. +- **Fig 2** ROC pre/post per scheme; **Fig 3** Pareto frontier; + **Table 3** quality; **Table 4** strength×length ablation; + **Table 5** multilingual (v1 extension); **Table 6** attack cost. +- Key claims: (i) paraphrase/back-translation collapse KGW-class + detection; (ii) SynthID-Text robust to token substitution but not + paraphrase; (iii) layering > single layers at equal quality cost; + (iv) even the strongest config (γ=.5, δ=4, 300 tok) drops below + usable TPR under adaptive rewrite; (v) multilingual (DE/FR/ES) is + systematically more fragile. +- **Fig 5** case study (redacted before/after + detector scores). + +### 6. Analysis & Case Study (~1 p) +- Cost of attack vs cost of defense (Table 6): attacker spends ~10-20¢ + and minutes; defender must raise γ/δ (quality cost) — asymmetry. +- Failure modes ranked; which scheme properties survive (semantic + preservation, n-gram robustness) and which don't (token-level stats). + +### 7. Policy Discussion (~0.75 p) +- Art. 50 mechanics; what a regulator can actually rely on; metadata + + platform provenance as the robust complement; honest limits of our + study (no vendor black-box measurement). + +### 8. Limitations (~0.5 p) +- Same-config ≠ vendor detection; small open-weight generators + (opt-1.3b; Qwen2.5-1.5B for DE/FR/ES as an explicit model holdout — + not frontier LLMs); API probe limits; our rewrite oracle is stronger + than naive users (we show both adaptive and single-pass numbers). + +### 9. Ethics Statement (~0.5 p) +- Draft text in 04 §3; adapt to venue template. + +### 10. Conclusion (~0.25 p) + +## 6. Exact tables & figures spec + +### Tables +| # | Content | Why reviewers need it | +| --- | --- | --- | +| T1 | Attack taxonomy: family × mechanism × implementation × prior-art anchor | reproducibility + novelty of layering | +| T2 | **AUROC / TPR@1%FPR matrix: rows = scheme-config (7) × attack (8); cols = pre-attack, A-only, B-only, A+B** (core findings) | money table | +| T3 | Quality per attack at collapse point: PPL Δ, BERTScore, ROUGE-L, SBERT, length drift, num/URL survival | preempts "destroys text" | +| T4 | Ablation: TPR@1%FPR × (γ,δ) × length × temp | strength/length dependence | +| T5 | Multilingual EN/DE/FR/ES (v1 extension) | known weak spot, cheap win | +| T6 | Attack cost: tokens in/out, wall time, USD per 1k words per attack | practical-asymmetry argument | +| T7 | Comparison with published baselines (cite-and-compare where numbers are reported in the same metric) | situates vs literature | + +### Figures +| # | Content | +| --- | --- | +| F1 | Pipeline/system diagram (2 layers + detection-feedback loop) | +| F2 | ROC curves pre/post per scheme (the money figure) | +| F3 | Quality–detectability Pareto frontier (PPL vs AUROC across attacks) | +| F4 | TPR@1%FPR vs watermark strength, length as line style | +| F5 | Case study: redacted before/after text with detector scores | +| F6 | (policy/position only) Art. 50 timeline vs measured collapse — include, small (policy) | + +## 7. Citation placement map + +| Section | Cites (from 03) | +| --- | --- | +| 2 watermark families | KGW, SynthID-Text, SIR, UPV, survey, MarkLLM | +| 2 attacks | X-SIR translation, fragility-of-multilingual, random-walk impossibility, watermark stealing, black-box watermarking | +| 2 metadata (one sentence) | C2PA spec | +| 5 results | X-SIR (cross-lingual numbers), fragility paper (paraphrase numbers) | +| 7 policy | EU AI Act Art. 50 (Regulation (EU) 2024/1689) | + +## 8. Writing phases (solo) + +1. **W1:** core experiment runs; Tables 2–4 first drafts (numbers only). +2. **W2:** Figs 2–3, T6 cost, §4–5 prose; **arXiv v1** (target ~2–4 + weeks from core results). +3. **W3+ (post-v1):** future venue per §3; extend if anything remains. + +## 9. Reviewer-bait checklist (preempt in text) + +- [ ] "Detector is strawman" → best-effort tuned detector + strongest config column (T2 includes δ=4) +- [ ] "You destroyed the text" → T3 at collapse point +- [ ] "Unrealistic oracle" → report both adaptive (early-stop) and single-pass numbers +- [ ] "No negative control" → unwatermarked control through full pipeline (`--restamp-control`) +- [ ] "Not reproducible" → configs + seeds + manifest; MarkLLM commit pinned +- [ ] "Ethics" → 04 file; ethics statement §9 diff --git a/research/03-related-work.md b/research/03-related-work.md new file mode 100644 index 0000000..a9fbffb --- /dev/null +++ b/research/03-related-work.md @@ -0,0 +1,80 @@ +# 03 — Related Work (verified citation list) + +Every entry below was checked against the arXiv API on **2026-08-18** +(title/ID queries); all entries marked ✅ resolved to the exact paper. +**Re-verify every ID and venue label before the camera-ready / at +submission time.** + +Four candidate citations were dropped on 2026-08-18 because they could +not be verified (see v0.1 §F): re-verify before ever re-adding any of +them (Blinov et al. "Who Wrote this?", Zhao et al. provable-watermark +OpenReview claim, the "Fragility of Multilingual LLMs" deletion-paper +claim, "Stronger Watermarks for Language Models"). + +Venue labels are from the papers'/MarkLLM's own claims where available; +re-verify venue before the camera-ready. + +--- + +## A. Watermarking methods (the attack surface we test) + +| ✅ | Cite | First author | ID / DOI | Date | Notes & where used in paper | +| --- | --- | --- | --- | --- | --- | +| ✅ | A Watermark for Large Language Models (KGW) | John Kirchenbauer | arXiv **2301.10226** (ICML 2023) | 2023-01 | Primary scheme under test; §2, §5 | +| ✅ | Undetectable Watermarks for Language Models | Miranda Christ | arXiv **2306.09194** (COLT 2024) | 2023-05 | Christ-family (PF line); §2 robustness theory | +| ✅ | Robust Distortion-free Watermarks for Language Models | Rohith Kuditipudi | arXiv **2307.15593** (TMLR) | 2023-07 | Sampling-free family; §2 | +| ✅ | A Semantic Invariant Robust Watermark (SIR) | Aiwei Liu | arXiv **2310.06356** (ICLR 2024) | 2023-10 | Strongest defense class; Tier-2 scheme; §2, §5 | +| ✅ | An Unforgeable Publicly Verifiable Watermark (UPV) | Aiwei Liu | arXiv **2307.16230** (ICLR 2024) | 2023-07 | Provable family; §2 | +| ✅ | Permute-and-Flip: an optimally stable and watermarkable decoder | Xuandong Zhao | arXiv **2402.05864** | 2024-02 | §2 | +| ✅ | Unbiased Watermark for Large Language Models | Zhengmian Hu | arXiv **2310.10669** | 2023-10 | §2 | +| ✅ | A Resilient and Accessible Distribution-Preserving Watermark (DiPmark) | Yihan Wu | arXiv **2310.07710** | 2023-10 | §2 | +| ✅ | Token-Specific Watermarking (TS-Watermark) | Mingjia Huo | arXiv **2402.18059** (ICML 2024) | 2024-02 | §2 | +| ✅ | Adaptive Text Watermark | Yepeng Liu | arXiv **2401.13927** | 2024-01 | §2 | +| ✅ | SemStamp: A Semantic Watermark with Paraphrastic Robustness | Abe Bohan Hou | arXiv **2310.03991** | 2023-10 | semantic family; §2 | +| ✅ | k-SemStamp: A Clustering-Based Semantic Watermark | Abe Bohan Hou | arXiv **2402.11399** | 2024-02 | §2 | +| ✅ | Invisible Entropy (IE): Safe and Efficient Low-Entropy LLM Watermarking | Tianle Gu | arXiv **2505.14112** | 2025-05 | recent method; §2 | +| ✅ | MorphMark: Flexible Adaptive Watermarking | Zongqi Wang | arXiv **2505.11541** | 2025-05 | §2 | +| ✅ | Watermarking Text Generated by Black-Box LLMs | Xi Yang | arXiv **2305.08883** (NAACL 2024) | 2023-05 | black-box line; §2 | + +## B. Robustness, attacks, and limits (our direct neighbors — positioning) + +| ✅ | Cite | First author | ID / DOI | Date | Notes | +| --- | --- | --- | --- | --- | --- | +| ✅ | On the Reliability of **Watermarks** for Large Language Models (note: title is "Watermarks", not "Watermarking") | John Kirchenbauer | arXiv **2306.04634** (NeurIPS 2023) | 2023-06 | foundational robustness study; §2, §5 compare | +| ✅ | Can Watermarks Survive Translation? (X-SIR) | Zhiwei He | arXiv **2402.14007** (ACL 2024) | 2024-02 | cross-lingual attack; our A4 baseline anchor; §2, §5 | +| ✅ | Watermarks in the Sand: Impossibility of Strong Watermarking | Hanlin Zhang | arXiv **2311.04378** (ICML 2024) | 2023-11 | Random Walk attack; impossibility theory; §2, §7 | +| ✅ | Watermark Stealing in Large Language Models | Nikola Jovanović | arXiv **2402.19361** | 2024-02 | key-extraction threat; §2 | +| ✅ | WaterSeeker: Efficient Detection of Watermarked Segments | Leyi Pan | arXiv **2409.05112** (NAACL 2025 Findings) | 2024-09 | detector side; §2 | +| ✅ | An Entropy-based Text Watermarking Detection Method (EWD) | Yijian Lu | arXiv **2403.13485** (ACL 2024) | 2024-03 | detector side; §2 | +| ✅ | Can Watermarked LLMs be Identified by Users via Crafted Prompts? | Aiwei Liu | arXiv **2410.03168** (ICLR 2025 Spotlight) | 2024-10 | §2 | +| ✅ | Can LLM Watermarks Robustly Prevent Unauthorized Knowledge Distillation? | Leyi Pan | arXiv **2502.11598** (ACL 2025) | 2025-02 | §2 | + +## C. Closest recent neighbors (2025-2026 measurement/forensics — cite these!) + +| ✅ | Cite | First author | ID / DOI | Date | Notes | +| --- | --- | --- | --- | --- | --- | +| ✅ | Robustness Assessment and Enhancement of Text Watermarking for Google's SynthID | Xia Han | arXiv **2508.20228** | 2025-08 | closest method-specific robustness work; must cite & differentiate in §2/§5 | +| ✅ | On Google's SynthID-Text LLM Watermarking System: Theoretical Analysis and Empirical Validation | Romina Omidi | arXiv **2603.03410** | 2026-03 | theoretical analysis of SynthID-Text; §2 | +| ✅ | AI Watermark Evidence Fails Forensic Readiness: An Empirical Evaluation | Saifur Rahman Tamim | arXiv **2607.16010** | 2026-07 | directly supports our policy finding; §7 | +| ✅ | Sandcastles in the Storm: Revisiting the (Im)possibility of Strong Watermarking | Fabrice Y Harel-Canada | arXiv **2505.06827** | 2025-05 | impossibility revisited; §2 | + +## D. Tools & surveys + +| ✅ | Cite | First author | ID / DOI | Date | Notes | +| --- | --- | --- | --- | --- | --- | +| ✅ | MarkLLM: An Open-Source Toolkit for LLM Watermarking | Leyi Pan | arXiv **2405.10051** (EMNLP 2024 Demo) | 2024-05 | our harness; acknowledge THU-BPM | +| ✅ | A Survey of Text Watermarking in the Era of LLMs | Aiwei Liu | arXiv **2312.07913** (ACM Computing Surveys 2025) | 2023-12 | §2 overview + taxonomy | + +## E. Non-arXiv sources (correct IDs; cite directly) + +- **SynthID-Text** — Google DeepMind, *Nature* **638**, 625-632 (2024); DOI [**10.1038/s41586-024-08025-4**](https://www.nature.com/articles/s41586-024-08025-4). (arXiv tech-report id exists but was not resolvable in our checks — cite the Nature DOI as primary.) +- **C2PA** — Coalition for Content Provenance and Authenticity, specification, https://c2pa.org/specifications/specifications/2.1/specs/C2PA_Specification.html +- **EU AI Act** — Regulation (EU) 2024/1689, Art. 50 (transparency obligations), in force for GPAI since 2026-08-02. + +## Positioning summary (for §2 last paragraph) + +- vs **X-SIR** (2402.14007): they test translation robustness of *their own* scheme; we test the *deployed-class* schemes users actually meet, with a layered pipeline and ROC metrics. +- vs **2508.20228 / 2603.03410** (SynthID robustness): they assess/enhance SynthID specifically; we add KGW, multi-layer attacks, quality Pareto, and the policy measurement. +- vs **2607.16010** (forensic readiness): they measure evidence-grade failure; we measure the removal side end-to-end — complementary. +- vs **2306.04634** (Reliability): they established fragility of watermarking; we add the layered attack + empirical-null FPR + quality-at-collapse, on 2024-2026 schemes. +- Novelty claim rests on: **layered attack (formatting+statistical in v1; metadata layer deferred to v2), systematic ROC measurement across schemes under one protocol, quality Pareto frontier, and Art. 50 policy measurement** — none of the above does all four. diff --git a/research/04-ethics-and-legal.md b/research/04-ethics-and-legal.md new file mode 100644 index 0000000..b7b1715 --- /dev/null +++ b/research/04-ethics-and-legal.md @@ -0,0 +1,124 @@ +# 04 — Ethics & Legal Framing + +Version: v0.1 (2026-08-18). This file feeds the paper's Ethics Statement +and your own position (see `~/wired_interview_prep.md` — the two should +stay consistent). + +--- + +## 1. Dual-use framing (the core stance) + +This is a **robustness/measurement study of deployed mechanisms**, not a +how-to for harming third parties. Published precedent is strong: attack +and robustness papers on watermarking are routine at NeurIPS, ICLR, +ACL, IEEE S&P, USENIX Security (see 03: random-walk impossibility, +watermark stealing, fragility studies — all published, all attacker- +facing). The field treats "measure the mechanism before regulators +mandate it" as legitimate, even necessary, research. + +Frame, consistently: + +1. **The tool edits text the user generated with their own account.** + It is not a forgery or circumvention tool (watermark = label, not + lock — no access control is bypassed). +2. **The contribution is measurement.** We quantify what the literature + already suspected (fragility under paraphrase/translation) and what + the public deployment showed at scale — with controlled experiments + instead of anecdotes. +3. **We do not attack third-party content.** Corpus is self-generated. +4. **We help defenders too.** Detection-feedback loops, empirical-null + FPR calibration, and the Pareto analysis are directly useful for + building watermarking that survives reality (or for regulators to + pick mechanisms that do). + +## 2. EU AI Act analysis (for §7 of the paper) + +- Regulation (EU) 2024/1689, **Article 50** (transparency obligations + for providers and deployers of certain AI systems), general + application for GPAI systems: 2 August 2026 (per your notes and the + public timeline). +- Key point for the paper: Art. 50 obligations fall on **providers and + deployers**, not on end users editing their own documents. The Act + does not require end users to preserve machine-readable provenance + marks; it requires providers to make machine-readable output + detectable. +- Therefore a user stripping a watermark from their own AI-assisted + writing is not violating Art. 50. The *policy problem* the paper + documents is upstream: if the mandated detection mechanism collapses + under ordinary editing, the transparency obligation is not met in + practice — that is a compliance/effectiveness finding, not a + user-facing prohibition. +- Cite: Regulation (EU) 2024/1689, Art. 50; recitals on transparency. + Verify exact recital numbers before submission. + +## 3. Ethics statement draft (adapt to venue template) + +> All text used in this study was generated by the authors using +> open-weight models (opt-1.3b) via the MarkLLM toolkit. No +> third-party content, user data, or live vendor outputs were used; +> no watermarked content produced by commercial providers was +> collected or altered. The removal pipeline evaluated here operates +> on text generated by the same user who owns it; the study does not +> enable or endorse alteration of third-party content. Watermarking +> is a label, not an access-control mechanism, so no security control +> is circumvented. We disclose our findings to support (i) realistic +> expectations for regulators relying on watermarking for transparency +> obligations (Art. 50, Regulation (EU) 2024/1689), and (ii) the +> design of more robust provenance mechanisms. Detectors are +> same-config open-source implementations; commercial detectors were +> not probed. We do not provide live removal services or weights +> tuned for evasion of specific vendor detectors beyond what is +> reported. The authors' tooling is public (github.com/guillaumemeyer/ +> watermarks-remover) and was deployed publicly before this study +> began; this paper formalizes measurements of mechanisms already in +> production use. + +## 4. Data policy + +- **v1: synthetic data only.** No user data from the viral deployment. + If usage telemetry is ever included: aggregate, anonymize, obtain + consent, and get IRB/ethics review. +- No scraping of watermarked third-party text (avoids both legal and + ethical surface). +- Release: corpus, configs, results JSONL (all synthetic). + +## 5. Legal notes (know your lines; don't overclaim) + +- MarkLLM is Apache-2.0 → attribution required, no restriction on + research use. Credit THU-BPM in acknowledgments. +- C2PA stripping: your position is privacy/hygiene on owned content. + Note that anti-circumvention regimes (e.g., US DMCA 1201) generally + target access controls, not labels; watermark-as-label framing keeps + you clear, but do not give legal opinions in the paper — one + sentence of framing, then cite nothing you haven't verified. +- **Never claim 100% effectiveness** anywhere in the paper (reviewers + and journalists will test it). Use ROC numbers and honest confidence + intervals. +- Disclosure status: the tool is already public and widely reported; + no embargo or coordinated-disclosure obligation applies to the + measurement. State this if a venue asks about disclosure. + +## 6. Venue-specific notes + +- **arXiv v1 (active target):** arXiv itself has no ethics checklist; + include the §3 statement as submitted and the §5 disclosure note + (tool already public, no embargo applies). Pick a license at upload. +- Future venues (post-v1, one line): NeurIPS/ACL want an ethics + checklist + responsible-NLP/broader-impact paragraph (§3 covers both); + security venues (S&P/USENIX) expect a stronger adversary framing plus + §7 defender takeaways; TIFS wants extended related work + a more + formal threat model. + +## 7. Defender takeaways (include in any version) + +1. Token-level statistical watermarks do not survive paraphrase; don't + bet Art. 50 compliance on them alone. +2. N-gram/tournament methods (SynthID-Text) survive substitution but + not semantic rewriting — combine with semantic-invariant schemes + (SIR line) if rewriting is the threat. +3. Robust detection needs empirical-null calibration, not normal + assumptions. +4. Platform-level provenance + metadata (C2PA) is complementary, not a + substitute for text-layer robustness. +5. Publish attack benchmarks openly so defenders can measure the + Pareto frontier instead of assuming worst-case strength. diff --git a/research/05-arxiv-readiness.md b/research/05-arxiv-readiness.md new file mode 100644 index 0000000..15352ef --- /dev/null +++ b/research/05-arxiv-readiness.md @@ -0,0 +1,85 @@ +# 05 — arXiv v1 Readiness: Gap Analysis & Submission Checklist + +Version: v0.1 (2026-08-18). Scope: arXiv v1 of *"How Fragile Are +Deployed Text Watermarks? An Empirical Study of Layered Watermark +Removal under Realistic User-Side Editing"* (locked scope in +research/README.md decision log). + +This file itemizes **everything missing** between the current repo and +a submittable arXiv v1. It is a hand-off to the implementation phase — +**as of PR #174 all code/analysis gaps (A1-A7, B1-B3) and the paper +skeleton/bibliography (C1/C2) are implemented**; the experiment run +(D) and publishing logistics (E) remain. Items are marked with the +stage of the pipeline they block (generate → attack → detect → +evaluate → paper → publish). + +--- + +## A. Verified code gaps (experiment harness) + +Each row: what exists today / what's missing / effort / where it lands. +(All "exists" claims verified 2026-08-18.) + +| # | Gap | What exists today | What's missing | Effort | Blocks | +| --- | --- | --- | --- | --- | --- | +| A1 | Multi-scheme **generation** | `service/scripts/bench_synthid_text.py` hardcodes `SCHEME = "synthid"` (line 63); MarkLLM checkout already ships `config/{KGW,SynthID,EXP,Unigram,SIR}.json` | `--scheme kgw|synthid|exp|unigram|sir` + `--config` override on the bench; 3 KGW strength JSONs (`research/configs/KGW-d1/d2/d4.json`) mirroring the checkout's `KGW.json` (γ=.25 δ=1; γ=.5 δ=2; γ=.5 δ=4) | small refactor (per 01 §10) | generate | +| A2 | Multi-scheme **detection** | `service/scripts/detect_text_watermark.py` scheme map (lines 41–45) covers `kgw`, `synthid` only | Add `exp`, `unigram`, `sir` entries + config wiring; same-config detection guarantee (01 §3: identical JSON for gen and detect) | small | detect | +| A3 | A7 cheap baselines | nothing | `research/scripts/attacks/cheap.py`: synonym substitution, 5–10% random word deletion, sentence reorder; deterministic + seeded | small | attack | +| A4 | Orchestrator stage wiring | `research/scripts/run_experiments.py` stages `generate/attack/detect/evaluate/report` all `raise NotImplementedError` (design/constants already aligned to the locked matrix) | Wire each stage to the repo scripts; emit documented results layout (`results/manifest.json`, per-cell `generated/attacked/scores.jsonl`, `metrics.json`, `report.md`); resume-able checkpoints | medium | all | +| A5 | Multilingual generator | opt-1.3b is English-only; **no GPU available** (verified 2026-08-18) | CPU path for `Qwen/Qwen2.5-1.5B-Instruct` (fallback 0.5B) via MarkLLM `TransformersConfig`; 5-prompt × 1-seed pilot quality check per language (01 §9) before the 750-cell grid | medium | generate (de/fr/es) | +| A6 | Corpus | `benchmarks/corpus/` has **8 files** (research/README previously said 9); factual, neutral, 50–90 words | 17 new EN prompts + 3×25 translated DE/FR/ES sets → `research/corpus/` (self-written, checkable claims) | medium | generate | +| A7 | Config pinning & manifest | `requirements-markllm.txt` pinned; MarkLLM checkout at `/home/guillaume/MarkLLM` | Record MarkLLM commit (`git -C /home/guillaume/MarkLLM rev-parse HEAD`), HF revisions (opt-1.3b, Qwen2.5-1.5B, gpt2-large, deberta-xlarge-mnli, all-MiniLM-L6-v2), `pip freeze` for MarkLLM + quality envs, `manifest.json` writer | small | reproducibility | + +## B. Analysis gaps (nothing exists — verified by grep, 2026-08-18) + +| # | Gap | Spec (from 01 §5) | Effort | +| --- | --- | --- | --- | +| B1 | ROC module | `research/scripts/analyze_roc.py`: AUROC, TPR@FPR ∈ {0.1%, 1%, 10%}, full ROC data per cell, **empirical null** from unwatermarked controls (never assume normal z-scores), 95% bootstrap CIs (10k resamples) | medium | +| B2 | Quality metrics | `research/scripts/evaluate_quality.py`: PPL (`gpt2-large`, never the generator), BERTScore (`deberta-xlarge-mnli`), ROUGE-L, SBERT cosine (`all-MiniLM-L6-v2`), Levenshtein %, length drift, number/URL survival; `research/requirements-quality.txt` (bert-score, rouge-score, sentence-transformers); stratified subset ~2–4k texts incl. every collapse point | medium | +| B3 | Table/figure generators | T1–T7 + F1–F6 per 02 §6: F2 ROC curves, F3 Pareto (PPL vs AUROC), F4 strength×length, F1 pipeline TikZ, F5 case study, F6 policy timeline | medium | + +## C. Paper gaps + +| # | Gap | Notes | +| --- | --- | --- | +| C1 | LaTeX skeleton | `research/paper/`, ACL-style template (fits venue ladder); abstract draft (02 §2), ethics draft (04 §3), acknowledgments (MarkLLM/THU-BPM) exist as markdown | +| C2 | Bibliography | `.bib` from 03 A–E; **§F candidates dropped** (see 03 header); re-verify every ID/venue at submission | +| C3 | Tables 1–7, Figures 1–6 | Specs in 02 §6; **require real numbers from the run** — no placeholders | +| C4 | Full prose §1–§10 | Skeleton in 02 §5; writing phase W2 | + +## D. The experiment run itself (the largest single item) + +- 3,500-cell locked matrix (01 §2): 7,000 generations, 56,000 attack + outputs, ~65,000 detections + ~2,000 null controls. +- Budget: ~$50–120 API rewrites (model + version recorded), ~2.5–4 + weeks wall on one 8-core CPU box (01 §6). +- Order: smoke test (01 §9) → EN core → temp/length axes → multilingual + pilot → multilingual grid → analysis (B1–B3) → tables/figures (C3). + +## E. Publishing logistics (user-owned) + +| # | Item | Who | Notes | +| --- | --- | --- | --- | +| E1 | arXiv account + endorsement | user | Solo, no affiliation (allowed); new authors in cs.CL typically need endorsement from an existing author — **start in week 1**, it can take days | +| E2 | Categories | user | cs.CL primary; cs.CR + cs.LG secondary | +| E3 | Release package | agent+user | Repo URL (github.com/guillaumemeyer/watermarks-remover), MIT license, corpus/configs/results JSONL; Zenodo DOI optional | +| E4 | Final upload | user | After the checklist below passes; arXiv license selection at upload | + +## F. arXiv v1 submission checklist + +- [ ] Title/abstract/authors finalized (02 §1-2; decision log) +- [ ] All Tables 2–7 contain real numbers (no placeholders) +- [ ] All Figures 1–6 rendered from real data +- [ ] Ethics statement (04 §3) + disclosure note (04 §5) in the PDF +- [ ] Acknowledgments (MarkLLM / THU-BPM) present +- [ ] Citations re-verified (03; IDs + venue labels re-checked at submission) +- [ ] Data-availability + reproducibility statements (01 §7-8) written +- [ ] Release links live (corpus, configs, results JSONL, harness) +- [ ] E1/E2 done; E4 performed + +## G. Deferred (v2, explicitly out of arXiv v1) + +- Human eval (20 × 3 annotators) — journal/ACL requirement. +- File/metadata mini-study (C2PA/EXIF strip; v0.1 01 §4.3) — separate + mini-experiment. +- Venue ladder beyond arXiv (NeurIPS 2026 workshop → ACL 2027 / TIFS). diff --git a/research/README.md b/research/README.md new file mode 100644 index 0000000..e46291f --- /dev/null +++ b/research/README.md @@ -0,0 +1,84 @@ +# research/ — Watermark Removal: Measurement Study & Paper Kit + +Working area for turning the `watermarks-remover` project into a +publishable research paper. As of the arXiv-v1 implementation PR +(https://github.com/guillaumemeyer/watermarks-remover/pull/174), the +full code/analysis gap list (05-A1..A7, B1..B3, C1/C2) is implemented +here; the multi-week experiment run (05-D) and publishing logistics +(05-E) remain. Generated run data stays gitignored (`research/results/`). + +## What is this paper about (one line) + +*How robust are deployed-class LLM text watermarking schemes (KGW, +SynthID-Text, EXP, Unigram, SIR) to realistic user-side editing, and +what does a layered (formatting + statistical) removal pipeline do to +detection-rate and text quality?* + +## File map + +| File | Contents | +| --- | --- | +| `01-experiment-protocol.md` | Full experimental protocol: locked v1 factorial, factors, attacks, detection & quality metrics, statistics, budget, smoke test, risk register | +| `02-paper-outline.md` | Locked title, abstract draft, venue plan (arXiv v1 active), section-by-section skeleton, exact tables/figures spec, writing phases | +| `03-related-work.md` | Verified citation list (arXiv IDs checked), grouped by theme, with "where we position vs each" notes | +| `04-ethics-and-legal.md` | Dual-use framing, EU AI Act Art. 50 analysis, ethics statement draft, data policy, disclosure notes | +| `05-arxiv-readiness.md` | **Gap analysis for arXiv v1**: every missing code/analysis/paper/logistics item, budget, submission checklist — what must be built before publishing | +| `scripts/run_experiments.py` | Orchestrator: locked factorial design, `--plan` budget mode, and fully wired generate/attack/detect/evaluate/report stages with resume markers (gap 05-A4) | +| `scripts/pins.py` | Reproducibility pins: MarkLLM commit, repo commit, HF revisions, pip freeze (gap 05-A7) | +| `scripts/multilingual_gen.py` | CPU Qwen2.5-1.5B-Instruct generator for DE/FR/ES cells (model holdout, gap 05-A5) | +| `scripts/attacks/cheap.py` | Deterministic cheap baselines: synonym / word-deletion / sentence-reorder (gap 05-A3) | +| `scripts/analyze_roc.py` | AUROC, TPR@FPR (empirical null), bootstrap CIs (gap 05-B1) | +| `scripts/evaluate_quality.py` | PPL/BERTScore/ROUGE-L/SBERT/Levenshtein + survival metrics (gap 05-B2) | +| `scripts/make_tables.py`, `scripts/make_figures.py` | Paper tables T1-T7 and figures F1-F6 (gap 05-B3) | +| `configs/` | Pinned scheme configs incl. KGW-d1/d2/d4 strength JSONs (same JSON for gen and detect) | +| `corpus/` | 25 EN + 3x25 DE/FR/ES factual prompts (gap 05-A6) | +| `tests/` | pytest suite for the research harness (`make research-check`) | +| `paper/` | arXiv v1 LaTeX skeleton + verified bibliography (gaps C1/C2) | + +## What already exists in the repo (reuse, don't rebuild) + +| Capability | Where | Notes | +| --- | --- | --- | +| Watermarked corpus generation (SynthID only) | `service/scripts/bench_synthid_text.py` | MarkLLM `facebook/opt-1.3b`, 300 tok default, `--seeds`, `--docs`, `--max-new-tokens`; multi-scheme support is gap 05-A1 | +| Layer A (Unicode/invisible chars) | `service/scripts/text_unicode.py` via `clean_text.py` | deterministic | +| Layer B rewrites | `service/scripts/rewrite_text.py` | strengths: `paraphrase`, `humanize`, `backtranslate`, `structural`, `code`; transports: `ollama`, `openai-compatible`; evaluation-loop w/ early stop | +| Detection (same-config, KGW/SynthID) | `service/scripts/detect_text_watermark.py`, `text_detectors.py` | EXP/Unigram/SIR detection is gap 05-A2 | +| Existing metrics | `bench_synthid_text.py` | clear rate, score suppression, lexical divergence, length drift, number/URL survival, token/USD cost | +| Corpus seeds | `benchmarks/corpus/` (**8 files** — README previously said 9) | factual, neutral, 50-90 words; 25 EN + 75 multilingual prompts needed (gap 05-A6) | +| How-to doc | `docs/synthid-text-benchmark.md` | | + +## Gap to close (this is the paper work) + +Everything missing between today and an arXiv v1 is itemized in +**`05-arxiv-readiness.md`** (verified code gaps, analysis gaps, paper +gaps, publishing logistics, submission checklist, budget). Nothing in +that list is implemented yet — it is the hand-off to the implementation +phase. + +## Status checklist + +- [x] Decide framing — **measurement study** (not "attack tool" paper); title locked in 02 §1 +- [x] Scope locked for arXiv v1 — Tier 1 core + multilingual (DE/FR/ES) + EXP/Unigram/SIR + length 500 + temp 1.0; attacks A0–A8; policy §7 in; human eval & file/metadata mini-study **deferred** (decision log below) +- [ ] Core experiment run (01 §2 matrix: 3,500 cells) — generate → attack → detect → evaluate +- [ ] Tables 1-7 + Figures 1-6 (02 §6) +- [ ] arXiv preprint (target: ~2-4 weeks after core results) +- [ ] Post-v1: workshop submission (NeurIPS 2026 workshops) → journal (TIFS) or ACL 2027 + +## Deferred (v2, not in arXiv v1) + +- Human eval (20 × 3 annotators) — journal requirement. +- File/metadata mini-study (C2PA/EXIF strip; v0.1 01 §4.3). + +## Decision log + +| Date | Decision | +| --- | --- | +| 2026-08-18 | Scoped paper as *empirical robustness measurement* with layered-attack contribution; primary metrics ROC-based; venue ladder = arXiv → NeurIPS 2026 workshop → TIFS. Research dir created, gitignored, not committed. | +| 2026-08-18 | **v1 scope locked** (Q&A): measurement framing; title 2 in 02 §1; matrix = 7 schemes (KGW×3, SynthID, EXP, Unigram, SIR) × lengths 100/300/500 × temps 0.7/1.0 × langs en/de/fr/es with restricted subsets = **3,500 cells**; attacks A0–A8; policy §7 in; **deferred**: human eval, file/metadata mini-study; rewrite backend = OpenAI-compatible API; solo author; full run first (~2-4 weeks). Multi-scheme bench/detector support, ROC + quality metrics, cheap.py, corpus expansion, and paper artifacts are tracked as gaps in 05-arxiv-readiness.md (not yet implemented). | + +## Git note + +The arXiv-v1 implementation PR adds `!/research/` allow rules to +`.gitignore`, so the paper kit is tracked from that point on. Only +`research/results/` (generated JSONL data, released via Zenodo) stays +ignored. diff --git a/research/configs/EXP.json b/research/configs/EXP.json new file mode 100644 index 0000000..bf08ceb --- /dev/null +++ b/research/configs/EXP.json @@ -0,0 +1,7 @@ +{ + "algorithm_name": "EXP", + "prefix_length": 4, + "hash_key": 15485863, + "threshold": 1e-4, + "sequence_length": 200 +} diff --git a/research/configs/KGW-d1.json b/research/configs/KGW-d1.json new file mode 100644 index 0000000..51dc8c3 --- /dev/null +++ b/research/configs/KGW-d1.json @@ -0,0 +1,10 @@ +{ + "algorithm_name": "KGW", + "gamma": 0.25, + "delta": 1.0, + "hash_key": 15485863, + "prefix_length": 1, + "z_threshold": 4.0, + "f_scheme": "time", + "window_scheme": "left" +} diff --git a/research/configs/KGW-d2.json b/research/configs/KGW-d2.json new file mode 100644 index 0000000..f42a6ad --- /dev/null +++ b/research/configs/KGW-d2.json @@ -0,0 +1,10 @@ +{ + "algorithm_name": "KGW", + "gamma": 0.5, + "delta": 2.0, + "hash_key": 15485863, + "prefix_length": 1, + "z_threshold": 4.0, + "f_scheme": "time", + "window_scheme": "left" +} diff --git a/research/configs/KGW-d4.json b/research/configs/KGW-d4.json new file mode 100644 index 0000000..3af7b5a --- /dev/null +++ b/research/configs/KGW-d4.json @@ -0,0 +1,10 @@ +{ + "algorithm_name": "KGW", + "gamma": 0.5, + "delta": 4.0, + "hash_key": 15485863, + "prefix_length": 1, + "z_threshold": 4.0, + "f_scheme": "time", + "window_scheme": "left" +} diff --git a/research/configs/SIR.json b/research/configs/SIR.json new file mode 100644 index 0000000..917b2bd --- /dev/null +++ b/research/configs/SIR.json @@ -0,0 +1,11 @@ +{ + "algorithm_name": "SIR", + "delta": 1.0, + "chunk_length": 10, + "scale_dimension": 300, + "z_threshold": 0.2, + "transform_model_input_dim": 1024, + "transform_model_name": "watermark/sir/model/transform_model_cbert.pth", + "embedding_model_path": "watermark/sir/model/compositional-bert-large-uncased/", + "mapping_name": "watermark/sir/mapping/300_mapping_50272.json" +} diff --git a/research/configs/SynthID.json b/research/configs/SynthID.json new file mode 100644 index 0000000..b903f1f --- /dev/null +++ b/research/configs/SynthID.json @@ -0,0 +1,16 @@ +{ + "algorithm_name": "SynthID", + "ngram_len": 5, + "keys": [ + 654, 400, 836, 123, 340, 443, 597, 160, 57, 29, + 590, 639, 13, 715, 468, 990, 966, 226, 324, 585, + 118, 504, 421, 521, 129, 669, 732, 225, 90, 960 + ], + "sampling_table_size": 65536, + "sampling_table_seed": 0, + "watermark_mode": "non-distortionary", + "num_leaves": 2, + "context_history_size": 1024, + "detector_type": "mean", + "threshold": 0.52 +} diff --git a/research/configs/Unigram.json b/research/configs/Unigram.json new file mode 100644 index 0000000..2dfa9c7 --- /dev/null +++ b/research/configs/Unigram.json @@ -0,0 +1,7 @@ +{ + "algorithm_name": "Unigram", + "gamma": 0.5, + "delta": 2.0, + "hash_key": 15485863, + "z_threshold": 4.0 +} diff --git a/research/corpus/de/01.txt b/research/corpus/de/01.txt new file mode 100644 index 0000000..fff02f9 --- /dev/null +++ b/research/corpus/de/01.txt @@ -0,0 +1 @@ +Cloud-Computing ermöglicht es Organisationen, Rechenleistung, Speicher und Netzwerkressourcen zu mieten, statt eigene physische Server zu besitzen. Die Hauptmodelle sind Infrastructure as a Service, Platform as a Service und Software as a Service, die jeweils unterschiedlich viel Betriebsverantwortung an den Anbieter abgeben. Die Abrechnung erfolgt meist nutzungsbasiert, was Start-ups hilft, ohne große Anfangsinvestitionen zu wachsen. Zu den Abwägungen gehören Anbieterbindung, Datenresidenz-Vorschriften und die Notwendigkeit, Kosten bei unerwartet wachsenden Arbeitslasten sorgfältig zu überwachen. diff --git a/research/corpus/de/02.txt b/research/corpus/de/02.txt new file mode 100644 index 0000000..5b78cf9 --- /dev/null +++ b/research/corpus/de/02.txt @@ -0,0 +1 @@ +Ein gleichmäßiger Kaffee hängt von einigen Variablen ab: Mahlgrad, Wassertemperatur, Dosis und Brühzeit. Feinere Mahlgrade extrahieren schneller, können aber bitter werden, wenn das Wasser zu heiß ist. Ein übliches Ausgangsverhältnis sind sechzig Gramm Kaffee pro Liter Wasser, angepasst an den eigenen Geschmack. Aufgusmethoden brauchen eine ruhige Hand und ein ebenes Kaffeebett, während Immersionsmethoden wie die French Press verzeihender sind. Bohnen in einem luftdichten Behälter und fern von Licht aufbewahrt, halten ihr Aroma länger als in der Originaltüte. diff --git a/research/corpus/de/03.txt b/research/corpus/de/03.txt new file mode 100644 index 0000000..bffd9ef --- /dev/null +++ b/research/corpus/de/03.txt @@ -0,0 +1 @@ +Eine Tagestour in den Bergen erfordert mehr als Wasser und Snacks. Regenkleidung, eine Karte, eine Stirnlampe und ein Erste-Hilfe-Set gehören in jeden Rucksack, auch an klaren Morgen. Das Wetter in großer Höhe kann sich innerhalb einer Stunde ändern, und oberhalb der Baumgrenze sind Wege oft unmarkiert. Teilen Sie jemandem Ihre geplante Route und die voraussichtliche Rückkehrzeit mit, bevor Sie starten. Prüfen Sie die Vorhersage zweimal, packen Sie zusätzliche Schichten ein und kehren Sie früh um, wenn die Bedingungen instabil wirken, statt zum Gipfel weiterzugehen. diff --git a/research/corpus/de/04.txt b/research/corpus/de/04.txt new file mode 100644 index 0000000..df50672 --- /dev/null +++ b/research/corpus/de/04.txt @@ -0,0 +1 @@ +Viele populäre Ernährungsbehauptungen werden nur teilweise durch Belege gestützt. Fett zu essen führt nicht automatisch zu Gewichtszunahme; die gesamte Kalorienzufuhr zählt mehr als ein einzelner Makronährstoff. Das Auslassen des Frühstücks verlangsamt bei den meisten Menschen den Stoffwechsel nicht, trotz des verbreiteten Rats, früh zu essen. Bioprodukte reduzieren die Pestizidbelastung, sind aber nicht messbar nährstoffreicher als konventionelle Erzeugnisse. Das konsistenteste Forschungsergebnis ist, dass eine abwechslungsreiche Ernährung mit viel Gemüse jedes einzelne Superfood oder Nahrungsergänzungsmittel übertrifft. diff --git a/research/corpus/de/05.txt b/research/corpus/de/05.txt new file mode 100644 index 0000000..6b859fd --- /dev/null +++ b/research/corpus/de/05.txt @@ -0,0 +1 @@ +Open-Source-Lizenzen unterscheiden sich vor allem darin, was sie bei der Weitergabe von Code verlangen. Permissive Lizenzen wie MIT und Apache 2.0 erlauben fast jede Nutzung, einschließlich proprietärer Ableger, solange der Urheberrechtshinweis erhalten bleibt. Copyleft-Lizenzen wie die GPL verlangen, dass abgeleitete Werke unter denselben Bedingungen veröffentlicht werden. Apache 2.0 ergänzt eine ausdrückliche Patentlizenz, die MIT nicht enthält. Projekte sollten auch Beitragsvereinbarungen bedenken, denn das Annehmen fremden Codes ohne Lizenz kann rechtliche Unklarheiten über die Urheberschaft des Beitrags schaffen. diff --git a/research/corpus/de/06.txt b/research/corpus/de/06.txt new file mode 100644 index 0000000..5881770 --- /dev/null +++ b/research/corpus/de/06.txt @@ -0,0 +1 @@ +Erneuerbare Energien erzeugen inzwischen etwa dreißig Prozent des weltweiten Stroms. Solar- und Windkraft sind im letzten Jahrzehnt schneller gewachsen als jede andere Quelle, während die Kosten für Batteriespeicher stark gefallen sind. Netzbetreiber stehen weiterhin vor der Herausforderung, Angebot und Nachfrage auszugleichen, wenn die Sonne nicht scheint und der Wind nicht weht. Mehrere Länder investieren in Langzeitspeicher und grenzüberschreitende Verbindungen, um diese Lücken zu glätten und die Abhängigkeit von fossilen Brennstoffen in Spitzenzeiten zu verringern. diff --git a/research/corpus/de/07.txt b/research/corpus/de/07.txt new file mode 100644 index 0000000..723e502 --- /dev/null +++ b/research/corpus/de/07.txt @@ -0,0 +1 @@ +Kleine Unternehmen verfolgen oft vor allem drei Kennzahlen: verfügbares Bargeld, monatliche Verbrauchsrate und Bruttomarge. Die Barreserven bestimmen, wie viele Monate das Unternehmen ohne neue Einnahmen überleben kann. Die Verbrauchsrate zeigt, wie schnell diese Reserven aufgebraucht werden, und die Bruttomarge zeigt, wie viel von jedem Verkauf die Fixkosten deckt. Kreditgeber und Investoren verlangen in der Regel zwölf Monate Finanzhistorie, eine aktuelle Bilanz und eine realistische Prognose, bevor sie Kapital in ein junges Unternehmen stecken. diff --git a/research/corpus/de/08.txt b/research/corpus/de/08.txt new file mode 100644 index 0000000..6f4fa72 --- /dev/null +++ b/research/corpus/de/08.txt @@ -0,0 +1 @@ +Venedig wurde im fünften Jahrhundert auf sumpfigen Inseln gegründet und wuchs bis zum Mittelalter zu einer bedeutenden Handelsrepublik heran. Seine Flotte kontrollierte Routen über das östliche Mittelmeer, und seine Kaufleute finanzierten den Handel mit Gewürzen, Seide und Glas. Das politische System der Stadt war bewusst komplex, mit gewählten Dogen und Räten, die verhindern sollten, dass eine einzelne Familie dominiert. Die heute erhaltenen Kanäle und Paläste ziehen jedes Jahr Millionen von Besuchern an, obwohl steigende Wasserstände Teile des historischen Zentrums bedrohen. diff --git a/research/corpus/de/09.txt b/research/corpus/de/09.txt new file mode 100644 index 0000000..e37b8c7 --- /dev/null +++ b/research/corpus/de/09.txt @@ -0,0 +1 @@ +Der Mars hat zwei kleine Monde, Phobos und Deimos, beide benannt nach Figuren der griechischen Mythologie. Der Planet Jupiter hat mehr als neunzig bestätigte Monde, der größte ist Ganymed, der größer ist als der Planet Merkur. Die Ringe des Saturn bestehen hauptsächlich aus Eisteilchen, von winzigen Körnern bis zu hausgroßen Blöcken. Das James-Webb-Weltraumteleskop, gestartet 2021, beobachtet Infrarotlicht aus fernen Galaxien. Sonnenfinsternisse entstehen, wenn der Mond direkt zwischen Sonne und Erde steht. diff --git a/research/corpus/de/10.txt b/research/corpus/de/10.txt new file mode 100644 index 0000000..6168e90 --- /dev/null +++ b/research/corpus/de/10.txt @@ -0,0 +1 @@ +Die Erdkruste ist in tektonische Platten unterteilt, die sich einige Zentimeter pro Jahr bewegen. Die meisten Erdbeben und Vulkane treten an Plattengrenzen auf. Der Himalaya entstand, als die Indische Platte vor etwa fünfzig Millionen Jahren mit der Eurasischen Platte kollidierte. Der Gesteinskreislauf beschreibt, wie magmatische, sedimentäre und metamorphe Gesteine sich im Lauf der Zeit ineinander umwandeln. Basalt ist das häufigste vulkanische Gestein der Erde und bedeckt weite Teile des Meeresbodens. Verwitterung und Erosion formen Landschaften, indem sie Gesteinsoberflächen abbauen. diff --git a/research/corpus/de/11.txt b/research/corpus/de/11.txt new file mode 100644 index 0000000..7c9da43 --- /dev/null +++ b/research/corpus/de/11.txt @@ -0,0 +1 @@ +Der Afrikanische Elefant ist das größte Landtier, erwachsene Männchen wiegen bis zu sechstausend Kilogramm. Geparden sind die schnellsten Landsäugetiere und erreichen in kurzen Sprints etwa hundert Kilometer pro Stunde. Honigbienen teilen den Standort von Nahrung durch Schwänzeltänze mit. Eisbären haben unter ihrem weißen Fell schwarze Haut, um Sonnenlicht aufzunehmen. Kraken haben drei Herzen und blaues Blut. Viele Vogelarten wandern jedes Jahr tausende Kilometer zwischen Brut- und Überwinterungsgebieten. diff --git a/research/corpus/de/12.txt b/research/corpus/de/12.txt new file mode 100644 index 0000000..48f7b41 --- /dev/null +++ b/research/corpus/de/12.txt @@ -0,0 +1 @@ +Ludwig van Beethoven komponierte neun Sinfonien, die letzte vollendete er, als er fast völlig taub war. Johann Sebastian Bach schrieb mehr als zweihundert Kantaten, während er als Kirchenmusiker in Leipzig arbeitete. Im achtzehnten Jahrhundert ersetzte das Klavier das Cembalo als wichtigstes Tasteninstrument. Thomas Edison erfand 1877 den Phonographen, das erste Gerät, das Klang aufnehmen und wiedergeben konnte. Der später entwickelte Grammophon nutzte flache Schallplatten statt Zylinder, was sich schließlich zum Standardformat entwickelte. diff --git a/research/corpus/de/13.txt b/research/corpus/de/13.txt new file mode 100644 index 0000000..cd3e91a --- /dev/null +++ b/research/corpus/de/13.txt @@ -0,0 +1 @@ +Lebensmittelsicherheitsregeln empfehlen, verderbliche Lebensmittel aus dem Temperaturbereich zwischen vier und sechzig Grad Celsius herauszuhalten, der oft als Gefahrenzone bezeichnet wird. In diesem Bereich vermehren sich Bakterien am schnellsten. Gekochtes Fleisch sollte eine Kerntemperatur von mindestens siebzig Grad Celsius erreichen, um schädliche Krankheitserreger abzutöten. Reste sollten innerhalb von zwei Stunden nach dem Kochen gekühlt werden. Schneidebretter für rohes Fleisch sollten gründlich gewaschen werden, bevor Gemüse zubereitet wird. Einfrieren stoppt das Bakterienwachstum, tötet aber nicht alle Mikroorganismen ab. diff --git a/research/corpus/de/14.txt b/research/corpus/de/14.txt new file mode 100644 index 0000000..d286842 --- /dev/null +++ b/research/corpus/de/14.txt @@ -0,0 +1 @@ +Containerschiffe befördern den größten Teil der weltweit gehandelten Güter, die größten Schiffe transportieren mehr als zwanzigtausend Container. Die ersten modernen Autobahnsysteme wurden in den 1930er Jahren in Deutschland gebaut. Hochgeschwindigkeitszüge in Japan und Frankreich überschreiten im Regelbetrieb dreihundert Kilometer pro Stunde. Elektrobusse erzeugen am Einsatzort keine Abgasemissionen, ihre Umweltwirkung hängt jedoch von der Stromquelle ab. Kreisverkehre verringern die Schwere von Kreuzungsunfällen im Vergleich zu herkömmlichen Kreuzungen. diff --git a/research/corpus/de/15.txt b/research/corpus/de/15.txt new file mode 100644 index 0000000..12b392d --- /dev/null +++ b/research/corpus/de/15.txt @@ -0,0 +1 @@ +Fruchtwechsel ist die Praxis, in aufeinanderfolgenden Saisons verschiedene Pflanzen auf demselben Feld anzubauen, um die Bodenfruchtbarkeit zu erhalten. Hülsenfrüchte wie Erbsen und Bohnen reichern den Boden mit Stickstoff an und verringern so den Bedarf an synthetischen Düngemitteln. Tröpfchenbewässerung führt Wasser direkt an die Pflanzenwurzeln und kann den Wasserverbrauch gegenüber der Flutbewässerung senken. Weizen, Reis und Mais sind die drei weltweit am häufigsten angebauten Getreidearten. Die moderne Pflanzenzüchtung hat Sorten mit höheren Erträgen und besserer Resistenz gegen Schädlinge und Krankheiten entwickelt. diff --git a/research/corpus/de/16.txt b/research/corpus/de/16.txt new file mode 100644 index 0000000..05bd2c9 --- /dev/null +++ b/research/corpus/de/16.txt @@ -0,0 +1 @@ +Die Industrielle Revolution begann im späten achtzehnten Jahrhundert in Großbritannien und breitete sich über Europa und Nordamerika aus. Dampfmaschinen trieben Fabriken und Eisenbahnen an und veränderten Produktion und Verkehr. Die Einführung des Fließbands durch Henry Ford im Jahr 1913 verkürzte die Bauzeit eines Autos erheblich. Die Weltwirtschaftskrise der 1930er Jahre verursachte weltweit Massenarbeitslosigkeit und Bankenzusammenbrüche. Nach dem Zweiten Weltkrieg führten viele Länder politische Maßnahmen ein, um ihre Volkswirtschaften wieder aufzubauen und den internationalen Handel auszuweiten. diff --git a/research/corpus/de/17.txt b/research/corpus/de/17.txt new file mode 100644 index 0000000..bc8ee6e --- /dev/null +++ b/research/corpus/de/17.txt @@ -0,0 +1 @@ +Der Buchdruck, um 1450 von Johannes Gutenberg eingeführt, machte Bücher billiger und verbreitete die Lesefähigkeit in ganz Europa. Der Roman entwickelte sich im achtzehnten Jahrhundert zur wichtigsten literarischen Form, mit Werken von Autoren wie Daniel Defoe und Jane Austen. Shakespeare schrieb etwa siebenunddreißig Theaterstücke, darunter Tragödien, Komödien und Historiendramen. Lyrik nutzt Versmaß, Reim und Bildsprache, um Bedeutung zu erzeugen. Bibliotheken bewahren schriftliche Werke und machen sie der Öffentlichkeit zugänglich, was Bildung und Forschung unterstützt. diff --git a/research/corpus/de/18.txt b/research/corpus/de/18.txt new file mode 100644 index 0000000..bf3587e --- /dev/null +++ b/research/corpus/de/18.txt @@ -0,0 +1 @@ +Impfungen trainieren das Immunsystem, einen Krankheitserreger zu erkennen, bevor er eine Krankheit auslöst. Edward Jenner führte 1796 die erste Impfung durch, indem er Kuhpocken gegen Pocken einsetzte. Sauberes Trinkwasser und Abwasserbehandlung haben mehr Leben gerettet als jede einzelne medizinische Behandlung. Regelmäßige Kinderimpfungen verhindern jedes Jahr Millionen von Todesfällen. Händewaschen mit Seife verringert die Übertragung von Atemwegs- und Durchfallerkrankungen. Regelmäßige körperliche Aktivität senkt das Risiko für Herzkrankheiten, Schlaganfall und Typ-2-Diabetes. diff --git a/research/corpus/de/19.txt b/research/corpus/de/19.txt new file mode 100644 index 0000000..49cd096 --- /dev/null +++ b/research/corpus/de/19.txt @@ -0,0 +1 @@ +Batterien speichern elektrische Energie als chemische Energie und geben sie bei Bedarf wieder ab. Lithium-Ionen-Batterien, Anfang der 1990er Jahre erstmals kommerzialisiert, versorgen die meisten tragbaren Elektronikgeräte und Elektrofahrzeuge. Pumpspeicherkraftwerke sind weltweit die größte Netzspeichertechnologie; sie bewegen Wasser zwischen Stauseen auf unterschiedlichen Höhen. Druckluftspeicher und Flow-Batterien werden für Langzeitanwendungen entwickelt. Netzspeicher helfen, Angebot und Nachfrage auszugleichen, wenn die erneuerbare Erzeugung schwankt. diff --git a/research/corpus/de/20.txt b/research/corpus/de/20.txt new file mode 100644 index 0000000..75298f0 --- /dev/null +++ b/research/corpus/de/20.txt @@ -0,0 +1 @@ +Stadtplanung gestaltet das Wachstum von Städten und bringt Wohnen, Verkehr und öffentlichen Raum in Einklang. Die Zonierung trennt Wohn-, Gewerbe- und Industriegebiete. Gemischt genutzte Viertel bringen Wohnungen, Geschäfte und Arbeitsplätze näher zusammen und verringern die Abhängigkeit vom Auto. Grünflächen in Städten senken die Temperaturen im Sommer und absorbieren Regenwasser. Öffentliche Verkehrssysteme befördern viele Fahrgäste mit weniger Platz und Energie pro Person als private Autos. Fußgängerfreundliche Straßen verbessern die Sicherheit und fördern Gehen und Radfahren. diff --git a/research/corpus/de/21.txt b/research/corpus/de/21.txt new file mode 100644 index 0000000..ac739cd --- /dev/null +++ b/research/corpus/de/21.txt @@ -0,0 +1 @@ +Korallenriffe gehören zu den artenreichsten Ökosystemen der Erde und beherbergen etwa ein Viertel der Meeresarten. Riffe entstehen, wenn Korallenpolypen Kalziumkarbonat-Skelette absondern. Der tiefste bekannte Punkt des Ozeans ist der Marianengraben, mehr als zehntausend Meter unter dem Meeresspiegel. Wale wandern über weite Strecken zwischen Nahrungs- und Brutgebieten. Überfischung und steigende Wassertemperaturen sind große Bedrohungen für die Gesundheit der Riffe. Meeresschutzgebiete geben Lebensräumen Zeit, sich von menschlichen Belastungen zu erholen. diff --git a/research/corpus/de/22.txt b/research/corpus/de/22.txt new file mode 100644 index 0000000..bea8f86 --- /dev/null +++ b/research/corpus/de/22.txt @@ -0,0 +1 @@ +Meteorologie ist die Wissenschaft vom Wetter und der Atmosphäre. Wettervorhersagen stützen sich auf Computermodelle, die atmosphärische Bedingungen simulieren. Warmfronten und Kaltfronten markieren Grenzen zwischen Luftmassen unterschiedlicher Temperatur. Gewitter entstehen, wenn warme, feuchte Luft schnell aufsteigt und kondensiert. Die Hurrikansaison variiert je nach Ozeanbecken; im Atlantik dauert sie von Juni bis November. Satellitenbeobachtungen haben die Genauigkeit von Wetterwarnungen vor schweren Ereignissen erheblich verbessert. diff --git a/research/corpus/de/23.txt b/research/corpus/de/23.txt new file mode 100644 index 0000000..bea69e0 --- /dev/null +++ b/research/corpus/de/23.txt @@ -0,0 +1 @@ +Sprachen werden nach gemeinsamer Abstammung in Familien eingeteilt. Englisch, Deutsch, Französisch, Spanisch und Russisch gehören alle zur indoeuropäischen Familie. Sprachwissenschaftler rekonstruieren ältere Stufen wie das Indogermanische, indem sie moderne Sprachen vergleichen. Schriftsysteme entwickelten sich unabhängig in Mesopotamien, China und Mesoamerika. Das lateinische Alphabet, das viele europäische Sprachen verwenden, stammt vom griechischen Alphabet ab. Sprachwandel ist allmählich, und verwandte Sprachen entfernen sich über lange Trennungszeiten voneinander. diff --git a/research/corpus/de/24.txt b/research/corpus/de/24.txt new file mode 100644 index 0000000..b3b2271 --- /dev/null +++ b/research/corpus/de/24.txt @@ -0,0 +1 @@ +Die Materialwissenschaft untersucht die Struktur und die Eigenschaften von Materialien und wie sie zusammenhängen. Stahl, eine Legierung aus Eisen und Kohlenstoff, ist sowohl auf Zug als auch auf Druck stark, was ihn zentral für das Bauwesen macht. Beton hält Druck gut aus, ist aber auf Zug schwach, daher wird Stahlbeton mit Stahlstäben kombiniert. Polymere wie Polyethylen sind leicht und korrosionsbeständig. Verbundwerkstoffe kombinieren Fasern und eine Matrix, um bei geringem Gewicht hohe Festigkeit zu erreichen, etwa in Flugzeugen und Sportgeräten. diff --git a/research/corpus/de/25.txt b/research/corpus/de/25.txt new file mode 100644 index 0000000..2dbe579 --- /dev/null +++ b/research/corpus/de/25.txt @@ -0,0 +1 @@ +Die Sportwissenschaft untersucht, wie der Körper auf Belastung und Training reagiert. Ausdauertraining erhöht die Fähigkeit des Herzens, Blut zu pumpen, und verbessert die Sauerstoffversorgung der Muskeln. Krafttraining mit progressiv schwereren Lasten baut Muskelkraft und -größe auf. Dehnen vor oder nach dem Sport hat im Vergleich zu einem richtigen Aufwärmen bescheidene Auswirkungen auf das Verletzungsrisiko. Ausreichend Schlaf und Ernährung sind wichtig für die Erholung nach intensivem Training. Sportler nutzen Herzfrequenzmesser, um die Trainingsintensität zu überwachen und Übertraining zu vermeiden. diff --git a/research/corpus/en/01.txt b/research/corpus/en/01.txt new file mode 100644 index 0000000..a77ad00 --- /dev/null +++ b/research/corpus/en/01.txt @@ -0,0 +1 @@ +Cloud computing lets organizations rent compute, storage, and networking capacity instead of owning physical servers. The main models are infrastructure as a service, platform as a service, and software as a service, each shifting a different amount of operational responsibility to the provider. Pricing is usually usage-based, which helps startups scale without large upfront capital spending. The trade-offs include vendor lock-in, data residency rules, and the need for careful cost monitoring when workloads grow unexpectedly. \ No newline at end of file diff --git a/research/corpus/en/02.txt b/research/corpus/en/02.txt new file mode 100644 index 0000000..f6f0874 --- /dev/null +++ b/research/corpus/en/02.txt @@ -0,0 +1 @@ +A consistent cup of coffee depends on a few variables: grind size, water temperature, dose, and brew time. Finer grinds extract faster but can turn bitter if the water is too hot. A common starting ratio is sixty grams of coffee per liter of water, adjusted to taste. Pour-over methods need a steady hand and a flat bed of grounds, while immersion methods like the French press are more forgiving. Storing beans in an airtight container away from light preserves flavor longer than leaving them in the bag. \ No newline at end of file diff --git a/research/corpus/en/03.txt b/research/corpus/en/03.txt new file mode 100644 index 0000000..08e5cb0 --- /dev/null +++ b/research/corpus/en/03.txt @@ -0,0 +1 @@ +A day hike in the mountains calls for more than water and snacks. Rain gear, a map, a headlamp, and a basic first aid kit should be in every pack, even on clear mornings. Weather in high terrain can change within an hour, and trails are often unmarked above the treeline. Tell someone your planned route and expected return time before you start. Check the forecast twice, carry extra layers, and turn back early if conditions look unstable rather than pushing to the summit. \ No newline at end of file diff --git a/research/corpus/en/04.txt b/research/corpus/en/04.txt new file mode 100644 index 0000000..7b73c45 --- /dev/null +++ b/research/corpus/en/04.txt @@ -0,0 +1 @@ +Many popular nutrition claims are only partly supported by evidence. Eating fat does not automatically cause weight gain; total calorie intake matters more than any single macronutrient. Skipping breakfast does not slow metabolism for most people, despite the common advice to eat early. Organic produce reduces pesticide exposure but is not measurably more nutritious than conventional crops. The most consistent finding in the research is that a varied diet with plenty of vegetables beats any single superfood or supplement. \ No newline at end of file diff --git a/research/corpus/en/05.txt b/research/corpus/en/05.txt new file mode 100644 index 0000000..e5046fb --- /dev/null +++ b/research/corpus/en/05.txt @@ -0,0 +1 @@ +Open source licenses differ mainly in what they require when code is redistributed. Permissive licenses like MIT and Apache 2.0 allow almost any use, including proprietary forks, as long as the copyright notice is kept. Copyleft licenses like GPL require derivative works to be released under the same terms. Apache 2.0 adds an explicit patent grant that MIT lacks. Projects should also consider contributor agreements, because accepting outside code without a license can create legal ambiguity about who owns the contribution. \ No newline at end of file diff --git a/research/corpus/en/06.txt b/research/corpus/en/06.txt new file mode 100644 index 0000000..7841072 --- /dev/null +++ b/research/corpus/en/06.txt @@ -0,0 +1 @@ +Renewable energy now accounts for about thirty percent of global electricity generation. Solar and wind capacity have grown faster than any other source over the past decade, while battery storage costs have fallen sharply. Grid operators still face challenges balancing supply when the sun does not shine and the wind does not blow. Several countries are investing in long-duration storage and cross-border interconnectors to smooth those gaps and reduce reliance on fossil fuels during peak demand. \ No newline at end of file diff --git a/research/corpus/en/07.txt b/research/corpus/en/07.txt new file mode 100644 index 0000000..c1c0d48 --- /dev/null +++ b/research/corpus/en/07.txt @@ -0,0 +1 @@ +Small businesses often track three numbers above all others: cash on hand, monthly burn rate, and gross margin. Cash reserves determine how many months the company can survive without new revenue. The burn rate shows how quickly those reserves are spent, and gross margin reveals how much of each sale covers fixed costs. Lenders and investors typically ask for twelve months of financial history, a current balance sheet, and a realistic forecast before committing capital to a young company. \ No newline at end of file diff --git a/research/corpus/en/08.txt b/research/corpus/en/08.txt new file mode 100644 index 0000000..b14dfff --- /dev/null +++ b/research/corpus/en/08.txt @@ -0,0 +1 @@ +Venice was founded on marshy islands in the fifth century and grew into a major trading republic by the Middle Ages. Its fleet controlled routes across the eastern Mediterranean, and its merchants financed trade in spices, silk, and glass. The city's political system was deliberately complex, with elected doges and councils designed to prevent any single family from dominating. The canals and palaces that survive today draw millions of visitors each year, although rising water levels now threaten parts of the historic center. \ No newline at end of file diff --git a/research/corpus/en/09.txt b/research/corpus/en/09.txt new file mode 100644 index 0000000..c063e86 --- /dev/null +++ b/research/corpus/en/09.txt @@ -0,0 +1 @@ +Mars has two small moons, Phobos and Deimos, both named after figures from Greek mythology. The planet Jupiter has more than ninety confirmed moons, the largest being Ganymede, which is bigger than the planet Mercury. Saturn's rings consist mainly of ice particles ranging from tiny grains to house-sized boulders. The James Webb Space Telescope, launched in 2021, observes infrared light from distant galaxies. Solar eclipses occur when the Moon passes directly between the Sun and the Earth. diff --git a/research/corpus/en/10.txt b/research/corpus/en/10.txt new file mode 100644 index 0000000..baa8ec7 --- /dev/null +++ b/research/corpus/en/10.txt @@ -0,0 +1 @@ +The Earth's crust is divided into tectonic plates that move a few centimeters per year. Most earthquakes and volcanoes occur along plate boundaries. The Himalayas formed when the Indian plate collided with the Eurasian plate about fifty million years ago. The rock cycle describes how igneous, sedimentary, and metamorphic rocks transform into one another over time. Basalt is the most common volcanic rock on Earth, covering much of the ocean floor. Weathering and erosion shape landscapes by breaking down rock surfaces. diff --git a/research/corpus/en/11.txt b/research/corpus/en/11.txt new file mode 100644 index 0000000..61ec671 --- /dev/null +++ b/research/corpus/en/11.txt @@ -0,0 +1 @@ +The African elephant is the largest land animal, with adult males weighing up to six thousand kilograms. Cheetahs are the fastest land mammals, reaching speeds of about one hundred kilometers per hour in short bursts. Honeybees communicate the location of food through waggle dances. Polar bears have black skin beneath their white fur to absorb sunlight. Octopuses have three hearts and blue blood. Many bird species migrate thousands of kilometers each year between breeding and wintering grounds. diff --git a/research/corpus/en/12.txt b/research/corpus/en/12.txt new file mode 100644 index 0000000..f09ee42 --- /dev/null +++ b/research/corpus/en/12.txt @@ -0,0 +1 @@ +Ludwig van Beethoven composed nine symphonies, the last completed when he was almost completely deaf. Johann Sebastian Bach wrote more than two hundred cantatas while working as church musician in Leipzig. The piano replaced the harpsichord as the main keyboard instrument during the eighteenth century. Thomas Edison invented the phonograph in 1877, the first device able to record and replay sound. The gramophone, developed later, used flat discs instead of cylinders, which eventually became the standard format. diff --git a/research/corpus/en/13.txt b/research/corpus/en/13.txt new file mode 100644 index 0000000..305c9bd --- /dev/null +++ b/research/corpus/en/13.txt @@ -0,0 +1 @@ +Food safety rules recommend keeping perishable food out of the temperature range between four and sixty degrees Celsius, often called the danger zone. Bacteria multiply most quickly in this range. Cooked meat should reach an internal temperature of at least seventy degrees Celsius to kill harmful pathogens. Leftovers should be refrigerated within two hours of cooking. Cutting boards used for raw meat should be washed thoroughly before preparing vegetables. Freezing food stops bacterial growth but does not kill all microorganisms. diff --git a/research/corpus/en/14.txt b/research/corpus/en/14.txt new file mode 100644 index 0000000..82608ac --- /dev/null +++ b/research/corpus/en/14.txt @@ -0,0 +1 @@ +Container ships carry the majority of globally traded goods, with the largest vessels transporting more than twenty thousand containers. The first modern highway systems were built in Germany in the 1930s. High-speed rail trains in Japan and France routinely exceed three hundred kilometers per hour. Electric buses produce no exhaust emissions at the point of use, though their environmental impact depends on the electricity source. Roundabouts reduce the severity of intersection crashes compared with traditional crossings. diff --git a/research/corpus/en/15.txt b/research/corpus/en/15.txt new file mode 100644 index 0000000..7f058b1 --- /dev/null +++ b/research/corpus/en/15.txt @@ -0,0 +1 @@ +Crop rotation is the practice of growing different crops in the same field in successive seasons to maintain soil fertility. Legumes such as peas and beans add nitrogen to the soil, reducing the need for synthetic fertilizers. Drip irrigation delivers water directly to plant roots and can reduce water use compared with flood irrigation. Wheat, rice, and maize are the three most widely cultivated cereal crops in the world. Modern plant breeding has developed varieties with higher yields and better resistance to pests and diseases. diff --git a/research/corpus/en/16.txt b/research/corpus/en/16.txt new file mode 100644 index 0000000..8f99acd --- /dev/null +++ b/research/corpus/en/16.txt @@ -0,0 +1 @@ +The Industrial Revolution began in Britain in the late eighteenth century and spread across Europe and North America. Steam engines powered factories and railways, transforming manufacturing and transport. The introduction of the assembly line by Henry Ford in 1913 greatly reduced the time needed to build a car. The Great Depression of the 1930s caused mass unemployment and bank failures worldwide. After the Second World War, many countries adopted policies to rebuild their economies and expand international trade. diff --git a/research/corpus/en/17.txt b/research/corpus/en/17.txt new file mode 100644 index 0000000..b9cb11c --- /dev/null +++ b/research/corpus/en/17.txt @@ -0,0 +1 @@ +The printing press, introduced by Johannes Gutenberg around 1450, made books cheaper and spread literacy across Europe. The novel developed as a major literary form in the eighteenth century, with works by writers such as Daniel Defoe and Jane Austen. Shakespeare wrote about thirty-seven plays, including tragedies, comedies, and histories. Poetry uses meter, rhyme, and imagery to create meaning. Libraries preserve written works and make them accessible to the public, supporting education and research. diff --git a/research/corpus/en/18.txt b/research/corpus/en/18.txt new file mode 100644 index 0000000..0a7bc36 --- /dev/null +++ b/research/corpus/en/18.txt @@ -0,0 +1 @@ +Vaccination works by training the immune system to recognize a pathogen before it causes disease. Edward Jenner performed the first vaccination in 1796, using cowpox to protect against smallpox. Clean drinking water and sewage treatment have saved more lives than any single medical treatment. Routine childhood immunization prevents millions of deaths each year. Handwashing with soap reduces the transmission of respiratory and diarrheal diseases. Regular physical activity lowers the risk of heart disease, stroke, and type 2 diabetes. diff --git a/research/corpus/en/19.txt b/research/corpus/en/19.txt new file mode 100644 index 0000000..3a1dc2a --- /dev/null +++ b/research/corpus/en/19.txt @@ -0,0 +1 @@ +Batteries store electrical energy as chemical energy and release it when needed. Lithium-ion batteries, first commercialized in the early 1990s, power most portable electronics and electric vehicles. Pumped hydroelectric storage is the largest grid storage technology worldwide, moving water between reservoirs at different heights. Compressed air energy storage and flow batteries are being developed for long-duration applications. Grid-scale batteries help balance supply and demand when renewable generation fluctuates. diff --git a/research/corpus/en/20.txt b/research/corpus/en/20.txt new file mode 100644 index 0000000..51d15d5 --- /dev/null +++ b/research/corpus/en/20.txt @@ -0,0 +1 @@ +Urban planning shapes how cities grow, balancing housing, transport, and public space. Zoning separates residential, commercial, and industrial areas. Mixed-use neighborhoods place homes, shops, and workplaces close together, reducing car dependence. Green spaces in cities lower temperatures in summer and absorb rainwater. Public transit systems carry many passengers with less space and energy per person than private cars. Pedestrian-friendly streets improve safety and encourage walking and cycling. diff --git a/research/corpus/en/21.txt b/research/corpus/en/21.txt new file mode 100644 index 0000000..9dcace1 --- /dev/null +++ b/research/corpus/en/21.txt @@ -0,0 +1 @@ +Coral reefs are among the most biodiverse ecosystems on Earth, supporting roughly a quarter of marine species. Reefs form when coral polyps secrete calcium carbonate skeletons. The deepest known point in the ocean is the Mariana Trench, more than ten thousand meters below sea level. Whales migrate over long distances between feeding and breeding areas. Overfishing and rising water temperatures are major threats to reef health. Marine protected areas give habitats time to recover from human pressures. diff --git a/research/corpus/en/22.txt b/research/corpus/en/22.txt new file mode 100644 index 0000000..6e1d62d --- /dev/null +++ b/research/corpus/en/22.txt @@ -0,0 +1 @@ +Meteorology is the study of weather and the atmosphere. Weather forecasts rely on computer models that simulate atmospheric conditions. Warm fronts and cold fronts mark boundaries between air masses with different temperatures. Thunderstorms form when warm, moist air rises quickly and condenses. Hurricane seasons vary by ocean basin, with the Atlantic season running from June to November. Satellite observations have greatly improved the accuracy of weather warnings for severe events. diff --git a/research/corpus/en/23.txt b/research/corpus/en/23.txt new file mode 100644 index 0000000..6e6676c --- /dev/null +++ b/research/corpus/en/23.txt @@ -0,0 +1 @@ +Languages are grouped into families based on shared ancestry. English, German, French, Spanish, and Russian all belong to the Indo-European family. Linguists reconstruct older stages such as Proto-Indo-European by comparing modern languages. Writing systems developed independently in Mesopotamia, China, and Mesoamerica. The Latin alphabet, used by many European languages, derives from the Greek alphabet. Language change is gradual, and related languages diverge over long periods of separation. diff --git a/research/corpus/en/24.txt b/research/corpus/en/24.txt new file mode 100644 index 0000000..e9a35e7 --- /dev/null +++ b/research/corpus/en/24.txt @@ -0,0 +1 @@ +Materials science studies the structure and properties of materials and how they relate. Steel, an alloy of iron and carbon, is strong in both tension and compression, making it central to construction. Concrete handles compression well but is weak in tension, so reinforced concrete combines it with steel bars. Polymers such as polyethylene are lightweight and corrosion-resistant. Composites combine fibers and a matrix to achieve high strength at low weight, as used in aircraft and sporting equipment. diff --git a/research/corpus/en/25.txt b/research/corpus/en/25.txt new file mode 100644 index 0000000..a90e916 --- /dev/null +++ b/research/corpus/en/25.txt @@ -0,0 +1 @@ +Sports science studies how the body responds to exercise and training. Endurance training increases the heart's ability to pump blood and improves oxygen delivery to muscles. Resistance training with progressively heavier loads builds muscle strength and size. Stretching before or after exercise has modest effects on injury risk compared with proper warm-up. Adequate sleep and nutrition are important for recovery after intense training. Athletes use heart-rate monitors to track training intensity and avoid overtraining. diff --git a/research/corpus/es/01.txt b/research/corpus/es/01.txt new file mode 100644 index 0000000..dfc1f67 --- /dev/null +++ b/research/corpus/es/01.txt @@ -0,0 +1 @@ +La computación en la nube permite a las organizaciones alquilar cómputo, almacenamiento y red en vez de poseer servidores. Los modelos principales son infraestructura como servicio, plataforma como servicio y software como servicio, cada uno con distinta responsabilidad operativa para el proveedor. Los precios suelen basarse en el uso, lo que ayuda a las empresas jóvenes a crecer sin grandes gastos iniciales. Las contrapartidas incluyen la dependencia del proveedor, las normas de residencia de datos y la necesidad de vigilar los costos cuando las cargas de trabajo crecen inesperadamente. diff --git a/research/corpus/es/02.txt b/research/corpus/es/02.txt new file mode 100644 index 0000000..b8d61b3 --- /dev/null +++ b/research/corpus/es/02.txt @@ -0,0 +1 @@ +Una taza de café constante depende de algunas variables: tamaño de molido, temperatura del agua, dosis y tiempo de infusión. Los molidos más finos extraen más rápido pero pueden volverse amargos si el agua está demasiado caliente. Una proporción inicial común es sesenta gramos de café por litro de agua. El vertido requiere mano firme y lecho plano, mientras que la inmersión como la prensa francesa es más tolerante. Guardar los granos en un recipiente hermético y a oscuras preserva su aroma más que la bolsa. diff --git a/research/corpus/es/03.txt b/research/corpus/es/03.txt new file mode 100644 index 0000000..626f33b --- /dev/null +++ b/research/corpus/es/03.txt @@ -0,0 +1 @@ +Una caminata en la montaña exige más que agua y refrigerios. Impermeable, mapa, linterna frontal y botiquín de primeros auxilios van en cada mochila, incluso en días despejados. El clima en altitud puede cambiar en una hora, y los senderos suelen estar sin marcar por encima de la línea de árboles. Avisa a alguien de tu ruta y hora de regreso antes de salir. Revisa el pronóstico dos veces, lleva capas y da la vuelta temprano si las condiciones parecen inestables en vez de seguir a la cumbre. diff --git a/research/corpus/es/04.txt b/research/corpus/es/04.txt new file mode 100644 index 0000000..ee8ceee --- /dev/null +++ b/research/corpus/es/04.txt @@ -0,0 +1 @@ +Muchas afirmaciones populares sobre nutrición solo están respaldadas en parte por la evidencia. Comer grasa no causa automáticamente aumento de peso; el aporte calórico total importa más que cualquier macronutriente. Saltarse el desayuno no ralentiza el metabolismo de la mayoría de las personas, pese al consejo habitual de comer temprano. Los productos orgánicos reducen la exposición a pesticidas pero no son notablemente más nutritivos que los cultivos convencionales. El hallazgo más constante de la investigación es que una dieta variada con muchas verduras supera a cualquier superalimento o suplemento. diff --git a/research/corpus/es/05.txt b/research/corpus/es/05.txt new file mode 100644 index 0000000..4c1c09c --- /dev/null +++ b/research/corpus/es/05.txt @@ -0,0 +1 @@ +Las licencias de código abierto se diferencian sobre todo en lo que exigen al redistribuir. Las licencias permisivas como MIT y Apache 2.0 permiten casi cualquier uso, incluidos los forks propietarios, siempre que se conserve el aviso de copyright. Las licencias copyleft como GPL exigen que las obras derivadas se publiquen bajo los mismos términos. Apache 2.0 añade una concesión explícita de patente que MIT no tiene. Los proyectos deberían prever acuerdos de contribución, porque aceptar código externo sin licencia puede crear ambigüedad legal sobre la propiedad de la contribución. diff --git a/research/corpus/es/06.txt b/research/corpus/es/06.txt new file mode 100644 index 0000000..47f368a --- /dev/null +++ b/research/corpus/es/06.txt @@ -0,0 +1 @@ +Las energías renovables representan hoy alrededor del treinta por ciento de la generación eléctrica mundial. La capacidad solar y eólica ha crecido más rápido que cualquier otra fuente en la última década, mientras los costos del almacenamiento en baterías han caído con fuerza. Los operadores de red aún deben equilibrar la oferta cuando el sol no brilla y el viento no sopla. Varios países invierten en almacenamiento de larga duración e interconexiones transfronterizas para suavizar esas brechas y reducir la dependencia de los combustibles fósiles en horas punta. diff --git a/research/corpus/es/07.txt b/research/corpus/es/07.txt new file mode 100644 index 0000000..65710eb --- /dev/null +++ b/research/corpus/es/07.txt @@ -0,0 +1 @@ +Las pequeñas empresas suelen seguir tres cifras por encima de todas las demás: efectivo disponible, tasa de consumo mensual y margen bruto. Las reservas de efectivo determinan cuántos meses puede sobrevivir la empresa sin nuevos ingresos. La tasa de consumo muestra qué tan rápido se gastan esas reservas, y el margen bruto revela qué parte de cada venta cubre los costos fijos. Los prestamistas e inversores suelen pedir doce meses de historial financiero, un balance actual y un pronóstico realista antes de comprometer capital en una empresa joven. diff --git a/research/corpus/es/08.txt b/research/corpus/es/08.txt new file mode 100644 index 0000000..2d27dc6 --- /dev/null +++ b/research/corpus/es/08.txt @@ -0,0 +1 @@ +Venecia fue fundada en islas pantanosas en el siglo quinto y se convirtió en una importante república mercantil durante la Edad Media. Su flota controlaba las rutas del Mediterráneo oriental, y sus mercaderes financiaban el comercio de especias, seda y vidrio. El sistema político de la ciudad era deliberadamente complejo, con dogos elegidos y consejos diseñados para impedir que una sola familia dominara. Los canales y palacios que sobreviven hoy atraen a millones de visitantes cada año, aunque la subida del nivel del agua amenaza ahora partes del centro histórico. diff --git a/research/corpus/es/09.txt b/research/corpus/es/09.txt new file mode 100644 index 0000000..ee84ea5 --- /dev/null +++ b/research/corpus/es/09.txt @@ -0,0 +1 @@ +Marte tiene dos lunas pequeñas, Fobos y Deimos, ambas nombradas por figuras de la mitología griega. El planeta Júpiter tiene más de noventa lunas confirmadas, la mayor es Ganímedes, que es más grande que el planeta Mercurio. Los anillos de Saturno consisten principalmente en partículas de hielo, desde granos diminutos hasta bloques del tamaño de una casa. El telescopio espacial James Webb, lanzado en 2021, observa luz infrarroja de galaxias lejanas. Los eclipses solares ocurren cuando la Luna pasa directamente entre el Sol y la Tierra. diff --git a/research/corpus/es/10.txt b/research/corpus/es/10.txt new file mode 100644 index 0000000..d8ca072 --- /dev/null +++ b/research/corpus/es/10.txt @@ -0,0 +1 @@ +La corteza terrestre está dividida en placas tectónicas que se mueven centímetros por año. La mayoría de los terremotos y volcanes ocurren en los límites de placas. El Himalaya se formó cuando la India chocó con Eurasia hace unos cincuenta millones de años. El ciclo de las rocas describe cómo las rocas ígneas, sedimentarias y metamórficas se transforman entre sí. El basalto es la roca volcánica más común de la Tierra y cubre gran parte del fondo oceánico. La meteorización y la erosión modelan los paisajes degradando la roca. diff --git a/research/corpus/es/11.txt b/research/corpus/es/11.txt new file mode 100644 index 0000000..7f13fbc --- /dev/null +++ b/research/corpus/es/11.txt @@ -0,0 +1 @@ +El elefante africano es el animal terrestre más grande; los machos adultos pesan hasta seis mil kilogramos. Los guepardos son los mamíferos terrestres más rápidos y alcanzan unos cien kilómetros por hora en ráfagas cortas. Las abejas comunican la ubicación del alimento mediante danzas de meneo. Los osos polares tienen piel negra bajo su pelaje blanco para absorber la luz solar. Los pulpos tienen tres corazones y sangre azul. Muchas especies de aves migran miles de kilómetros cada año entre sus zonas de cría e invernada. diff --git a/research/corpus/es/12.txt b/research/corpus/es/12.txt new file mode 100644 index 0000000..91c0c4d --- /dev/null +++ b/research/corpus/es/12.txt @@ -0,0 +1 @@ +Ludwig van Beethoven compuso nueve sinfonías, la última completada cuando estaba casi completamente sordo. Johann Sebastian Bach escribió más de doscientas cantatas mientras trabajaba como músico de iglesia en Leipzig. El piano reemplazó al clavecín como principal instrumento de teclado durante el siglo dieciocho. Thomas Edison inventó el fonógrafo en 1877, el primer dispositivo capaz de grabar y reproducir sonido. El gramófono, desarrollado después, usaba discos planos en lugar de cilindros, lo que finalmente se convirtió en el formato estándar. diff --git a/research/corpus/es/13.txt b/research/corpus/es/13.txt new file mode 100644 index 0000000..5ae4659 --- /dev/null +++ b/research/corpus/es/13.txt @@ -0,0 +1 @@ +Las normas de seguridad alimentaria recomiendan mantener los alimentos perecederos fuera del rango de temperatura entre cuatro y sesenta grados Celsius, llamada zona de peligro. Las bacterias se multiplican más rápido en ese rango. La carne cocida debe alcanzar al menos setenta grados Celsius internos para matar patógenos. Las sobras deben refrigerarse dentro de las dos horas posteriores a la cocción. Las tablas de cortar usadas para carne cruda deben lavarse antes de preparar verduras. Congelar los alimentos detiene el crecimiento bacteriano pero no mata todos los microorganismos. diff --git a/research/corpus/es/14.txt b/research/corpus/es/14.txt new file mode 100644 index 0000000..0a3c582 --- /dev/null +++ b/research/corpus/es/14.txt @@ -0,0 +1 @@ +Los buques portacontenedores transportan la mayoría de las mercancías del mundo; los más grandes llevan más de veinte mil contenedores. Los primeros sistemas modernos de autopistas se construyeron en Alemania en los años treinta. Los trenes de alta velocidad en Japón y Francia superan habitualmente los trescientos kilómetros por hora. Los autobuses eléctricos no producen emisiones de escape en el punto de uso, aunque su impacto ambiental depende de la fuente de electricidad. Las rotondas reducen la gravedad de los accidentes en cruces en comparación con los cruces tradicionales. diff --git a/research/corpus/es/15.txt b/research/corpus/es/15.txt new file mode 100644 index 0000000..21ff452 --- /dev/null +++ b/research/corpus/es/15.txt @@ -0,0 +1 @@ +La rotación de cultivos consiste en sembrar plantas distintas en el mismo campo en temporadas sucesivas para mantener la fertilidad. Las leguminosas como guisantes y frijoles añaden nitrógeno al suelo, reduciendo la necesidad de fertilizantes sintéticos. El riego por goteo lleva el agua a las raíces y reduce el uso de agua frente al riego por inundación. El trigo, el arroz y el maíz son los tres cultivos de cereales más sembrados del mundo. El mejoramiento vegetal moderno ha desarrollado variedades más productivas y resistentes a plagas y enfermedades. diff --git a/research/corpus/es/16.txt b/research/corpus/es/16.txt new file mode 100644 index 0000000..3bf96f9 --- /dev/null +++ b/research/corpus/es/16.txt @@ -0,0 +1 @@ +La Revolución Industrial comenzó en Gran Bretaña a fines del siglo dieciocho y se extendió por Europa y América del Norte. Las máquinas de vapor alimentaban fábricas y ferrocarriles, transformando la manufactura y el transporte. La línea de ensamblaje de Henry Ford en 1913 redujo mucho el tiempo para construir un automóvil. La Gran Depresión de los años treinta causó desempleo masivo y quiebras bancarias en todo el mundo. Después de la Segunda Guerra Mundial, muchos países adoptaron políticas para reconstruir sus economías y expandir el comercio internacional. diff --git a/research/corpus/es/17.txt b/research/corpus/es/17.txt new file mode 100644 index 0000000..4957837 --- /dev/null +++ b/research/corpus/es/17.txt @@ -0,0 +1 @@ +La imprenta, introducida por Johannes Gutenberg hacia 1450, abarató los libros y difundió la alfabetización por Europa. La novela se desarrolló como forma literaria importante en el siglo dieciocho, con obras de escritores como Daniel Defoe y Jane Austen. Shakespeare escribió unas treinta y siete obras, incluidas tragedias, comedias y piezas históricas. La poesía usa metro, rima e imágenes para crear significado. Las bibliotecas conservan las obras escritas y las hacen accesibles al público, apoyando la educación y la investigación. diff --git a/research/corpus/es/18.txt b/research/corpus/es/18.txt new file mode 100644 index 0000000..72d7af7 --- /dev/null +++ b/research/corpus/es/18.txt @@ -0,0 +1 @@ +La vacunación funciona entrenando al sistema inmunitario para reconocer un patógeno antes de que cause una enfermedad. Edward Jenner realizó la primera vacunación en 1796, usando la viruela bovina contra la viruela. El agua potable limpia y el tratamiento de aguas residuales han salvado más vidas que cualquier tratamiento médico. La inmunización infantil sistemática previene millones de muertes cada año. Lavarse las manos con jabón reduce la transmisión de enfermedades respiratorias y diarreicas. La actividad física regular reduce el riesgo de enfermedades cardíacas, accidentes cerebrovasculares y diabetes tipo 2. diff --git a/research/corpus/es/19.txt b/research/corpus/es/19.txt new file mode 100644 index 0000000..1747458 --- /dev/null +++ b/research/corpus/es/19.txt @@ -0,0 +1 @@ +Las baterías almacenan energía eléctrica como energía química y la liberan cuando se necesita. Las baterías de litio, comercializadas a principios de los años noventa, alimentan la mayoría de los dispositivos electrónicos portátiles y vehículos eléctricos. El almacenamiento por bombeo hidroeléctrico es la mayor tecnología de almacenamiento de red del mundo, moviendo agua entre embalses a distintas alturas. El aire comprimido y las baterías de flujo se desarrollan para la larga duración. Las baterías de red ayudan a equilibrar oferta y demanda cuando fluctúa la generación renovable. diff --git a/research/corpus/es/20.txt b/research/corpus/es/20.txt new file mode 100644 index 0000000..dfd6571 --- /dev/null +++ b/research/corpus/es/20.txt @@ -0,0 +1 @@ +La planificación urbana moldea cómo crecen las ciudades, equilibrando vivienda, transporte y espacio público. La zonificación separa las áreas residenciales, comerciales e industriales. Los barrios de uso mixto acercan hogares, tiendas y lugares de trabajo, reduciendo la dependencia del automóvil. Los espacios verdes urbanos bajan las temperaturas en verano y absorben el agua de lluvia. Los sistemas de transporte público llevan a muchos pasajeros con menos espacio y energía por persona que los autos privados. Las calles amigables con los peatones mejoran la seguridad y fomentan caminar y el ciclismo. diff --git a/research/corpus/es/21.txt b/research/corpus/es/21.txt new file mode 100644 index 0000000..c1378fe --- /dev/null +++ b/research/corpus/es/21.txt @@ -0,0 +1 @@ +Los arrecifes de coral están entre los ecosistemas más biodiversos de la Tierra y albergan un cuarto de las especies marinas. Se forman cuando los pólipos secretan esqueletos de carbonato de calcio. El punto más profundo del océano es la fosa de las Marianas, a más de diez mil metros bajo el mar. Las ballenas migran lejos entre áreas de alimentación y reproducción. La sobrepesca y el aumento de la temperatura del agua amenazan los arrecifes. Las áreas marinas protegidas dan a los hábitats tiempo para recuperarse. diff --git a/research/corpus/es/22.txt b/research/corpus/es/22.txt new file mode 100644 index 0000000..276af02 --- /dev/null +++ b/research/corpus/es/22.txt @@ -0,0 +1 @@ +La meteorología es el estudio del tiempo y la atmósfera. Los pronósticos del tiempo dependen de modelos informáticos que simulan las condiciones atmosféricas. Los frentes cálidos y fríos marcan los límites entre masas de aire de distintas temperaturas. Las tormentas se forman cuando el aire cálido y húmedo asciende rápidamente y se condensa. Las temporadas de huracanes varían según la cuenca oceánica; la del Atlántico va de junio a noviembre. Las observaciones satelitales han mejorado enormemente la precisión de las alertas meteorológicas para eventos severos. diff --git a/research/corpus/es/23.txt b/research/corpus/es/23.txt new file mode 100644 index 0000000..0681915 --- /dev/null +++ b/research/corpus/es/23.txt @@ -0,0 +1 @@ +Las lenguas se agrupan en familias según su ascendencia común. El inglés, el alemán, el francés, el español y el ruso pertenecen todos a la familia indoeuropea. Los lingüistas reconstruyen etapas más antiguas como el protoindoeuropeo comparando lenguas modernas. Los sistemas de escritura se desarrollaron de forma independiente en Mesopotamia, China y Mesoamérica. El alfabeto latino, usado por muchas lenguas europeas, deriva del alfabeto griego. El cambio lingüístico es gradual, y las lenguas emparentadas divergen durante largos períodos de separación. diff --git a/research/corpus/es/24.txt b/research/corpus/es/24.txt new file mode 100644 index 0000000..50dbbb1 --- /dev/null +++ b/research/corpus/es/24.txt @@ -0,0 +1 @@ +La ciencia de materiales estudia la estructura y las propiedades de los materiales. El acero, una aleación de hierro y carbono, resiste tracción y compresión, lo que lo hace central para la construcción. El hormigón soporta bien la compresión pero es débil a la tracción, por lo que el hormigón armado añade barras de acero. Los polímeros como el polietileno son ligeros y resistentes a la corrosión. Los compuestos combinan fibras y una matriz para lograr alta resistencia con bajo peso, como en aviones y equipos deportivos. diff --git a/research/corpus/es/25.txt b/research/corpus/es/25.txt new file mode 100644 index 0000000..67b9fea --- /dev/null +++ b/research/corpus/es/25.txt @@ -0,0 +1 @@ +La ciencia del deporte estudia cómo responde el cuerpo al ejercicio y al entrenamiento. El entrenamiento de resistencia aumenta la capacidad cardíaca y mejora el oxígeno a los músculos. El entrenamiento de fuerza con cargas crecientes desarrolla fuerza y masa muscular. Estirar antes o después del ejercicio tiene efecto modesto sobre el riesgo de lesiones frente a un buen calentamiento. Un sueño y una nutrición suficientes son importantes para recuperarse de un entrenamiento intenso. Los atletas usan monitores de frecuencia cardíaca para controlar la intensidad y evitar sobreentrenarse. diff --git a/research/corpus/fr/01.txt b/research/corpus/fr/01.txt new file mode 100644 index 0000000..3b30533 --- /dev/null +++ b/research/corpus/fr/01.txt @@ -0,0 +1 @@ +L'informatique en nuage permet aux organisations de louer de la puissance de calcul, du stockage et des ressources réseau au lieu de posséder des serveurs physiques. Les principaux modèles sont l'infrastructure, la plateforme et le logiciel en tant que service, chacun déplaçant une part différente de la responsabilité opérationnelle vers le fournisseur. La tarification repose sur l'usage, ce qui aide les jeunes entreprises à croître sans gros investissements initiaux. Les compromis incluent la dépendance au fournisseur, la résidence des données et la surveillance des coûts quand les charges augmentent. diff --git a/research/corpus/fr/02.txt b/research/corpus/fr/02.txt new file mode 100644 index 0000000..8fcf203 --- /dev/null +++ b/research/corpus/fr/02.txt @@ -0,0 +1 @@ +Une tasse de café dépend de quelques variables : mouture, température, dose et temps d'infusion. Les moutures plus fines extraient plus vite mais deviennent amères si l'eau est trop chaude. Un ratio courant est de soixante grammes de café par litre d'eau. Les méthodes par versement exigent une main stable et un lit de café plat, tandis que les méthodes par immersion comme la presse française sont plus indulgentes. Des grains conservés à l'abri de la lumière dans un contenant hermétique gardent mieux leur arôme que dans le sachet. diff --git a/research/corpus/fr/03.txt b/research/corpus/fr/03.txt new file mode 100644 index 0000000..9c63b79 --- /dev/null +++ b/research/corpus/fr/03.txt @@ -0,0 +1 @@ +Une randonnée en montagne exige plus que de l'eau et des en-cas. Vêtements de pluie, carte, lampe frontale et trousse de secours doivent être dans chaque sac, même par beau temps. La météo en altitude peut changer en une heure, et les sentiers sont souvent non balisés au-dessus de la limite des arbres. Prévenez quelqu'un de votre itinéraire et de l'heure de retour avant de partir. Vérifiez deux fois les prévisions, emportez des couches et faites demi-tour tôt si les conditions semblent instables plutôt que de continuer vers le sommet. diff --git a/research/corpus/fr/04.txt b/research/corpus/fr/04.txt new file mode 100644 index 0000000..9891631 --- /dev/null +++ b/research/corpus/fr/04.txt @@ -0,0 +1 @@ +De nombreuses affirmations nutritionnelles populaires ne sont que partiellement étayées. Manger du gras ne provoque pas automatiquement une prise de poids ; l'apport calorique total compte plus que n'importe quel macronutriment. Sauter le petit-déjeuner ne ralentit pas le métabolisme pour la plupart, malgré le conseil courant de manger tôt. Les produits bio réduisent l'exposition aux pesticides mais ne sont pas nettement plus nutritifs que les cultures conventionnelles. Le résultat le plus constant des recherches est qu'une alimentation variée avec beaucoup de légumes bat n'importe quel superaliment ou complément. diff --git a/research/corpus/fr/05.txt b/research/corpus/fr/05.txt new file mode 100644 index 0000000..ad46f68 --- /dev/null +++ b/research/corpus/fr/05.txt @@ -0,0 +1 @@ +Les licences open source diffèrent par ce qu'elles exigent lors de la redistribution du code. Les licences permissives comme MIT et Apache 2.0 autorisent tout usage, y compris les forks propriétaires, à condition de conserver la mention de copyright. Les licences copyleft comme la GPL exigent que les œuvres dérivées soient publiées sous les mêmes conditions. Apache 2.0 ajoute une concession de brevet explicite absente chez MIT. Les projets devraient prévoir des accords de contribution, car accepter du code externe sans licence peut créer une ambiguïté juridique sur la propriété. diff --git a/research/corpus/fr/06.txt b/research/corpus/fr/06.txt new file mode 100644 index 0000000..fb2daab --- /dev/null +++ b/research/corpus/fr/06.txt @@ -0,0 +1 @@ +Les énergies renouvelables représentent désormais environ trente pour cent de la production mondiale d'électricité. Les capacités solaire et éolienne ont crû plus vite que toute autre source depuis dix ans, tandis que les coûts du stockage par batteries ont fortement chuté. Les gestionnaires de réseau doivent équilibrer l'offre quand le soleil ne brille pas et que le vent ne souffle pas. Plusieurs pays investissent dans le stockage longue durée et les interconnexions transfrontalières pour combler ces écarts et réduire la dépendance aux combustibles fossiles en période de pointe. diff --git a/research/corpus/fr/07.txt b/research/corpus/fr/07.txt new file mode 100644 index 0000000..29606f8 --- /dev/null +++ b/research/corpus/fr/07.txt @@ -0,0 +1 @@ +Les petites entreprises suivent souvent trois chiffres avant tout : trésorerie disponible, taux de consommation mensuel et marge brute. Les réserves de trésorerie déterminent combien de mois l'entreprise peut survivre sans nouveaux revenus. Le taux de consommation montre à quelle vitesse ces réserves sont dépensées, et la marge brute révèle quelle part de chaque vente couvre les coûts fixes. Les prêteurs et investisseurs demandent généralement douze mois d'historique financier, un bilan actuel et une prévision réaliste avant d'engager du capital dans une jeune entreprise. diff --git a/research/corpus/fr/08.txt b/research/corpus/fr/08.txt new file mode 100644 index 0000000..19bbfa2 --- /dev/null +++ b/research/corpus/fr/08.txt @@ -0,0 +1 @@ +Venise, fondée sur des îles marécageuses au cinquième siècle, devint une grande république marchande au Moyen Âge. Sa flotte contrôlait les routes de la Méditerranée orientale, et ses marchands finançaient le commerce des épices, de la soie et du verre. Le système politique de la ville était délibérément complexe, avec des doges élus et des conseils conçus pour éviter qu'une famille ne domine. Les canaux et palais qui subsistent attirent des millions de visiteurs chaque année, même si la montée des eaux menace des parties du centre historique. diff --git a/research/corpus/fr/09.txt b/research/corpus/fr/09.txt new file mode 100644 index 0000000..d58dee4 --- /dev/null +++ b/research/corpus/fr/09.txt @@ -0,0 +1 @@ +Mars possède deux petites lunes, Phobos et Deimos, toutes deux nommées d'après des figures de la mythologie grecque. Jupiter compte plus de quatre-vingt-dix lunes confirmées, la plus grande étant Ganymède, plus grande que la planète Mercure. Les anneaux de Saturne sont composés principalement de particules de glace, des minuscules grains aux blocs de la taille d'une maison. Le télescope spatial James Webb, lancé en 2021, observe la lumière infrarouge de galaxies lointaines. Les éclipses solaires se produisent quand la Lune passe directement entre le Soleil et la Terre. diff --git a/research/corpus/fr/10.txt b/research/corpus/fr/10.txt new file mode 100644 index 0000000..719667b --- /dev/null +++ b/research/corpus/fr/10.txt @@ -0,0 +1 @@ +La croûte terrestre est divisée en plaques tectoniques qui se déplacent de quelques centimètres par an. La plupart des séismes et volcans se produisent le long des limites de plaques. L'Himalaya s'est formé quand l'Inde a heurté l'Eurasie il y a environ cinquante millions d'années. Le cycle des roches décrit comment les roches magmatiques, sédimentaires et métamorphiques se transforment entre elles. Le basalte est la roche volcanique la plus commune sur Terre, couvrant la majeure partie du fond océanique. Altération et érosion sculptent les paysages en dégradant la roche. diff --git a/research/corpus/fr/11.txt b/research/corpus/fr/11.txt new file mode 100644 index 0000000..73e7e58 --- /dev/null +++ b/research/corpus/fr/11.txt @@ -0,0 +1 @@ +L'éléphant d'Afrique est le plus grand animal terrestre, les mâles adultes pesant jusqu'à six mille kilogrammes. Les guépards sont les mammifères terrestres les plus rapides, atteignant environ cent kilomètres par heure en courtes accélérations. Les abeilles communiquent l'emplacement de la nourriture par des danses frétillantes. Les ours polaires ont une peau noire sous leur fourrure blanche pour absorber la lumière du soleil. Les pieuvres ont trois cœurs et du sang bleu. De nombreuses espèces d'oiseaux migrent chaque année sur des milliers de kilomètres entre leurs aires de reproduction et d'hivernage. diff --git a/research/corpus/fr/12.txt b/research/corpus/fr/12.txt new file mode 100644 index 0000000..d0af8cc --- /dev/null +++ b/research/corpus/fr/12.txt @@ -0,0 +1 @@ +Ludwig van Beethoven a composé neuf symphonies, la dernière achevée alors qu'il était presque complètement sourd. Jean-Sébastien Bach a écrit plus de deux cents cantates en travaillant comme musicien d'église à Leipzig. Le piano a remplacé le clavecin comme principal instrument à clavier au cours du dix-huitième siècle. Thomas Edison a inventé le phonographe en 1877, premier appareil capable d'enregistrer et de rejouer le son. Le gramophone, développé plus tard, utilisait des disques plats au lieu de cylindres, ce qui devint finalement le format standard. diff --git a/research/corpus/fr/13.txt b/research/corpus/fr/13.txt new file mode 100644 index 0000000..97c0f20 --- /dev/null +++ b/research/corpus/fr/13.txt @@ -0,0 +1 @@ +Les règles de sécurité alimentaire recommandent de garder les aliments périssables hors de la plage entre quatre et soixante degrés Celsius, dite zone de danger. Les bactéries s'y multiplient le plus vite. La viande cuite doit atteindre au moins soixante-dix degrés Celsius à cœur pour tuer les pathogènes. Les restes doivent être réfrigérés dans les deux heures après la cuisson. Les planches à découper pour la viande crue doivent être lavées avant de préparer des légumes. Congeler les aliments stoppe la croissance bactérienne mais ne tue pas tous les micro-organismes. diff --git a/research/corpus/fr/14.txt b/research/corpus/fr/14.txt new file mode 100644 index 0000000..1f8c823 --- /dev/null +++ b/research/corpus/fr/14.txt @@ -0,0 +1 @@ +Les porte-conteneurs transportent la majorité des marchandises échangées dans le monde, les plus grands navires transportant plus de vingt mille conteneurs. Les premiers réseaux autoroutiers modernes ont été construits en Allemagne dans les années 1930. Les trains à grande vitesse au Japon et en France dépassent régulièrement trois cents kilomètres par heure. Les bus électriques ne produisent pas d'émissions d'échappement sur place, mais leur impact environnemental dépend de la source d'électricité. Les ronds-points réduisent la gravité des accidents de carrefour par rapport aux croisements traditionnels. diff --git a/research/corpus/fr/15.txt b/research/corpus/fr/15.txt new file mode 100644 index 0000000..450a483 --- /dev/null +++ b/research/corpus/fr/15.txt @@ -0,0 +1 @@ +La rotation des cultures consiste à cultiver différentes plantes dans le même champ en saisons successives pour maintenir la fertilité du sol. Les légumineuses comme pois et haricots ajoutent de l'azote au sol, réduisant le besoin d'engrais synthétiques. L'irrigation au goutte-à-goutte amène l'eau directement aux racines et réduit la consommation d'eau face à l'irrigation par submersion. Le blé, le riz et le maïs sont les trois cultures céréalières les plus cultivées au monde. La sélection végétale moderne a développé des variétés plus productives et plus résistantes aux ravageurs et maladies. diff --git a/research/corpus/fr/16.txt b/research/corpus/fr/16.txt new file mode 100644 index 0000000..783f48d --- /dev/null +++ b/research/corpus/fr/16.txt @@ -0,0 +1 @@ +La révolution industrielle a commencé en Grande-Bretagne à la fin du dix-huitième siècle et s'est répandue en Europe et en Amérique du Nord. Les machines à vapeur alimentaient les usines et les chemins de fer, transformant fabrication et transports. La chaîne de montage de Henry Ford en 1913 réduisit fortement le temps de construction d'une voiture. La Grande Dépression des années 1930 causa chômage massif et faillites bancaires mondialement. Après la Seconde Guerre mondiale, beaucoup de pays ont adopté des politiques pour reconstruire leurs économies et le commerce international. diff --git a/research/corpus/fr/17.txt b/research/corpus/fr/17.txt new file mode 100644 index 0000000..9bc3157 --- /dev/null +++ b/research/corpus/fr/17.txt @@ -0,0 +1 @@ +La presse à imprimer, introduite par Johannes Gutenberg vers 1450, a rendu les livres moins chers et a diffusé l'alphabétisation en Europe. Le roman s'est développé comme forme littéraire majeure au dix-huitième siècle, avec des œuvres d'écrivains comme Daniel Defoe et Jane Austen. Shakespeare a écrit environ trente-sept pièces, dont des tragédies, des comédies et des pièces historiques. La poésie utilise le mètre, la rime et l'imagerie pour créer du sens. Les bibliothèques préservent les œuvres écrites et les rendent accessibles au public, soutenant l'éducation et la recherche. diff --git a/research/corpus/fr/18.txt b/research/corpus/fr/18.txt new file mode 100644 index 0000000..97d67e3 --- /dev/null +++ b/research/corpus/fr/18.txt @@ -0,0 +1 @@ +La vaccination entraîne le système immunitaire à reconnaître un agent pathogène avant qu'il ne provoque une maladie. Edward Jenner a pratiqué la première vaccination en 1796, utilisant la vaccine contre la variole. L'eau potable et le traitement des eaux usées ont sauvé plus de vies que tout traitement médical. La vaccination systématique des enfants prévient des millions de décès chaque année. Se laver les mains au savon réduit la transmission des maladies respiratoires et diarrhéiques. L'activité physique réduit le risque de maladies cardiaques, d'AVC et de diabète de type 2. diff --git a/research/corpus/fr/19.txt b/research/corpus/fr/19.txt new file mode 100644 index 0000000..9b35655 --- /dev/null +++ b/research/corpus/fr/19.txt @@ -0,0 +1 @@ +Les batteries stockent l'énergie électrique sous forme d'énergie chimique et la libèrent en cas de besoin. Les batteries lithium-ion, commercialisées au début des années 90, alimentent la plupart des appareils électroniques portables et des véhicules électriques. Le pompage-turbinage est la plus grande technologie de stockage sur réseau au monde, déplaçant l'eau entre des réservoirs à différentes hauteurs. Le stockage d'énergie par air comprimé et les batteries à flux sont développés pour la longue durée. Les batteries réseau aident à équilibrer l'offre et la demande quand la production renouvelable fluctue. diff --git a/research/corpus/fr/20.txt b/research/corpus/fr/20.txt new file mode 100644 index 0000000..cb5a532 --- /dev/null +++ b/research/corpus/fr/20.txt @@ -0,0 +1 @@ +L'urbanisme façonne la croissance des villes en équilibrant logement, transports et espace public. Le zonage sépare les zones résidentielles, commerciales et industrielles. Les quartiers à usage mixte rapprochent habitations, commerces et lieux de travail, réduisant la dépendance à la voiture. Les espaces verts urbains abaissent les températures en été et absorbent l'eau de pluie. Les systèmes de transport en commun transportent de nombreux passagers avec moins d'espace et d'énergie par personne que les voitures privées. Les rues adaptées aux piétons améliorent la sécurité et encouragent la marche et le vélo. diff --git a/research/corpus/fr/21.txt b/research/corpus/fr/21.txt new file mode 100644 index 0000000..d1d45a1 --- /dev/null +++ b/research/corpus/fr/21.txt @@ -0,0 +1 @@ +Les récifs coralliens comptent parmi les écosystèmes les plus riches de la Terre, abritant environ un quart des espèces marines. Ils se forment quand les polypes sécrètent des squelettes de carbonate de calcium. Le point océanique le plus profond connu est la fosse des Mariannes, à plus de dix mille mètres sous la mer. Les baleines migrent loin entre aires d'alimentation et de reproduction. La surpêche et la hausse des températures de l'eau menacent les récifs. Les aires marines protégées laissent aux habitats le temps de se remettre. diff --git a/research/corpus/fr/22.txt b/research/corpus/fr/22.txt new file mode 100644 index 0000000..34bac63 --- /dev/null +++ b/research/corpus/fr/22.txt @@ -0,0 +1 @@ +La météorologie est l'étude du temps et de l'atmosphère. Les prévisions météo reposent sur des modèles informatiques qui simulent les conditions atmosphériques. Les fronts chauds et froids marquent les limites entre des masses d'air de températures différentes. Les orages se forment quand de l'air chaud et humide s'élève rapidement et se condense. Les saisons des ouragans varient selon les bassins océaniques, celle de l'Atlantique s'étendant de juin à novembre. Les observations par satellite ont considérablement amélioré la précision des alertes météo pour les événements violents. diff --git a/research/corpus/fr/23.txt b/research/corpus/fr/23.txt new file mode 100644 index 0000000..ca22adb --- /dev/null +++ b/research/corpus/fr/23.txt @@ -0,0 +1 @@ +Les langues sont regroupées en familles selon leur ascendance commune. L'anglais, l'allemand, le français, l'espagnol et le russe appartiennent tous à la famille indo-européenne. Les linguistes reconstruisent des stades plus anciens comme le proto-indo-européen en comparant les langues modernes. Les systèmes d'écriture se sont développés indépendamment en Mésopotamie, en Chine et en Mésoamérique. L'alphabet latin, utilisé par de nombreuses langues européennes, dérive de l'alphabet grec. Le changement linguistique est graduel, et les langues apparentées divergent sur de longues périodes de séparation. diff --git a/research/corpus/fr/24.txt b/research/corpus/fr/24.txt new file mode 100644 index 0000000..1cf2ef4 --- /dev/null +++ b/research/corpus/fr/24.txt @@ -0,0 +1 @@ +La science des matériaux étudie la structure et les propriétés des matériaux. L'acier, un alliage de fer et de carbone, est résistant en traction et en compression, ce qui le rend central pour la construction. Le béton supporte bien la compression mais est faible en traction, donc le béton armé ajoute des barres d'acier. Les polymères comme le polyéthylène sont légers et résistants à la corrosion. Les composites combinent des fibres et une matrice pour obtenir une grande résistance à faible poids, comme dans les avions et les équipements sportifs. diff --git a/research/corpus/fr/25.txt b/research/corpus/fr/25.txt new file mode 100644 index 0000000..243b537 --- /dev/null +++ b/research/corpus/fr/25.txt @@ -0,0 +1 @@ +La science du sport étudie la réaction du corps à l'exercice et à l'entraînement. L'entraînement d'endurance renforce le cœur et améliore l'apport d'oxygène aux muscles. L'entraînement en résistance avec des charges croissantes développe force et masse musculaires. Les étirements avant ou après l'exercice ont un effet modeste sur le risque de blessure face à un échauffement. Un sommeil et une nutrition suffisants comptent pour récupérer d'un entraînement intense. Les athlètes utilisent des moniteurs de fréquence cardiaque pour suivre l'intensité et éviter le surentraînement. diff --git a/research/paper/README.md b/research/paper/README.md new file mode 100644 index 0000000..6e15c54 --- /dev/null +++ b/research/paper/README.md @@ -0,0 +1,56 @@ +# research/paper/ -- arXiv v1 skeleton (gaps C1/C2) + +Status: **skeleton only** -- placeholder prose + TODOs, **no real +experiment numbers**. The generated tables/figures do not exist yet. + +## Files + +| File | Contents | Source | +| --- | --- | --- | +| `main.tex` | Article-class skeleton; sections 1--10 (02 §5); table/figure placeholders (02 §6); ACL style drop-in note | `research/02-paper-outline.md` | +| `refs.bib` | 32 verified references, 03 A--E (@misc + eprint for arXiv; venue labels in `note`; `% re-verify at submission`) | `research/03-related-work.md` | +| `abstract.tex` | ~150-word abstract draft (02 §2); TODO: recheck numbers after the run | `research/02-paper-outline.md` §2 | +| `ethics.tex` | Ethics statement (04 §3 adapted to a paragraph; disclosure note 04 §5) | `research/04-ethics-and-legal.md` | +| `acknowledgments.tex` | MarkLLM / THU-BPM (Generative-Watermark-Toolkits) + HF models | `research/04-ethics-and-legal.md` §5, 03 §D | + +## Build + +Requires a TeX distribution with `natbib` (pdflatex + bibtex). From +this directory: + +```bash +pdflatex main +bibtex main +pdflatex main +pdflatex main +``` + +`main.tex` will **not compile until the generated tables and figures +exist** (`tables/t1.tex`--`tables/t7.tex`, +`figures/f1.pdf`--`figures/f6.pdf`). The dev environment here has no +pdflatex; syntax is checked with a brace-balance script instead +(research/tests/test_paper_skeleton.py). + +## Missing before submission (see research/05-arxiv-readiness.md) + +- **Tables/figures (gap C3)**: run `research/scripts/make_tables.py` + and `research/scripts/make_figures.py` (to be implemented; gaps + B3/C3) to emit `tables/` and `figures/` from the run results -- + they are the real-number inputs referenced by `main.tex`. +- **Real numbers**: replace every TODO in §4--§6 with the actual run + output (gaps A1--A7, B1--B3, C4); abstract numbers re-checked (W2). +- **ACL style**: drop in `acl2024.sty` and switch the preamble to the + ACL template at submission (gap C1); swap + `\bibliographystyle{plainnat}` if the venue requires its own style. +- **Citations**: re-verify every ID/venue at submission (03 header; + `% re-verify at submission` in refs.bib); every bib entry must be + cited (or pruned) before upload. +- **Reproducibility**: manifest, pins, release URLs (gaps A7/E3) and + the data-availability / reproducibility statements (01 §7--8). + +## Notes + +- `research/` is gitignored; use `git add -f` if you want the paper + tracked. +- Keep `ethics.tex` in sync with `research/04-ethics-and-legal.md` + and the abstract with `research/02-paper-outline.md` §2. diff --git a/research/paper/abstract.tex b/research/paper/abstract.tex new file mode 100644 index 0000000..1700d0e --- /dev/null +++ b/research/paper/abstract.tex @@ -0,0 +1,25 @@ +% abstract.tex -- abstract draft for the arXiv v1 paper. +% Source: research/02-paper-outline.md §2 (v0.2, 2026-08-18), copied +% verbatim (LaTeX-escaped). ~150 words. +% +% TODO (W2): recheck every number against the final run before +% submission -- the 3,500/3,500 cell counts and the metric list must +% match the released results (research/01-experiment-protocol.md §2; +% 05-arxiv-readiness.md checklist F). + +Text watermarking is the primary mechanism proposed for EU AI Act +Art.~50 transparency obligations on machine-generated content. We +measure its robustness against realistic user-side editing. Using +same-config detection over 3,500 watermarked and 3,500 unwatermarked +generations (KGW, SynthID-Text, EXP, Unigram, and SIR schemes, in +English and German/French/Spanish), we evaluate a layered removal +pipeline that combines formatting-layer cleanup (invisible Unicode, +bidi) with statistical rewriting driven by detection feedback, and we +report ROC-based metrics (AUROC, TPR@1\%FPR) and quality metrics +(perplexity, BERTScore, ROUGE-L) at the point of detection collapse. We +find that quality-preserving paraphrase and translation round-trip +collapse KGW-class detection to near chance, that SynthID-Text resists +token substitution but not paraphrase, that layering dominates single +attacks at equal quality cost, and that multilingual texts are +systematically more fragile. We discuss implications for Art.~50 +compliance and release our corpus, configs, and harness. diff --git a/research/paper/acknowledgments.tex b/research/paper/acknowledgments.tex new file mode 100644 index 0000000..fb0bcb4 --- /dev/null +++ b/research/paper/acknowledgments.tex @@ -0,0 +1,12 @@ +% acknowledgments.tex -- acknowledgments for the arXiv v1 paper. +% Source: research/04-ethics-and-legal.md §5 (MarkLLM is Apache-2.0; +% attribution required) and research/03-related-work.md §D (MarkLLM: +% Pan et al., EMNLP 2024 Demo). + +We thank the Tsinghua NLP group (THU-BPM) for releasing MarkLLM +\citep{pan2024markllm} -- an open-source, Apache-2.0 toolkit for LLM +watermarking -- via the Generative-Watermark-Toolkits repository, which +served as the generation and detection harness for this study. We also +thank the Hugging Face community for the open models used in our +experiments (opt-1.3b and Qwen2.5-1.5B-Instruct). +% TODO: add any further acknowledgments before submission. diff --git a/research/paper/ethics.tex b/research/paper/ethics.tex new file mode 100644 index 0000000..3430d94 --- /dev/null +++ b/research/paper/ethics.tex @@ -0,0 +1,30 @@ +% ethics.tex -- ethics statement for the arXiv v1 paper. +% Source: research/04-ethics-and-legal.md §3 (draft, adapted to a +% paragraph) and §5 (disclosure). Keep this file in sync with 04; adapt +% to the venue template at submission (arXiv has no ethics checklist -- +% 04 §6). Do not weaken the "no third-party content" and +% "watermark-as-label" framing. + +All text used in this study was generated by the authors using +open-weight models (opt-1.3b) via the MarkLLM toolkit +\citep{pan2024markllm}; no third-party content, user data, or live +vendor outputs were used, and no watermarked content produced by +commercial providers was collected or altered. The removal pipeline +evaluated here operates on text generated by the same user who owns it; +the study does not enable or endorse alteration of third-party content. +Watermarking is a label, not an access-control mechanism, so no +security control is circumvented. We disclose our findings to support +(i) realistic expectations for regulators relying on watermarking for +transparency obligations (Art.~50, Regulation (EU) 2024/1689 +\citep{euaiact2024}), and (ii) the design of more robust provenance +mechanisms. Detectors are same-config open-source implementations; +commercial detectors were not probed. We do not provide live removal +services or weights tuned for evasion of specific vendor detectors +beyond what is reported. The authors' tooling is public +(\url{https://github.com/guillaumemeyer/watermarks-remover}) and was +deployed publicly before this study began; this paper formalizes +measurements of mechanisms already in production use. + +% Disclosure (04 §5): the tool is already public and widely reported; +% no embargo or coordinated-disclosure obligation applies to this +% measurement -- state this if a venue asks about disclosure. diff --git a/research/paper/main.tex b/research/paper/main.tex new file mode 100644 index 0000000..b19a000 --- /dev/null +++ b/research/paper/main.tex @@ -0,0 +1,472 @@ +% main.tex -- arXiv v1 skeleton for research/paper/ (gaps C1/C2 in +% research/05-arxiv-readiness.md). +% +% Paper: +% "How Fragile Are Deployed Text Watermarks? An Empirical Study of +% Layered Watermark Removal under Realistic User-Side Editing" +% +% Sources of truth: research/02-paper-outline.md (title 01/02 §1, +% abstract §2, section skeleton §5, tables/figures spec §6, citation +% map §7), research/03-related-work.md (verified bibliography). +% +% Status: SKELETON ONLY. Placeholder prose + TODO comments; NO real +% experiment numbers yet (real values land after the W1 core run; see +% research/01-experiment-protocol.md and research/05-arxiv-readiness.md +% gaps A/B/C). +% +% Compile (no ACL style package required for the skeleton): +% pdflatex main +% bibtex main +% pdflatex main +% pdflatex main +% +% At submission: drop in acl2024.sty and switch the preamble to the ACL +% template; the \input{} files and the body carry over. Also swap +% \bibliographystyle{plainnat} for the ACL bib style if required. +% +% NOTE: tables/ and figures/ do not exist yet -- they are generated by +% research/scripts/make_tables.py and research/scripts/make_figures.py +% (gap C3). main.tex will not compile until those files are emitted. + +\documentclass[10pt,letterpaper]{article} + +% --- packages ------------------------------------------------------------ +% Standard packages only; the skeleton deliberately avoids venue-specific +% styles (ACL style is dropped in at submission, see the header comment). +\usepackage[T1]{fontenc} % UTF-8 is the default for modern engines +\usepackage{graphicx} % \includegraphics{figures/fN} (make_figures.py) +\usepackage{booktabs} % tables generated by make_tables.py +\usepackage{natbib} % \citet / \citep (author-year by default) +\usepackage[hidelinks]{hyperref} + +% --- metadata ------------------------------------------------------------ +% Title locked in research/02-paper-outline.md §1. +\title{How Fragile Are Deployed Text Watermarks? An Empirical Study of +Layered Watermark Removal under Realistic User-Side Editing} + +% Solo author, no affiliation (allowed on arXiv; see +% research/05-arxiv-readiness.md E1). Affiliation is optional at +% submission -- uncomment and fill in if desired, e.g.: +% \author{Guillaume Meyer\\ \small{Affiliation}} +\author{Guillaume Meyer} +\date{} % arXiv stamps its own submission date. + +\begin{document} + +\maketitle + +% --- abstract ------------------------------------------------------------ +% Draft from research/02-paper-outline.md §2 lives in abstract.tex. +% TODO: recheck all numbers against the final run (writing phase W2). +\begin{abstract} +\input{abstract} +\end{abstract} + +% ===================================================================== +% 1. Introduction +% ===================================================================== +\section{Introduction} +% Hook (02 §5.1): Art. 50 in force 2026-08-02; vendor rollouts (Claude, +% Gemini); Google retired the SynthID-Text API in Aug 2026 -- the market +% is consolidating on methods that do not survive editing. TODO: one +% sentence of this market context here, no metrics needed. + +The EU AI Act's transparency obligations for machine-generated content +(Regulation (EU) 2024/1689, Art.~50) \citep{euaiact2024} entered force +on 2~August 2026, and text watermarking is the primary mechanism +proposed to meet them. % TODO: add the vendor-rollout / SynthID-Text +% retirement context (one sentence). + +% Motivation (02 §5.1): the public deployment of our own removal tool -- +% users edit their own output; do watermarks survive? (1 short +% paragraph, no metrics needed.) +The public deployment of a watermark-removal tool in this project showed +that ordinary users routinely edit text they generated themselves +before publishing it. That raises the basic question we study: do +deployed text watermarks survive realistic user-side editing? + +% Contributions (02 §4) -- TODO: verify the four bullets at submission. +Our contributions are as follows: +\begin{itemize} + \item \textbf{Measurement.} The first ROC-based robustness study of + deployed-class text watermarking (KGW, SynthID-Text, EXP, Unigram, + SIR) under realistic, layered user-side editing, with empirical-null + FPR calibration. + \item \textbf{Method.} A layered removal pipeline (formatting + + statistical) with detection-feedback rewriting; we show it dominates + single-layer baselines at equal quality cost (Pareto). + \item \textbf{Resource.} An open corpus (25 EN + 75 multilingual + prompts), a config-pinned harness (MarkLLM-based), and results + (JSONL) for reproducible attack/defense benchmarking. + \item \textbf{Policy measurement.} Evidence on whether Art.~50 + transparency obligations can rely on watermarking, with concrete + recommendations (platform-level provenance, metadata, + robust-but-invisible schemes, honest failure modes). +\end{itemize} + +Figure~\ref{fig:f1} illustrates the layered pipeline we evaluate; +Section~\ref{sec:background} situates the work relative to the +watermarking literature. + +% ===================================================================== +% 2. Background and Related Work +% ===================================================================== +\section{Background and Related Work} +\label{sec:background} +% Watermarking families (02 §5.2; citation map 02 §7; full list 03 A). +Sampling-based watermarking biases the token distribution of the +generator under a secret key, starting with KGW +\citep{kirchenbauer2023kwg} and a large family of successors +\citep{christ2023undetectable,kuditipudi2023robust,hu2023unbiased, +wu2023dipmark,zhao2024permute,huo2024tswatermark,liu2024adaptive}. +Tournament-based methods such as SynthID-Text +\citep{deepmind2024synthidtext,omidi2026synthid} pair tokens into +tournaments to avoid distribution distortion. Semantic schemes +watermark in embedding space \citep{liu2023sir,hou2023semstamp, +hou2024ksemstamp}, provable families aim for statistical guarantees +\citep{liu2023upv}, and recent work adds entropy-aware and adaptive +designs \citep{gu2025invisible,wang2025morphmark} as well as +black-box watermarking \citep{yang2023blackbox}. We follow the +taxonomy of \citet{liu2023survey} and build on the MarkLLM toolkit +\citep{pan2024markllm}. + +% Attack literature (02 §5.2; citation map 02 §7; full list 03 B/C). +Robustness studies established that watermarks are fragile under +paraphrasing and other edits \citep{kirchenbauer2023reliability}, that +strong watermarking faces impossibility limits under random-walk +attacks \citep{zhang2023sand,harelcanada2025sandcastles}, and that +translation round-trips defeat several schemes \citep{he2024xsir}; +watermark keys can also be stolen from a black-box detector +\citep{jovanovic2024stealing}. Closest to us, recent work assesses +SynthID-Text robustness specifically \citep{han2025synthid, +omidi2026synthid} and the forensic readiness of watermark evidence +\citep{tamim2026forensic}. Detector-side methods improve segment +detection \citep{pan2024waterseeker,lu2024ewd}, users can sometimes +identify watermarked outputs through crafted prompts +\citep{liu2024crafted}, and watermarks may not robustly prevent +knowledge distillation \citep{pan2025distillation}. Metadata +provenance standards such as C2PA \citep{c2pa2024spec} are +complementary; a metadata mini-study is deferred to v2. + +% Positioning (02 §5.2; 03 "Positioning summary"). +Relative to this literature, we add (i) a \emph{layered} attack that +combines a formatting layer with statistical rewriting, (ii) a +systematic ROC measurement of deployed-class schemes under a single +protocol, (iii) a quality--detectability Pareto analysis, and (iv) an +Art.~50 policy measurement; none of the closest neighbors +\citep{he2024xsir,han2025synthid,omidi2026synthid,tamim2026forensic, +kirchenbauer2023reliability} combines all four. + +% ===================================================================== +% 3. Threat Model and System +% ===================================================================== +\section{Threat Model and System} +% Who (02 §5.3): end users editing text they generated with their own +% account -- explicitly NOT third-party content (see ethics.tex). +We consider users who edit text they generated themselves with their +own account before publishing it, for reasons of privacy or style. We +do not attack third-party content, and we do not frame the study as an +attacker--victim setting. + +% What (02 §5.3): watermark-as-label (not access control); detector = +% same-config MarkLLM detector (standard in the literature; vendor +% detectors are black-box and out of scope -- Section 8). +Watermarking is a label, not a lock: no access control is bypassed. The +detector is the same-config open-source MarkLLM implementation +\citep{pan2024markllm} used for generation, following standard +practice in the robustness literature; vendor detectors are not probed +(Section~\ref{sec:limitations}). + +% System (02 §5.3): two layers for v1 -- A formatting / B statistical -- +% with a detection-feedback rewrite loop (Fig. 1). +Our v1 pipeline has two layers: Layer~A strips formatting-layer marks +(invisible Unicode, bidi, tag cleanup) deterministically, and Layer~B +applies statistical rewriting (paraphrase, back-translation, structural +rewrite, humanization) driven by a detection-feedback loop that stops +once detection collapses. Figure~\ref{fig:f1} shows the system. + +\begin{figure}[t] + \centering + \includegraphics[width=\linewidth]{figures/f1} + \caption{The layered removal pipeline: Layer~A (formatting cleanup) + followed by Layer~B (statistical rewriting with a detection-feedback + loop).} + \label{fig:f1} +\end{figure} +% TODO: figures/f1.pdf is generated by research/scripts/make_figures.py +% (pipeline diagram; gap C3) -- until it exists the skeleton will not +% compile. + +% Definitions box (02 §5.3). +\begin{quote} +\textbf{Definitions.} \emph{TPR@FPR} -- true-positive rate at a fixed +false-positive rate (we report TPR@1\%FPR); \emph{AUROC} -- area +under the ROC curve; \emph{detection collapse} -- detection at +near-chance levels (AUROC $\approx 0.5$); \emph{quality cost} -- +drift in perplexity, BERTScore, and ROUGE-L relative to the unedited +output. % TODO: freeze exact metric definitions against 01 §5 once B2 +% lands. +\end{quote} + +% ===================================================================== +% 4. Experimental Setup +% ===================================================================== +\section{Experimental Setup} +% Design summary (02 §5.4 -> 01 §2): locked v1 factorial. +We follow the locked v1 factorial design of +\emph{research/01-experiment-protocol.md}: seven scheme configurations +(KGW $\gamma{=}.25,\delta{=}1$; KGW $\gamma{=}.5,\delta{=}2$; KGW +$\gamma{=}.5,\delta{=}4$; SynthID-Text; EXP; Unigram; SIR) across +lengths $\{100,300,500\}$, temperatures $\{0.7,1.0\}$, and +languages en, de, fr, es (restricted subsets) -- 3,500 watermarked and +3,500 unwatermarked generations (7,000 texts). +% TODO: cross-check the cell counts against the final run (01 §2). + +% Generation protocol (01 §3). +English texts are generated with \texttt{facebook/opt-1.3b} via +MarkLLM; the multilingual cells use \texttt{Qwen/Qwen2.5-1.5B-Instruct} +as an explicit model holdout. Watermark configs are pinned in +\texttt{research/configs/} and the same JSON is used for generation +and detection; seeds are fixed and recorded. + +% Attack cells A0-A8 (01 §4) + T1 taxonomy table (02 §6). +Each text passes through the attack cells A0--A8 of the protocol +(Table~\ref{tab:t1}): no attack, Layer~A only, single-pass paraphrase, +adaptive paraphrase with early stopping, back-translation round-trip, +structural rewrite, humanization, cheap token-level baselines, and the +full layered pipeline (A1 then A2/A3). + +% Detection + quality (01 §5): empirical null; quality metrics. +Detection uses the same-config MarkLLM detectors with an empirical null +distribution estimated from unwatermarked controls rather than normal +assumptions; quality is measured with perplexity (gpt2-large), +BERTScore, ROUGE-L, and length drift on a stratified subset. + +% Reproducibility (01 §7-8; 05 A7/E3). +We release the corpus, configs, seeds, harness pins (MarkLLM commit, HF +revisions, pip freeze), and results as JSONL. +% TODO: manifest.json + release URLs before submission (gaps A7/E3). + +\begin{table}[t] + \centering + \caption{Attack taxonomy: family $\times$ mechanism $\times$ + implementation $\times$ prior-art anchor.} + \label{tab:t1} + \input{tables/t1} +\end{table} +% TODO: tables/t1.tex is generated by research/scripts/make_tables.py +% (real rows from the run; gap C3). + +% ===================================================================== +% 5. Results +% ===================================================================== +\section{Results} +% Money table (02 §6 T2). +Table~\ref{tab:t2} reports the core AUROC / TPR@1\%FPR matrix: rows +are scheme configurations (7) $\times$ attacks (8); columns are +pre-attack, A-only, B-only, and A+B. +% TODO: fill Table 2 with real numbers from the run via +% research/scripts/make_tables.py, and confirm the five key claims +% (02 §5.5): +% (i) paraphrase/back-translation collapse KGW-class detection; +% (ii) SynthID-Text resists token substitution but not paraphrase; +% (iii) layering > single layers at equal quality cost; +% (iv) even the strongest config (gamma=.5, delta=4, 300 tok) drops +% below usable TPR under adaptive rewrite; +% (v) multilingual (de/fr/es) is systematically more fragile. + +Figure~\ref{fig:f2} shows ROC curves before and after removal per +scheme, and Figure~\ref{fig:f3} shows the quality--detectability +Pareto frontier. Table~\ref{tab:t3} reports quality metrics +(perplexity delta, BERTScore, ROUGE-L, length drift, number/URL +survival) at the point of detection collapse; Table~\ref{tab:t4} and +Figure~\ref{fig:f4} ablate watermark strength ($\gamma,\delta$) by +length and temperature; Table~\ref{tab:t5} covers the multilingual +grid; Table~\ref{tab:t6} reports attack cost (tokens in/out, wall +time, USD per 1k words); Table~\ref{tab:t7} compares with published +baselines where numbers are reported in the same metric. +Figure~\ref{fig:f5} shows a redacted before/after case study with +detector scores. +% TODO: replace the placeholder prose above with result-specific +% discussion once the run lands (writing phase W2). + +\begin{table}[t] + \centering + \caption{AUROC / TPR@1\%FPR matrix (the money table).} + \label{tab:t2} + \input{tables/t2} +\end{table} +% TODO: tables/t2.tex from research/scripts/make_tables.py. + +\begin{figure}[t] + \centering + \includegraphics[width=\linewidth]{figures/f2} + \caption{ROC curves pre/post removal per scheme (the money figure).} + \label{fig:f2} +\end{figure} +% TODO: figures/f2.pdf from research/scripts/make_figures.py. + +\begin{figure}[t] + \centering + \includegraphics[width=\linewidth]{figures/f3} + \caption{Quality--detectability Pareto frontier (PPL vs.\ AUROC + across attacks).} + \label{fig:f3} +\end{figure} +% TODO: figures/f3.pdf from research/scripts/make_figures.py. + +\begin{table}[t] + \centering + \caption{Quality per attack at the collapse point: PPL delta, + BERTScore, ROUGE-L, length drift, number/URL survival.} + \label{tab:t3} + \input{tables/t3} +\end{table} +% TODO: tables/t3.tex from research/scripts/make_tables.py. + +\begin{table}[t] + \centering + \caption{Ablation: TPR@1\%FPR $\times$ ($\gamma,\delta$) $\times$ + length $\times$ temperature.} + \label{tab:t4} + \input{tables/t4} +\end{table} +% TODO: tables/t4.tex from research/scripts/make_tables.py. + +\begin{figure}[t] + \centering + \includegraphics[width=\linewidth]{figures/f4} + \caption{TPR@1\%FPR vs.\ watermark strength, length as line + style.} + \label{fig:f4} +\end{figure} +% TODO: figures/f4.pdf from research/scripts/make_figures.py. + +\begin{table}[t] + \centering + \caption{Multilingual results (en/de/fr/es; v1 extension).} + \label{tab:t5} + \input{tables/t5} +\end{table} +% TODO: tables/t5.tex from research/scripts/make_tables.py. + +\begin{table}[t] + \centering + \caption{Attack cost: tokens in/out, wall time, USD per 1k words.} + \label{tab:t6} + \input{tables/t6} +\end{table} +% TODO: tables/t6.tex from research/scripts/make_tables.py. + +\begin{table}[t] + \centering + \caption{Comparison with published baselines (cite-and-compare where + numbers are reported in the same metric).} + \label{tab:t7} + \input{tables/t7} +\end{table} +% TODO: tables/t7.tex from research/scripts/make_tables.py. + +\begin{figure}[t] + \centering + \includegraphics[width=\linewidth]{figures/f5} + \caption{Case study: redacted before/after text with detector + scores.} + \label{fig:f5} +\end{figure} +% TODO: figures/f5.pdf from research/scripts/make_figures.py. + +% ===================================================================== +% 6. Analysis and Case Study +% ===================================================================== +\section{Analysis and Case Study} +% Cost of attack vs cost of defense (02 §5.6; Table 6). +The cost asymmetry between attack and defense (Table~\ref{tab:t6}) is +central to the policy argument: an attacker spends minutes and a few +cents of API credit per document, while the defender must raise +watermark strength at a measurable quality cost. +% TODO: quote the real cost figures and quality deltas from the run. + +% Failure modes ranked (02 §5.6). +Ranking the failure modes, token-level statistical watermarks (the KGW +line) collapse under semantic rewriting, n-gram/tournament methods +(SynthID-Text) survive token substitution but not paraphrase, and +semantic-invariant properties survive best. % TODO: replace with the +% ranked, number-backed analysis once results exist; the redacted case +% study appears in Figure~\ref{fig:f5}. + +% ===================================================================== +% 7. Policy Discussion +% ===================================================================== +\section{Policy Discussion} +% Art. 50 mechanics (04 §2; 02 §5.7; citation map 02 §7). +Article~50 of Regulation (EU) 2024/1689 \citep{euaiact2024} places +transparency obligations on providers and deployers of general-purpose +AI systems, not on end users editing their own documents: the +obligation is that machine-readable output be detectable, and our +measurements ask whether that detection survives ordinary editing. +% TODO: verify exact recital numbers before submission (04 §2). + +Recent forensic-readiness evidence \citep{tamim2026forensic} is +consistent with our findings. We argue that platform-level provenance +and metadata (C2PA) \citep{c2pa2024spec} are the robust complement to +text-layer watermarking, and that regulators should not rely on +token-level schemes alone. Figure~\ref{fig:f6} places the measured +collapse against the Art.~50 timeline. % TODO: honest limits of the +% study here (no vendor black-box measurement; see Section 8). + +\begin{figure}[t] + \centering + \includegraphics[width=\linewidth]{figures/f6} + \caption{Art.~50 timeline vs.\ measured detection collapse (policy + figure).} + \label{fig:f6} +\end{figure} +% TODO: figures/f6.pdf from research/scripts/make_figures.py. + +% ===================================================================== +% 8. Limitations +% ===================================================================== +\section{Limitations} +\label{sec:limitations} +% 02 §5.8. +Our detector is same-config and open-source; vendor detectors are +black-box and were not probed. Generators are small open-weight models +(opt-1.3b; Qwen2.5-1.5B-Instruct for de/fr/es as an explicit model +holdout) rather than frontier LLMs. API-probe limits bound the rewrite +loop, and our detection-feedback oracle is stronger than a naive user; +we therefore report both adaptive and single-pass numbers. +% TODO: add any additional limitations surfaced during the run. + +% ===================================================================== +% 9. Ethics Statement +% ===================================================================== +\section{Ethics Statement} +% Draft from research/04-ethics-and-legal.md §3, adapted to a paragraph; +% adapt to the venue template at submission (arXiv has no ethics +% checklist -- 04 §6). +\input{ethics} + +% ===================================================================== +% 10. Conclusion +% ===================================================================== +\section{Conclusion} +We measured the robustness of deployed-class text watermarking under +realistic, layered user-side editing. Our results indicate that +quality-preserving rewriting collapses KGW-class detection, that +layering dominates single attacks at equal quality cost, and that +multilingual text is systematically more fragile. % TODO: summarize +% the final numbers and the Art. 50 recommendation here (02 §5.10). + +% --- acknowledgments ----------------------------------------------------- +\section*{Acknowledgments} +% Source: research/04-ethics-and-legal.md §5 + research/03-related-work.md §D. +\input{acknowledgments} + +% --- bibliography --------------------------------------------------------- +\bibliographystyle{plainnat} +\bibliography{refs} +% TODO: every entry in refs.bib must be cited (or pruned) before +% submission; re-verify all IDs and venue labels (03 header). + +\end{document} diff --git a/research/paper/refs.bib b/research/paper/refs.bib new file mode 100644 index 0000000..be31c66 --- /dev/null +++ b/research/paper/refs.bib @@ -0,0 +1,388 @@ +% refs.bib -- verified bibliography for the arXiv v1 paper (gap C2). +% Source: research/03-related-work.md. Every entry was checked against +% the arXiv API on 2026-08-18 (see the 03 header); all entries in +% sections A-E below resolve to the exact paper. +% +% % re-verify at submission: every ID, venue label, and URL below must be +% re-checked before upload (03 header; 05-arxiv-readiness.md checklist F). +% Full author lists were not re-verified -- expand "and others" then. +% +% Conventions: +% - ID = first-author-lastname + arXiv year (+ short tag). +% - arXiv items are @misc with eprint/archivePrefix (and primaryClass). +% - Venue labels are from the 03 table and kept in the note field. +% +% Four candidates were dropped from 03 on 2026-08-18 (unverifiable) and +% are intentionally NOT here; re-verify before ever re-adding any of them +% (see 03 header, "Four candidate citations were dropped..."). + +% --------------------------------------------------------------------- +% A. Watermarking methods (the attack surface we test) +% --------------------------------------------------------------------- + +@misc{kirchenbauer2023kwg, + title = {A Watermark for Large Language Models}, + author = {Kirchenbauer, John and others}, + year = {2023}, + eprint = {2301.10226}, + archivePrefix = {arXiv}, + primaryClass = {cs.CL}, + note = {ICML 2023}, +} + +@misc{christ2023undetectable, + title = {Undetectable Watermarks for Language Models}, + author = {Christ, Miranda and others}, + year = {2023}, + eprint = {2306.09194}, + archivePrefix = {arXiv}, + primaryClass = {cs.LG}, + note = {COLT 2024}, +} + +@misc{kuditipudi2023robust, + title = {Robust Distortion-free Watermarks for Language Models}, + author = {Kuditipudi, Rohith and others}, + year = {2023}, + eprint = {2307.15593}, + archivePrefix = {arXiv}, + primaryClass = {cs.LG}, + note = {TMLR}, +} + +@misc{liu2023sir, + title = {A Semantic Invariant Robust Watermark for Large + Language Models}, + author = {Liu, Aiwei and others}, + year = {2023}, + eprint = {2310.06356}, + archivePrefix = {arXiv}, + primaryClass = {cs.CL}, + note = {ICLR 2024}, +} + +@misc{liu2023upv, + title = {An Unforgeable Publicly Verifiable Watermark for + Large Language Models}, + author = {Liu, Aiwei and others}, + year = {2023}, + eprint = {2307.16230}, + archivePrefix = {arXiv}, + primaryClass = {cs.CL}, + note = {ICLR 2024}, +} + +@misc{zhao2024permute, + title = {Permute-and-Flip: An Optimally Stable and + Watermarkable Decoder}, + author = {Zhao, Xuandong and others}, + year = {2024}, + eprint = {2402.05864}, + archivePrefix = {arXiv}, + primaryClass = {cs.LG}, + note = {no venue label in 03 -- re-verify}, +} + +@misc{hu2023unbiased, + title = {Unbiased Watermark for Large Language Models}, + author = {Hu, Zhengmian and others}, + year = {2023}, + eprint = {2310.10669}, + archivePrefix = {arXiv}, + primaryClass = {cs.CL}, + note = {no venue label in 03 -- re-verify}, +} + +@misc{wu2023dipmark, + title = {A Resilient and Accessible Distribution-Preserving + Watermark for Large Language Models}, + author = {Wu, Yihan and others}, + year = {2023}, + eprint = {2310.07710}, + archivePrefix = {arXiv}, + primaryClass = {cs.CL}, + note = {no venue label in 03 -- re-verify}, +} + +@misc{huo2024tswatermark, + title = {Token-Specific Watermarking with Enhanced Detectability + and Semantic Coherence for Large Language Models}, + author = {Huo, Mingjia and others}, + year = {2024}, + eprint = {2402.18059}, + archivePrefix = {arXiv}, + primaryClass = {cs.CL}, + note = {ICML 2024}, +} + +@misc{liu2024adaptive, + title = {Adaptive Text Watermark for Large Language Models}, + author = {Liu, Yepeng and others}, + year = {2024}, + eprint = {2401.13927}, + archivePrefix = {arXiv}, + primaryClass = {cs.CL}, + note = {no venue label in 03 -- re-verify}, +} + +@misc{hou2023semstamp, + title = {SemStamp: A Semantic Watermark with Paraphrastic + Robustness for Text Generation}, + author = {Hou, Abe Bohan and others}, + year = {2023}, + eprint = {2310.03991}, + archivePrefix = {arXiv}, + primaryClass = {cs.CL}, + note = {no venue label in 03 -- re-verify}, +} + +@misc{hou2024ksemstamp, + title = {k-SemStamp: A Clustering-Based Semantic Watermark + for Generated Text}, + author = {Hou, Abe Bohan and others}, + year = {2024}, + eprint = {2402.11399}, + archivePrefix = {arXiv}, + primaryClass = {cs.CL}, + note = {no venue label in 03 -- re-verify}, +} + +@misc{gu2025invisible, + title = {Invisible Entropy: Safe and Efficient Low-Entropy + LLM Watermarking}, + author = {Gu, Tianle and others}, + year = {2025}, + eprint = {2505.14112}, + archivePrefix = {arXiv}, + primaryClass = {cs.CL}, + note = {no venue label in 03 -- re-verify}, +} + +@misc{wang2025morphmark, + title = {MorphMark: Flexible Adaptive Watermarking for + Large Language Models}, + author = {Wang, Zongqi and others}, + year = {2025}, + eprint = {2505.11541}, + archivePrefix = {arXiv}, + primaryClass = {cs.CL}, + note = {no venue label in 03 -- re-verify}, +} + +@misc{yang2023blackbox, + title = {Watermarking Text Generated by Black-Box Language + Models}, + author = {Yang, Xi and others}, + year = {2023}, + eprint = {2305.08883}, + archivePrefix = {arXiv}, + primaryClass = {cs.CL}, + note = {NAACL 2024}, +} + +% --------------------------------------------------------------------- +% B. Robustness, attacks, and limits (our direct neighbors) +% --------------------------------------------------------------------- + +@misc{kirchenbauer2023reliability, + title = {On the Reliability of Watermarks for Large Language + Models}, + author = {Kirchenbauer, John and others}, + year = {2023}, + eprint = {2306.04634}, + archivePrefix = {arXiv}, + primaryClass = {cs.CL}, + note = {NeurIPS 2023; note title is "Watermarks", not + "Watermarking" (03 header)}, +} + +@misc{he2024xsir, + title = {Can Watermarks Survive Translation? On the + Cross-lingual Consistency of Text Watermark for Large + Language Models}, + author = {He, Zhiwei and others}, + year = {2024}, + eprint = {2402.14007}, + archivePrefix = {arXiv}, + primaryClass = {cs.CL}, + note = {ACL 2024; cross-lingual attack; our A4 baseline + anchor}, +} + +@misc{zhang2023sand, + title = {Watermarks in the Sand: Impossibility of Strong + Watermarking for Generative Models}, + author = {Zhang, Hanlin and others}, + year = {2023}, + eprint = {2311.04378}, + archivePrefix = {arXiv}, + primaryClass = {cs.LG}, + note = {ICML 2024; Random Walk attack; impossibility theory}, +} + +@misc{jovanovic2024stealing, + title = {Watermark Stealing in Large Language Models}, + author = {Jovanovi\'{c}, Nikola and others}, + year = {2024}, + eprint = {2402.19361}, + archivePrefix = {arXiv}, + primaryClass = {cs.CR}, + note = {key-extraction threat; no venue label in 03 -- + re-verify}, +} + +@misc{pan2024waterseeker, + title = {WaterSeeker: Efficient Detection of Watermarked + Segments in Large Documents}, + author = {Pan, Leyi and others}, + year = {2024}, + eprint = {2409.05112}, + archivePrefix = {arXiv}, + primaryClass = {cs.CL}, + note = {NAACL 2025 Findings; detector side}, +} + +@misc{lu2024ewd, + title = {An Entropy-based Text Watermarking Detection + Method}, + author = {Lu, Yijian and others}, + year = {2024}, + eprint = {2403.13485}, + archivePrefix = {arXiv}, + primaryClass = {cs.CL}, + note = {ACL 2024; detector side}, +} + +@misc{liu2024crafted, + title = {Can Watermarked LLMs be Identified by Users via + Crafted Prompts?}, + author = {Liu, Aiwei and others}, + year = {2024}, + eprint = {2410.03168}, + archivePrefix = {arXiv}, + primaryClass = {cs.CL}, + note = {ICLR 2025 Spotlight}, +} + +@misc{pan2025distillation, + title = {Can LLM Watermarks Robustly Prevent Unauthorized + Knowledge Distillation?}, + author = {Pan, Leyi and others}, + year = {2025}, + eprint = {2502.11598}, + archivePrefix = {arXiv}, + primaryClass = {cs.CL}, + note = {ACL 2025}, +} + +% --------------------------------------------------------------------- +% C. Closest recent neighbors (2025-2026 measurement / forensics) +% --------------------------------------------------------------------- + +@misc{han2025synthid, + title = {Robustness Assessment and Enhancement of Text + Watermarking for Google's SynthID}, + author = {Han, Xia and others}, + year = {2025}, + eprint = {2508.20228}, + archivePrefix = {arXiv}, + primaryClass = {cs.CL}, + note = {closest method-specific robustness work; must cite + and differentiate in Sections 2/5}, +} + +@misc{omidi2026synthid, + title = {On Google's SynthID-Text LLM Watermarking System: + Theoretical Analysis and Empirical Validation}, + author = {Omidi, Romina and others}, + year = {2026}, + eprint = {2603.03410}, + archivePrefix = {arXiv}, + primaryClass = {cs.LG}, + note = {theoretical analysis of SynthID-Text}, +} + +@misc{tamim2026forensic, + title = {AI Watermark Evidence Fails Forensic Readiness: An + Empirical Evaluation}, + author = {Tamim, Saifur Rahman and others}, + year = {2026}, + eprint = {2607.16010}, + archivePrefix = {arXiv}, + primaryClass = {cs.CL}, + note = {directly supports our policy finding (Section 7)}, +} + +@misc{harelcanada2025sandcastles, + title = {Sandcastles in the Storm: Revisiting the + (Im)possibility of Strong Watermarking}, + author = {Harel-Canada, Fabrice Y and others}, + year = {2025}, + eprint = {2505.06827}, + archivePrefix = {arXiv}, + primaryClass = {cs.LG}, + note = {impossibility revisited}, +} + +% --------------------------------------------------------------------- +% D. Tools & surveys +% --------------------------------------------------------------------- + +@misc{pan2024markllm, + title = {MarkLLM: An Open-Source Toolkit for LLM Watermarking}, + author = {Pan, Leyi and others}, + year = {2024}, + eprint = {2405.10051}, + archivePrefix = {arXiv}, + primaryClass = {cs.CL}, + note = {EMNLP 2024 Demo; our harness; THU-BPM}, +} + +@misc{liu2023survey, + title = {A Survey of Text Watermarking in the Era of Large + Language Models}, + author = {Liu, Aiwei and others}, + year = {2023}, + eprint = {2312.07913}, + archivePrefix = {arXiv}, + primaryClass = {cs.CL}, + note = {ACM Computing Surveys 2025; overview + taxonomy}, +} + +% --------------------------------------------------------------------- +% E. Non-arXiv sources (correct IDs; cite directly) +% --------------------------------------------------------------------- + +@article{deepmind2024synthidtext, + title = {SynthID-Text: Text Watermarking for LLMs with + Tournament-based Sampling}, + author = {{Google DeepMind}}, + journal = {Nature}, + year = {2024}, + volume = {638}, + pages = {625--632}, + doi = {10.1038/s41586-024-08025-4}, + note = {cite the Nature DOI as primary; the arXiv tech-report id + was not resolvable in our checks (03 header)}, +} + +@misc{c2pa2024spec, + title = {C2PA Specification}, + author = {{Coalition for Content Provenance and Authenticity}}, + year = {2024}, + howpublished = {Specification 2.1}, + url = {https://c2pa.org/specifications/specifications/2.1/specs/C2PA_Specification.html}, + note = {verify spec version and year at submission}, +} + +@misc{euaiact2024, + title = {Regulation (EU) 2024/1689 of the European Parliament + and of the Council of 13 June 2024 laying down + harmonised rules on artificial intelligence + (Artificial Intelligence Act)}, + author = {{European Parliament and Council of the European Union}}, + year = {2024}, + url = {https://eur-lex.europa.eu/eli/reg/2024/1689/oj}, + note = {Art.~50 (transparency obligations); in force for GPAI + since 2026-08-02; verify exact recitals at submission}, +} diff --git a/research/pins-quality.txt b/research/pins-quality.txt new file mode 100644 index 0000000..ae8b522 --- /dev/null +++ b/research/pins-quality.txt @@ -0,0 +1,77 @@ +absl-py==2.5.0 +annotated-doc==0.0.5 +anyio==4.14.2 +bert-score==0.3.13 +certifi==2026.7.22 +charset-normalizer==3.5.1 +click==8.4.2 +contourpy==1.3.3 +cuda-bindings==13.3.1 +cuda-pathfinder==1.6.1 +cuda-toolkit==13.0.3.0 +cycler==0.12.1 +defusedxml==0.7.1 +filelock==3.32.3 +fonttools==4.63.0 +fsspec==2026.7.0 +h11==0.16.0 +hf-xet==1.6.0 +httpcore==1.0.9 +httpx==0.28.1 +huggingface_hub==1.28.0 +idna==3.19 +Jinja2==3.1.6 +joblib==1.5.3 +kiwisolver==1.5.0 +markdown-it-py==4.2.0 +MarkupSafe==3.0.3 +matplotlib==3.11.1 +mdurl==0.1.2 +mpmath==1.3.0 +narwhals==2.24.0 +networkx==3.6.1 +nltk==3.10.3 +numpy==2.5.2 +nvidia-cublas==13.1.1.3 +nvidia-cuda-cupti==13.0.85 +nvidia-cuda-nvrtc==13.0.88 +nvidia-cuda-runtime==13.0.96 +nvidia-cudnn-cu13==9.20.0.48 +nvidia-cufft==12.0.0.61 +nvidia-cufile==1.15.1.6 +nvidia-curand==10.4.0.35 +nvidia-cusolver==12.0.4.66 +nvidia-cusparse==12.6.3.3 +nvidia-cusparselt-cu13==0.8.1 +nvidia-nccl-cu13==2.29.7 +nvidia-nvjitlink==13.3.33 +nvidia-nvshmem-cu13==3.4.5 +nvidia-nvtx==13.0.85 +packaging==26.3 +pandas==3.0.5 +pillow==12.3.0 +Pygments==2.21.0 +pyparsing==3.3.2 +python-dateutil==2.9.0.post0 +PyYAML==6.0.3 +regex==2026.7.19 +requests==2.34.2 +rich==15.0.0 +rouge_score==0.1.2 +safetensors==0.8.0 +scikit-learn==1.9.0 +scipy==1.18.0 +sentence-transformers==6.0.0 +setuptools==84.0.0 +shellingham==1.5.4 +six==1.17.0 +sympy==1.14.0 +threadpoolctl==3.6.0 +tokenizers==0.22.2 +torch==2.13.0 +tqdm==4.70.0 +transformers==5.15.0 +triton==3.7.1 +typer==0.27.1 +typing_extensions==4.16.0 +urllib3==2.7.0 diff --git a/research/requirements-quality.txt b/research/requirements-quality.txt new file mode 100644 index 0000000..3003b17 --- /dev/null +++ b/research/requirements-quality.txt @@ -0,0 +1,34 @@ +# research/requirements-quality.txt +# ================================= +# +# SEPARATE quality-metric environment for research/scripts/evaluate_quality.py +# (research/01-experiment-protocol.md §5.3, gap 05-B2). +# +# This env MUST stay isolated from the MarkLLM environment (the service/ +# MarkLLM harness): it pins a different torch/transformers/tokenizers stack +# and adds the metric libraries (sentence-transformers, bert-score, +# rouge-score). Installing this file into the MarkLLM env would break that +# env's pinned versions -- create a dedicated venv instead: +# +# python3 -m venv .venv-quality +# .venv-quality/bin/pip install -r research/requirements-quality.txt +# +# PPL is scored with gpt2-large and NEVER with the generator (opt-1.3b / +# Qwen2.5), so the MarkLLM stack is not needed here (protocol §5.3, "never +# score with the generator"). Record `pip freeze` of this env in the run +# manifest (protocol §7). +# +# --- pinned core stack (exact versions, do not float) --- +torch==2.13.0.* +transformers==5.15.0 +tokenizers==0.22.2 + +# --- general numeric / hub dependencies --- +numpy +scipy +huggingface_hub + +# --- metric libraries: pin exact versions at setup time --- +sentence-transformers +bert-score +rouge-score diff --git a/research/scripts/analyze_roc.py b/research/scripts/analyze_roc.py new file mode 100644 index 0000000..1e4628d --- /dev/null +++ b/research/scripts/analyze_roc.py @@ -0,0 +1,371 @@ +#!/usr/bin/env python3 +"""ROC-based detection metrics with an empirical null (gap 05-B1). + +Computes AUROC and TPR@FPR for watermark detection scores, calibrating +every FPR threshold against the *empirical* null score distribution and +never against a parametric (e.g. standard-normal) assumption +(research/01-experiment-protocol.md §5.1-5.2). SynthID tournament +scores, in particular, are not standard normal, so z-score calibration +would be wrong; the empirical null is mandatory (protocol §5.2). + +CLI +--- + python3 research/scripts/analyze_roc.py --scores scores.jsonl \ + --out metrics.json [--n-bootstrap 10000] [--seed 1] \ + [--fpr-targets 0.001,0.01,0.1] [--attack NAME] + +scores.jsonl rows (one JSON object per line): + + {"condition": str, "attack": str, "kind": "watermarked"|"control"|"attacked", + "score": float, "is_watermarked": bool, "ok": bool} + +Rows with ok != true or a missing/non-finite score are skipped. For each +attack the signal distribution is kind == "watermarked" when the attack +is "none" (untouched originals) and kind == "attacked" otherwise; the +null distribution is kind == "control" under the same attack +(unwatermarked texts run through the same attack pipeline). + +Conventions +----------- +- AUROC is rank-based (Mann-Whitney U with tie-averaged ranks), so no + scikit-learn dependency is needed; ties are handled exactly. The value + is None (plus a warning) when scores are degenerate (all identical) or + a group is empty. +- For a target FPR alpha, the operating threshold is read off the + empirical null: the smallest null-derived score t with + P(null >= t) <= alpha — the ROC point whose FPR is the largest one not + exceeding the budget (ties count toward the positive side). If no null + score reaches FPR <= alpha (coarse null), the strictest threshold (max + null score) is used and a warning is emitted. +- 95% bootstrap CIs (percentile interval) are computed over + --n-bootstrap resamples of both signal and null with a seeded RNG; + thresholds are re-derived from the resampled null, so the CI covers + null-sampling variability too. +""" + +from __future__ import annotations + +import argparse +import json +import math +import sys +from pathlib import Path +from typing import Any + +import numpy as np + + +def _rankdata_average(values: np.ndarray) -> np.ndarray: + """1-based average ranks of *values*; tied entries share the mean rank.""" + order = np.argsort(values, kind="mergesort") + sorted_vals = values[order] + ranks = np.empty(values.size, dtype=float) + n = values.size + i = 0 + while i < n: + j = i + while j + 1 < n and sorted_vals[j + 1] == sorted_vals[i]: + j += 1 + ranks[order[i : j + 1]] = (i + j) / 2.0 + 1.0 + i = j + 1 + return ranks + + +def _auroc(signal: np.ndarray, null: np.ndarray) -> float | None: + """Rank-based AUROC; None when undefined (empty group or constant scores).""" + if signal.size == 0 or null.size == 0: + return None + if np.unique(np.concatenate([signal, null])).size <= 1: + return None + combined = np.concatenate([signal, null]) + ranks = _rankdata_average(combined) + n_s, n_n = signal.size, null.size + sum_signal_ranks = ranks[:n_s].sum() + auc = (sum_signal_ranks - n_s * (n_s + 1) / 2.0) / (n_s * n_n) + return float(auc) + + +def _fpr_thresholds(null: np.ndarray, alphas: list[float]) -> list[float]: + """One operating threshold per target FPR, read off the empirical null. + + For each alpha the threshold is the smallest score t taken from the + null itself with P(null >= t) <= alpha (ties count positive). When no + null score fits the budget, the strictest null score (its max) is + used, giving the smallest achievable FPR. + """ + if null.size == 0: + return [0.0] * len(alphas) + sorted_null = np.sort(null) + unique = np.unique(sorted_null) + counts = null.size - np.searchsorted(sorted_null, unique, side="left") + thresholds: list[float] = [] + for alpha in alphas: + budget = math.floor(alpha * null.size) + ok = counts <= budget + if not ok.any(): + thresholds.append(float(unique[-1])) + else: + thresholds.append(float(unique[int(np.flatnonzero(ok)[0])])) + return thresholds + + +def _roc_points(signal: np.ndarray, null: np.ndarray) -> list[list[float]]: + """Empirical ROC points [[fpr, tpr], ...] (monotone, endpoints included).""" + if signal.size == 0 or null.size == 0: + return [[0.0, 0.0], [1.0, 1.0]] + thresholds = np.unique(np.concatenate([signal, null]))[::-1] + points: list[list[float]] = [[0.0, 0.0]] + for t in thresholds: + point = [float((null >= t).mean()), float((signal >= t).mean())] + if point != points[-1]: + points.append(point) + if points[-1] != [1.0, 1.0]: + points.append([1.0, 1.0]) + return points + + +def _as_score(row: dict[str, Any]) -> float | None: + """Parse the row's score; None when missing or non-finite.""" + if "score" not in row: + return None + try: + score = float(row["score"]) + except (TypeError, ValueError): + return None + return score if math.isfinite(score) else None + + +def compute_roc_metrics( + signal_scores: list[float], + null_scores: list[float], + fpr_targets: list[float], + n_bootstrap: int = 10000, + seed: int = 1, +) -> dict[str, Any]: + """ROC metrics (AUROC, TPR@FPR, bootstrap CIs) for one attack. + + Args: + signal_scores: detector scores of watermarked (or attacked) texts. + null_scores: detector scores of unwatermarked controls, used as the + empirical null for FPR calibration (protocol §5.2). + fpr_targets: FPR targets in (0, 1] at which TPR is reported. + n_bootstrap: number of bootstrap resamples for the 95% CIs. + seed: RNG seed for the bootstrap resamples. + + Returns: + A dict with n_signal, n_null, auroc, tpr_at_fpr (keyed by str(fpr)), + auroc_ci95, tpr_ci95, roc_points and warnings. + """ + if not fpr_targets: + raise ValueError("fpr_targets must not be empty") + for fpr in fpr_targets: + if not 0.0 < fpr <= 1.0: + raise ValueError(f"fpr target must be in (0, 1], got {fpr!r}") + + signal = np.asarray(signal_scores, dtype=float) + null = np.asarray(null_scores, dtype=float) + n_s, n_n = int(signal.size), int(null.size) + + warnings: list[str] = [] + if n_n < 10: + warnings.append(f"n_null={n_n} < 10: empirical FPR calibration is coarse") + if n_s == 0: + warnings.append("no signal scores") + if n_n == 0: + warnings.append("no null scores; FPR cannot be calibrated from the empirical null") + if n_s > 0 and n_n > 0 and np.unique(np.concatenate([signal, null])).size <= 1: + warnings.append("degenerate: all scores identical; AUROC undefined (None)") + + auroc = _auroc(signal, null) + if auroc is not None and not 0.0 <= auroc <= 1.0: + warnings.append(f"auroc={auroc:.6f} outside [0, 1] (numerical noise)") + + tpr_at_fpr: dict[str, float | None] = {} + if n_s > 0 and n_n > 0: + for fpr, threshold in zip(fpr_targets, _fpr_thresholds(null, fpr_targets), strict=True): + achieved = float((null >= threshold).mean()) + if achieved > fpr: + warnings.append( + f"FPR target {fpr}: achieved FPR {achieved:.4f} " + "(null too coarse; strictest null threshold used)" + ) + tpr_at_fpr[str(fpr)] = float((signal >= threshold).mean()) + else: + tpr_at_fpr = {str(fpr): None for fpr in fpr_targets} + + auroc_ci95: list[float | None] = [None, None] + tpr_ci95: dict[str, list[float | None]] = {str(fpr): [None, None] for fpr in fpr_targets} + if n_bootstrap >= 2 and n_s > 0 and n_n > 0: + rng = np.random.default_rng(seed) + aucs: list[float] = [] + tpr_boot: dict[str, list[float]] = {str(fpr): [] for fpr in fpr_targets} + for _ in range(n_bootstrap): + sb = signal[rng.integers(0, n_s, size=n_s)] + nb = null[rng.integers(0, n_n, size=n_n)] + boot_auc = _auroc(sb, nb) + if boot_auc is not None: + aucs.append(boot_auc) + for fpr, threshold in zip(fpr_targets, _fpr_thresholds(nb, fpr_targets), strict=True): + tpr_boot[str(fpr)].append(float((sb >= threshold).mean())) + if len(aucs) >= 2: + auroc_ci95 = [float(x) for x in np.percentile(aucs, [2.5, 97.5])] + else: + warnings.append("bootstrap AUROC CI unavailable (resamples degenerate)") + for fpr in fpr_targets: + vals = tpr_boot[str(fpr)] + if len(vals) >= 2: + tpr_ci95[str(fpr)] = [float(x) for x in np.percentile(vals, [2.5, 97.5])] + elif n_bootstrap < 2: + warnings.append("bootstrap skipped (n_bootstrap < 2)") + + return { + "n_signal": n_s, + "n_null": n_n, + "auroc": auroc, + "tpr_at_fpr": tpr_at_fpr, + "auroc_ci95": auroc_ci95, + "tpr_ci95": tpr_ci95, + "roc_points": _roc_points(signal, null), + "warnings": warnings, + } + + +def analyze_scores( + rows: list[dict[str, Any]], + fpr_targets: list[float], + n_bootstrap: int = 10000, + seed: int = 1, + attack: str | None = None, +) -> dict[str, Any]: + """Group score rows by attack and compute ROC metrics per attack. + + Rows with ok != true or a missing/non-finite score are skipped. For + each attack, signal is kind == "watermarked" (attack "none") or + kind == "attacked" (any other attack); null is kind == "control" under + the same attack. When *attack* is given, only that attack is reported. + """ + condition: str | None = None + valid: list[dict[str, Any]] = [] + for row in rows: + if condition is None and row.get("condition") is not None: + condition = row["condition"] + if row.get("ok") is not True: + continue + score = _as_score(row) + if score is None: + continue + parsed = dict(row) + parsed["score"] = score + valid.append(parsed) + + groups: dict[str, list[dict[str, Any]]] = {} + for row in valid: + groups.setdefault(str(row.get("attack")), []).append(row) + if attack is not None: + groups = {name: group for name, group in groups.items() if name == attack} + + per_attack: dict[str, dict[str, Any]] = {} + for name, group in groups.items(): + signal_kind = "watermarked" if name == "none" else "attacked" + signal = [row["score"] for row in group if row.get("kind") == signal_kind] + null = [row["score"] for row in group if row.get("kind") == "control"] + per_attack[name] = compute_roc_metrics( + signal, + null, + fpr_targets, + n_bootstrap=n_bootstrap, + seed=seed, + ) + return {"condition": condition, "per_attack": per_attack} + + +def _parse_fpr_targets(raw: str) -> list[float]: + """Parse a comma-separated --fpr-targets value into validated floats.""" + parts = [part.strip() for part in raw.split(",") if part.strip()] + targets = [float(part) for part in parts] + if not targets: + raise ValueError("--fpr-targets must contain at least one value") + for target in targets: + if not 0.0 < target <= 1.0: + raise ValueError(f"FPR target {target} must be in (0, 1]") + return targets + + +def _load_rows(path: str) -> list[dict[str, Any]]: + """Load scores.jsonl into a list of dicts; raise on malformed lines.""" + rows: list[dict[str, Any]] = [] + with open(path, encoding="utf-8") as fh: + for line_number, line in enumerate(fh, start=1): + content = line.strip() + if not content: + continue + try: + rows.append(json.loads(content)) + except json.JSONDecodeError as exc: + raise ValueError(f"{path}:{line_number}: malformed JSON: {exc}") from exc + return rows + + +def main(argv: list[str] | None = None) -> int: + """CLI entry point: compute per-attack ROC metrics and write JSON.""" + parser = argparse.ArgumentParser( + description="ROC-based detection metrics with an empirical null (protocol §5.1-5.2)." + ) + parser.add_argument("--scores", required=True, help="scores.jsonl input file") + parser.add_argument("--out", required=True, help="JSON output file path") + parser.add_argument( + "--n-bootstrap", + type=int, + default=10000, + help="number of bootstrap resamples for the 95 percent CIs (default: 10000)", + ) + parser.add_argument("--seed", type=int, default=1, help="bootstrap RNG seed") + parser.add_argument( + "--fpr-targets", + default="0.001,0.01,0.1", + help="comma-separated FPR targets in (0, 1] (default: 0.001,0.01,0.1)", + ) + parser.add_argument( + "--attack", + default=None, + help="restrict analysis to one attack name (default: all)", + ) + args = parser.parse_args(argv) + if args.seed < 0: + print("error: --seed must be >= 0", file=sys.stderr) + return 1 + try: + fpr_targets = _parse_fpr_targets(args.fpr_targets) + rows = _load_rows(args.scores) + except (ValueError, OSError) as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + result = analyze_scores( + rows, + fpr_targets, + n_bootstrap=args.n_bootstrap, + seed=args.seed, + attack=args.attack, + ) + try: + Path(args.out).write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + except OSError as exc: + print(f"error: cannot write {args.out}: {exc}", file=sys.stderr) + return 1 + + print(f"condition: {result['condition']}") + for name, metrics in result["per_attack"].items(): + auroc = "n/a" if metrics["auroc"] is None else f"{metrics['auroc']:.4f}" + print( + f" {name!r}: n_signal={metrics['n_signal']} n_null={metrics['n_null']} auroc={auroc}" + ) + for fpr, tpr in metrics["tpr_at_fpr"].items(): + tpr_str = "n/a" if tpr is None else f"{tpr:.4f}" + print(f" TPR@{fpr} = {tpr_str}") + for warning in metrics["warnings"]: + print(f" warning: {warning}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/research/scripts/attacks/cheap.py b/research/scripts/attacks/cheap.py new file mode 100644 index 0000000..e01ccf8 --- /dev/null +++ b/research/scripts/attacks/cheap.py @@ -0,0 +1,338 @@ +#!/usr/bin/env python3 +"""Cheap, deterministic, local text attacks (protocol §4.1, attack A7). + +Implements the three "cheap baseline" attacks from +research/01-experiment-protocol.md §4.1 row A7 (prior-art anchor: Random +Walk / impossibility, ICML 2024), tracked as gap 05-A3 in +research/05-arxiv-readiness.md: + +- ``synonym``: deterministic synonym substitution. Uses nltk WordNet when + it is importable *and* its corpus data is present locally (the probe + never downloads anything); otherwise falls back to a small built-in map + of ~40 common English word pairs embedded in this file. +- ``delete``: random deletion of ~5-10% of words (``--delete-ratio``, + clamped to [0.02, 0.20]). +- ``sentence-reorder``: shuffle sentences. Sentences split on ``[.!?]`` + followed by whitespace + a capital letter (fallback: split on ``". "``). + +Every attack is fully deterministic given its seed: each function creates +a private ``random.Random(seed)`` and never touches the global RNG. Numbers +(``\\d+``) and URLs (``https?://\\S+``) are tokenized apart and are never +substituted, deleted, or split. + +CLI:: + + python3 research/scripts/attacks/cheap.py \ + --input FILE --output FILE \ + --attack {synonym,delete,sentence-reorder} [--seed N] [--delete-ratio F] + +Word tokens are ASCII letter runs (optionally with internal apostrophes or +hyphens); non-ASCII words pass through every attack untouched. +""" + +from __future__ import annotations + +import argparse +import random +import re +import sys +from functools import lru_cache +from pathlib import Path + +# Fraction of eligible (substitutable) word tokens the synonym attack +# actually replaces. Deterministic given the seed. +SYNONYM_RATE = 0.5 + +# Bounds for --delete-ratio / random_word_delete(ratio=...). +MIN_DELETE_RATIO = 0.02 +MAX_DELETE_RATIO = 0.20 + +# Small curated map used when WordNet is unavailable: ~40 common English +# word pairs (same part of speech, common register, single-word targets). +_BUILTIN_SYNONYMS: dict[str, str] = { + "happy": "glad", + "sad": "unhappy", + "angry": "mad", + "afraid": "scared", + "big": "large", + "small": "little", + "fast": "quick", + "slow": "sluggish", + "quick": "rapid", + "beautiful": "pretty", + "ugly": "hideous", + "good": "fine", + "bad": "poor", + "great": "excellent", + "nice": "pleasant", + "awful": "terrible", + "smart": "clever", + "strong": "powerful", + "weak": "feeble", + "rich": "wealthy", + "old": "ancient", + "new": "novel", + "young": "youthful", + "difficult": "hard", + "easy": "simple", + "important": "significant", + "interesting": "fascinating", + "buy": "purchase", + "begin": "start", + "end": "finish", + "help": "assist", + "think": "believe", + "see": "observe", + "look": "glance", + "talk": "speak", + "say": "state", + "get": "obtain", + "give": "grant", + "make": "create", + "use": "employ", + "show": "display", + "tell": "inform", + "ask": "inquire", + "answer": "reply", +} + +# Tokenizer used by the substitution and deletion attacks. Every character +# is matched by exactly one named alternative, so concatenating the values +# reproduces the input byte-for-byte; attacks can drop or rewrite tokens +# without losing anything else. +_TOKEN_RE = re.compile( + r"(?Phttps?://\S+)" + r"|(?P\d+(?:[.,]\d+)*)" + r"|(?P[A-Za-z]+(?:['-][A-Za-z]+)*)" + r"|(?P\s+)" + r"|(?P.)" +) + +# Sentence boundaries: whitespace after terminal punctuation, followed by +# a capital letter. The punctuation itself is kept with the sentence via a +# lookbehind, so splits never drop it. The loose fallback also accepts a +# lowercase start but never splits a digit-preceded period, so decimals +# like "3. 14" are left intact. +_SENT_BOUNDARY_RE = re.compile(r"(?<=[.!?])(\s+)(?=[A-Z])") +_SENT_BOUNDARY_LOOSE_RE = re.compile(r"(?<=(? list[tuple[str, str]]: + """Split *text* into ``(kind, value)`` tokens, preserving everything. + + Kinds: ``url``, ``number``, ``word``, ``space``, ``other`` (a single + punctuation/unknown character). Concatenating the values reproduces + *text* exactly. + """ + return [(match.lastgroup or "other", match.group()) for match in _TOKEN_RE.finditer(text)] + + +@lru_cache(maxsize=1) +def _wordnet_available() -> bool: + """True iff nltk WordNet is importable and its corpus data is present. + + Probes the local install only (``nltk.data.find``); never calls + ``nltk.download``, so this works offline. Callers fall back to + ``_BUILTIN_SYNONYMS`` when it returns False. + """ + try: + import nltk + except ImportError: + return False + try: + nltk.data.find("corpora/wordnet") + except LookupError: + return False + return True + + +@lru_cache(maxsize=4096) +def _wordnet_synonyms(word: str) -> list[str]: + """Sorted distinct lemma candidates for lowercase *word* (WordNet).""" + from nltk.corpus import wordnet + + candidates: set[str] = set() + for synset in wordnet.synsets(word): + for lemma in synset.lemmas(): + name = lemma.name().replace("_", " ") + if name.lower() != word: + candidates.add(name) + return sorted(candidates) + + +def _match_case(original: str, replacement: str) -> str: + """Apply *original*'s casing to *replacement* (upper/title/lower).""" + if original.isupper(): + return replacement.upper() + if original[:1].isupper(): + return replacement[:1].upper() + replacement[1:] + return replacement + + +def _synonym_for(word: str) -> str | None: + """Deterministic synonym candidate for *word*, or None if none exists. + + Checks the built-in map first, then WordNet when available. + """ + key = word.lower() + mapped = _BUILTIN_SYNONYMS.get(key) + if mapped is not None: + return mapped + if _wordnet_available(): + candidates = _wordnet_synonyms(key) + if candidates: + return candidates[0] + return None + + +def synonym_substitute(text: str, seed: int) -> str: + """Replace some substitutable words with synonyms (deterministic in seed). + + Each word token that has a synonym is replaced with probability + ``SYNONYM_RATE``, decided by the seeded RNG; numbers and URLs are never + touched. + """ + rng = _seeded_rng(seed) + out: list[str] = [] + for kind, value in _tokenize(text): + if kind == "word": + replacement = _synonym_for(value) + if replacement is not None and rng.random() < SYNONYM_RATE: + out.append(_match_case(value, replacement)) + continue + out.append(value) + return "".join(out) + + +def _clamp_delete_ratio(ratio: float) -> float: + """Clamp *ratio* into [MIN_DELETE_RATIO, MAX_DELETE_RATIO].""" + return max(MIN_DELETE_RATIO, min(MAX_DELETE_RATIO, ratio)) + + +# S311 is deliberate: this is a reproducible text attack, not cryptography. +def _seeded_rng(seed: int) -> random.Random: + """Private RNG seeded with *seed*; never the global RNG (determinism).""" + return random.Random(seed) # noqa: S311 + + +def random_word_delete(text: str, seed: int, ratio: float = 0.07) -> str: + """Delete a random subset of words (deterministic in seed). + + ``ratio`` is clamped to [0.02, 0.20]; the number deleted is + ``round(ratio * n_words)``. Numbers and URLs are never deleted. An + adjacent plain space is dropped with each deleted word so the text + stays tidy. + """ + ratio = _clamp_delete_ratio(ratio) + tokens = _tokenize(text) + word_indices = [i for i, (kind, _value) in enumerate(tokens) if kind == "word"] + n_words = len(word_indices) + if n_words == 0: + return text + rng = _seeded_rng(seed) + n_delete = round(ratio * n_words) + if n_delete <= 0: + return text + delete = set(rng.sample(word_indices, min(n_delete, n_words))) + for i in sorted(delete): + for j in (i - 1, i + 1): + if ( + 0 <= j < len(tokens) + and j not in delete + and tokens[j][0] == "space" + and tokens[j][1] == " " + ): + delete.add(j) + break + kept = [value for i, (_kind, value) in enumerate(tokens) if i not in delete] + return re.sub(r" {2,}", " ", "".join(kept)) + + +def _split_sentences(text: str) -> tuple[list[str], list[str]]: + """Split *text* into ``(sentences, separators)``. + + The first separator is ``""``; separator *i* is the whitespace that + preceded sentence *i* in the original text, so rejoining + ``sentences[0] + separators[1] + sentences[1] + ...`` reproduces *text* + exactly (up to sentence order). + """ + parts = _SENT_BOUNDARY_RE.split(text) + if len(parts) <= 1: + parts = _SENT_BOUNDARY_LOOSE_RE.split(text) + return parts[0::2], ["", *parts[1::2]] + + +def sentence_reorder(text: str, seed: int) -> str: + """Shuffle sentences (deterministic in seed), preserving every token. + + The multiset of characters - and therefore of word tokens - is + unchanged; only the order of sentences (and which inter-sentence + whitespace precedes which) is permuted. + """ + sentences, separators = _split_sentences(text) + if len(sentences) <= 1: + return text + rng = _seeded_rng(seed) + rng.shuffle(sentences) + parts = [sentences[0]] + for i in range(1, len(sentences)): + parts.append(separators[i]) + parts.append(sentences[i]) + return "".join(parts) + + +def apply_attack(text: str, attack: str, seed: int, delete_ratio: float = 0.07) -> str: + """Apply the named attack (``synonym`` | ``delete`` | ``sentence-reorder``).""" + if attack == "synonym": + return synonym_substitute(text, seed) + if attack == "delete": + return random_word_delete(text, seed, delete_ratio) + if attack == "sentence-reorder": + return sentence_reorder(text, seed) + raise ValueError( + f"unknown attack {attack!r}; expected one of synonym, delete, sentence-reorder" + ) + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + """Parse the CLI arguments (see module docstring).""" + parser = argparse.ArgumentParser( + description="Cheap deterministic text attacks (research/01-experiment-protocol.md §4.1 A7)." + ) + parser.add_argument("--input", required=True, help="UTF-8 text file to read") + parser.add_argument("--output", required=True, help="UTF-8 file to write the attacked text to") + parser.add_argument( + "--attack", + required=True, + choices=("synonym", "delete", "sentence-reorder"), + help="attack to apply", + ) + parser.add_argument("--seed", type=int, default=0, help="random seed (default: 0)") + parser.add_argument( + "--delete-ratio", + type=float, + default=0.07, + help="fraction of words to delete, clamped to [0.02, 0.20] (default: 0.07)", + ) + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + """CLI entry point: read --input, attack, write --output. 0 on success.""" + args = parse_args(argv) + try: + text = Path(args.input).read_text(encoding="utf-8") + except OSError as exc: + print(f"cheap.py: cannot read {args.input!r}: {exc}", file=sys.stderr) + return 1 + transformed = apply_attack(text, args.attack, args.seed, args.delete_ratio) + try: + Path(args.output).write_text(transformed, encoding="utf-8") + except OSError as exc: + print(f"cheap.py: cannot write {args.output!r}: {exc}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/research/scripts/evaluate_quality.py b/research/scripts/evaluate_quality.py new file mode 100644 index 0000000..27a7309 --- /dev/null +++ b/research/scripts/evaluate_quality.py @@ -0,0 +1,505 @@ +#!/usr/bin/env python3 +"""Text-quality metrics for the watermark-removal study (gap 05-B2). + +Implements research/01-experiment-protocol.md §5.3 for paired +(original, candidate) texts: + + ppl GPT-2 large mean-token perplexity of the *candidate* + (NEVER the generator -- the generator is opt-1.3b / + Qwen2.5, protocol §5.3 "never score with the + generator") + bertscore BERTScore F1, roberta-large (deberta-xlarge-mnli is + broken under transformers>=5: OverflowError in + set_truncation_and_padding; override via + WATERMARKS_BERTSCORE_MODEL), rescale_with_baseline=True + rouge_l ROUGE-L F1 (rouge-score lib, no stemmer / no nltk) + sbert_cosine SBERT cosine similarity, all-MiniLM-L6-v2 + levenshtein_pct edit distance % of original length (pure-python DP) + length_drift (len(candidate) - len(original)) / len(original) + numbers_preserved / urls_preserved regex set overlap, identical to + service/scripts/bench_synthid_text.py helpers + +Robustness rules (the run must never crash on a model problem): + + * every model/tool is lazy-loaded on first use and cached; + * each load is wrapped in its own try/except -- on failure that + metric is set to None for every row, a warning is recorded, and + the run continues; + * per-row computation failures degrade to None + warning; + * Levenshtein is capped: if len(original) > 4000 both texts are + truncated to 4000 chars and the row gains a "notes" entry. + +CLI: + + python3 research/scripts/evaluate_quality.py --input FILE --out FILE + [--limit N] [--device cpu] [--skip ppl,bertscore,rouge,sbert] + [--warnings-out FILE] + +Input JSONL rows: {"condition", "scheme", "seed", "prompt_idx", +"attack", "original", "candidate"}. Output rows = input row + the eight +metric keys above (None when skipped/unavailable) and, when relevant, a +"notes" list. + +The heavy dependencies (torch, transformers, bert-score, +sentence-transformers, rouge-score) are imported lazily inside the +loaders so the pure helpers and the CLI remain usable without them; +install them from research/requirements-quality.txt (a SEPARATE env +from the MarkLLM env, protocol §7). +""" + +from __future__ import annotations + +import argparse +import json +import math +import os +import re +import sys +from collections.abc import Callable +from pathlib import Path +from typing import Any + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +#: Output metric keys, in the order they are appended to each output row. +METRIC_KEYS: tuple[str, ...] = ( + "ppl", + "bertscore", + "rouge_l", + "sbert_cosine", + "levenshtein_pct", + "length_drift", + "numbers_preserved", + "urls_preserved", +) + +#: Model-backed metric names accepted by --skip (output key for rouge is +#: "rouge_l"; the skip name is "rouge"). +MODEL_METRICS: tuple[str, ...] = ("ppl", "bertscore", "rouge", "sbert") + +#: Cap for Levenshtein inputs: DP is O(n*m), 4000 chars keeps the +#: worst case at ~16M cell updates per pair (protocol §5.3, "edit +#: magnitude" on realistic user-edited text). +LEVENSHTEIN_MAX_CHARS: int = 4000 + +#: Module-level caches. _MODEL_CACHE holds loaded models/tools; _FAILED +#: holds metric names whose load already failed (metric -> None forever +#: in this process); _WARNINGS is the deduplicated warning log. +_MODEL_CACHE: dict[Any, Any] = {} +_FAILED: set[str] = set() +_WARNINGS: list[dict[str, str | None]] = [] + + +# --------------------------------------------------------------------------- +# Pure helpers (unit-testable without any model installed) +# --------------------------------------------------------------------------- + + +def levenshtein_distance(a: str, b: str) -> int: + """Character-level edit distance via space-optimized dynamic programming.""" + if a == b: + return 0 + if not a: + return len(b) + if not b: + return len(a) + prev = list(range(len(b) + 1)) + for i, ca in enumerate(a, start=1): + cur = [i] + for j, cb in enumerate(b, start=1): + cur.append(min(cur[-1] + 1, prev[j] + 1, prev[j - 1] + (ca != cb))) + prev = cur + return prev[-1] + + +def levenshtein_pct(original: str, candidate: str) -> float: + """Edit distance as a percentage of the original length. + + dist / max(len(original), 1) * 100 -- 0.0 when identical, 100.0 + when every original character must be replaced. + """ + return levenshtein_distance(original, candidate) / max(len(original), 1) * 100.0 + + +def truncate_pair( + original: str, candidate: str, max_chars: int = LEVENSHTEIN_MAX_CHARS +) -> tuple[str, str, bool]: + """Truncate *both* texts to *max_chars* when the original is longer. + + Returns (original', candidate', truncated). Keeps Levenshtein + cost bounded; the caller records a note when truncated is True. + """ + if len(original) <= max_chars: + return original, candidate, False + return original[:max_chars], candidate[:max_chars], True + + +def length_drift(original: str, candidate: str) -> float: + """Signed relative length change: (len(c) - len(o)) / max(len(o), 1).""" + return (len(candidate) - len(original)) / max(len(original), 1) + + +def numbers_preserved(original: str, candidate: str) -> float: + """Fraction of original numbers (regex \\d+) surviving in the candidate. + + Mirrors service/scripts/bench_synthid_text.py._numbers_preserved. + """ + a = set(re.findall(r"\d+", original)) + if not a: + return 1.0 + b = set(re.findall(r"\d+", candidate)) + return len(a & b) / len(a) + + +def urls_preserved(original: str, candidate: str) -> float: + """Fraction of original URLs (regex https?://\\S+) surviving. + + Mirrors service/scripts/bench_synthid_text.py._urls_preserved. + """ + a = set(re.findall(r"https?://\S+", original)) + if not a: + return 1.0 + b = set(re.findall(r"https?://\S+", candidate)) + return len(a & b) / len(a) + + +def ppl_from_loss(loss: float) -> float: + """Perplexity from a mean-token cross-entropy loss: exp(loss).""" + return float(math.exp(loss)) + + +# --------------------------------------------------------------------------- +# Metric math on top of loaded models (injected objects => unit-testable) +# --------------------------------------------------------------------------- + + +def compute_ppl( + model: Any, tokenizer: Any, text: str, *, device: str = "cpu", max_length: int = 1024 +) -> float: + """Mean-token perplexity of *text* under a causal-LM model. + + The tokenizer must accept (text, return_tensors="pt", + truncation=True, max_length=...) and return an input_ids / + attention_mask dict. Labels are shifted internally by + transformers, so the loss is the standard per-token NLL. + """ + import torch + + inputs = tokenizer(text, return_tensors="pt", truncation=True, max_length=max_length) + inputs = {key: value.to(device) for key, value in inputs.items()} + with torch.no_grad(): + outputs = model(**inputs, labels=inputs["input_ids"]) + return ppl_from_loss(float(outputs.loss)) + + +def compute_bertscore_f1(f1: Any) -> float: + """Scalar F1 out of the BERTScore F1 tensor (shape [batch]).""" + return float(f1.reshape(-1)[0].item()) + + +def compute_sbert_cosine(emb_a: Any, emb_b: Any) -> float: + """Cosine similarity between two sentence embeddings.""" + import torch + + a = emb_a.flatten().unsqueeze(0).float() + b = emb_b.flatten().unsqueeze(0).float() + return float(torch.nn.functional.cosine_similarity(a, b).item()) + + +def compute_rouge_l_fmeasure(result: Any) -> float: + """ROUGE-L F1 out of a rouge-score Score (has .fmeasure).""" + return float(result.fmeasure) + + +# --------------------------------------------------------------------------- +# Lazy loaders: one try/except each, cache on success, _FAILED on failure +# --------------------------------------------------------------------------- + + +def _get_ppl_models() -> tuple[Any, Any] | None: + """Load (tokenizer, model) for gpt2-large once; None on failure.""" + if "ppl" in _MODEL_CACHE: + return _MODEL_CACHE["ppl"] + if "ppl" in _FAILED: + return None + model_id = "gpt2-large" + try: + from transformers import AutoModelForCausalLM, AutoTokenizer + + tokenizer = AutoTokenizer.from_pretrained(model_id) + model = AutoModelForCausalLM.from_pretrained(model_id) + model.eval() + except Exception as exc: + _FAILED.add("ppl") + _warn("ppl", f"failed to load {model_id}: {exc}") + return None + _MODEL_CACHE["ppl"] = (model, tokenizer) + return _MODEL_CACHE["ppl"] + + +def _get_bert_score_fn() -> Callable[..., Any] | None: + """Load the bert_score score callable once; None on failure.""" + if "bertscore" in _MODEL_CACHE: + return _MODEL_CACHE["bertscore"] + if "bertscore" in _FAILED: + return None + try: + from bert_score import score as bert_score_score # type: ignore[import-not-found] + except Exception as exc: + _FAILED.add("bertscore") + _warn("bertscore", f"bert-score unavailable: {exc}") + return None + _MODEL_CACHE["bertscore"] = bert_score_score + return bert_score_score + + +def _get_rouge_scorer() -> Any | None: + """Load a ROUGE-L scorer once (pure python, no nltk); None on failure.""" + if "rouge" in _MODEL_CACHE: + return _MODEL_CACHE["rouge"] + if "rouge" in _FAILED: + return None + try: + from rouge_score import rouge_scorer # type: ignore[import-not-found] + + scorer = rouge_scorer.RougeScorer(["rougeL"], use_stemmer=False) + except Exception as exc: + _FAILED.add("rouge") + _warn("rouge", f"rouge-score unavailable: {exc}") + return None + _MODEL_CACHE["rouge"] = scorer + return scorer + + +def _get_sbert_model(device: str) -> Any | None: + """Load all-MiniLM-L6-v2 on *device* once (per device); None on failure.""" + cache_key: Any = ("sbert", device) + if cache_key in _MODEL_CACHE: + return _MODEL_CACHE[cache_key] + if "sbert" in _FAILED: + return None + try: + from sentence_transformers import SentenceTransformer # type: ignore[import-not-found] + + model = SentenceTransformer("all-MiniLM-L6-v2", device=device) + except Exception as exc: + _FAILED.add("sbert") + _warn("sbert", f"failed to load all-MiniLM-L6-v2: {exc}") + return None + _MODEL_CACHE[cache_key] = model + return model + + +# --------------------------------------------------------------------------- +# Per-metric compute wrappers +# --------------------------------------------------------------------------- + + +def _ppl_for_text(text: str, device: str) -> float | None: + loaded = _get_ppl_models() + if loaded is None: + return None + model, tokenizer = loaded + return compute_ppl(model, tokenizer, text, device=device) + + +# transformers >= 5.x breaks bert_score's deberta tokenizer path +# (OverflowError in set_truncation_and_padding); roberta-large is the +# bert_score default and works. Override with WATERMARKS_BERTSCORE_MODEL. +BERTSCORE_MODEL = os.environ.get("WATERMARKS_BERTSCORE_MODEL", "roberta-large") + + +def _bertscore_for_pair(original: str, candidate: str, device: str) -> float | None: + score_fn = _get_bert_score_fn() + if score_fn is None: + return None + _, _, f1 = score_fn( + [candidate], + [original], + lang="en", + model_type=BERTSCORE_MODEL, + rescale_with_baseline=True, + device=device, + ) + return compute_bertscore_f1(f1) + + +def _rouge_l_for_pair(original: str, candidate: str) -> float | None: + scorer = _get_rouge_scorer() + if scorer is None: + return None + result = scorer.score(original, candidate) + return compute_rouge_l_fmeasure(result["rougeL"]) + + +def _sbert_for_pair(original: str, candidate: str, device: str) -> float | None: + model = _get_sbert_model(device) + if model is None: + return None + embeddings = model.encode([original, candidate], convert_to_tensor=True) + return compute_sbert_cosine(embeddings[0], embeddings[1]) + + +# --------------------------------------------------------------------------- +# Warning log + row metric assembly +# --------------------------------------------------------------------------- + + +def _warn(metric: str | None, message: str) -> None: + """Record a deduplicated warning (also surfaced on stderr by main).""" + record: dict[str, str | None] = {"metric": metric, "message": message} + if record not in _WARNINGS: + _WARNINGS.append(record) + + +def _warnings() -> list[dict[str, str | None]]: + return list(_WARNINGS) + + +def _safe(metric: str, fn: Callable[[], Any]) -> Any: + """Run a model-backed computation; degrade to None instead of crashing.""" + try: + return fn() + except Exception as exc: + _warn(metric, f"{metric}: computation failed: {exc}") + return None + + +def _round4(value: float | None) -> float | None: + return round(value, 4) if value is not None else None + + +def compute_row_metrics( + row: dict[str, Any], skip: set[str], device: str +) -> tuple[dict[str, float | None], list[str]]: + """One input row's metrics plus any notes (e.g. Levenshtein truncation). + + skip holds MODEL_METRICS names to leave None; model metrics whose + load already failed are also None (see _FAILED). Never raises for + model problems -- every model-backed computation degrades via _safe. + """ + metrics: dict[str, float | None] = {key: None for key in METRIC_KEYS} + notes: list[str] = [] + original = row.get("original") + candidate = row.get("candidate") + if not isinstance(original, str) or not isinstance(candidate, str): + _warn(None, "row missing string 'original'/'candidate'; all metrics None") + notes.append("missing original/candidate; all metrics None") + return metrics, notes + + metrics["length_drift"] = _round4(length_drift(original, candidate)) + metrics["numbers_preserved"] = _round4(numbers_preserved(original, candidate)) + metrics["urls_preserved"] = _round4(urls_preserved(original, candidate)) + + o, c = original, candidate + if len(original) > LEVENSHTEIN_MAX_CHARS: + o, c, _ = truncate_pair(original, candidate, LEVENSHTEIN_MAX_CHARS) + notes.append(f"levenshtein: both texts truncated to {LEVENSHTEIN_MAX_CHARS} chars") + metrics["levenshtein_pct"] = _round4(levenshtein_pct(o, c)) + + if "ppl" not in skip: + metrics["ppl"] = _round4(_safe("ppl", lambda: _ppl_for_text(candidate, device))) + if "bertscore" not in skip: + metrics["bertscore"] = _round4( + _safe("bertscore", lambda: _bertscore_for_pair(original, candidate, device)) + ) + if "rouge" not in skip: + metrics["rouge_l"] = _round4(_safe("rouge", lambda: _rouge_l_for_pair(original, candidate))) + if "sbert" not in skip: + metrics["sbert_cosine"] = _round4( + _safe("sbert", lambda: _sbert_for_pair(original, candidate, device)) + ) + return metrics, notes + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +def main(argv: list[str] | None = None) -> int: + """Streaming CLI: read input JSONL, append metric keys, write output JSONL.""" + parser = argparse.ArgumentParser( + description=( + "Per-row text-quality metrics (research/01-experiment-protocol.md §5.3): " + "PPL (gpt2-large), BERTScore, ROUGE-L, SBERT cosine, Levenshtein %, " + "length drift, number/URL survival." + ) + ) + parser.add_argument("--input", required=True, help="input JSONL, one row per line") + parser.add_argument("--out", required=True, help="output JSONL (input row + metric keys)") + parser.add_argument("--limit", type=int, default=None, help="only process the first N rows") + parser.add_argument( + "--device", default="cpu", help="torch device for model inference (default: cpu)" + ) + parser.add_argument( + "--skip", + default="", + help="comma-separated model metrics to leave None: ppl,bertscore,rouge,sbert", + ) + parser.add_argument("--warnings-out", default=None, help="optional JSONL file for warnings") + args = parser.parse_args(argv) + + skip = {part.strip() for part in args.skip.split(",") if part.strip()} + unknown = skip - set(MODEL_METRICS) + if unknown: + parser.error( + f"unknown --skip values: {', '.join(sorted(unknown))} " + f"(valid: {', '.join(MODEL_METRICS)})" + ) + if args.limit is not None and args.limit < 0: + parser.error("--limit must be >= 0") + in_path = Path(args.input) + if not in_path.is_file(): + parser.error(f"input file not found: {in_path}") + + _WARNINGS.clear() + processed = 0 + with ( + in_path.open("r", encoding="utf-8") as src, + Path(args.out).open("w", encoding="utf-8") as dst, + ): + for line in src: + if args.limit is not None and processed >= args.limit: + break + stripped = line.strip() + if not stripped: + continue + try: + row = json.loads(stripped) + except json.JSONDecodeError as exc: + _warn(None, f"skipping malformed JSON line: {exc}") + continue + if not isinstance(row, dict): + _warn(None, "skipping non-object JSON row") + continue + metrics, notes = compute_row_metrics(row, skip, args.device) + out_row = dict(row) + out_row.update(metrics) + if notes: + existing = out_row.get("notes") + if isinstance(existing, list): + out_row["notes"] = existing + notes + elif existing is None: + out_row["notes"] = notes + else: + out_row["notes"] = [existing, *notes] + dst.write(json.dumps(out_row, ensure_ascii=False) + "\n") + processed += 1 + + records = _warnings() + for record in records: + metric = record["metric"] or "row" + print(f"warning: {metric}: {record['message']}", file=sys.stderr) + if args.warnings_out: + with Path(args.warnings_out).open("w", encoding="utf-8") as warn_file: + for record in records: + warn_file.write(json.dumps(record, ensure_ascii=False) + "\n") + print(f"processed {processed} rows; {len(records)} warnings", file=sys.stderr) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/research/scripts/make_figures.py b/research/scripts/make_figures.py new file mode 100644 index 0000000..9469806 --- /dev/null +++ b/research/scripts/make_figures.py @@ -0,0 +1,865 @@ +#!/usr/bin/env python3 +"""Paper figure generators (gap 05-B3): F1-F6 per research/02-paper-outline.md sec 6. + +CLI: + python3 research/scripts/make_figures.py --results-dir DIR --out-dir DIR \ + [--format png|pdf] [--dpi 200] + +Reads the same results layout as make_tables.py (one directory per cell): + + DIR//metrics.json per-attack AUROC/TPR + optional + "roc_points" per attack (F2) + DIR//quality.jsonl quality metrics per attack (F3) + DIR//attacked.jsonl before/after texts + cost (F5) + DIR//scores.jsonl detector scores (F5, optional) + +Writes out-dir/figures/f1. ... f6.. F1 (pipeline diagram) and F6 +(policy timeline) are static schematics; F2-F5 read data and degrade to a +"no data" figure when their inputs are missing. + +matplotlib is imported lazily inside main(): if it is not installed the +script prints a clear warning and exits 0 without writing any figures. +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +import textwrap +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +SCHEME_ORDER: tuple[str, ...] = ( + "kgw-d1", + "kgw-d2", + "kgw-d4", + "synthid", + "exp", + "unigram", + "sir", +) +SCHEME_LABELS: dict[str, str] = { + "kgw-d1": "KGW (gamma=.25, delta=1)", + "kgw-d2": "KGW (gamma=.5, delta=2)", + "kgw-d4": "KGW (gamma=.5, delta=4)", + "synthid": "SynthID-Text", + "exp": "EXP (Gumbel)", + "unigram": "Unigram", + "sir": "SIR", +} +ATTACK_LABELS: dict[str, str] = { + "none": "A0 none", + "layerA": "A1 layerA", + "paraphrase:1": "A2 paraphrase:1", + "paraphrase:3": "A3 paraphrase:3", + "backtranslate:de": "A4 backtranslate:de", + "structural": "A5 structural", + "humanize": "A6 humanize", + "cheap": "A7 cheap", + "layerA+paraphrase:3": "A8 layerA+paraphrase:3", +} +LAYERED_ATTACK = "layerA+paraphrase:3" + +_RESERVED_TOP: frozenset[str] = frozenset( + { + "roc_points", + "quality", + "config", + "meta", + "cell", + "condition", + "manifest", + "generated_at", + "timestamp", + } +) +_CONDITION_RE = re.compile( + r"^(?P.+?)-L(?P\d+)-T(?P[0-9]+(?:\.[0-9]+)?)" + r"-(?P[a-z]{2})-s(?P\d+)-p(?P\d+)$" +) + + +@dataclass(frozen=True) +class Condition: + """One results cell with the artifacts the figures need.""" + + dir: Path + name: str + scheme: str | None + length: int | None + temp: float | None + language: str | None + metrics: dict[str, dict[str, Any]] + roc_points: dict[str, tuple[list[float], list[float]]] + quality: list[dict[str, Any]] + attacked: list[dict[str, Any]] + scores: list[dict[str, Any]] + + +# --------------------------------------------------------------------------- +# Results loading (mirrors make_tables.py; also loads scores.jsonl) +# --------------------------------------------------------------------------- + + +def _scheme_from_name(name: str) -> str | None: + for scheme in SCHEME_ORDER: + if name == scheme or name.startswith(scheme + "-"): + return scheme + return None + + +def _num_in_name(name: str, prefix: str) -> int | None: + m = re.search(rf"{re.escape(prefix)}(\d+)", name) + return int(m.group(1)) if m else None + + +def _float_in_name(name: str, prefix: str) -> float | None: + m = re.search(rf"{re.escape(prefix)}([0-9]+(?:\.[0-9]+)?)", name) + return float(m.group(1)) if m else None + + +def _lang_in_name(name: str) -> str | None: + m = re.search(r"-(en|de|fr|es)-", name) + return m.group(1) if m else None + + +def _read_jsonl(path: Path) -> list[dict[str, Any]]: + if not path.is_file(): + return [] + rows: list[dict[str, Any]] = [] + try: + for raw_line in path.read_text(encoding="utf-8").splitlines(): + line = raw_line.strip() + if not line: + continue + try: + obj = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(obj, dict): + rows.append(obj) + except OSError: + return [] + return rows + + +def _normalize_metrics(raw: Any) -> dict[str, dict[str, Any]]: + if isinstance(raw, list): + out: dict[str, dict[str, Any]] = {} + for item in raw: + if isinstance(item, dict) and item.get("attack"): + out[str(item["attack"])] = {k: v for k, v in item.items() if k != "attack"} + return out + if not isinstance(raw, dict): + return {} + for wrapper in ("attacks", "results", "per_attack"): + if isinstance(raw.get(wrapper), dict): + return _normalize_metrics(raw[wrapper]) + return {str(k): v for k, v in raw.items() if isinstance(v, dict) and k not in _RESERVED_TOP} + + +def _roc_pair(pts: Any) -> tuple[list[float], list[float]] | None: + if isinstance(pts, dict): + fpr, tpr = pts.get("fpr"), pts.get("tpr") + if ( + isinstance(fpr, (list, tuple)) + and isinstance(tpr, (list, tuple)) + and len(fpr) == len(tpr) + and len(fpr) > 1 + ): + return [float(x) for x in fpr], [float(x) for x in tpr] + return None + if isinstance(pts, (list, tuple)) and len(pts) == 2: + a, b = pts[0], pts[1] + if ( + isinstance(a, (list, tuple)) + and isinstance(b, (list, tuple)) + and len(a) == len(b) + and len(a) > 1 + ): + return [float(x) for x in a], [float(x) for x in b] + if isinstance(pts, (list, tuple)): + try: + xs = [float(p[0]) for p in pts] + ys = [float(p[1]) for p in pts] + except (TypeError, ValueError, IndexError): + return None + if len(xs) == len(ys) and len(xs) > 1: + return xs, ys + return None + + +def _normalize_roc(raw: Any) -> dict[str, tuple[list[float], list[float]]]: + if not isinstance(raw, dict): + return {} + out: dict[str, tuple[list[float], list[float]]] = {} + for attack, pts in raw.items(): + pair = _roc_pair(pts) + if pair is not None: + out[str(attack)] = pair + return out + + +def _load_condition(path: Path) -> Condition | None: + name = path.name + m = _CONDITION_RE.match(name) + if m: + scheme = m.group("scheme") + length = int(m.group("length")) + temp = float(m.group("temp")) + language = m.group("language") + else: + scheme = _scheme_from_name(name) + length = _num_in_name(name, "L") + temp = _float_in_name(name, "T") + language = _lang_in_name(name) + + metrics: dict[str, dict[str, Any]] = {} + roc: dict[str, tuple[list[float], list[float]]] = {} + metrics_path = path / "metrics.json" + if metrics_path.is_file(): + try: + raw = json.loads(metrics_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + raw = None + if isinstance(raw, dict): + roc.update(_normalize_roc(raw.get("roc_points"))) + metrics = _normalize_metrics(raw) + for attack, attack_metrics in metrics.items(): + pair = _roc_pair(attack_metrics.get("roc_points")) + if pair is not None: + roc.setdefault(attack, pair) + + quality = _read_jsonl(path / "quality.jsonl") + attacked = _read_jsonl(path / "attacked.jsonl") + scores = _read_jsonl(path / "scores.jsonl") + if not metrics and not quality and not attacked and not scores: + return None + return Condition( + dir=path, + name=name, + scheme=scheme, + length=length, + temp=temp, + language=language, + metrics=metrics, + roc_points=roc, + quality=quality, + attacked=attacked, + scores=scores, + ) + + +def _load_conditions(results_dir: Path) -> list[Condition]: + if not results_dir.is_dir(): + return [] + conditions: list[Condition] = [] + for child in sorted(results_dir.iterdir()): + if not child.is_dir(): + continue + cond = _load_condition(child) + if cond is not None: + conditions.append(cond) + return conditions + + +# --------------------------------------------------------------------------- +# Metric helpers +# --------------------------------------------------------------------------- + + +def _auroc(m: Mapping[str, Any]) -> float | None: + for key in ("auroc", "auc", "roc_auc"): + v = m.get(key) + if isinstance(v, (int, float)) and not isinstance(v, bool): + return float(v) + sub = m.get("metrics") + if isinstance(sub, Mapping): + return _auroc(sub) + return None + + +def _fpr_in_key(key: str) -> float | None: + flat = re.sub(r"[\s_@]+", "", key).lower() + m = re.search(r"tprfpr([0-9.]+)", flat) + if m: + return float(m.group(1)) + m = re.search(r"tprat([0-9.]+)pct", flat) + if m: + return float(m.group(1)) / 100.0 + return None + + +def _tpr_at_fpr(m: Mapping[str, Any], fpr: float = 0.01) -> float | None: + t = m.get("tpr_at_fpr") + if isinstance(t, dict): + return _tpr_from_dict(t, fpr) + if isinstance(t, (int, float)) and not isinstance(t, bool): + return float(t) + for key, val in m.items(): + if not isinstance(val, (int, float)) or isinstance(val, bool): + continue + want = _fpr_in_key(str(key)) + if want is not None and abs(want - fpr) < 1e-9: + return float(val) + sub = m.get("metrics") + if isinstance(sub, Mapping): + return _tpr_at_fpr(sub, fpr) + return None + + +def _tpr_from_dict(d: Mapping[str, Any], fpr: float) -> float | None: + for key, val in d.items(): + if not isinstance(val, (int, float)) or isinstance(val, bool): + continue + k = str(key).strip() + pct = k.endswith("%") + num = k[:-1] if pct else k + try: + x = float(num) + except ValueError: + continue + if pct: + x = x / 100.0 + if abs(x - fpr) < 1e-9: + return float(val) + return None + + +def _mean_metric( + conds: Sequence[Condition], attack: str, metric: str, fpr: float = 0.01 +) -> float | None: + """Mean *metric* ('auroc' | 'tpr') for *attack* across *conds*.""" + values: list[float] = [] + for cond in conds: + m = cond.metrics.get(attack) + if not m: + continue + v = _auroc(m) if metric == "auroc" else _tpr_at_fpr(m, fpr) + if v is not None: + values.append(v) + return sum(values) / len(values) if values else None + + +def _quality_num(row: Mapping[str, Any], aliases: Sequence[str]) -> float | None: + sub = row.get("metrics") if isinstance(row.get("metrics"), Mapping) else None + for src in (row, sub): + if not src: + continue + for key in aliases: + v = src.get(key) + if isinstance(v, (int, float)) and not isinstance(v, bool): + return float(v) + return None + + +# --------------------------------------------------------------------------- +# Figure helpers +# --------------------------------------------------------------------------- + + +def _no_data_ax(ax: Any) -> None: + """Turn *ax* into a graceful 'no data' placeholder.""" + ax.text(0.5, 0.5, "no data", ha="center", va="center", fontsize=14, transform=ax.transAxes) + ax.set_xticks([]) + ax.set_yticks([]) + + +def _mean_roc(np: Any, pairs: Sequence[tuple[list[float], list[float]]]) -> tuple[Any, Any] | None: + """Interpolate ROC pairs onto a common FPR grid and average the TPRs.""" + if not pairs: + return None + grid = np.linspace(0.0, 1.0, 101) + tprs = [] + for fpr, tpr in pairs: + order = np.argsort(fpr) + f = np.asarray(fpr, dtype=float)[order] + t = np.asarray(tpr, dtype=float)[order] + tprs.append(np.interp(grid, f, t)) + return grid, np.mean(tprs, axis=0) + + +def _truncate(text: str, limit: int = 600) -> str: + """Collapse whitespace and truncate *text* to ~*limit* chars (word-safe).""" + text = " ".join(str(text).split()) + if len(text) <= limit: + return text + cut = text[:limit] + space = cut.rfind(" ") + if space > limit // 2: + cut = cut[:space] + return cut + " ... [truncated at ~600 chars; no redaction beyond truncation]" + + +def _wrap(text: str, width: int = 74) -> str: + return "\n".join(textwrap.wrap(text, width=width)) if text else "(empty)" + + +def _pick_case_row(cond: Condition) -> tuple[dict[str, Any], str]: + for attack in (LAYERED_ATTACK, "paraphrase:3"): + for row in cond.attacked: + if row.get("attack") == attack: + return row, attack + row = cond.attacked[0] + return row, str(row.get("attack", "unknown")) + + +def _detector_scores(cond: Condition, attack: str) -> tuple[str | None, str | None]: + """Best-effort before/after detector scores for *attack*.""" + + def scan(row: Mapping[str, Any]) -> tuple[str | None, str | None]: + before = after = None + for key in ("score_before", "before_score", "original_score", "score_original"): + v = row.get(key) + if isinstance(v, (int, float)) and not isinstance(v, bool): + before = f"{v:.3f}" + break + for key in ("score_after", "after_score", "candidate_score", "score_candidate"): + v = row.get(key) + if isinstance(v, (int, float)) and not isinstance(v, bool): + after = f"{v:.3f}" + break + if after is None: + for key in ("score", "detect_score"): + v = row.get(key) + if isinstance(v, (int, float)) and not isinstance(v, bool): + after = f"{v:.3f}" + break + return before, after + + for row in cond.attacked: + if row.get("attack") == attack: + before, after = scan(row) + if before is not None or after is not None: + return before, after + for row in cond.scores: + if row.get("attack") in (attack, None): + before, after = scan(row) + if before is not None or after is not None: + return before, after + return None, None + + +# --------------------------------------------------------------------------- +# Figure builders (each returns a matplotlib Figure) +# --------------------------------------------------------------------------- + + +def figure_f1(plt: Any) -> Any: + """F1 (static): pipeline diagram -- Layer A + Layer B + feedback loop.""" + from matplotlib.patches import FancyBboxPatch + + fig, ax = plt.subplots(figsize=(12, 5.5)) + ax.set_xlim(0, 1) + ax.set_ylim(0, 1) + ax.set_xticks([]) + ax.set_yticks([]) + ax.set_xlabel("pipeline stage (left to right)") + ax.set_ylabel("") + ax.set_title( + "F1 - Layered removal pipeline: Layer A formatting cleanup + " + "Layer B statistical rewrite with detection-feedback loop" + ) + + def box(x: float, y: float, w: float, h: float, text: str, fc: str, ec: str) -> None: + patch = FancyBboxPatch( + (x, y), w, h, boxstyle="round,pad=0.012", linewidth=1.5, edgecolor=ec, facecolor=fc + ) + ax.add_patch(patch) + ax.text(x + w / 2, y + h / 2, text, ha="center", va="center", fontsize=8.5) + + def arrow( + x1: float, y1: float, x2: float, y2: float, rad: float = 0.0, color: str = "black" + ) -> None: + ax.annotate( + "", + xy=(x2, y2), + xytext=(x1, y1), + arrowprops=dict( + arrowstyle="-|>", lw=1.5, color=color, connectionstyle=f"arc3,rad={rad}" + ), + ) + + box( + 0.02, + 0.62, + 0.17, + 0.22, + "Watermarked text\n(input; same-config\nscores known)", + "#e0f2fe", + "#0c4a6e", + ) + box( + 0.25, + 0.62, + 0.20, + 0.22, + "Layer A - formatting cleanup\nUnicode/invisible chars, bidi,\ntag removal (clean_text.py)", + "#dbeafe", + "#1e40af", + ) + box( + 0.51, + 0.62, + 0.20, + 0.22, + "Layer B - statistical rewrite\nparaphrase / back-translate /\nstructural / humanize", + "#dbeafe", + "#1e40af", + ) + box(0.77, 0.62, 0.20, 0.22, "Attacked text (candidate)\noutput", "#f0fdf4", "#166534") + box( + 0.40, + 0.10, + 0.32, + 0.22, + "Detector (same-config MarkLLM)\nwatermark score on candidate", + "#fef3c7", + "#92400e", + ) + arrow(0.19, 0.73, 0.25, 0.73) + arrow(0.45, 0.73, 0.51, 0.73) + arrow(0.71, 0.73, 0.77, 0.73) + arrow(0.87, 0.62, 0.60, 0.32, rad=0.22) # candidate -> detector + arrow(0.40, 0.32, 0.56, 0.62, rad=0.22) # detector -> Layer B feedback + ax.text( + 0.44, + 0.015, + "detection feedback: if still watermarked, rewrite again " + "(adaptive; early stop when detection passes)", + ha="center", + fontsize=8, + color="#92400e", + ) + fig.tight_layout() + return fig + + +def figure_f2(plt: Any, np: Any, conditions: Sequence[Condition]) -> Any: + """F2: ROC curves pre/post per scheme (mean over cells with data).""" + schemes = [s for s in SCHEME_ORDER if any(c.scheme == s for c in conditions)] + post = next( + (a for a in (LAYERED_ATTACK, "paraphrase:3") if any(a in c.roc_points for c in conditions)), + None, + ) + if not schemes: + fig, ax = plt.subplots(figsize=(8, 6)) + ax.set_title("F2 - ROC curves pre/post per scheme") + _no_data_ax(ax) + fig.tight_layout() + return fig + cols = 2 + rows = (len(schemes) + 1) // cols + fig, axes = plt.subplots(rows, cols, figsize=(11, 3.2 * rows), squeeze=False) + for i in range(rows * cols): + ax = axes.flat[i] + if i >= len(schemes): + ax.set_visible(False) + continue + scheme = schemes[i] + ax.set_title(SCHEME_LABELS[scheme], fontsize=10) + ax.plot([0, 1], [0, 1], "--", color="0.6", lw=0.8) + pre_pairs = [ + c.roc_points["none"] + for c in conditions + if c.scheme == scheme and "none" in c.roc_points + ] + post_pairs = [ + c.roc_points[post] + for c in conditions + if c.scheme == scheme and post and post in c.roc_points + ] + mean_pre = _mean_roc(np, pre_pairs) + mean_post = _mean_roc(np, post_pairs) + if mean_pre is None and mean_post is None: + _no_data_ax(ax) + else: + if mean_pre is not None: + ax.plot(mean_pre[0], mean_pre[1], "-", color="tab:blue", lw=1.8, label="pre-attack") + if mean_post is not None: + ax.plot( + mean_post[0], + mean_post[1], + "-", + color="tab:red", + lw=1.8, + label="post-attack (A+B)", + ) + ax.set_xlabel("false positive rate") + ax.set_ylabel("true positive rate") + ax.set_xlim(0, 1) + ax.set_ylim(0, 1) + handles, labels = axes.flat[0].get_legend_handles_labels() + fig.legend(handles, labels, loc="upper center", ncol=4, fontsize=9, frameon=False) + fig.suptitle( + "F2 - ROC curves pre/post attack per scheme (mean over cells; dashed = chance)", y=0.995 + ) + fig.tight_layout(rect=[0, 0, 1, 0.96]) + return fig + + +def figure_f3(plt: Any, conditions: Sequence[Condition]) -> Any: + """F3: quality-detectability Pareto frontier (PPL vs AUROC per attack).""" + fig, ax = plt.subplots(figsize=(8, 6)) + ax.set_title("F3 - Quality-detectability Pareto frontier (PPL vs AUROC)") + ax.set_xlabel("PPL (gpt2-large; lower is better)") + ax.set_ylabel("AUROC (higher is better)") + ref = next((c for c in conditions if c.quality and c.metrics), None) + if ref is None: + _no_data_ax(ax) + fig.tight_layout() + return fig + by_attack: dict[str, list[dict[str, Any]]] = {} + for row in ref.quality: + if isinstance(row, dict) and row.get("attack"): + by_attack.setdefault(str(row["attack"]), []).append(row) + points: list[tuple[str, float, float]] = [] + used_delta = False + for attack, rows in by_attack.items(): + m = ref.metrics.get(attack) + if not m: + continue + auroc = _auroc(m) + ppl: float | None = None + for r in rows: + v = _quality_num(r, ("ppl", "perplexity", "candidate_ppl")) + if v is None: + v = _quality_num(r, ("ppl_delta", "perplexity_delta")) + used_delta = v is not None + if v is not None: + ppl = v + break + if auroc is not None and ppl is not None: + points.append((attack, ppl, auroc)) + if not points: + _no_data_ax(ax) + else: + xs = [p[1] for p in points] + ys = [p[2] for p in points] + ax.scatter(xs, ys, s=60, color="tab:blue", zorder=3) + for attack, x, y in points: + ax.annotate( + ATTACK_LABELS.get(attack, attack), + (x, y), + fontsize=8, + xytext=(6, 6), + textcoords="offset points", + ) + ax.set_xlim(left=min(xs) * 0.9) + if used_delta: + ax.text( + 0.02, + 0.02, + "x = PPL delta when absolute PPL is not recorded", + transform=ax.transAxes, + fontsize=8, + color="0.4", + ) + fig.tight_layout() + return fig + + +def figure_f4(plt: Any, conditions: Sequence[Condition], attack: str) -> Any: + """F4: TPR@1%FPR vs KGW strength (delta), with length as line style.""" + fig, ax = plt.subplots(figsize=(8, 6)) + ax.set_title(f"F4 - TPR@1%FPR vs watermark strength (KGW), by length (attack: {attack})") + ax.set_xlabel("KGW strength delta (gamma=0.25 for delta=1, else 0.5)") + ax.set_ylabel("TPR@1%FPR (mean over cells)") + kgw = [ + c for c in conditions if c.scheme in ("kgw-d1", "kgw-d2", "kgw-d4") and c.language == "en" + ] + deltas = {"kgw-d1": 1, "kgw-d2": 2, "kgw-d4": 4} + styles = {100: ("-", "o"), 300: ("--", "s"), 500: (":", "^")} + plotted = False + for length in (100, 300, 500): + xs: list[float] = [] + ys: list[float] = [] + for scheme in ("kgw-d1", "kgw-d2", "kgw-d4"): + conds = [c for c in kgw if c.scheme == scheme and c.length == length and c.temp == 0.7] + if not conds: + conds = [c for c in kgw if c.scheme == scheme and c.length == length] + v = _mean_metric(conds, attack, "tpr") + if v is not None: + xs.append(float(deltas[scheme])) + ys.append(v) + if xs: + linestyle, marker = styles[length] + ax.plot(xs, ys, linestyle=linestyle, marker=marker, lw=1.8, label=f"length {length}") + plotted = True + if not plotted: + _no_data_ax(ax) + ax.set_xticks([1, 2, 4]) + ax.set_xticklabels(["delta=1\ngamma=.25", "delta=2\ngamma=.5", "delta=4\ngamma=.5"]) + ax.legend(fontsize=9) + fig.tight_layout() + return fig + + +def figure_f5(plt: Any, conditions: Sequence[Condition]) -> Any: + """F5: case study -- before/after text with detector scores.""" + fig = plt.figure(figsize=(13, 6)) + cond = next((c for c in conditions if c.attacked), None) + if cond is None: + ax = fig.add_subplot(111) + ax.set_title("F5 - Case study: before/after with detector scores") + _no_data_ax(ax) + fig.tight_layout() + return fig + row, attack = _pick_case_row(cond) + before = _truncate(row.get("original", ""), 600) + after = _truncate(row.get("candidate", ""), 600) + score_before, score_after = _detector_scores(cond, attack) + ax1, ax2 = fig.subplots(1, 2) + for ax in (ax1, ax2): + ax.axis("off") + ax1.set_title(f"Before (original, watermarked) - detector score: {score_before or 'n/a'}") + ax2.set_title(f"After ({attack}) - detector score: {score_after or 'n/a'}") + ax1.text( + 0.02, + 0.98, + _wrap(before), + transform=ax1.transAxes, + va="top", + ha="left", + fontsize=8.5, + family="monospace", + wrap=True, + ) + ax2.text( + 0.02, + 0.98, + _wrap(after), + transform=ax2.transAxes, + va="top", + ha="left", + fontsize=8.5, + family="monospace", + wrap=True, + ) + fig.suptitle( + "F5 - Case study: before/after (600-char window; no redaction " + "beyond truncation) with detector scores" + ) + fig.tight_layout() + return fig + + +def figure_f6(plt: Any) -> Any: + """F6 (static): policy timeline -- EU AI Act Art. 50 vs measured collapse.""" + from datetime import date + + import matplotlib.dates as mdates + + fig, ax = plt.subplots(figsize=(11, 5)) + events = [ + (date(2023, 1, 1), "KGW watermark proposed\n(arXiv 2301.10226)", "context", 1), + (date(2024, 5, 21), "EU AI Act adopted\n(Regulation (EU) 2024/1689)", "policy", 1), + (date(2026, 8, 2), "Art. 50 in force:\ntransparency obligations", "policy", 1), + (date(2026, 8, 2), "measured collapse:\nKGW-class under layered attack", "collapse", -1), + (date(2026, 8, 15), "SynthID-Text API\nretired (Google)", "context", -1), + ] + colors = {"policy": "tab:red", "collapse": "darkred", "context": "tab:blue"} + ax.axhline(0.0, color="0.25", lw=1.2) + for d, label, kind, side in events: + x = mdates.date2num(d) + color = colors[kind] + ax.plot([x, x], [0, side * 0.32], color=color, lw=1.8) + ax.scatter([x], [0], color=color, s=40, zorder=5) + ax.text( + x, + side * 0.36, + label, + ha="center", + va="bottom" if side > 0 else "top", + fontsize=8, + color=color, + ) + ax.axvspan( + mdates.date2num(date(2026, 7, 15)), + mdates.date2num(date(2026, 8, 20)), + color="red", + alpha=0.08, + label="Art. 50 in force + measured collapse", + ) + ax.set_xlim(mdates.date2num(date(2022, 9, 1)), mdates.date2num(date(2027, 6, 1))) + ax.set_ylim(-0.75, 0.85) + ax.set_yticks([]) + ax.set_xlabel("date") + ax.set_ylabel("policy / measurement") + ax.xaxis.set_major_formatter(mdates.DateFormatter("%Y-%m")) + ax.xaxis.set_major_locator(mdates.MonthLocator(interval=4)) + ax.grid(axis="x", alpha=0.3) + ax.legend(fontsize=8, loc="upper left") + ax.set_title( + "F6 - Policy timeline: EU AI Act Art. 50 (in force 2026-08-02) " + "vs measured watermark collapse (this work)" + ) + fig.tight_layout() + return fig + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +def build_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument( + "--results-dir", type=Path, required=True, help="results layout root: DIR//" + ) + p.add_argument( + "--out-dir", + type=Path, + required=True, + help="output root; figures are written to OUT/figures/", + ) + p.add_argument("--format", choices=("png", "pdf"), default="png") + p.add_argument( + "--dpi", type=int, default=200, help="raster resolution for PNG output (default: 200)" + ) + p.add_argument( + "--attack", default=LAYERED_ATTACK, help="attack used by F4 (default: %(default)s)" + ) + return p + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + try: + import matplotlib + except ImportError: + print( + "warning: matplotlib is not installed; skipping figure generation (F1-F6).", + file=sys.stderr, + ) + return 0 + matplotlib.use("Agg") + import matplotlib.pyplot as plt + import numpy as np + + figures_dir = args.out_dir / "figures" + figures_dir.mkdir(parents=True, exist_ok=True) + conditions = _load_conditions(args.results_dir) + suffix = f".{args.format}" + builders: list[tuple[str, Any]] = [ + ("f1", figure_f1(plt)), + ("f2", figure_f2(plt, np, conditions)), + ("f3", figure_f3(plt, conditions)), + ("f4", figure_f4(plt, conditions, args.attack)), + ("f5", figure_f5(plt, conditions)), + ("f6", figure_f6(plt)), + ] + for name, fig in builders: + fig.savefig(figures_dir / f"{name}{suffix}", dpi=args.dpi) + plt.close(fig) + print(f"wrote {len(builders)} figures to {figures_dir}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/research/scripts/make_tables.py b/research/scripts/make_tables.py new file mode 100644 index 0000000..ddcf6ca --- /dev/null +++ b/research/scripts/make_tables.py @@ -0,0 +1,1091 @@ +#!/usr/bin/env python3 +"""Paper table generators (gap 05-B3): T1-T7 per research/02-paper-outline.md sec 6. + +CLI: + python3 research/scripts/make_tables.py --results-dir DIR --out-dir DIR + +Reads the results layout emitted by research/scripts/run_experiments.py +(one directory per cell, named e.g. 'kgw-d2-L300-T0.7-en-s1-p0'): + + DIR//metrics.json per-attack {"auroc", "tpr_at_fpr"}, plus + optional per-attack "roc_points" + DIR//scores.jsonl detector scores per (doc, attack) + DIR//attacked.jsonl per-attack rows with fields attack, + original, candidate, stats, seconds, usd + DIR//quality.jsonl quality metrics per attack (optional) + +Writes out-dir/tables/t1.tex ... t7.tex and out-dir/tables/tables.md (every +table also in markdown). T1 (attack taxonomy) and T7 (published-baseline +template) are static; all data-dependent tables degrade to an explicit +"no data" row when their inputs are missing. +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from collections import defaultdict +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +# --------------------------------------------------------------------------- +# Paper constants (locked v1 matrix, research/01-experiment-protocol.md) +# --------------------------------------------------------------------------- + +SCHEME_ORDER: tuple[str, ...] = ( + "kgw-d1", + "kgw-d2", + "kgw-d4", + "synthid", + "exp", + "unigram", + "sir", +) + +SCHEME_MD: dict[str, str] = { + "kgw-d1": "KGW (gamma=0.25, delta=1)", + "kgw-d2": "KGW (gamma=0.5, delta=2)", + "kgw-d4": "KGW (gamma=0.5, delta=4)", + "synthid": "SynthID-Text", + "exp": "EXP (Gumbel)", + "unigram": "Unigram", + "sir": "SIR", +} + +SCHEME_TEX: dict[str, str] = { + "kgw-d1": "KGW ($\\gamma{=}0.25$, $\\delta{=}1$)", + "kgw-d2": "KGW ($\\gamma{=}0.5$, $\\delta{=}2$)", + "kgw-d4": "KGW ($\\gamma{=}0.5$, $\\delta{=}4$)", + "synthid": "SynthID-Text", + "exp": "EXP (Gumbel)", + "unigram": "Unigram", + "sir": "SIR", +} + +ATTACKS: tuple[str, ...] = ( + "none", + "layerA", + "paraphrase:1", + "paraphrase:3", + "backtranslate:de", + "structural", + "humanize", + "cheap", + "layerA+paraphrase:3", +) + +ATTACK_LABELS: dict[str, str] = { + "none": "A0 none", + "layerA": "A1 layerA", + "paraphrase:1": "A2 paraphrase:1", + "paraphrase:3": "A3 paraphrase:3", + "backtranslate:de": "A4 backtranslate:de", + "structural": "A5 structural", + "humanize": "A6 humanize", + "cheap": "A7 cheap", + "layerA+paraphrase:3": "A8 layerA+paraphrase:3", +} + +# Statistical-rewrite attacks map to the "B-only" pipeline column (T2). +B_FAMILY: frozenset[str] = frozenset( + {"paraphrase:1", "paraphrase:3", "backtranslate:de", "structural", "humanize", "cheap"} +) +LAYERED_ATTACK = "layerA+paraphrase:3" + +# T1: (id, attack, family, mechanism, implementation, prior-art anchor). +# Hardcoded from research/01-experiment-protocol.md section 4.1 -- no data +# needed; this is the reproducibility/novelty anchor of the attack design. +TAXONOMY: tuple[tuple[str, str, str, str, str, str], ...] = ( + ("A0", "None (control)", "---", "No attack (control)", "---", "---"), + ( + "A1", + "Layer A only", + "Formatting cleanup", + "Unicode/invisible-char, bidi, tag cleanup", + "clean_text.py (deterministic)", + "Formatting-layer marks (zero-width/steganography class)", + ), + ( + "A2", + "Paraphrase, single pass", + "Statistical rewrite", + "Paraphrase, single pass", + ("rewrite_text.py --strength paraphrase --candidates 1 --max-loops 1"), + "Paraphrase attacks in the watermark literature", + ), + ( + "A3", + "Paraphrase, adaptive", + "Statistical rewrite (adaptive)", + "Paraphrase with detection-feedback early stop (up to 3 loops)", + ( + "rewrite_text.py --strength paraphrase --candidates 3 " + "--max-loops 3 --markllm-scheme " + ), + "Same; our eval-loop is the 'oracle' version", + ), + ( + "A4", + "Back-translation round trip", + "Statistical rewrite", + "Translation round trip EN -> DE -> EN", + "rewrite_text.py --strength backtranslate --lang German", + "Can Watermarks Survive Translation? (X-SIR, ACL 2024)", + ), + ( + "A5", + "Structural: outline -> regenerate", + "Statistical rewrite", + "Outline -> regenerate", + "rewrite_text.py --strength structural", + "Summarization/outline attacks", + ), + ( + "A6", + "Humanize", + "Statistical rewrite", + "Style transfer to human-like prose", + "rewrite_text.py --strength humanize", + "Style-transfer attacks", + ), + ( + "A7", + "Cheap baselines", + "Heuristic / cheap", + ("Synonym substitution, random word deletion (5-10%), sentence reorder"), + "research/scripts/attacks/cheap.py (gap 05-A3)", + "Random Walk / impossibility (ICML 2024)", + ), + ( + "A8", + "Full pipeline (layered)", + "Layered (A + B)", + "Layer A cleanup then adaptive paraphrase (A1 -> A2/A3)", + "clean_text.py then rewrite_text.py", + "Our layered contribution (this work)", + ), +) + +# T7: (baseline, reported metric/setting, default citation key, our closest cell). +T7_DEFAULT_CITES: dict[str, str] = { + "kgw_robust": "kirchenbauer2023reliability", + "x_sir": "he2024canwatermarks", + "sand": "zhang2024watermarks", + "synthid": "han2025synthid", + "markllm": "pan2024markllm", +} +T7_ROWS: tuple[tuple[str, str, str, str], ...] = ( + ( + "KGW robustness (paraphrase)", + "KGW; paraphrase; AUROC / TPR@FPR", + "kgw_robust", + "kgw-d2, L300, en, paraphrase:3", + ), + ( + "X-SIR: can watermarks survive translation?", + "translation round trip; cross-lingual detection", + "x_sir", + "kgw-d2, L300, de, backtranslate:de", + ), + ( + "Watermarks in the Sand (impossibility)", + "random-walk removal; impossibility theory", + "sand", + "kgw-d4, L300, en, cheap + adaptive rewrite", + ), + ( + "SynthID-Text robustness assessment", + "SynthID under editing; detection rate", + "synthid", + "synthid, L300, en, layerA+paraphrase:3", + ), + ( + "MarkLLM toolkit (harness)", + "same-config detection; open-source toolkit", + "markllm", + "all cells (harness)", + ), +) + +# T3 quality columns: (header, candidate keys in quality.jsonl rows). +QUALITY_COLUMNS: tuple[tuple[str, tuple[str, ...]], ...] = ( + ("PPL delta", ("ppl_delta", "ppl_delta_pct", "perplexity_delta")), + ("BERTScore", ("bertscore", "bert_score")), + ("ROUGE-L", ("rouge_l", "rougeL", "rouge-l", "rouge")), + ("SBERT", ("sbert", "sbert_cosine", "sbert_sim", "sbert_score")), + ("length drift", ("length_drift", "length_drift_pct", "length_change")), + ("num survival", ("num_survival", "number_survival", "numbers_survival")), + ("URL survival", ("url_survival", "link_survival")), +) + +# Keys at the top level of metrics.json that are not attack names. +_RESERVED_TOP: frozenset[str] = frozenset( + { + "roc_points", + "quality", + "config", + "meta", + "cell", + "condition", + "manifest", + "generated_at", + "timestamp", + } +) + +_CONDITION_RE = re.compile( + r"^(?P.+?)-L(?P\d+)-T(?P[0-9]+(?:\.[0-9]+)?)" + r"-(?P[a-z]{2})-s(?P\d+)-p(?P\d+)$" +) + +_TEX_HEADER = ( + "% Generated by research/scripts/make_tables.py (gap 05-B3) -- do not " + "edit by hand.\n" + "% Requires the booktabs package; include with \\input{tables/.tex}.\n" +) + + +# --------------------------------------------------------------------------- +# Small rendering helpers +# --------------------------------------------------------------------------- + +_TEX_CHARS: dict[str, str] = { + "\\": r"\textbackslash{}", + "%": r"\%", + "&": r"\&", + "#": r"\#", + "$": r"\$", + "_": r"\_", + "{": r"\{", + "}": r"\}", + "~": r"\textasciitilde{}", + "^": r"\textasciicircum{}", + "<": r"\textless{}", + ">": r"\textgreater{}", + "\u2013": "--", # en dash + "\u2014": "---", # em dash +} + + +def _tex(text: str) -> str: + """Escape *text* for use inside a LaTeX table cell or caption. + + Single-pass character mapping: each character is escaped at most once, + so the braces introduced by \textbackslash{} are never re-escaped. + """ + return "".join(_TEX_CHARS.get(ch, ch) for ch in text) + + +def _fmt3(v: float | None) -> str: + return "---" if v is None else f"{v:.3f}" + + +def _fmt_int(v: float | None) -> str: + return "---" if v is None else f"{v:,.0f}" + + +def _fmt_sec(v: float | None) -> str: + return "---" if v is None else f"{v:.1f}" + + +def _fmt_usd(v: float | None) -> str: + return "---" if v is None else f"{v:.4f}" + + +def _render_tex( + title: str, + label: str, + headers: Sequence[str], + rows: Sequence[Sequence[str]], + note: str = "", + align: str | None = None, +) -> str: + """Render a complete LaTeX table snippet from raw (unescaped) cells.""" + n = len(headers) + align = align or ("l" + "c" * (n - 1)) + lines = [ + _TEX_HEADER, + "\\begin{table}[t]", + "\\centering", + f"\\caption{{{_tex(title)}}}", + f"\\label{{{label}}}", + "\\small", + f"\\begin{{tabular}}{{@{{{align}}}@{{}}}}", + "\\toprule", + " & ".join(_tex(h) for h in headers) + " \\\\", + "\\midrule", + ] + for row in rows: + lines.append(" & ".join(_tex(str(c)) for c in row) + " \\\\") + lines += ["\\bottomrule", "\\end{tabular}"] + if note: + lines.append(f"\\vspace{{2pt}}\\footnotesize{{{_tex(note)}}}") + lines += ["\\end{table}", ""] + return "\n".join(lines) + + +def _render_md( + heading: str, headers: Sequence[str], rows: Sequence[Sequence[str]], note: str = "" +) -> str: + """Render a markdown table block (dash placeholders become em dashes).""" + parts = [f"### {heading}", "", "| " + " | ".join(headers) + " |", "|" + "---|" * len(headers)] + for row in rows: + cells = [str(c).replace("---", "\u2014") for c in row] + parts.append("| " + " | ".join(cells) + " |") + if note: + parts += ["", f"*Note: {note}*"] + parts.append("") + return "\n".join(parts) + + +def _no_data_tex(title: str, label: str, headers: Sequence[str]) -> str: + n = len(headers) + return _render_tex( + title, + label, + headers, + [["no data" if i == 0 else "" for i in range(n)]], + note="Inputs not found in --results-dir; regenerate after the run.", + ) + + +def _no_data_md(heading: str, headers: Sequence[str]) -> str: + return _render_md( + heading, headers, [["no data" if i == 0 else "" for i in range(len(headers))]] + ) + + +# --------------------------------------------------------------------------- +# Results loading +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class Condition: + """One results cell: DIR// with its parsed artifacts.""" + + dir: Path + name: str + scheme: str | None + length: int | None + temp: float | None + language: str | None + metrics: dict[str, dict[str, Any]] + roc_points: dict[str, tuple[list[float], list[float]]] + quality: list[dict[str, Any]] + attacked: list[dict[str, Any]] + + +def _scheme_from_name(name: str) -> str | None: + for scheme in SCHEME_ORDER: + if name == scheme or name.startswith(scheme + "-"): + return scheme + return None + + +def _num_in_name(name: str, prefix: str) -> int | None: + m = re.search(rf"{re.escape(prefix)}(\d+)", name) + return int(m.group(1)) if m else None + + +def _float_in_name(name: str, prefix: str) -> float | None: + m = re.search(rf"{re.escape(prefix)}([0-9]+(?:\.[0-9]+)?)", name) + return float(m.group(1)) if m else None + + +def _lang_in_name(name: str) -> str | None: + m = re.search(r"-(en|de|fr|es)-", name) + return m.group(1) if m else None + + +def _read_jsonl(path: Path) -> list[dict[str, Any]]: + if not path.is_file(): + return [] + rows: list[dict[str, Any]] = [] + try: + for raw_line in path.read_text(encoding="utf-8").splitlines(): + line = raw_line.strip() + if not line: + continue + try: + obj = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(obj, dict): + rows.append(obj) + except OSError: + return [] + return rows + + +def _normalize_metrics(raw: Any) -> dict[str, dict[str, Any]]: + """Return {attack: metrics} from any supported metrics.json shape.""" + if isinstance(raw, list): + out: dict[str, dict[str, Any]] = {} + for item in raw: + if isinstance(item, dict) and item.get("attack"): + out[str(item["attack"])] = {k: v for k, v in item.items() if k != "attack"} + return out + if not isinstance(raw, dict): + return {} + for wrapper in ("attacks", "results", "per_attack"): + if isinstance(raw.get(wrapper), dict): + return _normalize_metrics(raw[wrapper]) + return {str(k): v for k, v in raw.items() if isinstance(v, dict) and k not in _RESERVED_TOP} + + +def _roc_pair(pts: Any) -> tuple[list[float], list[float]] | None: + """Normalize one attack's ROC points to (fpr, tpr) lists.""" + if isinstance(pts, dict): + fpr, tpr = pts.get("fpr"), pts.get("tpr") + if ( + isinstance(fpr, (list, tuple)) + and isinstance(tpr, (list, tuple)) + and len(fpr) == len(tpr) + and len(fpr) > 1 + ): + return [float(x) for x in fpr], [float(x) for x in tpr] + return None + if isinstance(pts, (list, tuple)) and len(pts) == 2: + a, b = pts[0], pts[1] + if ( + isinstance(a, (list, tuple)) + and isinstance(b, (list, tuple)) + and len(a) == len(b) + and len(a) > 1 + ): + return [float(x) for x in a], [float(x) for x in b] + if isinstance(pts, (list, tuple)): + try: + xs = [float(p[0]) for p in pts] + ys = [float(p[1]) for p in pts] + except (TypeError, ValueError, IndexError): + return None + if len(xs) == len(ys) and len(xs) > 1: + return xs, ys + return None + + +def _normalize_roc(raw: Any) -> dict[str, tuple[list[float], list[float]]]: + if not isinstance(raw, dict): + return {} + out: dict[str, tuple[list[float], list[float]]] = {} + for attack, pts in raw.items(): + pair = _roc_pair(pts) + if pair is not None: + out[str(attack)] = pair + return out + + +def _load_condition(path: Path) -> Condition | None: + """Parse one DIR// directory (missing files are tolerated).""" + name = path.name + m = _CONDITION_RE.match(name) + if m: + scheme = m.group("scheme") + length = int(m.group("length")) + temp = float(m.group("temp")) + language = m.group("language") + else: + scheme = _scheme_from_name(name) + length = _num_in_name(name, "L") + temp = _float_in_name(name, "T") + language = _lang_in_name(name) + + metrics: dict[str, dict[str, Any]] = {} + roc: dict[str, tuple[list[float], list[float]]] = {} + metrics_path = path / "metrics.json" + if metrics_path.is_file(): + try: + raw = json.loads(metrics_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + raw = None + if isinstance(raw, dict): + roc.update(_normalize_roc(raw.get("roc_points"))) + metrics = _normalize_metrics(raw) + for attack, attack_metrics in metrics.items(): + pair = _roc_pair(attack_metrics.get("roc_points")) + if pair is not None: + roc.setdefault(attack, pair) + + quality = _read_jsonl(path / "quality.jsonl") + attacked = _read_jsonl(path / "attacked.jsonl") + if not metrics and not quality and not attacked: + return None + return Condition( + dir=path, + name=name, + scheme=scheme, + length=length, + temp=temp, + language=language, + metrics=metrics, + roc_points=roc, + quality=quality, + attacked=attacked, + ) + + +def _load_conditions(results_dir: Path) -> list[Condition]: + if not results_dir.is_dir(): + return [] + conditions: list[Condition] = [] + for child in sorted(results_dir.iterdir()): + if not child.is_dir(): + continue + cond = _load_condition(child) + if cond is not None: + conditions.append(cond) + return conditions + + +# --------------------------------------------------------------------------- +# Metric helpers +# --------------------------------------------------------------------------- + + +def _auroc(m: Mapping[str, Any]) -> float | None: + for key in ("auroc", "auc", "roc_auc"): + v = m.get(key) + if isinstance(v, (int, float)) and not isinstance(v, bool): + return float(v) + sub = m.get("metrics") + if isinstance(sub, Mapping): + return _auroc(sub) + return None + + +def _fpr_in_key(key: str) -> float | None: + flat = re.sub(r"[\s_@]+", "", key).lower() + m = re.search(r"tprfpr([0-9.]+)", flat) + if m: + return float(m.group(1)) + m = re.search(r"tprat([0-9.]+)pct", flat) + if m: + return float(m.group(1)) / 100.0 + return None + + +def _tpr_at_fpr(m: Mapping[str, Any], fpr: float = 0.01) -> float | None: + """TPR at the given FPR, tolerating dict/number and key-name variants.""" + t = m.get("tpr_at_fpr") + if isinstance(t, dict): + return _tpr_from_dict(t, fpr) + if isinstance(t, (int, float)) and not isinstance(t, bool): + return float(t) + for key, val in m.items(): + if not isinstance(val, (int, float)) or isinstance(val, bool): + continue + want = _fpr_in_key(str(key)) + if want is not None and abs(want - fpr) < 1e-9: + return float(val) + sub = m.get("metrics") + if isinstance(sub, Mapping): + return _tpr_at_fpr(sub, fpr) + return None + + +def _tpr_from_dict(d: Mapping[str, Any], fpr: float) -> float | None: + for key, val in d.items(): + if not isinstance(val, (int, float)) or isinstance(val, bool): + continue + k = str(key).strip() + pct = k.endswith("%") + num = k[:-1] if pct else k + try: + x = float(num) + except ValueError: + continue + if pct: + x = x / 100.0 + if abs(x - fpr) < 1e-9: + return float(val) + return None + + +def _mean_metric( + conds: Sequence[Condition], attack: str, metric: str, fpr: float = 0.01 +) -> float | None: + """Mean *metric* ('auroc' | 'tpr') for *attack* across *conds*.""" + values: list[float] = [] + for cond in conds: + m = cond.metrics.get(attack) + if not m: + continue + v = _auroc(m) if metric == "auroc" else _tpr_at_fpr(m, fpr) + if v is not None: + values.append(v) + return sum(values) / len(values) if values else None + + +def _conds_for_scheme( + conditions: Sequence[Condition], scheme: str, prefer: str | None = None +) -> list[Condition]: + conds = [c for c in conditions if c.scheme == scheme] + if prefer: + preferred = [c for c in conds if prefer in c.name] + if preferred: + return preferred + return conds + + +# --------------------------------------------------------------------------- +# Table builders (each returns (latex, markdown)) +# --------------------------------------------------------------------------- + + +def table_t1() -> tuple[str, str]: + """T1: static attack taxonomy (family x mechanism x implementation x anchor).""" + headers = ("ID", "Attack", "Family", "Mechanism", "Implementation", "Prior-art anchor") + rows = [list(row) for row in TAXONOMY] + tex = _render_tex( + "Attack taxonomy (T1): family x mechanism x implementation x prior-art " + "anchor. Source: research/01-experiment-protocol.md section 4.1.", + "tab:t1", + headers, + rows, + align="clp{4.2cm}p{5.0cm}p{5.2cm}p{4.2cm}", + note="A0 = control; A1 = Layer A (formatting cleanup); A2-A7 = Layer B " + "(statistical rewrite); A8 = layered pipeline (our contribution).", + ) + md = _render_md( + "T1 -- Attack taxonomy (family x mechanism x implementation x prior-art anchor)", + headers, + rows, + note="Hardcoded from research/01-experiment-protocol.md section 4.1.", + ) + return tex, md + + +def _t2_scheme_rows(conditions: Sequence[Condition], scheme: str, metric: str) -> list[list[str]]: + conds = _conds_for_scheme(conditions, scheme, prefer="-L300-T0.7-en-") + rows: list[list[str]] = [] + for attack in ATTACKS[1:]: + pre = _mean_metric(conds, "none", metric) + aonly = _mean_metric(conds, "layerA", metric) + bonly = _mean_metric(conds, attack, metric) if attack in B_FAMILY else None + ab = _mean_metric(conds, LAYERED_ATTACK, metric) if attack == LAYERED_ATTACK else None + rows.append([ATTACK_LABELS[attack], _fmt3(pre), _fmt3(aonly), _fmt3(bonly), _fmt3(ab)]) + return rows + + +def table_t2(conditions: Sequence[Condition]) -> tuple[str, str]: + """T2: money table -- AUROC / TPR@1%FPR matrix (scheme x attack x stage).""" + headers = ("attack", "pre-attack", "A-only", "B-only", "A+B") + schemes = [s for s in SCHEME_ORDER if any(c.scheme == s for c in conditions)] + if not schemes: + return _no_data_tex("AUROC / TPR@1%FPR matrix (T2)", "tab:t2", headers), _no_data_md( + "T2 -- AUROC / TPR@1%FPR matrix", headers + ) + + note = ( + "Rows: scheme-config x attack. 'pre-attack' = A0 (no attack); " + "'A-only' = Layer A cleanup (A1); 'B-only' = the row's statistical " + "rewrite (A2-A7); 'A+B' = layered pipeline (A8). Cells: mean over " + "reference cells (L300, T0.7, en); '---' = not applicable." + ) + md_parts = [ + "### T2 -- AUROC / TPR@1%FPR matrix (scheme-config x attack x pipeline stage)", + ] + tex_blocks: list[str] = [] + for metric, short in (("auroc", "AUROC"), ("tpr", "TPR@1%FPR")): + ab = "a" if metric == "auroc" else "b" + tex_lines = [ + "\\begin{table}[t]", + "\\centering", + f"\\caption{{{short} by attack and pipeline stage (T2{ab}). {_tex(note)}}}", + f"\\label{{tab:t2{ab}}}", + "\\small", + "\\begin{tabular}{@{}lccccc@{}}", + "\\toprule", + " & ".join(headers) + " \\\\", + "\\midrule", + ] + md_lines: list[str] = [] + for scheme in schemes: + tex_lines.append(f"\\multicolumn{{5}}{{l}}{{\\textbf{{{SCHEME_TEX[scheme]}}}}} \\\\") + md_lines += [ + f"**{SCHEME_MD[scheme]}**", + "", + "| " + " | ".join(headers) + " |", + "|" + "---|" * len(headers), + ] + for row in _t2_scheme_rows(conditions, scheme, metric): + tex_lines.append(" & ".join(_tex(c) for c in row) + " \\\\") + md_lines.append("| " + " | ".join(c.replace("---", "\u2014") for c in row) + " |") + tex_lines += [ + "\\bottomrule", + "\\end{tabular}", + f"\\vspace{{2pt}}\\footnotesize{{{_tex(note)}}}", + "\\end{table}", + "", + ] + tex_blocks.append("\n".join(tex_lines)) + md_parts += md_lines + md_parts.append("") + return _TEX_HEADER + "\n".join(tex_blocks), "\n".join(md_parts) + + +def _quality_value(row: Mapping[str, Any], aliases: tuple[str, ...]) -> float | None: + sub = row.get("metrics") if isinstance(row.get("metrics"), Mapping) else None + for src in (row, sub): + if not src: + continue + for alias in aliases: + v = src.get(alias) + if isinstance(v, (int, float)) and not isinstance(v, bool): + return float(v) + return None + + +def table_t3(conditions: Sequence[Condition]) -> tuple[str, str]: + """T3: quality per attack at the detection-collapse operating point.""" + headers = ["attack"] + [col for col, _ in QUALITY_COLUMNS] + rows: list[dict[str, Any]] = [] + for cond in conditions: + for row in cond.quality: + if isinstance(row, dict) and row.get("attack"): + rows.append(row) + if not rows: + return _no_data_tex( + "Quality per attack at the collapse point (T3)", "tab:t3", headers + ), _no_data_md("T3 -- Quality per attack at collapse", headers) + by_attack: dict[str, list[dict[str, Any]]] = defaultdict(list) + for row in rows: + by_attack[str(row["attack"])].append(row) + out_rows: list[list[str]] = [] + for attack in ATTACKS: + bucket = by_attack.get(attack) + if not bucket: + continue + cells = [ATTACK_LABELS[attack]] + for col, aliases in QUALITY_COLUMNS: + values: list[float] = [] + for r in bucket: + v = _quality_value(r, aliases) + if v is not None: + values.append(v) + if not values: + cells.append("---") + else: + mean = sum(values) / len(values) + cells.append(f"{mean:.2f}" if col == "PPL delta" else f"{mean:.3f}") + out_rows.append(cells) + tex = _render_tex( + "Quality per attack at the detection-collapse operating point (T3). " + "PPL delta = perplexity increase vs. original (gpt2-large); BERTScore = " + "deberta-xlarge-mnli; SBERT = all-MiniLM-L6-v2; survival = fraction of " + "numbers/URLs preserved.", + "tab:t3", + headers, + out_rows, + note="Quality at the point where detection collapses (01 section 5.3); " + "'---' = metric not recorded for this attack.", + ) + md = _render_md( + "T3 -- Quality per attack at collapse (PPL delta, BERTScore, " + "ROUGE-L, SBERT, length drift, num/URL survival)", + headers, + out_rows, + ) + return tex, md + + +def table_t4(conditions: Sequence[Condition], attack: str) -> tuple[str, str]: + """T4: ablation -- TPR@1%FPR x (gamma, delta) x length (KGW, en).""" + kgw = [ + c for c in conditions if c.scheme in ("kgw-d1", "kgw-d2", "kgw-d4") and c.language == "en" + ] + col_specs = ((100, 0.7), (300, 0.7), (500, 0.7), (300, 1.0)) + headers = ["config (gamma, delta)"] + [f"L{length} / T{temp:g}" for length, temp in col_specs] + if not kgw: + return _no_data_tex( + "Ablation: TPR@1%FPR x (gamma, delta) x length (T4)", "tab:t4", headers + ), _no_data_md("T4 -- Ablation: TPR@1%FPR x (gamma, delta) x length", headers) + rows: list[list[str]] = [] + for scheme in ("kgw-d1", "kgw-d2", "kgw-d4"): + conds = [c for c in kgw if c.scheme == scheme] + cells: list[str] = [] + for length, temp in col_specs: + sub = [c for c in conds if c.length == length and c.temp == temp] + cells.append(_fmt3(_mean_metric(sub, attack, "tpr"))) + rows.append([SCHEME_MD[scheme], *cells]) + tex = _render_tex( + f"Ablation (T4): TPR@1%FPR x (gamma, delta) x length, EN cells. Attack: {attack}.", + "tab:t4", + headers, + rows, + note="Mean across seeds/prompts for the matching cells; '---' = cell " + "absent from the locked v1 matrix (01 section 2).", + ) + md = _render_md( + f"T4 -- Ablation: TPR@1%FPR x (gamma, delta) x length (attack: {attack})", headers, rows + ) + return tex, md + + +def table_t5(conditions: Sequence[Condition], attack: str) -> tuple[str, str]: + """T5: multilingual fragility -- EN/DE/FR/ES at L300, T0.7.""" + langs = ("en", "de", "fr", "es") + schemes = ("kgw-d2", "synthid") + headers = ["language"] + for scheme in schemes: + headers += [f"{SCHEME_MD[scheme]} AUROC", f"{SCHEME_MD[scheme]} TPR@1%FPR"] + rows: list[list[str]] = [] + any_data = False + for lang in langs: + cells = [lang] + for scheme in schemes: + conds = [ + c + for c in conditions + if c.scheme == scheme and c.language == lang and c.length == 300 and c.temp == 0.7 + ] + auroc = _mean_metric(conds, attack, "auroc") + tpr = _mean_metric(conds, attack, "tpr") + if auroc is not None or tpr is not None: + any_data = True + cells += [_fmt3(auroc), _fmt3(tpr)] + rows.append(cells) + if not any_data: + return _no_data_tex( + "Multilingual: AUROC / TPR@1%FPR by language (T5)", "tab:t5", headers + ), _no_data_md("T5 -- Multilingual (EN/DE/FR/ES)", headers) + tex = _render_tex( + f"Multilingual fragility (T5): AUROC / TPR@1%FPR by language at L300, " + f"T0.7. Attack: {attack}. DE/FR/ES use the Qwen2.5-1.5B holdout.", + "tab:t5", + headers, + rows, + note="Restricted multilingual grid (01 section 2): kgw-d2 + synthid, " + "L300, T0.7; '---' = cell absent.", + ) + md = _render_md(f"T5 -- Multilingual EN/DE/FR/ES (L300, T0.7; attack: {attack})", headers, rows) + return tex, md + + +def _first_num(*sources: Mapping[str, Any] | None, keys: Sequence[str]) -> float | None: + for src in sources: + if not src: + continue + for key in keys: + v = src.get(key) + if isinstance(v, (int, float)) and not isinstance(v, bool): + return float(v) + return None + + +def _word_count(text: Any) -> int | None: + if isinstance(text, str) and text.strip(): + return len(text.split()) + return None + + +def _attacked_cost( + row: Mapping[str, Any], +) -> tuple[str, float | None, float | None, float | None, float | None] | None: + """Normalize one attacked.jsonl row to per-1k-words cost figures.""" + attack = str(row.get("attack", "")).strip() + if not attack: + return None + stats = row.get("stats") if isinstance(row.get("stats"), Mapping) else {} + words = _first_num(stats, keys=("words", "words_in", "original_words")) + if words is None: + words = _word_count(row.get("original")) + if not words or words <= 0: + return None + tok_in = _first_num(stats, row, keys=("tokens_in", "input_tokens", "prompt_tokens")) + tok_out = _first_num(stats, row, keys=("tokens_out", "output_tokens", "completion_tokens")) + sec = _first_num(row, stats, keys=("seconds", "wall_time", "elapsed_seconds")) + usd = _first_num(row, stats, keys=("usd", "cost_usd", "cost")) + scale = 1000.0 / words + scaled = [None if v is None else v * scale for v in (tok_in, tok_out, sec, usd)] + return (attack, scaled[0], scaled[1], scaled[2], scaled[3]) + + +def table_t6(conditions: Sequence[Condition]) -> tuple[str, str]: + """T6: attack cost -- tokens in/out, wall time, USD per 1k words.""" + headers = ( + "attack", + "tokens in / 1k words", + "tokens out / 1k words", + "seconds / 1k words", + "USD / 1k words", + ) + per_attack: dict[str, dict[str, list[float]]] = defaultdict( + lambda: {"tin": [], "tout": [], "sec": [], "usd": []} + ) + for cond in conditions: + for row in cond.attacked: + parsed = _attacked_cost(row) + if parsed is None: + continue + attack, tin, tout, sec, usd = parsed + if attack == "none": + continue + bucket = per_attack[attack] + for name, val in (("tin", tin), ("tout", tout), ("sec", sec), ("usd", usd)): + if val is not None: + bucket[name].append(val) + if not per_attack: + return _no_data_tex( + "Attack cost per 1k words of original text (T6)", "tab:t6", headers + ), _no_data_md("T6 -- Attack cost per 1k words", headers) + rows: list[list[str]] = [] + for attack in ATTACKS[1:]: + bucket = per_attack.get(attack) + if not bucket: + continue + mean = {k: (sum(v) / len(v) if v else None) for k, v in bucket.items()} + rows.append( + [ + ATTACK_LABELS[attack], + _fmt_int(mean["tin"]), + _fmt_int(mean["tout"]), + _fmt_sec(mean["sec"]), + _fmt_usd(mean["usd"]), + ] + ) + tex = _render_tex( + "Attack cost (T6): tokens in/out, wall time, and USD per 1k words of " + "the original document, from attacked.jsonl (mean across cells).", + "tab:t6", + headers, + rows, + note="Rows normalized per 1,000 words of the original text; '---' = field not recorded.", + ) + md = _render_md( + "T6 -- Attack cost per 1k words (tokens in/out, wall time, USD)", + headers, + rows, + note="Mean across cells; normalized per 1k words of the original text.", + ) + return tex, md + + +def _bib_keys(results_dir: Path) -> set[str]: + """Citation keys from a refs.bib, searched in likely locations.""" + here = Path(__file__).resolve().parents[1] # research/ + candidates = [ + results_dir / "refs.bib", + results_dir.parent / "refs.bib", + results_dir / "paper" / "refs.bib", + here / "paper" / "refs.bib", + here / "refs.bib", + ] + keys: set[str] = set() + for path in candidates: + if not path.is_file(): + continue + try: + text = path.read_text(encoding="utf-8") + except OSError: + continue + keys.update(re.findall(r"@\w+\s*\{\s*([^,\s]+)", text)) + return keys + + +def table_t7(results_dir: Path) -> tuple[str, str]: + """T7: static comparison with published baselines (placeholders + cites).""" + bib_keys = _bib_keys(results_dir) + headers = ( + "Baseline", + "Reported metric / setting", + "Reported", + "Our closest cell", + "Ours", + "Status", + ) + tex_rows: list[list[str]] = [] + md_rows: list[list[str]] = [] + for name, setting, key_id, our_cell in T7_ROWS: + key = T7_DEFAULT_CITES[key_id] + placeholder = "TBD (re-verify at submission)" + tex_rows.append( + [f"{name} \\cite{{{key}}}", setting, placeholder, our_cell, placeholder, placeholder] + ) + md_rows.append( + [f"{name} [cite: {key}]", setting, placeholder, our_cell, placeholder, placeholder] + ) + note = ( + "All placeholder cells are 'TBD (re-verify at submission)': fill " + "from the original papers during the submission pass." + ) + if bib_keys: + note += f" Citation keys found in refs.bib: {', '.join(sorted(bib_keys))}." + else: + note += " No refs.bib found yet (gap C2); default keys listed." + tex = _render_tex( + "Comparison with published baselines (T7)", "tab:t7", headers, tex_rows, note=note + ) + # _render_tex escapes backslashes/braces; restore the \cite commands. + for key in T7_DEFAULT_CITES.values(): + escaped = r"\textbackslash{}cite\{" + key + r"\}" + tex = tex.replace(escaped, f"\\cite{{{key}}}") + md = _render_md("T7 -- Comparison with published baselines", headers, md_rows, note=note) + return tex, md + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +def build_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument( + "--results-dir", + type=Path, + required=True, + help="results layout root: DIR//{metrics,quality,attacked}.json[l]", + ) + p.add_argument( + "--out-dir", type=Path, required=True, help="output root; tables are written to OUT/tables/" + ) + p.add_argument( + "--attack", default=LAYERED_ATTACK, help="attack used by T4/T5 (default: %(default)s)" + ) + return p + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + if not args.results_dir.is_dir(): + print( + f"warning: results dir not found ({args.results_dir}); tables will contain 'no data'", + file=sys.stderr, + ) + conditions = _load_conditions(args.results_dir) + tables_dir = args.out_dir / "tables" + tables_dir.mkdir(parents=True, exist_ok=True) + + tables: list[tuple[str, tuple[str, str]]] = [ + ("t1", table_t1()), + ("t2", table_t2(conditions)), + ("t3", table_t3(conditions)), + ("t4", table_t4(conditions, args.attack)), + ("t5", table_t5(conditions, args.attack)), + ("t6", table_t6(conditions)), + ("t7", table_t7(args.results_dir)), + ] + + md_parts = [ + "# Research tables (gap 05-B3)", + "", + "Auto-generated by research/scripts/make_tables.py -- do not edit by hand.", + "", + ] + for name, (tex, md) in tables: + (tables_dir / f"{name}.tex").write_text(tex, encoding="utf-8") + md_parts += [md, ""] + (tables_dir / "tables.md").write_text("\n".join(md_parts), encoding="utf-8") + print(f"wrote {len(tables)} tables to {tables_dir}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/research/scripts/multilingual_gen.py b/research/scripts/multilingual_gen.py new file mode 100644 index 0000000..cc0ac25 --- /dev/null +++ b/research/scripts/multilingual_gen.py @@ -0,0 +1,562 @@ +#!/usr/bin/env python3 +"""CPU-feasible multilingual (DE/FR/ES) watermark generation (gap 05-A5). + +The EN core of the v1 study (research/01-experiment-protocol.md §2) uses +facebook/opt-1.3b, which is English-only. Multilingual cells (de/fr/es) +therefore use a CPU-feasible multilingual generator -- Qwen/Qwen2.5-1.5B- +Instruct (fallback: 0.5B) -- reported explicitly as a model-holdout factor +in the paper (§3): scheme x language, not confounded with the EN core. + +Like service/scripts/detect_text_watermark.py, this script does NOT vendor +upstream code: it imports AutoWatermark from a user-provided THU-BPM/MarkLLM +checkout at runtime (--markllm-dir / $MARKLLM_DIR) and mirrors that script's +_load_algorithm exactly, so multilingual generation and same-config detection +share identical MarkLLM wiring (scheme config + keys + TransformersConfig +defaults). + +Because Qwen2.5-Instruct tokenizers ship a chat template, each prompt is +formatted as a single user turn before generation +(tokenizer.apply_chat_template([{"role": "user", "content": prompt}], +tokenize=False, add_generation_prompt=True)); the generated text then +includes the (formatted) prompt prefix, consistent with the EN harness. +Tokenizers without a chat template receive the raw prompt. + +Subcommands: + watermark one-shot generation: emit a single JSON object on stdout + serve persistent JSON-lines stdin/stdout worker (model loaded once) + +Exit codes: 0 success, 1 runtime error, 2 bad input/usage, 3 unavailable +(no MarkLLM checkout / missing deps / bad config). +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from pathlib import Path +from typing import Any + +SCRIPTS_DIR = Path(__file__).resolve().parent +sys.path.insert(0, str(SCRIPTS_DIR)) + +import pins # noqa: E402 (reproducibility pins, gap 05-A7) + +# CLI scheme name -> MarkLLM algorithm name (config/{ALG}.json). The +# multilingual grid (01 §2) uses only KGW (gamma=.5, delta=2) and +# SynthID-Text; the mapping mirrors detect_text_watermark.SCHEMES. +SCHEMES = { + "kgw": "KGW", + "synthid": "SynthID", + "synthid-text": "SynthID", +} + +DEFAULT_MODEL = "Qwen/Qwen2.5-1.5B-Instruct" +LANGUAGES = ("de", "fr", "es") + +# Algorithm configs are ~200 B (KGW/SynthID). Cap well above that so a +# crafted or accidental huge file is refused before upstream reads it. +MAX_CONFIG_BYTES = 1 << 20 + + +class _Unavailable(RuntimeError): + """Backend present but unusable (unconfigured checkout, missing deps).""" + + +def resolve_upstream(raw: str | None) -> Path | None: + """Resolve a MarkLLM checkout path, or None when absent/invalid.""" + if not raw: + return None + upstream = Path(raw).expanduser().resolve() + if not upstream.is_dir(): + return None + return upstream + + +def resolve_device(raw: str | None) -> str: + """Resolve the 'auto' device hint to a concrete torch device.""" + if raw and raw != "auto": + return raw + try: + import torch + + if torch.cuda.is_available(): + return "cuda" + # Never auto-select mps: MarkLLM builds torch.Generator(device=...), + # which supports only cpu/cuda and raises RuntimeError on 'mps' + # (Apple Silicon). Fall through to cpu. Pass --device mps to override. + except Exception: # noqa: S110 - optional torch device detection + pass + return "cpu" + + +def _load_algorithm( + upstream: Path, + alg: str, + config: Path, + model: str, + device: str, + offline: bool = False, + temperature: float | None = None, + top_p: float | None = None, + torch_dtype: str = "auto", +): + """Import the MarkLLM checkout and build an AutoWatermark instance. + + Mirrors service/scripts/detect_text_watermark.py::_load_algorithm exactly + (same imports, TransformersConfig defaults, gen_kwargs folding) so + multilingual generation and the service detector share identical wiring. + The checkout is imported at runtime via sys.path -- never vendored. + + temperature/top_p (when not None) are folded into the generation kwargs + so the v1 study's temperature factor (01 §2) is reproducible. torch_dtype + mirrors the service flag ("auto"/"fp32"/"bf16") for CPU throughput. + """ + gen_kwargs_extra: dict[str, float] = {} + if temperature is not None: + gen_kwargs_extra["temperature"] = temperature + if top_p is not None: + gen_kwargs_extra["top_p"] = top_p + sys.path.insert(0, str(upstream)) + try: + from transformers import AutoModelForCausalLM, AutoTokenizer + from utils.transformers_config import TransformersConfig + from watermark.auto_watermark import AutoWatermark + except ImportError as e: + raise _Unavailable(f"MarkLLM dependencies missing: {e}") from e + + # --offline: never contact the HF hub. local_files_only makes transformers + # fail fast instead of hanging, and HF_HUB_OFFLINE covers the lower-level + # hub calls. Custom-code execution is not possible either way: transformers + # only honors auto_map/trust_remote_code when explicitly enabled, which is + # never done here. + if offline: + os.environ.setdefault("HF_HUB_OFFLINE", "1") + load_kwargs = {"local_files_only": True} if offline else {} + import torch + + dtype_map = {"fp32": torch.float32, "bf16": torch.bfloat16} + if torch_dtype in dtype_map: + load_kwargs["torch_dtype"] = dtype_map[torch_dtype] + + tokenizer = AutoTokenizer.from_pretrained(model, **load_kwargs) + lm = AutoModelForCausalLM.from_pretrained(model, **load_kwargs).to(device) + transformers_config = TransformersConfig( + model=lm, + tokenizer=tokenizer, + device=device, + max_new_tokens=200, + min_length=0, + do_sample=True, + no_repeat_ngram_size=4, + **gen_kwargs_extra, + ) + return AutoWatermark.load( + alg, + algorithm_config=str(config), + transformers_config=transformers_config, + ) + + +def _resolve_config(upstream: Path, alg: str, config: str | None) -> Path: + """Resolve the algorithm config JSON, with size + existence checks.""" + path = Path(config).expanduser().resolve() if config else upstream / "config" / f"{alg}.json" + if not path.is_file(): + raise _Unavailable(f"MarkLLM config not found: {path}") + try: + size = path.stat().st_size + except OSError as e: + raise _Unavailable(f"cannot stat MarkLLM config {path}: {e}") from e + if size > MAX_CONFIG_BYTES: + raise _Unavailable(f"MarkLLM config too large ({size} bytes > {MAX_CONFIG_BYTES}): {path}") + return path + + +def _threshold_from_config(config: Path) -> float | None: + """Detection threshold from the algorithm config (KGW/SynthID), if any.""" + try: + data = json.loads(config.read_text("utf-8")) + except (OSError, ValueError): + return None + for key in ("threshold", "z_threshold"): + value = data.get(key) + if isinstance(value, (int, float)): + return float(value) + return None + + +def apply_chat_template(tokenizer: Any, prompt: str) -> str: + """Format *prompt* as a single user turn when the tokenizer has a chat template. + + Qwen2.5-Instruct tokenizers ship a chat_template attribute; wrapping the + prompt in one user turn with the generation prompt appended matches how + the instruct model is normally driven (and mirrors the EN harness, where + opt-1.3b has no template and the raw prompt is used). + """ + template = getattr(tokenizer, "chat_template", None) + if not template: + return prompt + messages = [{"role": "user", "content": prompt}] + return tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) + + +def _generate( + wm: Any, + prompt: str, + seed: int | None, + max_new_tokens: int, + min_length: int = 0, + need_unwatermarked: bool = True, +) -> tuple[str, str | None]: + """Generate watermarked (and optionally unwatermarked) text for *prompt*.""" + if seed is not None: + import torch + + torch.manual_seed(seed) + wm.config.gen_kwargs["max_new_tokens"] = max_new_tokens + wm.config.gen_kwargs["min_length"] = min_length + watermarked = wm.generate_watermarked_text(prompt) + unwatermarked = wm.generate_unwatermarked_text(prompt) if need_unwatermarked else None + return watermarked, unwatermarked + + +def _read_prompt(path: str) -> str: + """Read the prompt file (or stdin for '-'); strip surrounding whitespace.""" + text = sys.stdin.read() if path == "-" else Path(path).read_text(encoding="utf-8") + return text.strip() + + +def _emit(payload: dict[str, Any]) -> None: + """Write one JSON object to stdout (single line, flushed).""" + print(json.dumps(payload), flush=True) + + +def _cmd_watermark(args: argparse.Namespace) -> int: + """One-shot watermark generation; emits a single JSON object on stdout. + + Success: {"ok": true, "doc_id", "lang", "seed", "model", "scheme", + "config", "temperature", "top_p", "watermarked", + "unwatermarked", "pins": {"markllm_commit", "hf_revision", ...}} + Failure: {"ok": false, "error": str, ...} (context keys included) + """ + context: dict[str, Any] = { + "doc_id": args.doc_id, + "lang": args.lang, + "seed": args.seed, + "model": args.model, + "scheme": args.scheme, + "config": str(args.config), + } + if args.prompt != "-" and not Path(args.prompt).is_file(): + _emit({**context, "ok": False, "error": f"not a file: {args.prompt}"}) + return 2 + try: + prompt = _read_prompt(args.prompt) + except (OSError, UnicodeDecodeError) as e: + _emit({**context, "ok": False, "error": f"cannot read prompt: {e}"}) + return 2 + + device = resolve_device(args.device) + try: + config = _resolve_config(args.upstream_dir, SCHEMES[args.scheme], args.config) + wm = _load_algorithm( + args.upstream_dir, + SCHEMES[args.scheme], + config, + args.model, + device, + offline=args.offline, + temperature=args.temperature, + top_p=args.top_p, + torch_dtype=args.torch_dtype, + ) + gen_prompt = apply_chat_template(wm.config.generation_tokenizer, prompt) + watermarked, unwatermarked = _generate( + wm, gen_prompt, args.seed, args.max_new_tokens, args.min_length + ) + except _Unavailable as e: + _emit({**context, "ok": False, "error": str(e)}) + return 3 + except Exception as e: + _emit({**context, "ok": False, "error": f"generation error: {e}"}) + return 1 + + _emit( + { + "ok": True, + **context, + "config": str(config), + "temperature": args.temperature, + "top_p": args.top_p, + "watermarked": watermarked, + "unwatermarked": unwatermarked, + "watermarked_chars": len(watermarked), + "unwatermarked_chars": len(unwatermarked) if unwatermarked is not None else None, + "pins": { + "markllm_commit": pins.markllm_commit(args.upstream_dir), + "hf_revision": pins.hf_revision(args.model), + "repo_commit": pins.repo_commit(), + }, + } + ) + return 0 + + +def _detect_payload(wm: Any, text: str, threshold: float | None) -> dict[str, Any]: + """Same-config detection payload (is_watermarked/score/threshold). + + Mirrors detect_text_watermark.py::_detect_payload: the loaded + AutoWatermark instance detects with the same scheme config + keys used at + generation, so a resident worker can score texts without a second model. + """ + result = wm.detect_watermark(text, return_dict=True) + is_watermarked = bool(result.get("is_watermarked", False)) + score = result.get("score") + try: + score = float(score) + except (TypeError, ValueError): + score = None + return { + "is_watermarked": is_watermarked, + "score": score, + "threshold": threshold, + } + + +def _handle_serve_request( + wm: Any, req: dict[str, Any], threshold: float | None = None +) -> dict[str, Any]: + """Handle one JSON-lines request; never raises (responds with ok:false). + + Supports watermark (generate) and detect ops; *threshold* is the + algorithm-config detection threshold read once at serve load. + """ + rid = req.get("id") + op = req.get("op") + if op == "exit": + return {"ok": True, "id": rid} + try: + if op == "watermark": + prompt = req.get("prompt") + if not isinstance(prompt, str) or not prompt: + raise ValueError("'prompt' must be a non-empty string") + # Per-request generation knobs (temperature factor of the v1 study). + for key in ("temperature", "top_p"): + value = req.get(key) + if isinstance(value, (int, float)): + wm.config.gen_kwargs[key] = float(value) + gen_prompt = apply_chat_template(wm.config.generation_tokenizer, prompt) + watermarked, unwatermarked = _generate( + wm, + gen_prompt, + req.get("seed"), + req.get("max_new_tokens", 200), + req.get("min_length", 0), + need_unwatermarked=True, + ) + return { + "ok": True, + "id": rid, + "watermarked": watermarked, + "unwatermarked": unwatermarked, + "watermarked_chars": len(watermarked), + "unwatermarked_chars": len(unwatermarked), + } + if op == "detect": + text = req.get("text") + if not isinstance(text, str) or not text: + raise ValueError("'text' must be a non-empty string") + det = _detect_payload(wm, text, threshold) + return {"ok": True, "id": rid, **det} + return {"ok": False, "id": rid, "error": f"unknown op {op!r}"} + except Exception as e: # a bad request must not kill the worker + return {"ok": False, "id": rid, "error": str(e)} + + +def _cmd_serve(args: argparse.Namespace) -> int: + """Serve watermark/detect requests over JSON-lines stdin/stdout. + + Loads the MarkLLM model once and keeps it resident so callers (e.g. the + v1 orchestrator) can generate AND detect many multilingual docs without + paying the torch + model load cost per call. Protocol (identical to + detect_text_watermark.py serve): + + first stdout line: {"ready": true, "scheme", "model", "device", ...} + request: {"op": "watermark", "id": N, "prompt": str, "seed": int|None, + "max_new_tokens": int, "min_length": int, + "temperature": float|None, "top_p": float|None} + {"op": "detect", "id": N, "text": str} + {"op": "exit", "id": N} + response: {"ok": true, "id": N, ...} | {"ok": false, "id": N, "error": str} + + Per-request "temperature"/"top_p" override the worker's generation kwargs + (the v1 study's temperature factor); omitted keys keep the current value. + Detect runs same-config detection on the resident model (the threshold is + read once from the algorithm config). Errors on one request never kill the + worker; {"op": "exit"} ends it. + """ + device = resolve_device(args.device) + try: + config = _resolve_config(args.upstream_dir, SCHEMES[args.scheme], args.config) + threshold = _threshold_from_config(config) + wm = _load_algorithm( + args.upstream_dir, + SCHEMES[args.scheme], + config, + args.model, + device, + offline=args.offline, + temperature=args.temperature, + top_p=args.top_p, + torch_dtype=args.torch_dtype, + ) + except _Unavailable as e: + _emit({"ready": False, "error": str(e)}) + return 3 + except Exception as e: + _emit({"ready": False, "error": f"serve load error: {e}"}) + return 1 + + _emit( + { + "ready": True, + "scheme": args.scheme, + "model": args.model, + "device": device, + "temperature": args.temperature, + "top_p": args.top_p, + } + ) + + for raw_line in sys.stdin: + line = raw_line.strip() + if not line: + continue + try: + req = json.loads(line) + except json.JSONDecodeError: + _emit({"ok": False, "error": "invalid JSON request"}) + continue + if not isinstance(req, dict): + _emit({"ok": False, "error": "request must be a JSON object"}) + continue + _emit(_handle_serve_request(wm, req, threshold)) + if req.get("op") == "exit": + return 0 + return 0 + + +def _add_common(p: argparse.ArgumentParser) -> None: + """Shared flags for every subcommand (MarkLLM + decoding options).""" + p.add_argument( + "--markllm-dir", + type=Path, + default=None, + help="MarkLLM checkout root (default: $MARKLLM_DIR)", + ) + p.add_argument( + "--scheme", + required=True, + choices=sorted(SCHEMES), + help="Watermark scheme to use (kgw, synthid)", + ) + p.add_argument( + "--config", + required=True, + help="Algorithm config JSON (e.g. research/configs/KGW-d2.json)", + ) + p.add_argument( + "--model", + default=os.environ.get("MARKLLM_MODEL", DEFAULT_MODEL), + help=f"HF causal LM for generation (default: $MARKLLM_MODEL or {DEFAULT_MODEL})", + ) + p.add_argument( + "--device", + default="auto", + help="auto|cpu|cuda|mps (default: auto)", + ) + p.add_argument( + "--torch-dtype", + default="auto", + choices=("auto", "fp32", "bf16"), + help="Model dtype: auto (fp32 default), fp32, or bf16 (faster on CPU)", + ) + p.add_argument( + "--offline", + action="store_true", + help="Never contact the HF hub: load the model from the local cache " + "only (fails fast if not cached)", + ) + p.add_argument( + "--temperature", + type=float, + default=None, + help="Generation temperature (default: unset -> MarkLLM/HF default)", + ) + p.add_argument( + "--top-p", + type=float, + default=None, + help="Generation nucleus-sampling top-p (default: unset -> MarkLLM/HF default)", + ) + + +def build_parser() -> argparse.ArgumentParser: + """Build the CLI parser (two subcommands: watermark, serve).""" + p = argparse.ArgumentParser(description=__doc__) + sub = p.add_subparsers(dest="cmd", required=True) + + wm = sub.add_parser( + "watermark", help="Generate watermarked + unwatermarked text for one prompt" + ) + wm.add_argument("--prompt", required=True, help="Prompt file, or - for stdin") + wm.add_argument("--seed", type=int, required=True, help="RNG seed (fixed 1..5 per cell, 01 §3)") + wm.add_argument( + "--max-new-tokens", type=int, required=True, help="Tokens to generate (length factor)" + ) + wm.add_argument( + "--min-length", type=int, default=0, help="Minimum total length in tokens (default: 0)" + ) + wm.add_argument("--lang", required=True, choices=LANGUAGES, help="Target language: de, fr, es") + wm.add_argument("--doc-id", required=True, help="Stable document id recorded in the manifest") + _add_common(wm) + wm.set_defaults(handler=_cmd_watermark) + + serve = sub.add_parser( + "serve", help="Persistent JSON-lines stdin/stdout worker (model loaded once)" + ) + _add_common(serve) + serve.set_defaults(handler=_cmd_serve) + + return p + + +def main(argv: list[str] | None = None) -> int: + """CLI entry point; returns the process exit code.""" + args = build_parser().parse_args(argv) + fail_key = "ready" if args.cmd == "serve" else "ok" + raw_upstream = args.markllm_dir or os.environ.get("MARKLLM_DIR") + upstream = resolve_upstream(str(raw_upstream) if raw_upstream else None) + if upstream is None: + _emit( + { + fail_key: False, + "error": "MarkLLM not configured: set MARKLLM_DIR or pass --markllm-dir", + } + ) + return 3 + if not (upstream / "watermark").is_dir(): + _emit( + { + fail_key: False, + "error": f"MarkLLM checkout incomplete (no watermark/ dir): {upstream}", + } + ) + return 3 + args.upstream_dir = upstream + return args.handler(args) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/research/scripts/pins.py b/research/scripts/pins.py new file mode 100644 index 0000000..f2546f7 --- /dev/null +++ b/research/scripts/pins.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +"""Reproducibility pin helpers (gap 05-A7). + +Records, at run time, the exact versions every result depends on so the +released JSONL can be reproduced (research/01-experiment-protocol.md §7): + + - the MarkLLM checkout commit (git rev-parse HEAD of --markllm-dir) + - the watermarks-remover repo commit + - HF hub revisions for every model used (opt-1.3b, Qwen2.5-1.5B-Instruct, + gpt2-large, deberta-xlarge-mnli, all-MiniLM-L6-v2) + - pip freeze of the active interpreter's environment + +Every helper fails soft: a missing git repo, hub, or pip returns None and +the caller records the gap rather than aborting the run. +""" + +from __future__ import annotations + +import os +import subprocess +import sys +from pathlib import Path +from typing import Any + + +def _git_commit(dir_path: Path | str | None, *, short: bool = True) -> str | None: + if not dir_path: + return None + try: + # S607: "git" is resolved from PATH by design (repo/MarkLLM checkouts + # are not guaranteed to be absolute); args are a fixed literal list. + r = subprocess.run( + ["git", "-C", str(dir_path), "rev-parse", "HEAD"], # noqa: S607 + capture_output=True, + text=True, + timeout=10, + check=False, + ) + if r.returncode != 0: + return None + head = r.stdout.strip() + return head[:12] if short and head else head or None + except (OSError, subprocess.SubprocessError): + return None + + +def repo_commit(*, short: bool = True) -> str | None: + """Commit of the watermarks-remover repo containing this file.""" + here = Path(__file__).resolve() + # research/scripts/pins.py -> repo root is three levels up. + root = here.parents[2] + return _git_commit(root, short=short) + + +def markllm_commit(upstream: str | Path | None, *, short: bool = True) -> str | None: + """Commit of the MarkLLM checkout (--markllm-dir).""" + return _git_commit(upstream, short=short) + + +def hf_revision(model_id: str) -> str | None: + """HF hub revision (commit sha) for *model_id*, or None if unavailable. + + Uses huggingface_hub when installed (it is, in the MarkLLM env); never + raises -- offline runs simply record None. + """ + if os.environ.get("HF_HUB_OFFLINE"): + return None + try: + from huggingface_hub import model_info # type: ignore[import-not-found] + + info = model_info(model_id) + return info.sha or None + except Exception: + return None + + +def hf_revisions(model_ids: list[str]) -> dict[str, str | None]: + return {mid: hf_revision(mid) for mid in model_ids} + + +def pip_freeze() -> list[str] | None: + """pip freeze of the active environment (MarkLLM or quality env).""" + try: + r = subprocess.run( + [sys.executable, "-m", "pip", "freeze"], + capture_output=True, + text=True, + timeout=120, + check=False, + ) + if r.returncode != 0: + return None + return sorted(line for line in r.stdout.splitlines() if line.strip()) + except (OSError, subprocess.SubprocessError): + return None + + +def env_summary( + upstream: str | Path | None = None, + model_ids: list[str] | None = None, +) -> dict[str, Any]: + """One dict of every pin the manifest needs (gap 05-A7).""" + model_ids = model_ids or [] + return { + "repo_commit": repo_commit(), + "markllm_commit": markllm_commit(upstream), + "python": sys.version.split()[0], + "executable": sys.executable, + "hf_revisions": hf_revisions(model_ids), + "pip_freeze": pip_freeze(), + } + + +if __name__ == "__main__": + import json + + print(json.dumps(env_summary(), indent=2)) diff --git a/research/scripts/run_experiments.py b/research/scripts/run_experiments.py new file mode 100644 index 0000000..8e8e449 --- /dev/null +++ b/research/scripts/run_experiments.py @@ -0,0 +1,1430 @@ +#!/usr/bin/env python3 +"""Experiment orchestrator for the watermark-removal study. + +Plans and drives the locked v1 factorial described in +research/01-experiment-protocol.md (3,500 cells, 7 schemes x lengths +100/300/500 x temps 0.7/1.0 x langs en/de/fr/es with the restricted +subsets encoded in cell_allowed()). It deliberately reuses the repo's +existing machinery (detect_text_watermark.py, rewrite_text.py, +clean_text.py, multilingual_gen.py, cheap.py) instead of reimplementing +anything, and shells out to it so the audit trail is the command history. + +Modes: + --plan print cell counts + budget, exit (fully implemented) + --dry-run print the exact stage commands for a sample of cells + --stage N run a single stage (generate|attack|detect|evaluate|report) + (no --stage) run every stage in order + +Results layout (release-ready, condition-grouped; every row carries the +full cell identity scheme/length/temp/lang/seed/prompt): + + results/ + manifest.json # pins, configs, seeds, timestamps (gap 05-A7) + /generated.jsonl # watermarked + control texts per (seed, prompt) + /attacked.jsonl # per-attack outputs + cost fields + /scores.jsonl # detector scores per (doc, attack) + /metrics.json # AUROC, TPR@FPR, quality metrics + /quality.jsonl # quality metrics on a stratified subset + report.md # human-readable summary + +Condition = (scheme, length, temp, language); there are 28 conditions +(14 EN core + 4 temp axis + 4 length axis + 6 multilingual). Resume is +per (condition, stage) via .done markers: a finished stage is +skipped unless --force is passed. +""" + +from __future__ import annotations + +import argparse +import contextlib +import json +import os +import queue +import re +import shlex +import subprocess +import sys +import threading +import time +from collections.abc import Iterator +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +# -------------------------------------------------------------------------- +# Design definition (mirrors 01-experiment-protocol.md, locked v1 matrix) +# -------------------------------------------------------------------------- + +SCHEMES = { + # orchestrator name: (display, markllm alg, research/configs JSON) + "kgw-d1": ("KGW", "KGW", "KGW-d1.json"), # gamma=.25, delta=1 + "kgw-d2": ("KGW", "KGW", "KGW-d2.json"), # gamma=.5, delta=2 + "kgw-d4": ("KGW", "KGW", "KGW-d4.json"), # gamma=.5, delta=4 + "synthid": ("SynthID", "SynthID", "SynthID.json"), + "exp": ("EXP", "EXP", "EXP.json"), + "unigram": ("Unigram", "Unigram", "Unigram.json"), + "sir": ("SIR", "SIR", "SIR.json"), +} + +# orchestrator scheme -> detector CLI scheme (detect_text_watermark.SCHEMES key) +SCHEME_CLI = { + "kgw-d1": "kgw", + "kgw-d2": "kgw", + "kgw-d4": "kgw", + "synthid": "synthid", + "exp": "exp", + "unigram": "unigram", + "sir": "sir", +} + +# The 4 core schemes used by the temp / length / multilingual axes (01 §2). +CORE4 = ("kgw-d1", "kgw-d2", "kgw-d4", "synthid") +# The 2 schemes used by the multilingual grid (model holdout, 01 §3). +MULTILINGUAL2 = ("kgw-d2", "synthid") + +ATTACKS = [ + "none", + "layerA", # clean_text.py (deterministic) + "paraphrase:1", # rewrite_text.py single pass + "paraphrase:3", # adaptive, early-stop on detection + "backtranslate:de", # --strength backtranslate --lang German + "structural", + "humanize", + "cheap", # expands to cheap:synonym / cheap:delete / cheap:reorder + "layerA+paraphrase:3", # full layered pipeline (our contribution) +] + +# cheap.py sub-attacks (01 §4.1 A7). +CHEAP_SUBATTACKS = ("synonym", "delete", "reorder") + +LENGTHS = [100, 300, 500] +TEMPS = [0.7, 1.0] +LANGUAGES = ["en", "de", "fr", "es"] +SEEDS = [1, 2, 3, 4, 5] +PROMPTS = 25 + +EN_MODEL = "facebook/opt-1.3b" +MULTILINGUAL_MODEL = "Qwen/Qwen2.5-1.5B-Instruct" +MULTILINGUAL_MODEL_FALLBACK = "Qwen/Qwen2.5-0.5B-Instruct" + +TOP_P = 0.95 # decoding protocol, 01 §3 +REWRITE_TEMPERATURE = 0.9 # rewrite backend temperature, 01 §4.1 +REWRITE_TIMEOUT = 300.0 +MARKLLM_TIMEOUT = 900.0 # CPU generation can take minutes per text + +MAX_CONFIG_BYTES = 1 << 20 + + +@dataclass(frozen=True) +class Cell: + scheme: str + length: int + temp: float + language: str + seed: int + prompt_idx: int + + @property + def id(self) -> str: + return ( + f"{self.scheme}-L{self.length}-T{self.temp}" + f"-{self.language}-s{self.seed}-p{self.prompt_idx}" + ) + + +@dataclass(frozen=True) +class Condition: + scheme: str + length: int + temp: float + language: str + + @property + def id(self) -> str: + return f"{self.scheme}-L{self.length}-T{self.temp}-{self.language}" + + +@dataclass +class Budget: + cells: int + generations: int + attacks: int + detections: int + rewrite_tokens: int + est_usd: float + + +def cell_allowed(scheme: str, length: int, temp: float, language: str) -> bool: + """Locked v1 matrix restrictions (research/01-experiment-protocol.md §2).""" + if language == "en" and temp == 0.7 and length in (100, 300): + return True + if language == "en" and temp == 1.0 and length == 300: + return scheme in CORE4 + if language == "en" and length == 500 and temp == 0.7: + return scheme in CORE4 + if language in ("de", "fr", "es") and length == 300 and temp == 0.7: + return scheme in MULTILINGUAL2 + return False + + +def iter_cells( + schemes: list[str], + lengths: list[int], + temps: list[float], + languages: list[str], + seeds: list[int], + prompts: int, +) -> Iterator[Cell]: + for scheme in schemes: + for length in lengths: + for temp in temps: + for language in languages: + if not cell_allowed(scheme, length, temp, language): + continue + for seed in seeds: + for pidx in range(prompts): + yield Cell(scheme, length, temp, language, seed, pidx) + + +def iter_conditions( + schemes: list[str], + lengths: list[int], + temps: list[float], + languages: list[str], +) -> Iterator[Condition]: + seen: set[tuple[str, int, float, str]] = set() + for cell in iter_cells(schemes, lengths, temps, languages, [1], 1): + key = (cell.scheme, cell.length, cell.temp, cell.language) + if key not in seen: + seen.add(key) + yield Condition(*key) + + +def compute_budget( + schemes: list[str], + lengths: list[int], + temps: list[float], + languages: list[str], + seeds: list[int], + prompts: int, + attacks: list[str], +) -> Budget: + cells = sum(1 for _ in iter_cells(schemes, lengths, temps, languages, seeds, prompts)) + generations = 2 * cells + n_attacks = len(attacks) - 1 + len(CHEAP_SUBATTACKS) # 'none' + others; cheap x3 + detections = 2 * cells * (1 + n_attacks) + tok_factor = { + "paraphrase:1": 1.3, + "paraphrase:3": 2.6, + "backtranslate:de": 4.0, + "structural": 2.0, + "humanize": 1.3, + "cheap": 0.1, + "layerA": 0.0, + "layerA+paraphrase:3": 2.6, + "none": 0.0, + } + avg_in = 250 + rewrite_tokens = int( + 2 + * cells + * avg_in + * sum(tok_factor[a] * (len(CHEAP_SUBATTACKS) if a == "cheap" else 1) for a in attacks) + ) + est_usd = rewrite_tokens * (0.5 + 1.5) / 1e6 * 1.4 + return Budget(cells, generations, n_attacks, detections, rewrite_tokens, est_usd) + + +# -------------------------------------------------------------------------- +# Runtime helpers (subprocess shell-out + JSON-lines workers) +# -------------------------------------------------------------------------- + + +class RunContext: + """Everything a stage needs beyond its cell: scripts, workers, options.""" + + def __init__(self, args: argparse.Namespace, upstream: Path) -> None: + self.args = args + self.upstream = upstream + self.python = str(_venv_python(upstream) or sys.executable) + # Quality metrics (evaluate_quality.py) need the isolated metrics env + # (research/requirements-quality.txt): repo .venv-quality when present, + # else an explicit --quality-python, else fall back to the MarkLLM + # python with a warning (metric loaders degrade to None + warnings). + self.quality_python = None + if getattr(args, "quality_python", None): + self.quality_python = str(args.quality_python) + else: + cand = Path(__file__).resolve().parents[2] / ".venv-quality" / "bin" / "python" + if cand.is_file(): + self.quality_python = str(cand) + if self.quality_python is None: + print( + "[warn] quality-metrics env not found (repo .venv-quality or --quality-python); " + "evaluate_quality will run with the MarkLLM python and may report metric warnings", + file=sys.stderr, + ) + self.quality_python = self.python + self.scripts = Path(__file__).resolve().parents[1] # research/ + self.service = self.scripts.parent / "service" / "scripts" + self.corpus_dir = Path(args.corpus_dir).resolve() + self.workers: dict[tuple, ServeWorker] = {} + # CLI smoke controls: --prompts / --seeds limit generation to a + # subset of the locked 25x5 matrix (budget/plan already use them; + # the generate loop must too, so small smokes stay cheap on CPU). + self.prompts = int(getattr(args, "prompts", PROMPTS) or PROMPTS) + self.seeds = [ + int(x) + for x in str( + getattr(args, "seeds", ",".join(str(s) for s in SEEDS)) + ).split(",") + if x.strip() + ] + self._stamp = time.strftime("%Y-%m-%dT%H:%M:%S%z") + + def config_path(self, scheme: str) -> Path: + cfg = self.scripts / "configs" / SCHEMES[scheme][2] + if not cfg.is_file(): + raise SystemExit(f"error: missing config {cfg} (run from the repo root)") + return cfg + + def prompt_text(self, language: str, prompt_idx: int) -> str: + p = self.corpus_dir / language / f"{prompt_idx:02d}.txt" + if not p.is_file(): + raise SystemExit(f"error: missing prompt {p}; build the corpus first (gap 05-A6)") + return p.read_text(encoding="utf-8", errors="surrogateescape").strip() + + def worker_for( + self, + *, + kind: str, + scheme: str, + config: Path, + model: str, + language: str = "en", + temperature: float | None = None, + ) -> ServeWorker: + """Persistent serve worker keyed by (kind, scheme, config, model, lang).""" + key = (kind, scheme, str(config), model, language) + w = self.workers.get(key) + if w is not None: + return w + cli = SCHEME_CLI[scheme] + # bf16: opt-1.3b fp32 cached decode is ~8x slower on this aarch64 + # CPU-only box; bf16 keeps same-config gen/detect consistent and makes + # the EN core feasible (measured ~0.5 s/token vs ~4 s/token fp32). + if kind == "gen-multilingual": + script = self.scripts / "scripts" / "multilingual_gen.py" + cmd = [ + self.python, + str(script), + "serve", + "--markllm-dir", + str(self.upstream), + "--scheme", + cli, + "--config", + str(config), + "--model", + model, + "--lang", + language, + "--torch-dtype", + "bf16", + ] + else: # detect_text_watermark serve (EN gen + all detection) + script = self.service / "detect_text_watermark.py" + cmd = [ + self.python, + str(script), + "serve", + "--scheme", + cli, + "--config", + str(config), + "--model", + model, + "--upstream-dir", + str(self.upstream), + "--port", + "0", + "--torch-dtype", + "bf16", + ] + if temperature is not None: + cmd += ["--temperature", str(temperature), "--top-p", str(TOP_P)] + w = ServeWorker(cmd, timeout=MARKLLM_TIMEOUT, label=key[0] + " " + key[1]) + self.workers[key] = w + return w + + def close_workers(self) -> None: + for w in self.workers.values(): + with _suppress(Exception): + w.close() + self.workers.clear() + + +class ServeWorker: + """JSON-lines stdin/stdout worker (detect_text_watermark serve protocol). + + Speaks the ready-handshake + watermark/detect/exit protocol of + detect_text_watermark.py serve (and multilingual_gen.py serve). + """ + + def __init__(self, cmd: list[str], *, timeout: float, label: str) -> None: + self._timeout = timeout + self._label = label + self._proc = subprocess.Popen( + cmd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + bufsize=1, + ) + self._stderr_tail: list[str] = [] + threading.Thread(target=self._drain_stderr, daemon=True).start() + ready = self._read_line(timeout) + if ready is None or not ready.get("ready"): + self.close() + raise RuntimeError( + f"{label} worker did not become ready" + + (f": {ready.get('error')}" if ready else "") + + (f"; stderr: {' | '.join(self._stderr_tail[-2:])}" if self._stderr_tail else "") + ) + self.info = ready + self.port = ready.get("port") + + def _drain_stderr(self) -> None: + if self._proc.stderr is None: + return + for line in self._proc.stderr: + self._stderr_tail.append(line.rstrip()) + if len(self._stderr_tail) > 200: + self._stderr_tail.pop(0) + + def _read_line(self, timeout: float) -> dict[str, Any] | None: + q: queue.Queue[str] = queue.Queue() + + def _reader() -> None: + try: + line = self._proc.stdout.readline() if self._proc.stdout else "" + q.put(line) + except Exception as e: + q.put(f"__error__:{e}") + + t = threading.Thread(target=_reader, daemon=True) + t.start() + t.join(timeout) + if t.is_alive(): + raise RuntimeError(f"{self._label} worker response timed out") + line = q.get() + if line.startswith("__error__:"): + raise RuntimeError(line[len("__error__:") :]) + if not line: + raise RuntimeError(f"{self._label} worker closed (EOF)") + try: + data = json.loads(line) + except json.JSONDecodeError: + raise RuntimeError(f"{self._label} worker non-JSON: {line[:120]!r}") from None + return data if isinstance(data, dict) else None + + def request(self, payload: dict[str, Any]) -> dict[str, Any]: + if self._proc.stdin is None: + raise RuntimeError(f"{self._label} worker has no stdin") + try: + self._proc.stdin.write(json.dumps(payload) + "\n") + self._proc.stdin.flush() + resp = self._read_line(self._timeout) + except Exception as e: + hint = "; ".join(self._stderr_tail[-3:]) + raise RuntimeError(f"{self._label} worker failed: {e} ({hint})") from None + if not resp.get("ok"): + raise RuntimeError(resp.get("error") or f"{self._label} request failed") + return resp + + def watermark( + self, + prompt: str, + seed: int, + max_new_tokens: int, + *, + temperature: float | None, + top_p: float | None, + ) -> dict[str, Any]: + return self.request( + { + "op": "watermark", + "id": seed, + "prompt": prompt, + "seed": seed, + "max_new_tokens": max_new_tokens, + "temperature": temperature, + "top_p": top_p, + } + ) + + def detect(self, text: str) -> dict[str, Any]: + return self.request({"op": "detect", "id": 0, "text": text}) + + def close(self) -> None: + if self._proc.poll() is None: + try: + if self._proc.stdin is not None: + self._proc.stdin.write(json.dumps({"op": "exit"}) + "\n") + self._proc.stdin.flush() + self._proc.wait(timeout=10) + except Exception: + with _suppress(Exception): + self._proc.terminate() + self._proc.wait(timeout=5) + for stream in (self._proc.stdin, self._proc.stdout, self._proc.stderr): + with _suppress(Exception): + if stream is not None: + stream.close() + + +def _venv_python(upstream: Path) -> Path | None: + if os.name == "nt": + candidate = upstream / ".venv" / "Scripts" / "python.exe" + else: + candidate = upstream / ".venv" / "bin" / "python" + return candidate if candidate.is_file() else None + + +def _suppress(exc: type[BaseException]): + return contextlib.suppress(exc) + + +def _run_cmd(cmd: list[str], *, timeout: float) -> subprocess.CompletedProcess[str]: + return subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=timeout, + check=False, + ) + + +def _json_from_stderr(stderr: str) -> dict[str, Any] | None: + idx = stderr.find("{") + if idx < 0: + return None + try: + data = json.loads(stderr[idx:]) + except json.JSONDecodeError: + return None + return data if isinstance(data, dict) else None + + +def _tokens(text: str) -> int: + return max(1, int(len(text) / 4.0)) + + +def _numbers_preserved(original: str, candidate: str) -> float: + a = set(re.findall(r"\d+", original)) + if not a: + return 1.0 + return len(a & set(re.findall(r"\d+", candidate))) / len(a) + + +def _urls_preserved(original: str, candidate: str) -> float: + a = set(re.findall(r"https?://\S+", original)) + if not a: + return 1.0 + return len(a & set(re.findall(r"https?://\S+", candidate))) / len(a) + + +def _cell_dir(out_dir: Path, condition: Condition) -> Path: + return out_dir / condition.id + + +def _done(ctx: RunContext, cell_dir: Path, stage: str) -> bool: + return not ctx.args.force and (cell_dir / f"{stage}.done").exists() + + +def _mark_done(cell_dir: Path, stage: str) -> None: + (cell_dir / f"{stage}.done").write_text(time.strftime("%Y-%m-%dT%H:%M:%S%z"), encoding="utf-8") + + +def _write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None: + tmp = path.with_suffix(path.suffix + ".tmp") + with open(tmp, "w", encoding="utf-8") as f: + for row in rows: + f.write(json.dumps(row, ensure_ascii=False) + "\n") + tmp.replace(path) + + +def _read_jsonl(path: Path) -> list[dict[str, Any]]: + if not path.is_file(): + return [] + rows: list[dict[str, Any]] = [] + for raw in path.read_text(encoding="utf-8").splitlines(): + line = raw.strip() + if line: + rows.append(json.loads(line)) + return rows + + +# -------------------------------------------------------------------------- +# Stage: generate +# -------------------------------------------------------------------------- + + +def stage_generate( + condition: Condition, out_dir: Path, ctx: RunContext, dry_run: bool = False +) -> list[str]: + """Generate watermarked + control text for one condition (25 prompts x 5 seeds).""" + cell_dir = _cell_dir(out_dir, condition) + out_file = cell_dir / "generated.jsonl" + if dry_run: + cfg = ctx.config_path(condition.scheme) + model = EN_MODEL if condition.language == "en" else MULTILINGUAL_MODEL + cli = SCHEME_CLI[condition.scheme] + if condition.language == "en": + script = ctx.service / "detect_text_watermark.py" + cmd = [ + ctx.python, + str(script), + "serve", + "--scheme", + cli, + "--config", + str(cfg), + "--model", + model, + "--upstream-dir", + str(ctx.upstream), + "--temperature", + str(condition.temp), + "--top-p", + str(TOP_P), + "--port", + "0", + "--torch-dtype", + "bf16", + ] + else: + script = ctx.scripts / "scripts" / "multilingual_gen.py" + cmd = [ + ctx.python, + str(script), + "serve", + "--markllm-dir", + str(ctx.upstream), + "--scheme", + cli, + "--config", + str(cfg), + "--model", + model, + "--lang", + condition.language, + "--torch-dtype", + "bf16", + ] + return [ + " ".join(shlex.quote(c) for c in cmd), + f"# {ctx.prompts * len(ctx.seeds)} watermark requests over stdin -> {out_file}", + ] + if _done(ctx, cell_dir, "generate"): + print(f"[skip] generate {condition.id} (done)") + return [] + cell_dir.mkdir(parents=True, exist_ok=True) + cfg = ctx.config_path(condition.scheme) + model = EN_MODEL if condition.language == "en" else MULTILINGUAL_MODEL + worker = ctx.worker_for( + kind="gen-multilingual" if condition.language != "en" else "gen-en", + scheme=condition.scheme, + config=cfg, + model=model, + language=condition.language, + ) + rows: list[dict[str, Any]] = [] + for pidx in range(1, ctx.prompts + 1): + prompt = ctx.prompt_text(condition.language, pidx) + for seed in ctx.seeds: + row: dict[str, Any] = { + "condition": condition.id, + "scheme": condition.scheme, + "length": condition.length, + "temp": condition.temp, + "language": condition.language, + "seed": seed, + "prompt_idx": pidx, + "prompt": prompt, + "model": model, + "config": str(cfg), + "ok": False, + "error": None, + } + try: + resp = worker.watermark( + prompt, + seed, + condition.length, + temperature=condition.temp, + top_p=TOP_P, + ) + wm = resp.get("watermarked") or "" + plain = resp.get("unwatermarked") or "" + if len(wm.strip()) < 50: + raise RuntimeError("watermarked sample too short") + row.update( + { + "watermarked": wm, + "unwatermarked": plain, + "ok": True, + } + ) + except Exception as e: + row["error"] = str(e)[:300] + rows.append(row) + print(f"[generate] {condition.id} s{seed} p{pidx}: " + ("ok" if row["ok"] else "FAIL")) + _write_jsonl(out_file, rows) + _mark_done(cell_dir, "generate") + return [] + + +# -------------------------------------------------------------------------- +# Stage: attack +# -------------------------------------------------------------------------- + + +def _attack_rows(attack: str) -> list[str]: + if attack == "cheap": + return [f"cheap:{a}" for a in CHEAP_SUBATTACKS] + return [attack] + + +def stage_attack( + condition: Condition, attack: str, out_dir: Path, ctx: RunContext, dry_run: bool = False +) -> list[str]: + """Apply one attack condition to every generated (seed, prompt) row.""" + cell_dir = _cell_dir(out_dir, condition) + gen_file = cell_dir / "generated.jsonl" + out_file = cell_dir / "attacked.jsonl" + if dry_run: + return [ + "clean_text.py / rewrite_text.py / cheap.py invocation " + f"({condition.id}, {attack}) -> {out_file}" + ] + if not gen_file.is_file(): + print(f"[warn] attack {condition.id} {attack}: no generated.jsonl; run generate first") + return [] + if _done(ctx, cell_dir, f"attack:{attack}"): + print(f"[skip] attack {condition.id} {attack} (done)") + return [] + rows = _read_jsonl(out_file) + existing = {(r.get("seed"), r.get("prompt_idx"), r.get("attack")) for r in rows} + done_any = False + for gen in _read_jsonl(gen_file): + if not gen.get("ok"): + continue + seed, pidx = gen["seed"], gen["prompt_idx"] + original = gen["watermarked"] + for sub in _attack_rows(attack): + if (seed, pidx, sub) in existing: + continue + candidate, stats, err, seconds = _run_one_attack(condition, sub, original, seed, ctx) + row: dict[str, Any] = { + "condition": condition.id, + "scheme": condition.scheme, + "seed": seed, + "prompt_idx": pidx, + "attack": sub, + "original": original, + "candidate": candidate, + "ok": err is None, + "error": err, + "seconds": seconds, + "tokens_in": _tokens(original), + "tokens_out": _tokens(candidate) if candidate else None, + } + if stats: + row["stats"] = stats + row["usd"] = _usd_estimate(row) + rows.append(row) + done_any = True + print( + f"[attack] {condition.id} {sub} s{seed} p{pidx}: " + + ("ok" if err is None else f"FAIL {err[:120]}") + ) + if rows: + _write_jsonl(out_file, rows) + if done_any: + _mark_done(cell_dir, f"attack:{attack}") + return [] + + +def _run_one_attack( + condition: Condition, + attack: str, + original: str, + seed: int, + ctx: RunContext, +) -> tuple[str | None, dict[str, Any] | None, str | None, float]: + """Run one attack; returns (candidate, stats, error, seconds).""" + import tempfile + + started = time.monotonic() + with tempfile.TemporaryDirectory(prefix="wm-attack-") as raw_td: + td = Path(raw_td) + in_path = td / "in.txt" + in_path.write_text(original, encoding="utf-8") + + if attack == "none": + # Control row: the watermarked artifact unchanged (A0 baseline). + return original, None, None, time.monotonic() - started + + if attack == "layerA": + out_path = td / "out.txt" + cmd = [ + ctx.python, + str(ctx.service / "clean_text.py"), + str(in_path), + "-o", + str(out_path), + ] + proc = _run_cmd(cmd, timeout=REWRITE_TIMEOUT) + if proc.returncode != 0: + return ( + None, + None, + (proc.stderr or proc.stdout or "").strip()[:300], + time.monotonic() - started, + ) + return ( + out_path.read_text(encoding="utf-8", errors="surrogateescape"), + None, + None, + time.monotonic() - started, + ) + + if attack.startswith("cheap:"): + sub = attack.split(":", 1)[1] + out_path = td / "out.txt" + cmd = [ + ctx.python, + str(ctx.scripts / "scripts" / "attacks" / "cheap.py"), + "--input", + str(in_path), + "--output", + str(out_path), + "--attack", + sub, + "--seed", + str(seed), + ] + proc = _run_cmd(cmd, timeout=REWRITE_TIMEOUT) + if proc.returncode != 0: + return ( + None, + None, + (proc.stderr or proc.stdout or "").strip()[:300], + time.monotonic() - started, + ) + return ( + out_path.read_text(encoding="utf-8", errors="surrogateescape"), + None, + None, + time.monotonic() - started, + ) + + if attack.startswith("paraphrase:") or attack in ( + "backtranslate:de", + "structural", + "humanize", + ): + strength, _, cand_s = attack.partition(":") + candidates = int(cand_s) if cand_s else 1 + out_path = td / "out.txt" + lang = "German" if attack == "backtranslate:de" else "French" + cmd = [ + ctx.python, + str(ctx.service / "rewrite_text.py"), + str(in_path), + "-o", + str(out_path), + "--backend", + ctx.args.rewrite_backend, + "--model", + ctx.args.rewrite_model or "", + "--base-url", + ctx.args.rewrite_base_url, + "--strength", + strength, + "--lang", + lang, + "--candidates", + str(candidates), + "--max-loops", + str(candidates), + "--temperature", + str(REWRITE_TEMPERATURE), + "--timeout", + str(REWRITE_TIMEOUT), + "--markllm-scheme", + SCHEME_CLI[condition.scheme], + "--markllm-dir", + str(ctx.upstream), + "--markllm-model", + EN_MODEL if condition.language == "en" else MULTILINGUAL_MODEL, + "--json-stats", + ] + env = dict(os.environ) + if ctx.args.rewrite_api_key: + env["WATERMARKS_REWRITE_API_KEY"] = ctx.args.rewrite_api_key + proc = _run_cmd( + cmd, + timeout=REWRITE_TIMEOUT + MARKLLM_TIMEOUT + 60, + ) + if proc.returncode != 0: + return ( + None, + None, + (proc.stderr or proc.stdout or "").strip()[:300], + time.monotonic() - started, + ) + stats = _json_from_stderr(proc.stderr) + out_text = out_path.read_text(encoding="utf-8", errors="surrogateescape") + return out_text, stats, None, time.monotonic() - started + + if attack == "layerA+paraphrase:3": + cleaned = td / "cleaned.txt" + cmd = [ + ctx.python, + str(ctx.service / "clean_text.py"), + str(in_path), + "-o", + str(cleaned), + ] + proc = _run_cmd(cmd, timeout=REWRITE_TIMEOUT) + if proc.returncode != 0: + return None, None, (proc.stderr or "").strip()[:300], time.monotonic() - started + out_path = td / "out.txt" + cmd = [ + ctx.python, + str(ctx.service / "rewrite_text.py"), + str(cleaned), + "-o", + str(out_path), + "--backend", + ctx.args.rewrite_backend, + "--model", + ctx.args.rewrite_model or "", + "--base-url", + ctx.args.rewrite_base_url, + "--strength", + "paraphrase", + "--candidates", + "3", + "--max-loops", + "3", + "--temperature", + str(REWRITE_TEMPERATURE), + "--timeout", + str(REWRITE_TIMEOUT), + "--markllm-scheme", + SCHEME_CLI[condition.scheme], + "--markllm-dir", + str(ctx.upstream), + "--markllm-model", + EN_MODEL if condition.language == "en" else MULTILINGUAL_MODEL, + "--json-stats", + ] + proc = _run_cmd(cmd, timeout=REWRITE_TIMEOUT + MARKLLM_TIMEOUT + 60) + if proc.returncode != 0: + return None, None, (proc.stderr or "").strip()[:300], time.monotonic() - started + stats = _json_from_stderr(proc.stderr) + out_text = out_path.read_text(encoding="utf-8", errors="surrogateescape") + return out_text, stats, None, time.monotonic() - started + + return None, None, f"unknown attack {attack!r}", time.monotonic() - started + + +def _usd_estimate(row: dict[str, Any]) -> float: + stats = row.get("stats") or {} + in_tok = stats.get("tokens_in") or row.get("tokens_in") or 0 + out_tok = stats.get("tokens_out") or row.get("tokens_out") or 0 + return round(float(in_tok) / 1e6 * 0.5 + float(out_tok) / 1e6 * 1.5, 8) + + +# -------------------------------------------------------------------------- +# Stage: detect +# -------------------------------------------------------------------------- + + +def stage_detect( + condition: Condition, attack: str, out_dir: Path, ctx: RunContext, dry_run: bool = False +) -> list[str]: + """Same-config detection on originals + controls + attacked text.""" + cell_dir = _cell_dir(out_dir, condition) + gen_file = cell_dir / "generated.jsonl" + attack_file = cell_dir / "attacked.jsonl" + out_file = cell_dir / "scores.jsonl" + if dry_run: + return [ + f"detect_text_watermark.py serve ({condition.id}) -> {out_file}", + ] + if not gen_file.is_file(): + print(f"[warn] detect {condition.id}: no generated.jsonl") + return [] + if _done(ctx, cell_dir, f"detect:{attack}"): + print(f"[skip] detect {condition.id} {attack} (done)") + return [] + cfg = ctx.config_path(condition.scheme) + model = EN_MODEL if condition.language == "en" else MULTILINGUAL_MODEL + # Reuse the generation worker: the serve protocol supports detect, so the + # resident model (opt-1.3b for EN, Qwen for multilingual) serves both. + worker = ctx.worker_for( + kind="gen-multilingual" if condition.language != "en" else "gen-en", + scheme=condition.scheme, + config=cfg, + model=model, + language=condition.language, + ) + rows = _read_jsonl(out_file) + existing = {(r.get("seed"), r.get("prompt_idx"), r.get("attack"), r.get("kind")) for r in rows} + + def add(seed: int, pidx: int, kind: str, text: str, attack: str) -> None: + if (seed, pidx, attack, kind) in existing: + return + try: + resp = worker.detect(text) + row = { + "condition": condition.id, + "scheme": condition.scheme, + "seed": seed, + "prompt_idx": pidx, + "attack": attack, + "kind": kind, + "score": resp.get("score"), + "is_watermarked": resp.get("is_watermarked"), + "threshold": resp.get("threshold"), + "ok": True, + } + except Exception as e: + row = { + "condition": condition.id, + "scheme": condition.scheme, + "seed": seed, + "prompt_idx": pidx, + "attack": attack, + "kind": kind, + "ok": False, + "error": str(e)[:300], + } + rows.append(row) + + for gen in _read_jsonl(gen_file): + if not gen.get("ok"): + continue + seed, pidx = gen["seed"], gen["prompt_idx"] + if attack == "none": + add(seed, pidx, "watermarked", gen["watermarked"], "none") + add(seed, pidx, "control", gen["unwatermarked"], "none") + else: + for arow in _read_jsonl(attack_file): + if ( + arow.get("seed") == seed + and arow.get("prompt_idx") == pidx + and arow.get("attack") == attack + and arow.get("ok") + ): + add(seed, pidx, "attacked", arow["candidate"], attack) + if rows: + _write_jsonl(out_file, rows) + _mark_done(cell_dir, f"detect:{attack}") + return [] + + +# -------------------------------------------------------------------------- +# Stage: evaluate +# -------------------------------------------------------------------------- + + +def stage_evaluate( + condition: Condition, out_dir: Path, ctx: RunContext, dry_run: bool = False +) -> list[str]: + """ROC metrics (B1) + quality metrics (B2) for one condition.""" + cell_dir = _cell_dir(out_dir, condition) + scores_file = cell_dir / "scores.jsonl" + attack_file = cell_dir / "attacked.jsonl" + metrics_file = cell_dir / "metrics.json" + quality_file = cell_dir / "quality.jsonl" + if dry_run: + return [ + f"{ctx.python} analyze_roc.py --scores {scores_file} --out {metrics_file}", + f"{ctx.quality_python} evaluate_quality.py --input --out {quality_file}", + ] + if not scores_file.is_file(): + print(f"[warn] evaluate {condition.id}: no scores.jsonl") + return [] + if _done(ctx, cell_dir, "evaluate"): + print(f"[skip] evaluate {condition.id} (done)") + return [] + roc = ctx.scripts / "scripts" / "analyze_roc.py" + proc = _run_cmd( + [ + ctx.python, + str(roc), + "--scores", + str(scores_file), + "--out", + str(metrics_file), + "--n-bootstrap", + str(ctx.args.n_bootstrap), + "--seed", + "1", + ], + timeout=600, + ) + if proc.returncode != 0: + print(f"[warn] analyze_roc failed for {condition.id}: {(proc.stderr or '')[:300]}") + else: + _mark_done(cell_dir, "evaluate") + + # Quality on a stratified subset of attacked pairs (01 §5.3). + pairs = _quality_pairs(condition, attack_file, ctx.args.quality_per_condition) + if pairs: + pairs_file = cell_dir / "pairs.jsonl" + _write_jsonl(pairs_file, pairs) + qual = ctx.scripts / "scripts" / "evaluate_quality.py" + proc = _run_cmd( + [ + ctx.quality_python, + str(qual), + "--input", + str(pairs_file), + "--out", + str(quality_file), + "--device", + "cpu", + "--warnings-out", + str(cell_dir / "quality-warnings.json"), + ], + timeout=3600, + ) + if proc.returncode != 0: + print(f"[warn] evaluate_quality failed for {condition.id}: {(proc.stderr or '')[:300]}") + return [] + + +def _quality_pairs(condition: Condition, attack_file: Path, cap: int) -> list[dict[str, Any]]: + """Stratified subset: up to *cap* (seed, prompt, attack) rows per condition.""" + pairs: list[dict[str, Any]] = [] + for row in _read_jsonl(attack_file): + if not row.get("ok"): + continue + if row.get("attack") in ("none",): + continue + pairs.append( + { + "condition": condition.id, + "scheme": condition.scheme, + "seed": row["seed"], + "prompt_idx": row["prompt_idx"], + "attack": row["attack"], + "original": row["original"], + "candidate": row["candidate"], + } + ) + if len(pairs) >= cap: + break + return pairs + + +# -------------------------------------------------------------------------- +# Stage: report +# -------------------------------------------------------------------------- + + +def stage_report(out_dir: Path, ctx: RunContext) -> list[str]: + """Aggregate everything into manifest.json + report.md.""" + from pins import env_summary + + manifest_path = out_dir / "manifest.json" + model_ids = sorted({EN_MODEL, MULTILINGUAL_MODEL}) + manifest: dict[str, Any] = { + "study": "How Fragile Are Deployed Text Watermarks? (arXiv v1)", + "created": ctx._stamp, + "pins": env_summary(ctx.upstream, model_ids), + "design": { + "schemes": { + k: {"display": v[0], "algorithm": v[1], "config": v[2]} for k, v in SCHEMES.items() + }, + "attacks": ATTACKS, + "lengths": LENGTHS, + "temps": TEMPS, + "languages": LANGUAGES, + "seeds": SEEDS, + "prompts": PROMPTS, + "top_p": TOP_P, + "rewrite_temperature": REWRITE_TEMPERATURE, + "rewrite_backend": ctx.args.rewrite_backend, + "rewrite_model": ctx.args.rewrite_model, + }, + "command": " ".join(shlex.quote(a) for a in sys.argv), + } + conditions: list[dict[str, Any]] = [] + total: dict[str, int] = {} + for cond_dir in sorted(out_dir.glob("*-L*-T*-*")): + if not cond_dir.is_dir(): + continue + gen = _read_jsonl(cond_dir / "generated.jsonl") + att = _read_jsonl(cond_dir / "attacked.jsonl") + sc = _read_jsonl(cond_dir / "scores.jsonl") + metrics: dict[str, Any] = {} + if (cond_dir / "metrics.json").is_file(): + try: + metrics = json.loads((cond_dir / "metrics.json").read_text("utf-8")) + except ValueError: + metrics = {"error": "unparseable metrics.json"} + ok_gen = sum(1 for r in gen if r.get("ok")) + ok_att = sum(1 for r in att if r.get("ok")) + ok_sc = sum(1 for r in sc if r.get("ok")) + total["generated"] = total.get("generated", 0) + ok_gen + total["attacked"] = total.get("attacked", 0) + ok_att + total["scores"] = total.get("scores", 0) + ok_sc + conditions.append( + { + "condition": cond_dir.name, + "generated": ok_gen, + "attacked": ok_att, + "scores": ok_sc, + "metrics": metrics.get("per_attack", {}) if isinstance(metrics, dict) else {}, + "done": {s: (cond_dir / f"{s}.done").exists() for s in ("generate", "evaluate")}, + } + ) + manifest["conditions"] = conditions + manifest["totals"] = total + manifest_path.write_text(json.dumps(manifest, indent=2), encoding="utf-8") + + L: list[str] = [] + L.append("# watermarks-remover study — run summary") + L.append("") + L.append(f"- created: {ctx._stamp}") + L.append(f"- repo commit: {manifest['pins'].get('repo_commit') or 'unknown'}") + L.append(f"- MarkLLM commit: {manifest['pins'].get('markllm_commit') or 'unknown'}") + L.append( + f"- generated ok: {total.get('generated', 0)}, attacked ok: {total.get('attacked', 0)}, scores ok: {total.get('scores', 0)}" + ) + L.append("") + L.append("| condition | gen | att | scores | AUROC (none) | AUROC (layerA+paraphrase:3) |") + L.append("| --- | ---: | ---: | ---: | ---: | ---: |") + + def _auroc(pa: dict[str, Any], attack: str) -> str: + m = pa.get(attack) or {} + v = m.get("auroc") + return f"{v:.3f}" if isinstance(v, (int, float)) else "-" + + for c in conditions: + pa = c["metrics"] + L.append( + f"| {c['condition']} | {c['generated']} | {c['attacked']} | {c['scores']} " + f"| {_auroc(pa, 'none')} | {_auroc(pa, 'layerA+paraphrase:3')} |" + ) + L.append("") + L.append("Full per-condition data: JSONL per condition dir; manifest: manifest.json.") + L.append("") + (out_dir / "report.md").write_text("\n".join(L) + "\n", encoding="utf-8") + return [] + + +# -------------------------------------------------------------------------- +# CLI +# -------------------------------------------------------------------------- + + +def build_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument( + "--schemes", + default=",".join(SCHEMES), + help="comma list; see SCHEMES (default: locked v1 set, 7 schemes)", + ) + p.add_argument( + "--lengths", + default=",".join(str(x) for x in LENGTHS), + help="comma list of token lengths (default: locked v1)", + ) + p.add_argument( + "--temps", + default=",".join(str(x) for x in TEMPS), + help="comma list of temperatures (default: locked v1)", + ) + p.add_argument( + "--languages", + default=",".join(LANGUAGES), + help="comma list (default: locked v1: en,de,fr,es)", + ) + p.add_argument("--seeds", default="1,2,3,4,5") + p.add_argument("--prompts", type=int, default=PROMPTS) + p.add_argument("--attacks", default=",".join(ATTACKS)) + p.add_argument("--out-dir", type=Path, default=Path("results")) + p.add_argument( + "--markllm-dir", default=None, help="MarkLLM checkout (default: env MARKLLM_DIR)" + ) + p.add_argument( + "--corpus-dir", + type=Path, + default=Path("research/corpus"), + help="Prompt corpus dir with /01.txt..25.txt (default: research/corpus)", + ) + p.add_argument( + "--rewrite-backend", choices=("ollama", "openai-compatible"), default="openai-compatible" + ) + p.add_argument( + "--rewrite-model", default=None, help="Rewrite backend model (required for attacks)" + ) + p.add_argument("--rewrite-base-url", default="http://127.0.0.1:11434") + p.add_argument("--rewrite-api-key", default=None, help="Rewrite API key (env-only in children)") + p.add_argument( + "--rewrite-allow-remote", action="store_true", help="Allow non-loopback rewrite endpoints" + ) + p.add_argument("--n-bootstrap", type=int, default=10000, help="ROC bootstrap resamples (B1)") + p.add_argument( + "--quality-per-condition", + type=int, + default=8, + help="Quality pairs per condition (01 §5.3 stratified subset)", + ) + p.add_argument( + "--quality-python", + default=None, + help="Python interpreter for evaluate_quality.py " + "(default: repo .venv-quality/bin/python when present, else MarkLLM venv python)", + ) + p.add_argument("--plan", action="store_true", help="print budget and exit") + p.add_argument("--dry-run", action="store_true", help="print stage commands without running") + p.add_argument( + "--stage", + choices=["generate", "attack", "detect", "evaluate", "report"], + default=None, + help="run a single stage", + ) + p.add_argument("--force", action="store_true", help="re-run stages even if marked done") + return p + + +def main() -> int: + args = build_parser().parse_args() + schemes = [s.strip() for s in args.schemes.split(",") if s.strip()] + lengths = [int(x) for x in args.lengths.split(",") if x.strip()] + temps = [float(x) for x in args.temps.split(",") if x.strip()] + languages = [x.strip() for x in args.languages.split(",") if x.strip()] + seeds = [int(x) for x in args.seeds.split(",") if x.strip()] + attacks = [a.strip() for a in args.attacks.split(",") if a.strip()] + + for s in schemes: + if s not in SCHEMES: + print(f"error: unknown scheme {s!r}; known: {sorted(SCHEMES)}", file=sys.stderr) + return 2 + + budget = compute_budget(schemes, lengths, temps, languages, seeds, args.prompts, attacks) + + if args.plan: + print("=== Plan (locked v1 matrix, 01-experiment-protocol.md §2) ===") + print(f"schemes : {schemes}") + print(f"lengths : {lengths}") + print(f"temps : {temps}") + print(f"languages : {languages}") + print(f"seeds : {seeds}") + print(f"prompts : {args.prompts}") + print(f"attack cells : {attacks}") + print( + "restrictions : cell_allowed() - temp-1.0/length-500 on CORE4, multilingual on " + + str(list(MULTILINGUAL2)) + ) + print("---") + print(f"cells : {budget.cells}") + print(f"generations : {budget.generations} (watermarked + control)") + print( + f"attack runs : {budget.cells} docs x {budget.attacks} attacks = {budget.cells * budget.attacks}" + ) + print(f"detections : {budget.detections} (incl. controls + empirical null)") + print(f"rewrite tokens (est): {budget.rewrite_tokens:,} (wm + control attacked)") + print(f"rewrite cost (est) : {budget.est_usd:.2f}") + print("---") + print("CPU estimate (8 cores): gen ~35-50h, detect ~25-90h, quality ~10-20h;") + print("API rewrite budget ~$50-120 (record model + version per 01 §6)") + print( + "Conditions : " + + str(sum(1 for _ in iter_conditions(schemes, lengths, temps, languages))) + ) + return 0 + + if not args.markllm_dir: + args.markllm_dir = os.environ.get("MARKLLM_DIR") + if not args.markllm_dir: + print("error: --markllm-dir (or MARKLLM_DIR) is required", file=sys.stderr) + return 2 + upstream = Path(args.markllm_dir).expanduser().resolve() + if not (upstream / "watermark").is_dir(): + print( + f"error: MarkLLM checkout incomplete (no watermark/ dir): {upstream}", file=sys.stderr + ) + return 2 + + out_dir = args.out_dir.resolve() + out_dir.mkdir(parents=True, exist_ok=True) + ctx = RunContext(args, upstream) + + conditions = list(iter_conditions(schemes, lengths, temps, languages)) + if args.dry_run: + print(f"# {len(conditions)} conditions; sample commands:") + for cond in conditions[:2]: + for line in stage_generate(cond, out_dir, ctx, dry_run=True): + print(line) + for cond in conditions[:1]: + for a in attacks[:2]: + for line in stage_attack(cond, a, out_dir, ctx, dry_run=True): + print(line) + for line in stage_detect(cond, "none", out_dir, ctx, dry_run=True): + print(line) + for line in stage_evaluate(cond, out_dir, ctx, dry_run=True): + print(line) + return 0 + + try: + if args.stage == "generate": + for cond in conditions: + stage_generate(cond, out_dir, ctx) + elif args.stage == "attack": + for cond in conditions: + for a in attacks: + stage_attack(cond, a, out_dir, ctx) + elif args.stage == "detect": + for cond in conditions: + for a in attacks: + stage_detect(cond, a, out_dir, ctx) + elif args.stage == "evaluate": + for cond in conditions: + stage_evaluate(cond, out_dir, ctx) + elif args.stage == "report": + stage_report(out_dir, ctx) + else: + print(f"=== generate ({len(conditions)} conditions) ===") + for cond in conditions: + stage_generate(cond, out_dir, ctx) + print("=== attack ===") + for cond in conditions: + for a in attacks: + stage_attack(cond, a, out_dir, ctx) + print("=== detect ===") + for cond in conditions: + for a in attacks: + stage_detect(cond, a, out_dir, ctx) + print("=== evaluate ===") + for cond in conditions: + stage_evaluate(cond, out_dir, ctx) + print("=== report ===") + stage_report(out_dir, ctx) + finally: + ctx.close_workers() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/research/tests/test_analyze_roc.py b/research/tests/test_analyze_roc.py new file mode 100644 index 0000000..4cd4596 --- /dev/null +++ b/research/tests/test_analyze_roc.py @@ -0,0 +1,232 @@ +"""Tests for research/scripts/analyze_roc.py (gap 05-B1). + +Covers the rank-based AUROC, TPR@FPR read off the empirical null, seeded +bootstrap CIs, degenerate inputs, and attack grouping in analyze_scores. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import numpy as np + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) + +from analyze_roc import analyze_scores, compute_roc_metrics + + +def test_perfect_separation_auroc_is_one(): + signal = list(range(100, 200)) + null = list(range(100)) + res = compute_roc_metrics(signal, null, [0.1], n_bootstrap=10, seed=1) + assert res["auroc"] == 1.0 + assert res["tpr_at_fpr"]["0.1"] == 1.0 + assert res["roc_points"][-1] == [1.0, 1.0] + + +def test_identical_distributions_auroc_half(): + rng = np.random.default_rng(7) + scores = list(rng.normal(size=200)) + res = compute_roc_metrics(scores, scores, [0.1], n_bootstrap=10, seed=1) + # The two groups are the exact same multiset: tie-averaged ranks put + # half the rank mass on each side, so AUROC is exactly 0.5. + assert res["auroc"] == 0.5 + + +def test_tpr_at_fpr_read_off_empirical_null(): + # Hand-built tiny example: null = [0,1,2,3,4], signal = [1,2,3,4,5]. + # For target FPR alpha the threshold is the smallest null score t with + # P(null >= t) <= alpha (>=, ties positive): + # alpha=0.2 -> t=4 (fpr 0.2) -> tpr = P(signal >= 4) = 2/5 = 0.4 + # alpha=0.4 -> t=3 (fpr 0.4) -> tpr = P(signal >= 3) = 3/5 = 0.6 + # alpha=0.6 -> t=2 (fpr 0.6) -> tpr = P(signal >= 2) = 4/5 = 0.8 + signal = [1, 2, 3, 4, 5] + null = [0, 1, 2, 3, 4] + res = compute_roc_metrics(signal, null, [0.2, 0.4, 0.6], n_bootstrap=50, seed=1) + assert res["tpr_at_fpr"] == {"0.2": 0.4, "0.4": 0.6, "0.6": 0.8} + # Rank-based AUROC for this example is 17/25 = 0.68. + assert res["auroc"] == 0.68 + # ROC points are monotone and include both endpoints. + fprs = [point[0] for point in res["roc_points"]] + tprs = [point[1] for point in res["roc_points"]] + assert fprs == sorted(fprs) + assert tprs == sorted(tprs) + assert res["roc_points"][0] == [0.0, 0.0] + assert res["roc_points"][-1] == [1.0, 1.0] + + +def test_bootstrap_ci_bounds_and_containment(): + rng = np.random.default_rng(42) + signal = list(rng.normal(loc=1.0, size=200)) + null = list(rng.normal(size=200)) + res = compute_roc_metrics(signal, null, [0.1, 0.5], n_bootstrap=500, seed=3) + lo, hi = res["auroc_ci95"] + assert lo <= hi + assert lo <= res["auroc"] <= hi + for fpr in ("0.1", "0.5"): + flo, fhi = res["tpr_ci95"][fpr] + assert flo <= fhi + assert flo <= res["tpr_at_fpr"][fpr] <= fhi + + +def test_determinism_with_fixed_seed(): + rng = np.random.default_rng(11) + signal = list(rng.normal(loc=0.5, size=150)) + null = list(rng.normal(size=150)) + kwargs = {"n_bootstrap": 300, "seed": 9} + first = compute_roc_metrics(signal, null, [0.01, 0.1], **kwargs) + second = compute_roc_metrics(signal, null, [0.01, 0.1], **kwargs) + assert first["auroc"] == second["auroc"] + assert first["tpr_at_fpr"] == second["tpr_at_fpr"] + assert first["auroc_ci95"] == second["auroc_ci95"] + assert first["tpr_ci95"] == second["tpr_ci95"] + assert first["roc_points"] == second["roc_points"] + + +def test_degenerate_all_equal_scores_yields_none_and_warning(): + res = compute_roc_metrics([1.0, 1.0, 1.0], [1.0, 1.0, 1.0], [0.1], n_bootstrap=50, seed=1) + assert res["auroc"] is None + assert any("degenerate" in warning.lower() for warning in res["warnings"]) + + +def test_missing_null_yields_none_and_warning(): + res = compute_roc_metrics([1.0, 2.0, 3.0], [], [0.1], n_bootstrap=50, seed=1) + assert res["auroc"] is None + assert res["tpr_at_fpr"]["0.1"] is None + assert any("no null scores" in warning for warning in res["warnings"]) + + +def test_small_null_warns(): + res = compute_roc_metrics([1.0, 2.0, 3.0], [0.5, 1.5], [0.1], n_bootstrap=50, seed=1) + assert any("n_null=2 < 10" in warning for warning in res["warnings"]) + + +def test_analyze_scores_groups_attacks(): + rows = [ + { + "condition": "en-core", + "attack": "none", + "kind": "watermarked", + "score": 3.0, + "is_watermarked": True, + "ok": True, + }, + { + "condition": "en-core", + "attack": "none", + "kind": "watermarked", + "score": 4.0, + "is_watermarked": True, + "ok": True, + }, + { + "condition": "en-core", + "attack": "none", + "kind": "control", + "score": 0.0, + "is_watermarked": False, + "ok": True, + }, + { + "condition": "en-core", + "attack": "none", + "kind": "control", + "score": 1.0, + "is_watermarked": False, + "ok": True, + }, + # kind "attacked" under attack "none" must NOT count as signal there. + { + "condition": "en-core", + "attack": "none", + "kind": "attacked", + "score": 9.0, + "is_watermarked": True, + "ok": True, + }, + { + "condition": "en-core", + "attack": "paraphrase", + "kind": "attacked", + "score": 2.0, + "is_watermarked": True, + "ok": True, + }, + { + "condition": "en-core", + "attack": "paraphrase", + "kind": "attacked", + "score": 2.5, + "is_watermarked": True, + "ok": True, + }, + { + "condition": "en-core", + "attack": "paraphrase", + "kind": "control", + "score": 0.5, + "is_watermarked": False, + "ok": True, + }, + { + "condition": "en-core", + "attack": "paraphrase", + "kind": "control", + "score": 1.5, + "is_watermarked": False, + "ok": True, + }, + # kind "watermarked" under a non-none attack must NOT count as signal. + { + "condition": "en-core", + "attack": "paraphrase", + "kind": "watermarked", + "score": 5.0, + "is_watermarked": True, + "ok": True, + }, + # Invalid rows must be skipped: ok=false, missing score, bad score. + { + "condition": "en-core", + "attack": "none", + "kind": "control", + "score": 0.0, + "is_watermarked": False, + "ok": False, + }, + { + "condition": "en-core", + "attack": "none", + "kind": "control", + "is_watermarked": False, + "ok": True, + }, + { + "condition": "en-core", + "attack": "none", + "kind": "control", + "score": "oops", + "is_watermarked": False, + "ok": True, + }, + ] + res = analyze_scores(rows, [0.1], n_bootstrap=50, seed=1) + assert res["condition"] == "en-core" + assert set(res["per_attack"]) == {"none", "paraphrase"} + none_metrics = res["per_attack"]["none"] + assert none_metrics["n_signal"] == 2 + assert none_metrics["n_null"] == 2 + assert none_metrics["auroc"] == 1.0 + par_metrics = res["per_attack"]["paraphrase"] + assert par_metrics["n_signal"] == 2 + assert par_metrics["n_null"] == 2 + # The --attack filter restricts per_attack to one key. + filtered = analyze_scores(rows, [0.1], n_bootstrap=50, seed=1, attack="paraphrase") + assert set(filtered["per_attack"]) == {"paraphrase"} + + +def test_analyze_scores_empty_and_condition_missing(): + res = analyze_scores([], [0.1], n_bootstrap=10, seed=1) + assert res["condition"] is None + assert res["per_attack"] == {} diff --git a/research/tests/test_cheap.py b/research/tests/test_cheap.py new file mode 100644 index 0000000..9576459 --- /dev/null +++ b/research/tests/test_cheap.py @@ -0,0 +1,184 @@ +"""Tests for research/scripts/attacks/cheap.py (protocol §4.1, attack A7). + +All tests are fast and offline: nltk is never required (the module lazily +falls back to the built-in synonym map), and no network access happens. +""" + +from __future__ import annotations + +import re +import sys +from collections import Counter +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) + +import pytest +from attacks import cheap + +WORD_RE = re.compile(r"[A-Za-z]+") +PROTECTED_RE = re.compile(r"\d+(?:[.,]\d+)*|https?://\S+") + + +def words(text: str) -> list[str]: + """Letter-run tokens of *text* (the module counts the same way).""" + return WORD_RE.findall(text) + + +def protected(text: str) -> list[str]: + """All numbers and URLs embedded in *text*.""" + return PROTECTED_RE.findall(text) + + +# A few sentences with words that are in the built-in map, a number, and a +# URL, so every attack exercises its protected-token path too. +BASE_TEXT = ( + "The happy scientist saw the fast result and told the angry manager. " + "She could not believe the big number 42 in the https://example.com/x report. " + "Everything seemed easy and important that day." +) + +DELETION_TEXT = " ".join( + ["alpha", "bravo", "charlie", "delta", "echo", "foxtrot", "golf", "hotel", "india", "juliet"] + * 25 +) + +REORDER_TEXT = ( + "The first sentence has five words here. " + "The second sentence talks about the moon. " + "The third sentence mentions a large number 12345. " + "The fourth sentence ends with a question? " + "The fifth sentence closes the story." +) + +# 20 words that are all keys of the built-in map; with SYNONYM_RATE=0.5 a +# fixed seed virtually always rewrites at least one of them. +SYNONYM_TEXT = ( + "The happy happy happy happy happy dog ran fast fast fast fast fast. " + "The big big big big big house was old old old old old." +) + + +@pytest.mark.parametrize("attack", ["synonym", "delete", "sentence-reorder"]) +@pytest.mark.parametrize("seed", [0, 1, 42]) +def test_determinism(attack: str, seed: int) -> None: + """Same seed -> byte-identical output for every attack.""" + first = cheap.apply_attack(BASE_TEXT, attack, seed) + second = cheap.apply_attack(BASE_TEXT, attack, seed) + assert first == second + + +@pytest.mark.parametrize("ratio", [0.02, 0.07, 0.20]) +def test_delete_ratio_in_bounds(ratio: float) -> None: + """Surviving word fraction stays within ratio +- 0.02.""" + out = cheap.random_word_delete(DELETION_TEXT, seed=0, ratio=ratio) + survived = len(words(out)) / len(words(DELETION_TEXT)) + assert 1 - ratio - 0.02 <= survived <= 1 - ratio + 0.02 + + +def test_delete_ratio_clamped_to_max() -> None: + """Ratios above 0.20 are clamped, not honored literally.""" + out = cheap.random_word_delete(DELETION_TEXT, seed=0, ratio=0.9) + survived = len(words(out)) / len(words(DELETION_TEXT)) + assert 1 - 0.20 - 0.02 <= survived <= 1 - 0.20 + 0.02 + + +def test_sentence_reorder_preserves_word_multiset() -> None: + """Reorder shuffles sentences but keeps every character (and word).""" + out = cheap.sentence_reorder(REORDER_TEXT, seed=3) + assert Counter(out) == Counter(REORDER_TEXT) + assert out != REORDER_TEXT + + +def test_sentence_reorder_single_sentence_is_identity() -> None: + """A one-sentence text has nothing to reorder.""" + single = "Only one sentence lives here, and it stays put." + assert cheap.sentence_reorder(single, seed=0) == single + + +def test_synonym_changes_at_least_one_word(monkeypatch: pytest.MonkeyPatch) -> None: + """Built-in map path changes >= 1 word (WordNet probed off).""" + monkeypatch.setattr(cheap, "_wordnet_available", lambda: False) + out = cheap.synonym_substitute(SYNONYM_TEXT, seed=0) + assert out != SYNONYM_TEXT + assert cheap._wordnet_available() is False # patch in effect + + +@pytest.mark.parametrize("attack", ["synonym", "delete", "sentence-reorder"]) +def test_numbers_and_urls_preserved(attack: str) -> None: + """No attack alters or drops numbers or URLs.""" + out = cheap.apply_attack(BASE_TEXT, attack, seed=7) + for item in protected(BASE_TEXT): + assert item in out + + +def test_apply_attack_unknown_name_raises() -> None: + with pytest.raises(ValueError): + cheap.apply_attack("text", "bogus", seed=0) + + +def test_cli_round_trip(tmp_path: Path) -> None: + """CLI reads a file, attacks it, writes output; rerun is identical.""" + src = tmp_path / "in.txt" + dst = tmp_path / "out.txt" + src.write_text(BASE_TEXT, encoding="utf-8") + rc = cheap.main( + [ + "--input", + str(src), + "--output", + str(dst), + "--attack", + "delete", + "--seed", + "0", + "--delete-ratio", + "0.07", + ] + ) + assert rc == 0 + out_text = dst.read_text(encoding="utf-8") + assert out_text != BASE_TEXT + dst2 = tmp_path / "out2.txt" + rc2 = cheap.main( + [ + "--input", + str(src), + "--output", + str(dst2), + "--attack", + "delete", + "--seed", + "0", + ] + ) + assert rc2 == 0 + assert dst2.read_text(encoding="utf-8") == out_text + + +def test_cli_missing_input_returns_nonzero(tmp_path: Path) -> None: + """A missing input file is an error, not a crash.""" + dst = tmp_path / "out.txt" + rc = cheap.main( + [ + "--input", + str(tmp_path / "missing.txt"), + "--output", + str(dst), + "--attack", + "synonym", + "--seed", + "0", + ] + ) + assert rc == 1 + + +def test_parse_args_defaults() -> None: + """Defaults: seed 0, delete ratio 0.07.""" + args = cheap.parse_args(["--input", "a", "--output", "b", "--attack", "delete"]) + assert args.seed == 0 + assert args.delete_ratio == 0.07 + + with pytest.raises(SystemExit): + cheap.parse_args(["--input", "a", "--output", "b", "--attack", "bogus"]) diff --git a/research/tests/test_corpus.py b/research/tests/test_corpus.py new file mode 100644 index 0000000..e19496c --- /dev/null +++ b/research/tests/test_corpus.py @@ -0,0 +1,70 @@ +"""Corpus validation (gap 05-A6). + +research/corpus/ must hold 25 factual, neutral prompts per language +(en/de/fr/es), 50-90 words each, index-aligned across languages (same +index = same topic, translated). The English set is self-written and +unique; the 8 legacy prompts from benchmarks/corpus/ are copied verbatim +as en/01..08. +""" + +from __future__ import annotations + +from pathlib import Path + +CORPUS = Path(__file__).resolve().parents[1] / "corpus" +LANGUAGES = ("en", "de", "fr", "es") +N_PROMPTS = 25 +MIN_WORDS = 50 +MAX_WORDS = 90 + + +def _word_count(text: str) -> int: + return len([t for t in text.split() if t.strip()]) + + +def _load(lang: str) -> dict[int, str]: + d = CORPUS / lang + assert d.is_dir(), f"missing corpus dir {d}" + files = sorted(p for p in d.glob("*.txt") if p.is_file()) + assert len(files) == N_PROMPTS, f"{lang}: expected {N_PROMPTS} files, got {len(files)}" + out: dict[int, str] = {} + for f in files: + idx = int(f.stem) + out[idx] = f.read_text(encoding="utf-8").strip() + return out + + +def test_every_language_has_25_prompts() -> None: + for lang in LANGUAGES: + prompts = _load(lang) + assert set(prompts) == set(range(1, N_PROMPTS + 1)) + + +def test_word_counts_within_range() -> None: + for lang in LANGUAGES: + for idx, text in _load(lang).items(): + n = _word_count(text) + assert MIN_WORDS <= n <= MAX_WORDS, f"{lang}/{idx:02d}.txt: {n} words" + + +def test_english_prompts_are_unique() -> None: + en = _load("en") + texts = list(en.values()) + assert len(set(texts)) == len(texts), "duplicate EN prompts" + + +def test_translations_actually_differ_from_english() -> None: + en = _load("en") + for lang in ("de", "fr", "es"): + for idx, text in _load(lang).items(): + assert text != en[idx], f"{lang}/{idx:02d}.txt identical to EN (not a translation)" + + +def test_legacy_prompts_copied_verbatim() -> None: + """en/01..08 match benchmarks/corpus/ byte-for-byte.""" + legacy = CORPUS.parents[1] / "benchmarks" / "corpus" + legacy_files = sorted(p for p in legacy.glob("*.txt") if p.is_file()) + assert len(legacy_files) == 8 + en = _load("en") + for i, f in enumerate(legacy_files, start=1): + assert en[i] == f.read_text(encoding="utf-8").strip(), f"en/{i:02d}.txt != {f.name}" diff --git a/research/tests/test_corpus_gap05a6.py b/research/tests/test_corpus_gap05a6.py new file mode 100644 index 0000000..d26e20f --- /dev/null +++ b/research/tests/test_corpus_gap05a6.py @@ -0,0 +1,123 @@ +"""Validation for the gap 05-A6 prompt corpus (research/corpus/). + +Checks the deliverable contract from research/01-experiment-protocol.md +section 3: exactly 25 prompts per language (en/de/fr/es), 50-90 words +each, index-aligned topics across languages, the 8 legacy +benchmarks/corpus prompts copied verbatim as en/01..08, and the 17 new +English seeds covering distinct factual domains. +""" + +from __future__ import annotations + +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +CORPUS = REPO_ROOT / "research" / "corpus" +BENCH_CORPUS = REPO_ROOT / "benchmarks" / "corpus" +LANGS = ("en", "de", "fr", "es") +MIN_WORDS = 50 +MAX_WORDS = 90 +N_PROMPTS = 25 + +# en/01..08 must match benchmarks/corpus/ in this exact order (verbatim). +LEGACY_FILES: tuple[str, ...] = ( + "cloud-computing.txt", + "coffee-brewing.txt", + "hiking-checklist.txt", + "nutrition-myths.txt", + "open-source-licenses.txt", + "renewable-energy.txt", + "small-biz-finance.txt", + "venice-history.txt", +) + +# The 17 new English seeds (en/09..25) cover these domains, one each. +NEW_EN_DOMAINS: tuple[str, ...] = ( + "astronomy", + "geology", + "zoology", + "music history", + "food safety", + "transportation", + "agriculture", + "economic history", + "literature", + "public health", + "energy storage", + "urban planning", + "marine biology", + "meteorology", + "language origins", + "materials science", + "sports science", +) + + +def word_count(text: str) -> int: + """Return whitespace-separated token count of *text*.""" + return len(text.split()) + + +def read_prompt(lang: str, index: int) -> str: + """Read and strip corpus prompt *index* (1-based) for *lang*.""" + path = CORPUS / lang / f"{index:02d}.txt" + assert path.is_file(), f"missing corpus file: {path}" + return path.read_text(encoding="utf-8").strip() + + +def test_four_language_directories_exist() -> None: + """The corpus root contains exactly the en/de/fr/es directories.""" + dirs = sorted(p.name for p in CORPUS.iterdir() if p.is_dir()) + assert dirs == sorted(LANGS) + + +def test_each_language_has_exactly_25_prompt_files() -> None: + """Each language directory holds exactly indices 01..25.""" + for lang in LANGS: + files = sorted(p.name for p in (CORPUS / lang).glob("*.txt")) + expected = [f"{i:02d}.txt" for i in range(1, N_PROMPTS + 1)] + assert files == expected, f"{lang}: unexpected file set {files}" + + +def test_word_counts_within_50_to_90() -> None: + """Every one of the 100 prompts is 50-90 words (whitespace tokens).""" + for lang in LANGS: + for index in range(1, N_PROMPTS + 1): + n = word_count(read_prompt(lang, index)) + assert MIN_WORDS <= n <= MAX_WORDS, f"{lang}/{index:02d}.txt: {n} words" + + +def test_english_01_to_08_match_benchmarks_verbatim() -> None: + """en/01..08 are byte-identical to benchmarks/corpus/ sources.""" + for i, name in enumerate(LEGACY_FILES, start=1): + bench = (BENCH_CORPUS / name).read_text(encoding="utf-8").strip() + assert read_prompt("en", i) == bench, f"en/{i:02d}.txt != {name}" + + +def test_english_prompts_are_all_unique() -> None: + """No two English prompts share identical text.""" + texts = [read_prompt("en", i) for i in range(1, N_PROMPTS + 1)] + assert len(set(texts)) == len(texts), "duplicate English prompts" + + +def test_new_english_domains_are_distinct() -> None: + """The 17 new English seeds cover 17 distinct required domains.""" + assert len(set(NEW_EN_DOMAINS)) == len(NEW_EN_DOMAINS) + + +def test_translations_differ_from_english() -> None: + """Every translated file differs from its English counterpart.""" + for lang in ("de", "fr", "es"): + for index in range(1, N_PROMPTS + 1): + assert read_prompt(lang, index) != read_prompt("en", index), ( + f"{lang}/{index:02d}.txt identical to English seed" + ) + + +def test_same_index_is_same_topic_across_languages() -> None: + """Cross-language files sharing an index are mutually different.""" + for index in range(1, N_PROMPTS + 1): + texts = {lang: read_prompt(lang, index) for lang in LANGS} + assert len(set(texts.values())) == len(LANGS), ( + f"duplicate text across languages for index {index}" + ) diff --git a/research/tests/test_evaluate_quality.py b/research/tests/test_evaluate_quality.py new file mode 100644 index 0000000..2d50f4a --- /dev/null +++ b/research/tests/test_evaluate_quality.py @@ -0,0 +1,366 @@ +"""Unit tests for research/scripts/evaluate_quality.py (gap 05-B2). + +Pure helpers (Levenshtein, length drift, number/URL survival, row +assembly) are tested directly. Model-dependent code paths are gated on +torch being importable and never touch the network: they use tiny local +models, stubs, and monkeypatched fake modules only. +""" + +from __future__ import annotations + +import json +import math +import sys +import types +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) + +import evaluate_quality as eq +import pytest + +try: + import torch +except ImportError: + torch = None + +needs_torch = pytest.mark.skipif(torch is None, reason="torch not installed") + + +def _long_text(n: int, chunk: str = "abcdefghij ") -> str: + return (chunk * (n // len(chunk) + 1))[:n] + + +# --------------------------------------------------------------------------- +# Pure helpers +# --------------------------------------------------------------------------- + + +def test_levenshtein_distance_known_pairs() -> None: + assert eq.levenshtein_distance("", "") == 0 + assert eq.levenshtein_distance("abc", "abc") == 0 + assert eq.levenshtein_distance("kitten", "sitting") == 3 + assert eq.levenshtein_distance("a", "") == 1 + assert eq.levenshtein_distance("", "abc") == 3 + assert eq.levenshtein_distance("abc", "ab") == 1 + + +def test_levenshtein_pct_known_pairs() -> None: + assert eq.levenshtein_pct("", "") == 0.0 + assert eq.levenshtein_pct("abc", "abc") == 0.0 + assert eq.levenshtein_pct("kitten", "sitting") == pytest.approx(50.0) + assert eq.levenshtein_pct("a", "") == 100.0 + assert eq.levenshtein_pct("abc", "ab") == pytest.approx(100.0 / 3) + + +def test_truncate_pair_long_original() -> None: + original = _long_text(5000) + candidate = "short" + o, c, truncated = eq.truncate_pair(original, candidate, max_chars=4000) + assert truncated is True + assert len(o) == 4000 + assert o == original[:4000] + assert c == "short" # shorter text is cut to max_chars but never padded + + +def test_truncate_pair_short_noop() -> None: + o, c, truncated = eq.truncate_pair("hello", "world", max_chars=4000) + assert truncated is False + assert (o, c) == ("hello", "world") + + +def test_length_drift() -> None: + assert eq.length_drift("ab", "abcd") == 1.0 + assert eq.length_drift("ab", "a") == -0.5 + assert eq.length_drift("", "abc") == 3.0 + assert eq.length_drift("hello", "hello") == 0.0 + + +def test_numbers_preserved() -> None: + assert eq.numbers_preserved("no numbers here", "anything") == 1.0 + assert eq.numbers_preserved("a 42 b 7", "a 42 c") == 0.5 + assert eq.numbers_preserved("a 42 b 7", "x 99") == 0.0 + assert eq.numbers_preserved("12 34", "34 12") == 1.0 + + +def test_urls_preserved() -> None: + assert eq.urls_preserved("plain text", "anything") == 1.0 + original = "see https://example.com/a and http://x.y/z" + assert eq.urls_preserved(original, "see https://example.com/a") == 0.5 + assert eq.urls_preserved(original, original) == 1.0 + assert eq.urls_preserved(original, "nothing") == 0.0 + + +def test_ppl_from_loss() -> None: + assert eq.ppl_from_loss(0.0) == pytest.approx(1.0) + assert eq.ppl_from_loss(math.log(10.0)) == pytest.approx(10.0) + + +def test_compute_rouge_l_fmeasure() -> None: + class _FakeScore: + fmeasure = 0.75 + + assert eq.compute_rouge_l_fmeasure(_FakeScore()) == pytest.approx(0.75) + + +# --------------------------------------------------------------------------- +# Row assembly (no models involved: all model metrics skipped) +# --------------------------------------------------------------------------- + + +def test_compute_row_metrics_all_pure_metrics_present() -> None: + row = { + "condition": "c", + "scheme": "kgw", + "seed": 1, + "prompt_idx": 0, + "attack": "paraphrase", + "original": "the fox jumps over 42 lazy dogs", # len 31 + "candidate": "the fox leaps over 42 dogs", # len 26, lev dist 8 + } + metrics, notes = eq.compute_row_metrics( + row, skip={"ppl", "bertscore", "rouge", "sbert"}, device="cpu" + ) + assert set(metrics) == set(eq.METRIC_KEYS) + assert metrics["ppl"] is None + assert metrics["bertscore"] is None + assert metrics["rouge_l"] is None + assert metrics["sbert_cosine"] is None + # metrics are rounded to 4 decimals by the script + assert metrics["length_drift"] == pytest.approx(round(-5 / 31, 4)) + assert metrics["levenshtein_pct"] == pytest.approx(round(8 / 31 * 100, 4)) + assert metrics["numbers_preserved"] == 1.0 + assert metrics["urls_preserved"] == 1.0 + assert notes == [] + + +def test_compute_row_metrics_missing_texts_all_none() -> None: + metrics, notes = eq.compute_row_metrics( + {"original": None, "candidate": "x"}, skip=set(), device="cpu" + ) + assert set(metrics) == set(eq.METRIC_KEYS) + assert all(value is None for value in metrics.values()) + assert any("missing" in note for note in notes) + + +def test_compute_row_metrics_levenshtein_truncation_note() -> None: + row = {"original": _long_text(5000), "candidate": "short"} + metrics, notes = eq.compute_row_metrics( + row, skip={"ppl", "bertscore", "rouge", "sbert"}, device="cpu" + ) + assert any("truncated" in note for note in notes) + assert 0.0 <= metrics["levenshtein_pct"] <= 100.0 + + +# --------------------------------------------------------------------------- +# Lazy loaders: failure contract + caching, via monkeypatched imports +# (deterministic, no network, independent of installed packages) +# --------------------------------------------------------------------------- + + +def test_rouge_loader_records_warning_on_failure(monkeypatch: pytest.MonkeyPatch) -> None: + eq._MODEL_CACHE.pop("rouge", None) + eq._FAILED.discard("rouge") + monkeypatch.setitem(sys.modules, "rouge_score", None) # force ImportError + assert eq._get_rouge_scorer() is None + assert any(w["metric"] == "rouge" for w in eq._warnings()) + + +def test_ppl_loader_records_warning_on_failure(monkeypatch: pytest.MonkeyPatch) -> None: + eq._MODEL_CACHE.pop("ppl", None) + eq._FAILED.discard("ppl") + monkeypatch.setitem(sys.modules, "transformers", None) # force ImportError + assert eq._get_ppl_models() is None + assert any(w["metric"] == "ppl" for w in eq._warnings()) + + +def test_failed_loader_is_cached(monkeypatch: pytest.MonkeyPatch) -> None: + eq._MODEL_CACHE.pop("sbert", None) + eq._FAILED.discard("sbert") + monkeypatch.setitem(sys.modules, "sentence_transformers", None) + assert eq._get_sbert_model("cpu") is None + warning_count = len(eq._warnings()) + assert eq._get_sbert_model("cpu") is None # cached failure, no new warning + assert len(eq._warnings()) == warning_count + + +def test_sbert_loader_caches_on_success(monkeypatch: pytest.MonkeyPatch) -> None: + eq._MODEL_CACHE.pop("sbert", None) + eq._FAILED.discard("sbert") + calls: list[tuple[str, str]] = [] + + class _FakeST: + def __init__(self, model_id: str, device: str) -> None: + calls.append((model_id, device)) + + fake = types.SimpleNamespace(SentenceTransformer=_FakeST) + monkeypatch.setitem(sys.modules, "sentence_transformers", fake) + first = eq._get_sbert_model("cpu") + second = eq._get_sbert_model("cpu") + assert first is not None and first is second + assert calls == [("all-MiniLM-L6-v2", "cpu")] + + +def test_ppl_loader_caches_on_success(monkeypatch: pytest.MonkeyPatch) -> None: + eq._MODEL_CACHE.pop("ppl", None) + eq._FAILED.discard("ppl") + calls: list[str] = [] + + class _FakeModel: + def eval(self) -> None: + calls.append("eval") + + class _FakeTokenizer: + @classmethod + def from_pretrained(cls, model_id: str) -> _FakeTokenizer: + calls.append(model_id) + return cls() + + class _FakeAuto: + @classmethod + def from_pretrained(cls, model_id: str) -> _FakeModel: + calls.append(model_id) + return _FakeModel() + + fake = types.SimpleNamespace(AutoModelForCausalLM=_FakeAuto, AutoTokenizer=_FakeTokenizer) + monkeypatch.setitem(sys.modules, "transformers", fake) + first = eq._get_ppl_models() + second = eq._get_ppl_models() + assert first is not None and first is second + assert len(first) == 2 # (model, tokenizer) + assert calls.count("gpt2-large") == 2 # tokenizer + model + assert "eval" in calls + + +# --------------------------------------------------------------------------- +# CLI end-to-end (model metrics skipped: runs without torch) +# --------------------------------------------------------------------------- + + +def test_main_end_to_end(tmp_path: Path) -> None: + inp = tmp_path / "in.jsonl" + out = tmp_path / "out.jsonl" + warn = tmp_path / "warnings.jsonl" + rows = [ + { + "condition": "c", + "scheme": "kgw", + "seed": 1, + "prompt_idx": 0, + "attack": "none", + "original": "a 42 b", + "candidate": "a 42 c", + }, + { + "condition": "c", + "scheme": "kgw", + "seed": 2, + "prompt_idx": 1, + "attack": "none", + "original": "hello https://example.com/x", + "candidate": "hello", + }, + ] + inp.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8") + + rc = eq.main( + [ + "--input", + str(inp), + "--out", + str(out), + "--skip", + "ppl,bertscore,rouge,sbert", + "--warnings-out", + str(warn), + ] + ) + assert rc == 0 + + written = [json.loads(line) for line in out.read_text(encoding="utf-8").splitlines()] + assert len(written) == 2 + for row in written: + assert set(eq.METRIC_KEYS) <= set(row) + assert row["ppl"] is None + assert row["bertscore"] is None + assert row["rouge_l"] is None + assert row["sbert_cosine"] is None + assert written[0]["numbers_preserved"] == 1.0 + assert written[1]["numbers_preserved"] == 1.0 # no numbers -> preserved + assert written[1]["urls_preserved"] == 0.0 # URL dropped + # warnings file is valid JSONL (possibly empty) + assert all(line.strip() for line in warn.read_text(encoding="utf-8").splitlines()) + + +def test_main_respects_limit(tmp_path: Path) -> None: + inp = tmp_path / "in.jsonl" + out = tmp_path / "out.jsonl" + rows = [{"original": f"text {i}", "candidate": f"text {i}"} for i in range(3)] + inp.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8") + + rc = eq.main( + [ + "--input", + str(inp), + "--out", + str(out), + "--skip", + "ppl,bertscore,rouge,sbert", + "--limit", + "2", + ] + ) + assert rc == 0 + written = [json.loads(line) for line in out.read_text(encoding="utf-8").splitlines()] + assert len(written) == 2 + + +# --------------------------------------------------------------------------- +# Model-dependent math (torch-gated; offline: tiny local models / stubs) +# --------------------------------------------------------------------------- + + +@needs_torch +def test_compute_ppl_offline_small_model() -> None: + try: + from transformers import GPT2Config, GPT2LMHeadModel + except ImportError: + pytest.skip("transformers not installed") + + class _StubTokenizer: + """Maps chars to ids < vocab_size; mimics the HF tokenizer contract.""" + + def __init__(self, vocab_size: int) -> None: + self.vocab_size = vocab_size + + def __call__(self, text, return_tensors=None, truncation=None, max_length=None): + ids = [ord(ch) % self.vocab_size for ch in text][:max_length] + return { + "input_ids": torch.tensor([ids], dtype=torch.long), + "attention_mask": torch.ones(1, len(ids), dtype=torch.long), + } + + config = GPT2Config(vocab_size=64, n_positions=128, n_embd=16, n_layer=1, n_head=1) + model = GPT2LMHeadModel(config).eval() + ppl = eq.compute_ppl(model, _StubTokenizer(64), "hello world", device="cpu", max_length=16) + assert math.isfinite(ppl) + assert ppl > 0.0 + + +@needs_torch +def test_compute_bertscore_f1() -> None: + f1 = torch.tensor([0.847]) + assert eq.compute_bertscore_f1(f1) == pytest.approx(0.847) + + +@needs_torch +def test_compute_sbert_cosine_identical() -> None: + emb = torch.tensor([1.0, 2.0, 3.0]) + assert eq.compute_sbert_cosine(emb, emb) == pytest.approx(1.0) + + +@needs_torch +def test_compute_sbert_cosine_orthogonal() -> None: + a = torch.tensor([1.0, 0.0]) + b = torch.tensor([0.0, 1.0]) + assert eq.compute_sbert_cosine(a, b) == pytest.approx(0.0, abs=1e-6) diff --git a/research/tests/test_make_tables.py b/research/tests/test_make_tables.py new file mode 100644 index 0000000..11a25db --- /dev/null +++ b/research/tests/test_make_tables.py @@ -0,0 +1,178 @@ +"""Tests for research/scripts/make_tables.py and make_figures.py (gap 05-B3). + +Runs the generators against synthetic and empty results dirs using only +pytest + stdlib (no matplotlib required): the figures script is exercised +with the matplotlib import mocked to fail. + +Run with: python3 -m pytest research/tests/test_make_tables.py -q +""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +import pytest + +SCRIPTS = Path(__file__).resolve().parents[1] / "scripts" +sys.path.insert(0, str(SCRIPTS)) + +import make_figures +import make_tables + + +def _run_tables(results: Path, out: Path) -> int: + return make_tables.main(["--results-dir", str(results), "--out-dir", str(out)]) + + +def _write_metrics(cond: Path, attacks: dict) -> None: + cond.mkdir(parents=True, exist_ok=True) + (cond / "metrics.json").write_text(json.dumps(attacks), encoding="utf-8") + + +def _synthetic_metrics() -> dict: + """A minimal per-attack metrics dict covering all four T2 columns.""" + return { + "none": {"auroc": 0.990, "tpr_at_fpr": {"0.01": 0.900}}, + "layerA": {"auroc": 0.970, "tpr_at_fpr": {"0.01": 0.870}}, + "paraphrase:3": {"auroc": 0.610, "tpr_at_fpr": {"0.01": 0.400}}, + "layerA+paraphrase:3": {"auroc": 0.520, "tpr_at_fpr": {"0.01": 0.310}}, + } + + +def _make_synthetic_results(root: Path) -> None: + _write_metrics(root / "kgw-d2-L300-T0.7-en-s1-p0", _synthetic_metrics()) + _write_metrics(root / "synthid-L300-T0.7-en-s1-p0", _synthetic_metrics()) + + +def test_static_tables_with_empty_results_dir(tmp_path) -> None: + """T1 (taxonomy) and T7 (baseline template) need no results data.""" + results = tmp_path / "results" + results.mkdir() + out = tmp_path / "out" + assert _run_tables(results, out) == 0 + for name in ("t1", "t2", "t3", "t4", "t5", "t6", "t7"): + assert (out / "tables" / f"{name}.tex").is_file() + md = (out / "tables" / "tables.md").read_text(encoding="utf-8") + assert "### T1" in md and "### T7" in md + t1 = (out / "tables" / "t1.tex").read_text(encoding="utf-8") + assert "clean\\_text.py" in t1 # LaTeX-escaped underscore + assert "Layer A" in t1 + assert "layered contribution" in t1 # capitalized in the taxonomy row + assert all(f"A{i}" in t1 for i in range(9)) # A0..A8 present + assert "clean_text.py" in md # unescaped in markdown + t7 = (out / "tables" / "t7.tex").read_text(encoding="utf-8") + assert "TBD (re-verify at submission)" in t7 + assert "\\cite{" in t7 + + +def test_data_dependent_tables_degrade_to_no_data(tmp_path) -> None: + """T2-T6 emit an explicit 'no data' row when inputs are missing.""" + results = tmp_path / "results" + results.mkdir() + out = tmp_path / "out" + assert _run_tables(results, out) == 0 + for name in ("t2", "t3", "t4", "t5", "t6"): + tex = (out / "tables" / f"{name}.tex").read_text(encoding="utf-8") + assert "no data" in tex + md = (out / "tables" / "tables.md").read_text(encoding="utf-8") + assert "no data" in md + + +def test_t2_matrix_reads_metrics(tmp_path) -> None: + """T2 maps attacks to the pre/A-only/B-only/A+B columns from metrics.json.""" + results = tmp_path / "results" + _make_synthetic_results(results) + out = tmp_path / "out" + assert _run_tables(results, out) == 0 + t2 = (out / "tables" / "t2.tex").read_text(encoding="utf-8") + assert "0.990" in t2 and "0.900" in t2 + assert "KGW" in t2 and "SynthID-Text" in t2 + assert "pre-attack" in t2 and "A+B" in t2 + # B-family row: its own value lands in the B-only column. + assert "A3 paraphrase:3 & 0.990 & 0.970 & 0.610 & ---" in t2 + # Layered row: its own value lands in the A+B column. + assert "A8 layerA+paraphrase:3 & 0.990 & 0.970 & --- & 0.520" in t2 + md = (out / "tables" / "tables.md").read_text(encoding="utf-8") + assert "0.610" in md + + +def test_t6_attack_cost_from_attacked_jsonl(tmp_path) -> None: + """T6 normalizes tokens/seconds/USD per 1k words from attacked.jsonl.""" + results = tmp_path / "results" + cond = results / "kgw-d2-L300-T0.7-en-s1-p0" + cond.mkdir(parents=True, exist_ok=True) + rows = [ + { + "attack": "paraphrase:3", + "original": "word " * 100, + "candidate": "word " * 95, + "stats": {"tokens_in": 300, "tokens_out": 250}, + "seconds": 12.5, + "usd": 0.0015, + }, + ] + (cond / "attacked.jsonl").write_text( + "".join(json.dumps(r) + "\n" for r in rows), encoding="utf-8" + ) + out = tmp_path / "out" + assert _run_tables(results, out) == 0 + t6 = (out / "tables" / "t6.tex").read_text(encoding="utf-8") + assert "paraphrase:3" in t6 + assert "3,000" in t6 # 300 tokens / 100 words * 1000 + assert "0.0150" in t6 # 0.0015 USD / 100 words * 1000 + + +def test_figures_exit_zero_when_matplotlib_missing(tmp_path, monkeypatch, capsys) -> None: + """With the matplotlib import mocked, the figures script warns and exits 0.""" + monkeypatch.setitem(sys.modules, "matplotlib", None) + results = tmp_path / "results" + results.mkdir() + out = tmp_path / "out" + code = make_figures.main(["--results-dir", str(results), "--out-dir", str(out)]) + assert code == 0 + captured = capsys.readouterr() + assert "matplotlib" in (captured.out + captured.err).lower() + figs = list((out / "figures").glob("f*")) if (out / "figures").is_dir() else [] + assert figs == [] + + +def test_figures_render_with_synthetic_data(tmp_path) -> None: + """All six figures are emitted when matplotlib and data are available.""" + pytest.importorskip("matplotlib") + results = tmp_path / "results" + cond = results / "kgw-d2-L300-T0.7-en-s1-p0" + metrics = _synthetic_metrics() + metrics["none"]["roc_points"] = {"fpr": [0.0, 0.01, 1.0], "tpr": [0.0, 0.9, 1.0]} + metrics["layerA+paraphrase:3"]["roc_points"] = { + "fpr": [0.0, 0.01, 1.0], + "tpr": [0.0, 0.31, 1.0], + } + _write_metrics(cond, metrics) + (cond / "quality.jsonl").write_text( + json.dumps({"attack": "paraphrase:3", "ppl": 24.5, "ppl_delta": 2.1}) + + "\n" + + json.dumps({"attack": "none", "ppl": 22.4}) + + "\n", + encoding="utf-8", + ) + (cond / "attacked.jsonl").write_text( + json.dumps( + { + "attack": "layerA+paraphrase:3", + "original": "alpha beta " * 50, + "candidate": "gamma delta " * 40, + "score_before": 2.3, + "score_after": 0.1, + } + ) + + "\n", + encoding="utf-8", + ) + _write_metrics(results / "synthid-L300-T0.7-en-s1-p0", _synthetic_metrics()) + out = tmp_path / "out" + code = make_figures.main(["--results-dir", str(results), "--out-dir", str(out)]) + assert code == 0 + for i in range(1, 7): + assert (out / "figures" / f"f{i}.png").is_file() diff --git a/research/tests/test_multilingual_gen.py b/research/tests/test_multilingual_gen.py new file mode 100644 index 0000000..d940e7f --- /dev/null +++ b/research/tests/test_multilingual_gen.py @@ -0,0 +1,567 @@ +"""Unit tests for research/scripts/multilingual_gen.py (gap 05-A5). + +Covers the chat-template formatting (fake tokenizer), arg parsing, the +watermark/serve JSON schemas (MarkLLM backend monkeypatched away), and the +seeding path. Model/torch-dependent tests are guarded: torch imports happen +in try/except and torch-requiring tests are marked skipif so the suite runs +in the repo venv (pytest, no torch) without network access. +""" + +from __future__ import annotations + +import io +import json +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + +SCRIPTS = Path(__file__).resolve().parents[1] / "scripts" +sys.path.insert(0, str(SCRIPTS)) + +import multilingual_gen as mg + +try: + import torch # noqa: F401 + + TORCH_AVAILABLE = True +except Exception: # pragma: no cover - environment-dependent + TORCH_AVAILABLE = False + +requires_torch = pytest.mark.skipif( + not TORCH_AVAILABLE, reason="torch is not installed in this environment" +) + + +class _FakeTokenizer: + """Minimal stand-in: chat_template attribute + apply_chat_template.""" + + def __init__( + self, chat_template: str | None = "", formatted: str = "" + ): + self.chat_template = chat_template + self.formatted = formatted + self.calls: list[tuple[list, dict]] = [] + + def apply_chat_template(self, messages, tokenize=False, add_generation_prompt=True): + self.calls.append( + (messages, {"tokenize": tokenize, "add_generation_prompt": add_generation_prompt}) + ) + return self.formatted + + +class _FakeWM: + """Stand-in watermark instance: config.gen_kwargs + generate/detect methods.""" + + def __init__(self, tokenizer: object | None = None): + self.config = SimpleNamespace( + gen_kwargs={}, + generation_tokenizer=tokenizer + if tokenizer is not None + else _FakeTokenizer(chat_template=None), + ) + self.detect_calls: list[tuple[str, bool]] = [] + + def generate_watermarked_text(self, prompt): + return "WM:" + prompt + + def generate_unwatermarked_text(self, prompt): + return "UW:" + prompt + + def detect_watermark(self, text, return_dict=True): + self.detect_calls.append((text, return_dict)) + return {"is_watermarked": True, "score": 3.5} + + +def _make_upstream(tmp_path: Path) -> Path: + """A fake MarkLLM checkout (only the watermark/ dir main() checks).""" + upstream = tmp_path / "MarkLLM" + (upstream / "watermark").mkdir(parents=True) + return upstream + + +def _watermark_argv(upstream: Path, config: Path, prompt: Path, **overrides) -> list[str]: + argv = [ + "watermark", + "--markllm-dir", + str(upstream), + "--scheme", + "kgw", + "--config", + str(config), + "--prompt", + str(prompt), + "--seed", + "3", + "--max-new-tokens", + "300", + "--temperature", + "0.7", + "--top-p", + "0.95", + "--lang", + "de", + "--doc-id", + "de-1", + ] + for key, value in overrides.items(): + flag = "--" + key.replace("_", "-") + argv[argv.index(flag) + 1] = str(value) + return argv + + +def _patch_backend(monkeypatch: pytest.MonkeyPatch, captured: dict) -> None: + """Replace _load_algorithm/_generate so no torch/MarkLLM is needed.""" + + def fake_load( + upstream, + alg, + config, + model, + device, + offline=False, + temperature=None, + top_p=None, + torch_dtype="auto", + ): + captured["load"] = { + "alg": alg, + "model": model, + "device": device, + "offline": offline, + "temperature": temperature, + "top_p": top_p, + "torch_dtype": torch_dtype, + } + return _FakeWM(tokenizer=_FakeTokenizer()) + + def fake_generate(wm, prompt, seed, max_new_tokens, min_length=0, need_unwatermarked=True): + captured["generate"] = { + "prompt": prompt, + "seed": seed, + "max_new_tokens": max_new_tokens, + "min_length": min_length, + } + return "WM|" + prompt, "UW|" + prompt + + monkeypatch.setattr(mg, "_load_algorithm", fake_load) + monkeypatch.setattr(mg, "_generate", fake_generate) + + +# -------------------------------------------------------------------------- +# Chat-template formatting (no torch, no network) +# -------------------------------------------------------------------------- + + +def test_chat_template_formats_single_user_turn(): + tok = _FakeTokenizer( + chat_template="{% for m in messages %}{{ m['role'] }}: {{ m['content'] }}{% endfor %}" + ) + out = mg.apply_chat_template(tok, "Erzähle mir über Berlin.") + assert out == "" + (messages, kwargs) = tok.calls[0] + assert messages == [{"role": "user", "content": "Erzähle mir über Berlin."}] + assert kwargs == {"tokenize": False, "add_generation_prompt": True} + + +@pytest.mark.parametrize("template", [None, ""]) +def test_chat_template_returns_raw_prompt_when_unset(template): + tok = _FakeTokenizer(chat_template=template) + assert mg.apply_chat_template(tok, "raw prompt") == "raw prompt" + assert tok.calls == [] + + +def test_chat_template_missing_attribute_returns_raw_prompt(): + class _NoTemplate: + pass + + assert mg.apply_chat_template(_NoTemplate(), "raw prompt") == "raw prompt" + + +# -------------------------------------------------------------------------- +# Arg parsing +# -------------------------------------------------------------------------- + + +def test_watermark_parser_full_args(): + args = mg.build_parser().parse_args( + _watermark_argv(Path("/fake/MarkLLM"), Path("/fake/cfg.json"), Path("/fake/p.txt")) + ) + assert args.cmd == "watermark" + assert args.scheme == "kgw" + assert args.seed == 3 + assert args.max_new_tokens == 300 + assert args.temperature == 0.7 + assert args.top_p == 0.95 + assert args.lang == "de" + assert args.doc_id == "de-1" + assert args.device == "auto" + assert args.model == mg.DEFAULT_MODEL # --model omitted -> Qwen2.5-1.5B + + +def test_watermark_parser_model_override(): + argv = _watermark_argv(Path("/fake/MarkLLM"), Path("/fake/cfg.json"), Path("/fake/p.txt")) + argv += ["--model", "Qwen/Qwen2.5-0.5B-Instruct"] + args = mg.build_parser().parse_args(argv) + assert args.model == "Qwen/Qwen2.5-0.5B-Instruct" + + +@pytest.mark.parametrize( + "missing", + ["--scheme", "--config", "--prompt", "--seed", "--max-new-tokens", "--lang", "--doc-id"], +) +def test_watermark_parser_requires_args(missing): + argv = _watermark_argv(Path("/fake/MarkLLM"), Path("/fake/cfg.json"), Path("/fake/p.txt")) + argv.remove(missing) + with pytest.raises(SystemExit): + mg.build_parser().parse_args(argv) + + +def test_watermark_parser_rejects_non_multilingual_lang(): + argv = _watermark_argv(Path("/fake/MarkLLM"), Path("/fake/cfg.json"), Path("/fake/p.txt")) + argv[argv.index("--lang") + 1] = "en" + with pytest.raises(SystemExit): + mg.build_parser().parse_args(argv) + + +def test_serve_parser_common_args(): + args = mg.build_parser().parse_args( + [ + "serve", + "--markllm-dir", + "/fake/MarkLLM", + "--scheme", + "synthid", + "--config", + "/fake/c.json", + ] + ) + assert args.cmd == "serve" + assert args.scheme == "synthid" + assert args.model == mg.DEFAULT_MODEL + + +# -------------------------------------------------------------------------- +# watermark subcommand: JSON schema on stdout +# -------------------------------------------------------------------------- + + +def test_cmd_watermark_json_schema(monkeypatch, tmp_path, capsys): + upstream = _make_upstream(tmp_path) + config = tmp_path / "cfg.json" + config.write_text('{"algorithm_name": "KGW"}', encoding="utf-8") + prompt = tmp_path / "prompt.txt" + prompt.write_text("Erzähle mir über Berlin.\n", encoding="utf-8") + captured: dict = {} + _patch_backend(monkeypatch, captured) + monkeypatch.setenv("HF_HUB_OFFLINE", "1") # pins.hf_revision fails fast, no network + + rc = mg.main(_watermark_argv(upstream, config, prompt)) + + assert rc == 0 + lines = capsys.readouterr().out.splitlines() + assert len(lines) == 1 # exactly one JSON object on stdout + out = json.loads(lines[0]) + assert out["ok"] is True + assert out["doc_id"] == "de-1" + assert out["lang"] == "de" + assert out["seed"] == 3 + assert out["model"] == mg.DEFAULT_MODEL + assert out["scheme"] == "kgw" + assert out["config"] == str(config.resolve()) + assert out["temperature"] == 0.7 + assert out["top_p"] == 0.95 + assert out["watermarked"] == "WM|" # chat template applied + assert out["unwatermarked"] == "UW|" + assert isinstance(out["pins"], dict) + assert "markllm_commit" in out["pins"] + assert "hf_revision" in out["pins"] + assert out["pins"]["hf_revision"] is None # offline -> no hub call + # The MarkLLM prompt was the chat-formatted string, not the raw prompt. + assert captured["generate"]["prompt"] == "" + assert captured["generate"]["seed"] == 3 + assert captured["generate"]["max_new_tokens"] == 300 + # temperature/top_p were folded into the load call (like the service). + assert captured["load"]["temperature"] == 0.7 + assert captured["load"]["top_p"] == 0.95 + + +def test_cmd_watermark_omitted_temp_top_p_default_none(monkeypatch, tmp_path, capsys): + upstream = _make_upstream(tmp_path) + config = tmp_path / "cfg.json" + config.write_text('{"algorithm_name": "KGW"}', encoding="utf-8") + prompt = tmp_path / "prompt.txt" + prompt.write_text("Hola, ¿qué tal?", encoding="utf-8") + captured: dict = {} + _patch_backend(monkeypatch, captured) + monkeypatch.setenv("HF_HUB_OFFLINE", "1") + + argv = _watermark_argv(upstream, config, prompt) + for flag in ("--temperature", "--top-p"): + idx = argv.index(flag) + del argv[idx : idx + 2] # remove flag and its value + argv[argv.index("--lang") + 1] = "es" + argv[argv.index("--doc-id") + 1] = "es-1" + + rc = mg.main(argv) + assert rc == 0 + out = json.loads(capsys.readouterr().out) + assert out["ok"] is True + assert out["temperature"] is None + assert out["top_p"] is None + assert captured["load"]["temperature"] is None + assert captured["load"]["top_p"] is None + + +def test_cmd_watermark_missing_prompt_file(monkeypatch, tmp_path, capsys): + upstream = _make_upstream(tmp_path) + config = tmp_path / "cfg.json" + config.write_text('{"algorithm_name": "KGW"}', encoding="utf-8") + argv = _watermark_argv(upstream, config, tmp_path / "nope.txt") + + rc = mg.main(argv) + assert rc == 2 + out = json.loads(capsys.readouterr().out) + assert out["ok"] is False + assert "not a file" in out["error"] + assert out["doc_id"] == "de-1" + + +def test_main_missing_markllm_dir(monkeypatch, capsys): + monkeypatch.delenv("MARKLLM_DIR", raising=False) + argv = _watermark_argv(Path("/nonexistent/MarkLLM"), Path("/fake/c.json"), Path("/fake/p.txt")) + rc = mg.main(argv) + assert rc == 3 + out = json.loads(capsys.readouterr().out) + assert out["ok"] is False + assert "MarkLLM not configured" in out["error"] + + +# -------------------------------------------------------------------------- +# serve subcommand: JSON-lines protocol +# -------------------------------------------------------------------------- + + +def test_handle_serve_request_watermark_applies_kwargs(): + wm = _FakeWM() + resp = mg._handle_serve_request( + wm, + { + "op": "watermark", + "id": 7, + "prompt": "Bonjour Paris", + "seed": None, + "max_new_tokens": 100, + "min_length": 5, + "temperature": 0.8, + "top_p": 0.9, + }, + ) + assert resp["ok"] is True + assert resp["id"] == 7 + assert resp["watermarked"] == "WM:Bonjour Paris" + assert resp["unwatermarked"] == "UW:Bonjour Paris" + assert wm.config.gen_kwargs["max_new_tokens"] == 100 + assert wm.config.gen_kwargs["min_length"] == 5 + assert wm.config.gen_kwargs["temperature"] == 0.8 + assert wm.config.gen_kwargs["top_p"] == 0.9 + + +def test_handle_serve_request_bad_prompt_keeps_worker_alive(): + wm = _FakeWM() + resp = mg._handle_serve_request(wm, {"op": "watermark", "id": 9, "prompt": ""}) + assert resp == {"ok": False, "id": 9, "error": "'prompt' must be a non-empty string"} + + +def test_handle_serve_request_unknown_op(): + wm = _FakeWM() + resp = mg._handle_serve_request(wm, {"op": "frobnicate", "id": 1}) + assert resp["ok"] is False + assert resp["id"] == 1 + assert "unknown op" in resp["error"] + + +def test_handle_serve_request_exit(): + wm = _FakeWM() + assert mg._handle_serve_request(wm, {"op": "exit", "id": 0}) == {"ok": True, "id": 0} + + +def test_handle_serve_request_detect(): + wm = _FakeWM() + resp = mg._handle_serve_request( + wm, {"op": "detect", "id": 5, "text": "Hola Madrid"}, threshold=0.52 + ) + assert resp == { + "ok": True, + "id": 5, + "is_watermarked": True, + "score": 3.5, + "threshold": 0.52, + } + assert wm.detect_calls == [("Hola Madrid", True)] # return_dict=True + + +@pytest.mark.parametrize("bad", [None, "", 42]) +def test_handle_serve_request_detect_bad_text(bad): + wm = _FakeWM() + resp = mg._handle_serve_request(wm, {"op": "detect", "id": 6, "text": bad}) + assert resp["ok"] is False + assert resp["id"] == 6 + assert "non-empty string" in resp["error"] + + +def test_handle_serve_request_detect_score_not_float(): + class _OddWM(_FakeWM): + def detect_watermark(self, text, return_dict=True): + return {"is_watermarked": False, "score": "n/a"} + + resp = mg._handle_serve_request(_OddWM(), {"op": "detect", "id": 8, "text": "x"}) + assert resp == {"ok": True, "id": 8, "is_watermarked": False, "score": None, "threshold": None} + + +def test_threshold_from_config(tmp_path): + kgw = tmp_path / "kgw.json" + kgw.write_text('{"algorithm_name": "KGW", "z_threshold": 4.0}', encoding="utf-8") + assert mg._threshold_from_config(kgw) == 4.0 + synthid = tmp_path / "synthid.json" + synthid.write_text('{"algorithm_name": "SynthID", "threshold": 0.52}', encoding="utf-8") + assert mg._threshold_from_config(synthid) == 0.52 + no_threshold = tmp_path / "none.json" + no_threshold.write_text('{"algorithm_name": "KGW"}', encoding="utf-8") + assert mg._threshold_from_config(no_threshold) is None + assert mg._threshold_from_config(tmp_path / "missing.json") is None + + +def test_cmd_serve_jsonlines_protocol(monkeypatch, tmp_path, capsys): + upstream = _make_upstream(tmp_path) + config = tmp_path / "cfg.json" + config.write_text('{"algorithm_name": "SynthID"}', encoding="utf-8") + monkeypatch.setattr(mg, "_load_algorithm", lambda *a, **k: _FakeWM()) + + requests = [ + {"op": "watermark", "id": 1, "prompt": "Hola Madrid"}, + {"op": "watermark", "id": 2, "prompt": ""}, # bad request: worker must live on + {"op": "exit", "id": 3}, + ] + stream = "\n".join(json.dumps(r) for r in requests) + "\n" + monkeypatch.setattr(sys, "stdin", io.StringIO(stream)) + + argv = ["serve", "--markllm-dir", str(upstream), "--scheme", "synthid", "--config", str(config)] + rc = mg.main(argv) + + assert rc == 0 + lines = capsys.readouterr().out.splitlines() + assert len(lines) == 4 # ready + 3 responses + ready = json.loads(lines[0]) + assert ready["ready"] is True + assert ready["scheme"] == "synthid" + assert ready["model"] == mg.DEFAULT_MODEL + assert ready["device"] in ("cpu", "cuda") # auto; never mps + r1 = json.loads(lines[1]) + assert r1["ok"] is True and r1["id"] == 1 + assert r1["watermarked"] == "WM:Hola Madrid" + r2 = json.loads(lines[2]) + assert r2["ok"] is False and r2["id"] == 2 + r3 = json.loads(lines[3]) + assert r3["ok"] is True and r3["id"] == 3 + + +def test_cmd_serve_detect_request(monkeypatch, tmp_path, capsys): + upstream = _make_upstream(tmp_path) + config = tmp_path / "cfg.json" + config.write_text('{"algorithm_name": "SynthID", "threshold": 0.52}', encoding="utf-8") + monkeypatch.setattr(mg, "_load_algorithm", lambda *a, **k: _FakeWM()) + + requests = [ + {"op": "watermark", "id": 1, "prompt": "Hola Madrid"}, + {"op": "detect", "id": 2, "text": "Hola Madrid"}, + {"op": "detect", "id": 3, "text": ""}, # bad detect: worker must live on + {"op": "exit", "id": 4}, + ] + stream = "\n".join(json.dumps(r) for r in requests) + "\n" + monkeypatch.setattr(sys, "stdin", io.StringIO(stream)) + + argv = ["serve", "--markllm-dir", str(upstream), "--scheme", "synthid", "--config", str(config)] + rc = mg.main(argv) + + assert rc == 0 + lines = capsys.readouterr().out.splitlines() + assert len(lines) == 5 # ready + 4 responses + assert json.loads(lines[0])["ready"] is True + r1 = json.loads(lines[1]) + assert r1["ok"] is True and r1["id"] == 1 and r1["watermarked"] == "WM:Hola Madrid" + r2 = json.loads(lines[2]) + assert r2 == {"ok": True, "id": 2, "is_watermarked": True, "score": 3.5, "threshold": 0.52} + r3 = json.loads(lines[3]) + assert r3["ok"] is False and r3["id"] == 3 + r4 = json.loads(lines[4]) + assert r4 == {"ok": True, "id": 4} + + +def test_cmd_serve_invalid_json_line(monkeypatch, tmp_path, capsys): + upstream = _make_upstream(tmp_path) + config = tmp_path / "cfg.json" + config.write_text('{"algorithm_name": "KGW"}', encoding="utf-8") + monkeypatch.setattr(mg, "_load_algorithm", lambda *a, **k: _FakeWM()) + + stream = "this is not json\n" + json.dumps({"op": "exit", "id": 0}) + "\n" + monkeypatch.setattr(sys, "stdin", io.StringIO(stream)) + + argv = ["serve", "--markllm-dir", str(upstream), "--scheme", "kgw", "--config", str(config)] + rc = mg.main(argv) + + assert rc == 0 + lines = capsys.readouterr().out.splitlines() + assert json.loads(lines[0])["ready"] is True + assert json.loads(lines[1]) == {"ok": False, "error": "invalid JSON request"} + assert json.loads(lines[2]) == {"ok": True, "id": 0} + + +def test_main_serve_missing_markllm_dir_ready_false(monkeypatch, capsys): + monkeypatch.delenv("MARKLLM_DIR", raising=False) + argv = ["serve", "--markllm-dir", "/nonexistent", "--scheme", "kgw", "--config", "/fake/c.json"] + rc = mg.main(argv) + assert rc == 3 + out = json.loads(capsys.readouterr().out) + assert out["ready"] is False + + +# -------------------------------------------------------------------------- +# Generation internals +# -------------------------------------------------------------------------- + + +def test_generate_seeds_via_torch_and_sets_kwargs(monkeypatch): + calls: list[int] = [] + + class _FakeTorch: + @staticmethod + def manual_seed(seed): + calls.append(seed) + + monkeypatch.setitem(sys.modules, "torch", _FakeTorch()) + + wm = _FakeWM() + watermarked, unwatermarked = mg._generate(wm, "prompt", seed=7, max_new_tokens=50, min_length=5) + + assert watermarked == "WM:prompt" + assert unwatermarked == "UW:prompt" + assert calls == [7] + assert wm.config.gen_kwargs == {"max_new_tokens": 50, "min_length": 5} + + +@requires_torch +def test_generate_with_real_torch(monkeypatch): + """Seed path with real torch installed (skipped when torch is absent).""" + wm = _FakeWM() + watermarked, unwatermarked = mg._generate(wm, "prompt", seed=42, max_new_tokens=10) + assert (watermarked, unwatermarked) == ("WM:prompt", "UW:prompt") + assert wm.config.gen_kwargs["max_new_tokens"] == 10 + + +@requires_torch +def test_resolve_device_auto_never_mps(): + device = mg.resolve_device("auto") + assert device in ("cpu", "cuda") + assert device != "mps" diff --git a/research/tests/test_paper_skeleton.py b/research/tests/test_paper_skeleton.py new file mode 100644 index 0000000..3868c91 --- /dev/null +++ b/research/tests/test_paper_skeleton.py @@ -0,0 +1,293 @@ +"""Structural tests for the arXiv v1 paper skeleton (research/paper/, gaps C1/C2). + +These tests validate the *skeleton*: the exact title, the section order +from research/02-paper-outline.md section 5, the input{} wiring, the +table/figure placeholders from 02 section 6, brace balance, and full +reference coverage of research/03-related-work.md sections A--E. They +intentionally assert nothing about experiment numbers, which do not +exist yet. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +PAPER_DIR = Path(__file__).resolve().parents[1] / "paper" +RELATED_WORK = Path(__file__).resolve().parents[1] / "03-related-work.md" + +TITLE = ( + "How Fragile Are Deployed Text Watermarks? An Empirical Study of " + "Layered Watermark Removal under Realistic User-Side Editing" +) + +SECTIONS = [ + "Introduction", + "Background and Related Work", + "Threat Model and System", + "Experimental Setup", + "Results", + "Analysis and Case Study", + "Policy Discussion", + "Limitations", + "Ethics Statement", + "Conclusion", +] + +#: Every entry in research/03-related-work.md A--E as bib keys +#: (ID = first-author-lastname + arXiv year [+ short tag]). +EXPECTED_BIB_KEYS = [ + # A. Watermarking methods (the attack surface we test) + "kirchenbauer2023kwg", + "christ2023undetectable", + "kuditipudi2023robust", + "liu2023sir", + "liu2023upv", + "zhao2024permute", + "hu2023unbiased", + "wu2023dipmark", + "huo2024tswatermark", + "liu2024adaptive", + "hou2023semstamp", + "hou2024ksemstamp", + "gu2025invisible", + "wang2025morphmark", + "yang2023blackbox", + # B. Robustness, attacks, and limits + "kirchenbauer2023reliability", + "he2024xsir", + "zhang2023sand", + "jovanovic2024stealing", + "pan2024waterseeker", + "lu2024ewd", + "liu2024crafted", + "pan2025distillation", + # C. Closest recent neighbors + "han2025synthid", + "omidi2026synthid", + "tamim2026forensic", + "harelcanada2025sandcastles", + # D. Tools & surveys + "pan2024markllm", + "liu2023survey", + # E. Non-arXiv sources + "deepmind2024synthidtext", + "c2pa2024spec", + "euaiact2024", +] + +#: Non-arXiv anchors that must appear in refs.bib (03 section E). +NON_ARXIV_ANCHORS = [ + "10.1038/s41586-024-08025-4", # SynthID-Text, Nature 638, 625-632 + "c2pa.org/specifications", # C2PA specification URL + "2024/1689", # EU AI Act Regulation (EU) 2024/1689, Art. 50 +] + + +def _read(path: Path) -> str: + assert path.is_file(), f"missing file: {path}" + return path.read_text(encoding="utf-8") + + +def _strip_latex_comments(text: str) -> str: + """Drop full-line comments and truncate at the first unescaped percent.""" + kept_lines: list[str] = [] + for line in text.splitlines(): + if line.lstrip().startswith("%"): + continue + kept: list[str] = [] + for i, ch in enumerate(line): + if ch == "%" and (i == 0 or line[i - 1] != "\\"): + break + kept.append(ch) + kept_lines.append("".join(kept)) + return "\n".join(kept_lines) + + +def _brace_delta(text: str) -> int: + """Unbalanced-brace count on comment-stripped text (open minus close).""" + return text.count("{") - text.count("}") + + +def _bib_entries(text: str) -> dict[str, str]: + """Map bib key -> full entry body for every entry in *text*. + + Runs on comment-stripped text and scans brace-balanced spans so that + field values containing braces (e.g. double-braced organization + authors) do not truncate the body. + """ + entries: dict[str, str] = {} + for match in re.finditer(r"@(\w+)\s*\{([^,]+),", text): + key = match.group(2).strip() + start = match.start() + # Scan from the entry's own opening brace so the first field's + # closing brace does not end the capture early. + brace_pos = text.find("{", start) + depth = 0 + for i in range(brace_pos, len(text)): + if text[i] == "{": + depth += 1 + elif text[i] == "}": + depth -= 1 + if depth == 0: + entries[key] = text[start : i + 1] + break + return entries + + +# --------------------------------------------------------------------- +# Skeleton file inventory +# --------------------------------------------------------------------- + + +def test_required_paper_files_exist() -> None: + for name in ( + "main.tex", + "refs.bib", + "abstract.tex", + "ethics.tex", + "acknowledgments.tex", + "README.md", + ): + assert (PAPER_DIR / name).is_file(), f"missing research/paper/{name}" + + +# --------------------------------------------------------------------- +# main.tex structure +# --------------------------------------------------------------------- + + +def test_exact_title() -> None: + main = _read(PAPER_DIR / "main.tex") + match = re.search(r"\\title\{([^}]*)\}", main, re.DOTALL) + assert match is not None, "no \\title{...} found" + normalized = re.sub(r"\s+", " ", match.group(1)).strip() + assert normalized == TITLE, f"title mismatch: {normalized!r}" + + +def test_single_author_placeholder() -> None: + main = _read(PAPER_DIR / "main.tex") + assert "\\author{Guillaume Meyer}" in main + # No affiliation line should be active (only a commented example). + active = _strip_latex_comments(main) + assert "\\author{Guillaume Meyer}\\" not in active + + +def test_sections_in_order() -> None: + main = _read(PAPER_DIR / "main.tex") + found = re.findall(r"\\section\{([^}]*)\}", main) + assert found == SECTIONS, f"section mismatch: {found}" + + +def test_acl_style_dropin_comment() -> None: + main = _read(PAPER_DIR / "main.tex") + assert "acl2024.sty" in main, "missing 'drop in acl2024.sty' comment" + + +def test_inputs_wired() -> None: + main = _read(PAPER_DIR / "main.tex") + for name in ("abstract", "ethics", "acknowledgments"): + assert f"\\input{{{name}}}" in main, f"missing \\input{{{name}}}" + + +def test_bibliography_wired() -> None: + main = _read(PAPER_DIR / "main.tex") + assert "\\bibliographystyle{plainnat}" in main + assert "\\bibliography{refs}" in main + + +def test_table_placeholders() -> None: + main = _read(PAPER_DIR / "main.tex") + for number in range(1, 8): + assert f"\\input{{tables/t{number}}}" in main, f"missing tables/t{number}" + assert f"tab:t{number}" in main, f"missing label tab:t{number}" + + +def test_figure_placeholders() -> None: + main = _read(PAPER_DIR / "main.tex") + for number in range(1, 7): + assert f"figures/f{number}" in main, f"missing figures/f{number}" + assert f"fig:f{number}" in main, f"missing label fig:f{number}" + + +def test_todo_comments_present() -> None: + main = _read(PAPER_DIR / "main.tex") + assert main.count("TODO") >= 10, "expected TODO markers for real numbers" + + +# --------------------------------------------------------------------- +# refs.bib coverage +# --------------------------------------------------------------------- + + +def test_bib_keys_match_03_expected_set() -> None: + bib = _read(PAPER_DIR / "refs.bib") + entries = _bib_entries(_strip_latex_comments(bib)) + assert set(entries) == set(EXPECTED_BIB_KEYS), ( + f"bib keys diverge from 03: missing={set(EXPECTED_BIB_KEYS) - set(entries)} " + f"extra={set(entries) - set(EXPECTED_BIB_KEYS)}" + ) + + +def test_every_arxiv_id_from_03_present() -> None: + related = _read(RELATED_WORK) + bib = _read(PAPER_DIR / "refs.bib") + ids = re.findall(r"arXiv \*\*\d{4}\.\d{5}\*\*", related) + assert ids, "no arXiv IDs parsed from 03-related-work.md" + for arxiv_id in ids: + digits = re.sub(r"[^0-9.]", "", arxiv_id) + assert digits in bib, f"arXiv id {digits} missing from refs.bib" + + +def test_non_arxiv_anchors_present() -> None: + bib = _read(PAPER_DIR / "refs.bib") + for anchor in NON_ARXIV_ANCHORS: + assert anchor in bib, f"missing non-arXiv anchor {anchor!r}" + + +def test_arxiv_entries_use_eprint_fields() -> None: + bib = _read(PAPER_DIR / "refs.bib") + entries = _bib_entries(_strip_latex_comments(bib)) + arxiv_keys = [ + k + for k in EXPECTED_BIB_KEYS + if k not in ("deepmind2024synthidtext", "c2pa2024spec", "euaiact2024") + ] + for key in arxiv_keys: + assert "eprint" in entries[key], f"{key} is missing eprint" + assert "archivePrefix" in entries[key], f"{key} is missing archivePrefix" + + +def test_reverify_comment_present() -> None: + bib = _read(PAPER_DIR / "refs.bib") + assert "re-verify at submission" in bib + + +# --------------------------------------------------------------------- +# Braces and abstract length +# --------------------------------------------------------------------- + + +def test_braces_balanced() -> None: + for name in ("main.tex", "abstract.tex", "ethics.tex", "acknowledgments.tex", "refs.bib"): + text = _read(PAPER_DIR / name) + delta = _brace_delta(_strip_latex_comments(text)) + assert delta == 0, f"{name}: unbalanced braces (delta={delta})" + + +def test_abstract_word_count() -> None: + abstract = _strip_latex_comments(_read(PAPER_DIR / "abstract.tex")) + words = len(abstract.split()) + assert 120 <= words <= 190, f"abstract word count out of range: {words}" + + +# --------------------------------------------------------------------- +# README +# --------------------------------------------------------------------- + + +def test_readme_build_and_generators() -> None: + readme = _read(PAPER_DIR / "README.md") + assert "pdflatex" in readme and "bibtex" in readme + assert "make_tables.py" in readme and "make_figures.py" in readme + assert "acl2024.sty" in readme diff --git a/research/tests/test_run_experiments.py b/research/tests/test_run_experiments.py new file mode 100644 index 0000000..ef5bdde --- /dev/null +++ b/research/tests/test_run_experiments.py @@ -0,0 +1,272 @@ +"""Integration tests for the v1 experiment orchestrator (gap 05-A4). + +These exercise the design constants, the locked matrix, results helpers, +and the report stage without spawning MarkLLM/API workers (the full run +is a multi-week CPU/API job driven by the smoke test in 01 §9). +""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) + +import run_experiments as rx + + +def test_locked_matrix_sizes() -> None: + cells = list( + rx.iter_cells( + list(rx.SCHEMES), + rx.LENGTHS, + rx.TEMPS, + rx.LANGUAGES, + rx.SEEDS, + rx.PROMPTS, + ) + ) + assert len(cells) == 3500 + + +def test_conditions_count() -> None: + conds = list(rx.iter_conditions(list(rx.SCHEMES), rx.LENGTHS, rx.TEMPS, rx.LANGUAGES)) + # 14 EN core + 4 temp axis + 4 length axis + 6 multilingual. + assert len(conds) == 28 + + +def test_cell_allowed_restrictions() -> None: + # EN core: every scheme at length 100/300, temp 0.7. + for scheme in rx.SCHEMES: + assert rx.cell_allowed(scheme, 100, 0.7, "en") + assert rx.cell_allowed(scheme, 300, 0.7, "en") + # temp axis: core 4 only. + assert rx.cell_allowed("synthid", 300, 1.0, "en") + assert not rx.cell_allowed("sir", 300, 1.0, "en") + assert not rx.cell_allowed("exp", 300, 1.0, "en") + # length axis: core 4 only. + assert rx.cell_allowed("kgw-d1", 500, 0.7, "en") + assert not rx.cell_allowed("unigram", 500, 0.7, "en") + # multilingual: kgw-d2 + synthid only. + assert rx.cell_allowed("kgw-d2", 300, 0.7, "de") + assert rx.cell_allowed("synthid", 300, 0.7, "fr") + assert not rx.cell_allowed("kgw-d1", 300, 0.7, "es") + assert not rx.cell_allowed("sir", 300, 0.7, "de") + + +def test_scheme_cli_map_covers_all_schemes() -> None: + assert set(rx.SCHEME_CLI) == set(rx.SCHEMES) + assert rx.SCHEME_CLI["kgw-d1"] == "kgw" + assert rx.SCHEME_CLI["synthid"] == "synthid" + assert rx.SCHEME_CLI["sir"] == "sir" + + +def test_cheap_expands_to_three_subattacks() -> None: + assert rx._attack_rows("cheap") == [ + "cheap:synonym", + "cheap:delete", + "cheap:reorder", + ] + assert rx._attack_rows("layerA") == ["layerA"] + + +def test_configs_exist_for_every_scheme(tmp_path) -> None: + cfg_dir = Path(__file__).resolve().parents[1] / "configs" + for scheme in rx.SCHEMES: + cfg = cfg_dir / rx.SCHEMES[scheme][2] + assert cfg.is_file(), f"missing {cfg}" + data = json.loads(cfg.read_text("utf-8")) + assert data["algorithm_name"] == rx.SCHEMES[scheme][1] + + +def test_quality_pairs_skip_none_and_respect_cap(tmp_path) -> None: + rows = [ + { + "ok": True, + "seed": 1, + "prompt_idx": 1, + "attack": "none", + "original": "a", + "candidate": "a", + }, + { + "ok": True, + "seed": 2, + "prompt_idx": 2, + "attack": "layerA", + "original": "b", + "candidate": "c", + }, + { + "ok": True, + "seed": 3, + "prompt_idx": 3, + "attack": "paraphrase:3", + "original": "d", + "candidate": "e", + }, + { + "ok": False, + "seed": 4, + "prompt_idx": 4, + "attack": "humanize", + "original": "f", + "candidate": "g", + }, + ] + att = tmp_path / "attacked.jsonl" + att.write_text("\n".join(json.dumps(r) for r in rows), encoding="utf-8") + cond = rx.Condition("kgw-d2", 300, 0.7, "en") + pairs = rx._quality_pairs(cond, att, cap=2) + assert len(pairs) == 2 + assert {p["attack"] for p in pairs} == {"layerA", "paraphrase:3"} + assert all(p["condition"] == cond.id for p in pairs) + + +def test_report_writes_manifest_and_markdown(tmp_path) -> None: + class FakeArgs: + rewrite_backend = "openai-compatible" + rewrite_model = "m" + force = False + corpus_dir = "research/corpus" + + args = FakeArgs() + from run_experiments import RunContext + + ctx = RunContext(args, tmp_path / "markllm") # upstream not needed for report + ctx.upstream = tmp_path / "markllm" # avoid resolving the real checkout + out = tmp_path / "results" + out.mkdir() + rx.stage_report(out, ctx) + assert (out / "manifest.json").is_file() + assert (out / "report.md").is_file() + manifest = json.loads((out / "manifest.json").read_text("utf-8")) + assert manifest["design"]["prompts"] == 25 + assert set(manifest["pins"]) >= {"repo_commit", "markllm_commit"} + + +class _FakeArgs: + """Minimal argparse.Namespace stand-in for RunContext construction.""" + + rewrite_backend = "openai-compatible" + rewrite_model = "m" + force = False + corpus_dir = "research/corpus" + quality_python = None + + +def test_quality_python_explicit_flag_wins(tmp_path) -> None: + args = _FakeArgs() + args.quality_python = str(tmp_path / "custom" / "bin" / "python") + ctx = rx.RunContext(args, tmp_path / "markllm") + assert ctx.quality_python == str(tmp_path / "custom" / "bin" / "python") + + +def test_quality_python_default_resolves_repo_venv(monkeypatch, tmp_path) -> None: + repo_root = Path(rx.__file__).resolve().parents[2] + cand = repo_root / ".venv-quality" / "bin" / "python" + real_is_file = Path.is_file + monkeypatch.setattr( + Path, "is_file", lambda self: True if self == cand else real_is_file(self) + ) + ctx = rx.RunContext(_FakeArgs(), tmp_path / "markllm") + assert ctx.quality_python == str(cand) + + +def test_quality_python_falls_back_to_markllm_python(monkeypatch, tmp_path) -> None: + monkeypatch.setattr(Path, "is_file", lambda self: False) + ctx = rx.RunContext(_FakeArgs(), tmp_path / "markllm") + # No repo .venv-quality and no upstream venv -> the orchestrator's own + # interpreter (sys.executable), recorded via the RunContext python. + assert ctx.quality_python == ctx.python + + +def test_evaluate_dry_run_shows_quality_interpreter(tmp_path) -> None: + args = _FakeArgs() + args.quality_python = "/opt/quality/bin/python" + ctx = rx.RunContext(args, tmp_path / "markllm") + cond = rx.Condition("synthid", 300, 0.7, "en") + lines = rx.stage_evaluate(cond, tmp_path / "results", ctx, dry_run=True) + assert any( + line.startswith("/opt/quality/bin/python evaluate_quality.py") + for line in lines + ) + assert any( + line.split()[0] == ctx.python and "analyze_roc.py" in line for line in lines + ) + + +def test_runcontext_prompts_seeds_from_cli_args(tmp_path) -> None: + args = _FakeArgs() + args.prompts = 1 + args.seeds = "1" + ctx = rx.RunContext(args, tmp_path / "markllm") + assert ctx.prompts == 1 + assert ctx.seeds == [1] + + +def test_runcontext_prompts_seeds_defaults(tmp_path) -> None: + ctx = rx.RunContext(_FakeArgs(), tmp_path / "markllm") + assert ctx.prompts == rx.PROMPTS + assert ctx.seeds == rx.SEEDS + + +def test_generate_dry_run_counts_use_cli_subset(tmp_path) -> None: + args = _FakeArgs() + args.prompts = 1 + args.seeds = "1" + ctx = rx.RunContext(args, tmp_path / "markllm") + cond = rx.Condition("synthid", 100, 0.7, "en") + lines = rx.stage_generate(cond, tmp_path / "out", ctx, dry_run=True) + assert any("1 watermark requests over stdin" in line for line in lines) + # CPU-only boxes: the orchestrator forces bf16 on every MarkLLM worker + # (fp32 cached decode is ~8x slower on aarch64), so the dry-run commands + # must show it for both the EN and multilingual generators. + assert any("--torch-dtype" in line and "bf16" in line for line in lines) + mlines = rx.stage_generate( + rx.Condition("kgw-d2", 300, 0.7, "de"), tmp_path / "out", ctx, dry_run=True + ) + assert any("multilingual_gen.py" in line and "bf16" in line for line in mlines) + + +def test_none_attack_is_passthrough(tmp_path) -> None: + ctx = rx.RunContext(_FakeArgs(), tmp_path / "markllm") + cond = rx.Condition("synthid", 100, 0.7, "en") + cand, stats, err, seconds = rx._run_one_attack(cond, "none", "original text", 1, ctx) + assert cand == "original text" + assert err is None + assert stats is None + assert seconds >= 0 + + +def test_worker_for_forces_bf16(tmp_path) -> None: + args = _FakeArgs() + ctx = rx.RunContext(args, tmp_path / "markllm") + cfg = Path(rx.__file__).resolve().parents[1] / "configs" / "KGW-d2.json" + # worker_for spawns a real subprocess; intercept Popen to inspect the cmd. + captured: list[list[str]] = [] + + class FakeWorker: + def __init__(self, cmd, *, timeout, label): + captured.append(cmd) + + import run_experiments as rx_mod + + real_sw = rx_mod.ServeWorker + rx_mod.ServeWorker = FakeWorker + try: + ctx.worker_for( + kind="gen-en", scheme="kgw-d1", config=cfg, model="facebook/opt-1.3b" + ) + finally: + rx_mod.ServeWorker = real_sw + assert captured, "worker cmd not captured" + cmd = captured[0] + assert "--torch-dtype" in cmd + assert cmd[cmd.index("--torch-dtype") + 1] == "bf16" + + +# quality-python plumbing tests live below (added by the environment setup). +placeholder = True + diff --git a/service/scripts/detect_text_watermark.py b/service/scripts/detect_text_watermark.py index 8b888b7..b88c24b 100755 --- a/service/scripts/detect_text_watermark.py +++ b/service/scripts/detect_text_watermark.py @@ -97,8 +97,15 @@ def _load_algorithm( offline: bool = False, temperature: float | None = None, top_p: float | None = None, + torch_dtype: str = "auto", ): - """Import the checkout and build an ``AutoWatermark`` instance. + """Import the checkout and build an AutoWatermark instance. + + torch_dtype maps to torch_dtype= in from_pretrained: "auto" (default, + fp32), "fp32", or "bf16". On CPU-only boxes bf16 can be several times + faster than fp32 (see the research harness), at a small precision cost; + same-config generation/detection stay consistent because both sides use + the same dtype. ``temperature``/``top_p`` (when not None) are folded into the generation kwargs so callers can control sampling. @@ -110,6 +117,7 @@ def _load_algorithm( gen_kwargs_extra["top_p"] = top_p sys.path.insert(0, str(upstream)) try: + import torch from transformers import AutoModelForCausalLM, AutoTokenizer from utils.transformers_config import TransformersConfig from watermark.auto_watermark import AutoWatermark @@ -124,6 +132,9 @@ def _load_algorithm( if offline: os.environ.setdefault("HF_HUB_OFFLINE", "1") load_kwargs = {"local_files_only": True} if offline else {} + dtype_map = {"fp32": torch.float32, "bf16": torch.bfloat16} + if torch_dtype in dtype_map: + load_kwargs["torch_dtype"] = dtype_map[torch_dtype] tokenizer = AutoTokenizer.from_pretrained(model, **load_kwargs) lm = AutoModelForCausalLM.from_pretrained(model, **load_kwargs).to(device) @@ -225,6 +236,7 @@ def _cmd_detect(args: argparse.Namespace, upstream: Path, alg: str) -> int: offline=args.offline, temperature=args.temperature, top_p=args.top_p, + torch_dtype=args.torch_dtype, ) det = _detect_payload(wm, text, threshold) except _Unavailable as e: @@ -276,6 +288,7 @@ def _cmd_watermark(args: argparse.Namespace, upstream: Path, alg: str) -> int: offline=args.offline, temperature=args.temperature, top_p=args.top_p, + torch_dtype=args.torch_dtype, ) watermarked, unwatermarked = _generate( wm, @@ -398,6 +411,7 @@ def _cmd_serve(args: argparse.Namespace, upstream: Path, alg: str) -> int: offline=args.offline, temperature=args.temperature, top_p=args.top_p, + torch_dtype=args.torch_dtype, ) except _Unavailable as e: eprint(str(e)) @@ -530,6 +544,13 @@ def _add_common(p: argparse.ArgumentParser) -> None: default="auto", help="auto|cpu|cuda|mps (default: auto)", ) + p.add_argument( + "--torch-dtype", + default="auto", + choices=("auto", "fp32", "bf16"), + help="Model dtype: auto (fp32 default), fp32, or bf16 (faster on " + "CPU-only boxes; same-config gen/detect stay consistent)", + ) p.add_argument( "--offline", action="store_true",