mirror of
https://github.com/guillaumemeyer/watermarks-remover.git
synced 2026-08-22 13:11:57 +02:00
* feat: split skill from service, add HTTP API and Docker distribution The agent skill (skills/remove-ai-marks/) is now a code-free remote client: all implementation moved to service/scripts/ and runs behind a stdlib HTTP service (server.py) with /health, /capabilities, /inspect, /clean and a dynamically generated OpenAPI 3.0.3 spec at /openapi.json. - Move scripts/ and the backend Dockerfiles under service/ - server.py: JSON/base64 HTTP entrypoint with size caps, binary guard, atomic writes, loopback default, optional bearer auth - Core Dockerfile (exiftool/qpdf/c2patool preinstalled) and a GHCR publish workflow for the core/markllm/markdiffusion images - compose.yaml (wr-* services, harness/heavy profiles) + compose-check.sh to validate the running stack (exit code only) - Fix markllm image build (tokenizers 0.22.2, CPU-only torch) and ctrlregen build (python:3.11 base for the 2023-era research pins) - Fix markllm/markdiffusion harness images missing common.py at runtime * docs: add .env.example and service configuration guide * fix: disable chain-of-thought for openai-compatible Layer B rewrites deepseek-v4-flash is a reasoning model: a one-line paraphrase burned 9,894 reasoning tokens (~100s) and hit the default timeout. Send reasoning_effort=none by default for the openai-compatible backend (--reasoning-effort / WATERMARKS_REWRITE_REASONING_EFFORT; 'off' omits the parameter), cutting the same rewrite to ~1s / 12 tokens. Tested end-to-end against api.deepseek.com. * fix: sanitize client-supplied filename in HTTP service CodeQL 'uncontrolled data in path expression' (server.py): a name like '../../x' flowed into Path(tmpdir) / name, letting an upload escape the request temp dir on write. Sanitize name to its basename in _decode_input (_safe_name) and refuse any joined path whose parent is not the tmpdir at the write sites (_tmp_path). Tests cover traversal names. * chore: gitignore .env (contains local rewrite credentials) * chore: deny-by-default gitignore and dockerignore; document compose env config .gitignore and service/.dockerignore now exclude everything by default and explicitly allow only what is publishable/needed: tracked source, docs, tests, .github, and (for images) the service/scripts/ tree that every Dockerfile COPYs. Root .dockerignore documents that all builds use service/ as context. README Configuration section now covers .env setup for docker compose, host-side export for CLI runs, and the full variable table.
240 lines
7.7 KiB
Python
240 lines
7.7 KiB
Python
"""Tests for the HTTP service entrypoint (server.py)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import base64
|
|
import http.client
|
|
import json
|
|
import struct
|
|
import sys
|
|
import threading
|
|
import zlib
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
SCRIPTS = ROOT / "service" / "scripts"
|
|
sys.path.insert(0, str(SCRIPTS))
|
|
|
|
import server # noqa: E402
|
|
|
|
|
|
def _png_chunk(ctype: bytes, payload: bytes) -> bytes:
|
|
crc = zlib.crc32(ctype)
|
|
crc = zlib.crc32(payload, crc) & 0xFFFFFFFF
|
|
return struct.pack(">I", len(payload)) + ctype + payload + struct.pack(">I", crc)
|
|
|
|
|
|
def _watermarked_png() -> bytes:
|
|
"""1x1 PNG whose tEXt chunk carries C2PA markers (structure only)."""
|
|
sig = b"\x89PNG\r\n\x1a\n"
|
|
ihdr = struct.pack(">IIBBBBB", 1, 1, 8, 2, 0, 0, 0)
|
|
idat = zlib.compress(b"\x00\x00\x00")
|
|
text = b"Comment\x00c2pa test contentcredentials"
|
|
return (
|
|
sig
|
|
+ _png_chunk(b"IHDR", ihdr)
|
|
+ _png_chunk(b"tEXt", text)
|
|
+ _png_chunk(b"IDAT", idat)
|
|
+ _png_chunk(b"IEND", b"")
|
|
)
|
|
|
|
|
|
def _b64(data: bytes) -> str:
|
|
return base64.b64encode(data).decode("ascii")
|
|
|
|
|
|
def _post(conn: http.client.HTTPConnection, path: str, payload: dict) -> tuple[int, dict]:
|
|
conn.request(
|
|
"POST",
|
|
path,
|
|
body=json.dumps(payload).encode("utf-8"),
|
|
headers={"Content-Type": "application/json"},
|
|
)
|
|
resp = conn.getresponse()
|
|
data = resp.read()
|
|
return resp.status, json.loads(data) if data else {}
|
|
|
|
|
|
def _get(conn: http.client.HTTPConnection, path: str) -> tuple[int, dict]:
|
|
conn.request("GET", path)
|
|
resp = conn.getresponse()
|
|
data = resp.read()
|
|
return resp.status, json.loads(data) if data else {}
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def conn() -> http.client.HTTPConnection:
|
|
srv = server.ThreadingHTTPServer(("127.0.0.1", 0), server.Handler)
|
|
thread = threading.Thread(target=srv.serve_forever, daemon=True)
|
|
thread.start()
|
|
c = http.client.HTTPConnection("127.0.0.1", srv.server_address[1])
|
|
yield c
|
|
c.close()
|
|
srv.shutdown()
|
|
srv.server_close()
|
|
thread.join(timeout=5)
|
|
|
|
|
|
def test_health(conn):
|
|
status, body = _get(conn, "/health")
|
|
assert status == 200
|
|
assert body["ok"] is True
|
|
assert "version" in body
|
|
|
|
|
|
def test_capabilities(conn):
|
|
status, body = _get(conn, "/capabilities")
|
|
assert status == 200
|
|
assert set(body["tools"]) == {"c2patool", "exiftool", "qpdf"}
|
|
assert "pixel_backends" in body
|
|
assert "scorers" in body
|
|
assert "harnesses" in body
|
|
|
|
|
|
def test_openapi_spec_covers_all_endpoints(conn):
|
|
status, body = _get(conn, "/openapi.json")
|
|
assert status == 200
|
|
assert body["openapi"] == "3.0.3"
|
|
assert body["info"]["title"] == "watermarks-remover service"
|
|
expected = {
|
|
"/health": {"get"},
|
|
"/capabilities": {"get"},
|
|
"/openapi.json": {"get"},
|
|
"/inspect": {"post"},
|
|
"/clean": {"post"},
|
|
}
|
|
for path, methods in expected.items():
|
|
assert path in body["paths"]
|
|
for method in methods:
|
|
assert method in body["paths"][path]
|
|
assert "responses" in body["paths"][path][method]
|
|
|
|
|
|
def test_openapi_spec_describes_request_bodies(conn):
|
|
status, body = _get(conn, "/openapi.json")
|
|
assert status == 200
|
|
clean = body["paths"]["/clean"]["post"]
|
|
assert clean["requestBody"]["required"] is True
|
|
schema = clean["requestBody"]["content"]["application/json"]["schema"]
|
|
assert "file" in schema["properties"]
|
|
assert "options" in schema["properties"]
|
|
inspect = body["paths"]["/inspect"]["post"]
|
|
assert "file" in inspect["requestBody"]["content"]["application/json"]["schema"]["properties"]
|
|
|
|
|
|
def test_openapi_spec_reflects_auth(conn, monkeypatch):
|
|
monkeypatch.setattr(server, "API_KEY", "sekret")
|
|
conn.request("GET", "/openapi.json", headers={"Authorization": "Bearer sekret"})
|
|
resp = conn.getresponse()
|
|
body = json.loads(resp.read())
|
|
assert resp.status == 200
|
|
assert "bearerAuth" in body["components"]["securitySchemes"]
|
|
assert body["security"] == [{"bearerAuth": []}]
|
|
|
|
|
|
def test_inspect_text_finds_watermark(conn):
|
|
data = "Hello\u200bWorld\u00ad!".encode("utf-8")
|
|
status, body = _post(conn, "/inspect", {"file": _b64(data), "name": "note.txt"})
|
|
assert status == 200
|
|
assert body["kind"] == "text"
|
|
assert body["suspicious"] is True
|
|
assert body["report"]["suspicious_total"] == 2
|
|
|
|
|
|
def test_clean_text_roundtrip(conn):
|
|
data = "Hello\u200bWorld\u00ad!".encode("utf-8")
|
|
status, body = _post(conn, "/clean", {"file": _b64(data), "name": "note.txt"})
|
|
assert status == 200
|
|
assert body["kind"] == "text"
|
|
cleaned = base64.b64decode(body["cleaned"]).decode("utf-8")
|
|
assert cleaned == "HelloWorld!"
|
|
assert body["report"]["stats"]["removed_count"] == 2
|
|
|
|
|
|
def test_clean_png_strips_metadata(conn):
|
|
data = _watermarked_png()
|
|
status, body = _post(conn, "/clean", {"file": _b64(data), "name": "shot.png"})
|
|
assert status == 200
|
|
assert body["kind"] == "image"
|
|
cleaned = base64.b64decode(body["cleaned"])
|
|
assert b"c2pa" not in cleaned.lower()
|
|
assert body["report"]["format"] == "png"
|
|
assert any("tEXt" in a for a in body["report"]["actions"]) or "actions" in body["report"]
|
|
|
|
|
|
def test_clean_markdown_container(conn):
|
|
data = (ROOT / "tests" / "fixtures" / "sample_ai.md").read_bytes()
|
|
status, body = _post(conn, "/clean", {"file": _b64(data), "name": "note.md"})
|
|
assert status == 200
|
|
assert body["kind"] == "container"
|
|
cleaned = base64.b64decode(body["cleaned"]).decode("utf-8")
|
|
assert "generator: Claude" not in cleaned
|
|
assert body["report"]["format"] == "markdown"
|
|
|
|
|
|
def test_unknown_option_rejected(conn):
|
|
status, body = _post(conn, "/clean", {"file": _b64(b"x"), "name": "x.txt", "options": {"nope": 1}})
|
|
assert status == 400
|
|
assert "unknown option" in body["error"]
|
|
|
|
|
|
def test_bad_base64_rejected(conn):
|
|
status, body = _post(conn, "/inspect", {"file": "!!!not-base64!!!"})
|
|
assert status == 400
|
|
|
|
|
|
def test_binary_named_as_text_rejected(conn):
|
|
status, body = _post(conn, "/clean", {"file": _b64(_watermarked_png()), "name": "x.txt"})
|
|
assert status == 400
|
|
|
|
|
|
def test_missing_file_field_rejected(conn):
|
|
status, body = _post(conn, "/inspect", {"name": "x.txt"})
|
|
assert status == 400
|
|
|
|
|
|
def test_safe_name_sanitizes_traversal():
|
|
assert server._safe_name("../../etc/passwd") == "passwd"
|
|
assert server._safe_name("a/b/c/notes.md") == "notes.md"
|
|
assert server._safe_name("..\\..\\win.txt") == "win.txt"
|
|
assert server._safe_name("..") == "input"
|
|
assert server._safe_name(".") == "input"
|
|
assert server._safe_name("") == "input"
|
|
assert server._safe_name("report.docx") == "report.docx"
|
|
|
|
|
|
def test_traversal_name_does_not_escape(conn, tmp_path):
|
|
data = "Hello\u200bWorld!".encode("utf-8")
|
|
status, body = _post(conn, "/clean", {"file": _b64(data), "name": "../../escape.txt"})
|
|
assert status == 200
|
|
assert body["kind"] == "text"
|
|
cleaned = base64.b64decode(body["cleaned"]).decode("utf-8")
|
|
assert cleaned == "HelloWorld!"
|
|
assert not (tmp_path / "escape.txt").exists()
|
|
|
|
|
|
def test_oversized_body_413(conn, monkeypatch):
|
|
monkeypatch.setattr(server, "MAX_BODY_BYTES", 64)
|
|
data = "x" * 200
|
|
status, body = _post(conn, "/inspect", {"file": _b64(data.encode())})
|
|
assert status == 413
|
|
|
|
|
|
def test_auth_required(conn, monkeypatch):
|
|
monkeypatch.setattr(server, "API_KEY", "sekret")
|
|
status, _ = _post(conn, "/health", {"anything": 1})
|
|
assert status == 401
|
|
conn.request("GET", "/health", headers={"Authorization": "Bearer sekret"})
|
|
resp = conn.getresponse()
|
|
assert resp.status == 200
|
|
resp.read()
|
|
|
|
|
|
def test_404(conn):
|
|
status, body = _get(conn, "/nope")
|
|
assert status == 404
|
|
status, body = _post(conn, "/nope", {"file": _b64(b"x")})
|
|
assert status == 404
|