mirror of
https://github.com/guillaumemeyer/watermarks-remover.git
synced 2026-08-22 13:11:57 +02:00
feat: add a lightweight Cursor text skill (#35)
* feat: add lightweight Cursor and Codex text skill Package the text-only workflow with safe cross-platform installation, optional persistent instructions, and focused tests so users can adopt it without the media tooling. Co-authored-by: Cursor <cursoragent@cursor.com> * test: keep the lightweight skill independently reviewable Avoid coupling the packaging PR to the separate Unicode safety change so either pull request can merge on its own. Co-authored-by: Cursor <cursoragent@cursor.com> * test: force UTF-8 for subprocess stdin in Windows CI --------- Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Guillaume Meyer (The Opinionated Man) <1385518+guillaumemeyer@users.noreply.github.com>
This commit is contained in:
Zhaohan Wang
co-authored by
Cursor
Guillaume Meyer
Cursor
Guillaume Meyer
parent
c7200d20c2
commit
8d79155ad2
@@ -3,7 +3,7 @@
|
||||
smoke-markllm bootstrap-markllm docker-markllm-build docker-markllm-help \
|
||||
smoke-markdiffusion bootstrap-markdiffusion docker-markdiffusion-build docker-markdiffusion-help \
|
||||
docker-core-build docker-core-help serve compose-up compose-up-heavy compose-check \
|
||||
install-skill clean
|
||||
install-skill install-cursor-text-skill clean
|
||||
|
||||
SCRIPTS := service/scripts
|
||||
PYTHON ?= $(shell if [ -x .venv/bin/python ]; then echo .venv/bin/python; else echo python3; fi)
|
||||
@@ -110,6 +110,9 @@ install-skill:
|
||||
ln -sfn $(CURDIR)/skills/remove-ai-marks $(HOME)/.grok/skills/remove-ai-marks
|
||||
@echo "linked -> $(HOME)/.grok/skills/remove-ai-marks"
|
||||
|
||||
install-cursor-text-skill:
|
||||
$(PYTHON) install_skill.py
|
||||
|
||||
clean:
|
||||
find . -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true
|
||||
rm -rf .pytest_cache .venv
|
||||
|
||||
@@ -45,6 +45,36 @@ ln -sfn "$(pwd)/skills/remove-ai-marks" ~/.grok/skills/remove-ai-marks
|
||||
|
||||
Invoke with `/remove-ai-marks` or ask to “strip AI watermarks / C2PA / Claude marks / SynthID-class text.”
|
||||
|
||||
### Optional Cursor text-only skill
|
||||
|
||||
[`skills/clean-user-facing-text/`](skills/clean-user-facing-text/) is a
|
||||
self-contained Cursor skill for authorized manuscripts, documentation, and web
|
||||
copy. It excludes image, C2PA, service, and external-model tooling.
|
||||
|
||||
Install it into `~/.cursor/skills/clean-user-facing-text`:
|
||||
|
||||
```bash
|
||||
python3 install_skill.py
|
||||
```
|
||||
|
||||
On Windows, use `py install_skill.py`. The `install-skill.sh` wrapper is
|
||||
provided for macOS/Linux shells. Existing installations are preserved unless
|
||||
you pass `--force`; replacement is staged first and the previous install is
|
||||
kept as a uniquely named backup.
|
||||
|
||||
Skill invocation is model-selected. Projects that explicitly adopt this
|
||||
workflow can also copy the optional rule:
|
||||
|
||||
```bash
|
||||
mkdir -p /path/to/project/.cursor/rules
|
||||
cp integrations/cursor/clean-user-facing-text.mdc \
|
||||
/path/to/project/.cursor/rules/clean-user-facing-text.mdc
|
||||
```
|
||||
|
||||
For all projects, put the same instruction in Cursor **User Rules** instead.
|
||||
Rules improve consistency but remain model instructions; Cursor does not expose
|
||||
a deterministic pre-send filter for final chat responses.
|
||||
|
||||
### Start the service
|
||||
|
||||
The fastest path is a local HTTP server (Python 3.10+ stdlib only — no deps, no Docker):
|
||||
|
||||
Executable
+5
@@ -0,0 +1,5 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
exec python3 "$ROOT/install_skill.py" "$@"
|
||||
@@ -0,0 +1,102 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Install the lightweight text skill for Cursor."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
SKILL_NAME = "clean-user-facing-text"
|
||||
ROOT = Path(__file__).resolve().parent
|
||||
SOURCE = ROOT / "skills" / SKILL_NAME
|
||||
|
||||
|
||||
def _present(path: Path) -> bool:
|
||||
return path.exists() or path.is_symlink()
|
||||
|
||||
|
||||
def _remove_path(path: Path) -> None:
|
||||
if path.is_symlink() or path.is_file():
|
||||
path.unlink()
|
||||
elif path.exists():
|
||||
shutil.rmtree(path)
|
||||
|
||||
|
||||
def _stage(skills_dir: Path) -> tuple[Path, Path]:
|
||||
skills_dir.mkdir(parents=True, exist_ok=True)
|
||||
staging_root = Path(
|
||||
tempfile.mkdtemp(prefix=f".{SKILL_NAME}.staging.", dir=skills_dir)
|
||||
)
|
||||
staged_skill = staging_root / SKILL_NAME
|
||||
try:
|
||||
shutil.copytree(SOURCE, staged_skill)
|
||||
if not (staged_skill / "SKILL.md").is_file():
|
||||
raise RuntimeError("staged skill is missing SKILL.md")
|
||||
except BaseException:
|
||||
shutil.rmtree(staging_root, ignore_errors=True)
|
||||
raise
|
||||
return staging_root, staged_skill
|
||||
|
||||
|
||||
def _install(destination: Path, force: bool) -> tuple[bool, Path | None]:
|
||||
if _present(destination) and not force:
|
||||
print(f"already exists: {destination}", file=sys.stderr)
|
||||
print(
|
||||
"No changes made. Re-run with --force to back up and replace.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return False, None
|
||||
|
||||
staging_root, staged_skill = _stage(destination.parent)
|
||||
backup: Path | None = None
|
||||
try:
|
||||
if _present(destination):
|
||||
backup = destination.with_name(
|
||||
f"{destination.name}.backup.{uuid.uuid4().hex[:12]}"
|
||||
)
|
||||
os.replace(destination, backup)
|
||||
try:
|
||||
os.replace(staged_skill, destination)
|
||||
except BaseException:
|
||||
if backup is not None and not _present(destination):
|
||||
os.replace(backup, destination)
|
||||
raise
|
||||
finally:
|
||||
shutil.rmtree(staging_root, ignore_errors=True)
|
||||
|
||||
return True, backup
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--force",
|
||||
action="store_true",
|
||||
help="Back up and replace an existing Cursor installation",
|
||||
)
|
||||
parser.add_argument("--cursor-home", help="Override Cursor home (default: ~/.cursor)")
|
||||
args = parser.parse_args()
|
||||
|
||||
cursor_home = Path(
|
||||
args.cursor_home
|
||||
or os.environ.get("CURSOR_HOME", Path.home() / ".cursor")
|
||||
).expanduser()
|
||||
destination = cursor_home / "skills" / SKILL_NAME
|
||||
|
||||
installed, backup = _install(destination, args.force)
|
||||
if not installed:
|
||||
return 1
|
||||
if backup is not None:
|
||||
print(f"Cursor: backed up existing skill to {backup}")
|
||||
print(f"Cursor: installed {destination}")
|
||||
print("Start a new Cursor session if the skill does not appear automatically.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,15 @@
|
||||
---
|
||||
description: Apply text hygiene to natural-language content intended for readers
|
||||
alwaysApply: true
|
||||
---
|
||||
|
||||
# Clean user-facing text
|
||||
|
||||
Before finalizing substantial natural-language content intended for readers, use the `clean-user-facing-text` skill.
|
||||
|
||||
- Apply this only to content the user owns or is authorized to process, and preserve required disclosures.
|
||||
- Apply it to articles, papers, reports, documentation, emails, product copy, UI text, Markdown prose, and HTML prose.
|
||||
- Preserve facts, numbers, names, citations, requirements, language, tone, and formatting.
|
||||
- Do not modify fenced or inline code, commands, paths, URLs, identifiers, APIs, formulas, or verbatim quotations.
|
||||
- Treat statistical-watermark reduction as best-effort; never claim the result is certified undetectable or proves human authorship.
|
||||
- Skip the skill for code-only tasks.
|
||||
@@ -0,0 +1,64 @@
|
||||
---
|
||||
name: clean-user-facing-text
|
||||
description: Clean and finalize authorized natural-language text intended for readers by auditing suspicious invisible Unicode and rewriting prose while preserving facts and meaning. Use in Cursor when the user asks to clean, humanize, polish, or finalize articles, manuscripts, reports, documentation, emails, product copy, UI text, Markdown, or HTML prose, or when an installed Cursor Rule explicitly requires this workflow. Do not use for code-only tasks or undisclosed authorship evasion; leave code, commands, identifiers, paths, APIs, formulas, citations, required disclosures, and verbatim quotations unchanged.
|
||||
---
|
||||
|
||||
# Clean user-facing text
|
||||
|
||||
Apply a final text-hygiene pass to prose the user owns or is authorized to process. Treat Unicode cleanup as deterministic and statistical-watermark reduction as best-effort; never claim that a rewrite proves human authorship or is undetectable. Preserve required academic, legal, platform, and regulatory disclosures.
|
||||
|
||||
## Workflow
|
||||
|
||||
1. Identify the prose that readers will see.
|
||||
2. Protect non-prose spans:
|
||||
- fenced and inline code
|
||||
- commands, paths, URLs, identifiers, API names, and exact values
|
||||
- formulas, citations, and text the user asks to quote verbatim
|
||||
3. Preserve every claim, fact, number, name, citation, and requirement.
|
||||
4. Rewrite the remaining prose once:
|
||||
- vary clause order, sentence boundaries, rhythm, connectors, and function words
|
||||
- replace formulaic transitions and filler with direct, natural wording
|
||||
- preserve the requested language, tone, structure, and formatting; never translate unless asked
|
||||
- for non-English text, use fluent constructions native to that language rather than English sentence patterns
|
||||
- do not add or remove claims merely to increase variation
|
||||
5. For text artifacts or supplied text files, run the deterministic Unicode pass after rewriting.
|
||||
6. Return only the polished result unless the user asks for an audit or explanation.
|
||||
|
||||
## Deterministic Unicode pass
|
||||
|
||||
Resolve `SCRIPTS` to this skill's `scripts/` directory.
|
||||
Use the available Python 3 launcher for the platform. Replace `PYTHON` below
|
||||
with `python3` on most macOS/Linux systems, `py` on Windows, or another verified
|
||||
Python 3 command.
|
||||
|
||||
Inspect first when editing an existing file:
|
||||
|
||||
```bash
|
||||
PYTHON "$SCRIPTS/inspect_text.py" --json INPUT
|
||||
PYTHON "$SCRIPTS/clean_text.py" INPUT -o OUTPUT --stats --no-normalize-spaces
|
||||
PYTHON "$SCRIPTS/inspect_text.py" --json OUTPUT
|
||||
```
|
||||
|
||||
Use `-` for stdin. Prefer a new `*.cleaned.*` output unless the user explicitly requests in-place editing.
|
||||
|
||||
Use `--no-normalize-spaces` by default so NBSP, narrow no-break spaces, figure spaces, and CJK ideographic spaces retain their layout semantics. Normalize spaces only when the user requests it.
|
||||
|
||||
Do not use `--aggressive-homoglyphs`, `--nfkc`, or `--strip-emoji-glue` unless the user requests aggressive normalization and accepts possible changes to multilingual text, emoji, directionality, or typography.
|
||||
|
||||
The scripts support plain text, source text, Markdown, and HTML source as text. For mixed Markdown or HTML, inspect hit positions first. If a hit falls inside protected code, attributes, or another non-prose span, do not run whole-file cleanup; clean only the prose segments or leave that hit unchanged. Do not pass binary containers such as PDF, DOCX, images, or archives.
|
||||
|
||||
For a chat-only response that is not written to a file, perform the rewrite workflow directly. Do not claim that the chat response received a deterministic post-send Unicode filter.
|
||||
|
||||
## Code boundary
|
||||
|
||||
When prose and code are mixed, rewrite prose only. Never rename variables, alter string literals, reformat code, or change executable output as part of this skill. If a Markdown or HTML file contains executable snippets, preserve those spans byte-for-byte whenever practical.
|
||||
|
||||
## Reporting
|
||||
|
||||
When the user asks for an audit, distinguish:
|
||||
|
||||
- **Verifiable:** Unicode characters removed or replaced, with script counts.
|
||||
- **Best-effort:** prose was rewritten to alter token and syntax patterns.
|
||||
- **Not established:** official detector evasion, human authorship, or removal of a vendor's secret-key watermark.
|
||||
|
||||
For technical background, read `references/watermark-notes.md`. For misuse or disclosure questions, read `references/responsible-use.md`.
|
||||
@@ -0,0 +1,7 @@
|
||||
# Responsible use
|
||||
|
||||
Use this skill for content the user owns or is authorized to process, including privacy, publishing hygiene, accessibility, and research.
|
||||
|
||||
Do not present cleaned text as proof that no AI assistance occurred. Do not help misrepresent authorship or bypass disclosure requirements in academic, legal, platform, or regulatory settings.
|
||||
|
||||
When intent is ambiguous, perform ordinary writing and Unicode hygiene without making detector-evasion claims. When disclosure is required, preserve or add the appropriate disclosure.
|
||||
@@ -0,0 +1,20 @@
|
||||
# Text watermark notes
|
||||
|
||||
## Deterministic marks
|
||||
|
||||
Invisible Unicode, bidirectional controls, tag characters, exotic spaces, and selected confusables can carry machine-readable signals or cause broken copy, search, and diffs.
|
||||
|
||||
`clean_text.py` removes or normalizes known carriers and reports exact counts. By default it preserves contextual characters used by emoji, joining scripts, Mongolian selectors, Khmer inherent vowels, Hangul fillers, and Arabic/Syriac orthography. The lightweight workflow also disables space normalization by default so multilingual typography remains intact. Aggressive flags can damage intentional text and should remain opt-in.
|
||||
|
||||
## Statistical marks
|
||||
|
||||
Token-sampling watermarks such as green-list or tournament-sampling schemes live in word choice and token sequences rather than metadata. A substantial rewrite can weaken such signals by changing syntax and vocabulary.
|
||||
|
||||
This is best-effort:
|
||||
|
||||
- no bundled detector has the vendor's secret key
|
||||
- a rewrite generated by the same provider may introduce a new signal
|
||||
- short or predictable text provides little statistical evidence either way
|
||||
- detector behavior can change independently of this skill
|
||||
|
||||
Never describe a successful rewrite as certified, undetectable, or proof of human authorship.
|
||||
+84
@@ -0,0 +1,84 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Strip invisible Unicode / normalize space homoglyphs (Layer A)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
|
||||
from common import backup_path, cleaned_path, eprint, read_text_input, write_text_output # noqa: E402
|
||||
from text_unicode import clean_text # noqa: E402
|
||||
|
||||
|
||||
def main() -> int:
|
||||
p = argparse.ArgumentParser(description=__doc__)
|
||||
p.add_argument("path", nargs="?", default="-", help="Input text file, or - for stdin")
|
||||
p.add_argument("-o", "--output", help="Output path (default: stdout or *.cleaned.*)")
|
||||
p.add_argument("--nfkc", action="store_true", help="Apply Unicode NFKC after scrub")
|
||||
p.add_argument(
|
||||
"--aggressive-homoglyphs",
|
||||
action="store_true",
|
||||
help="Map Cyrillic/fullwidth Latin confusables to ASCII Latin",
|
||||
)
|
||||
p.add_argument(
|
||||
"--no-normalize-spaces",
|
||||
action="store_true",
|
||||
help="Do not rewrite exotic spaces to U+0020",
|
||||
)
|
||||
p.add_argument(
|
||||
"--strip-emoji-glue",
|
||||
action="store_true",
|
||||
help="Paranoid: strip all load-bearing invisibles too (emoji glue, script joiners, flag tags, same-script fillers/selectors, orthographic Cf)",
|
||||
)
|
||||
p.add_argument("--stats", action="store_true", help="Print stats JSON to stderr")
|
||||
p.add_argument(
|
||||
"--force-text",
|
||||
action="store_true",
|
||||
help="Clean even when the input looks like a binary container "
|
||||
"(this rewrites the bytes and will corrupt the file)",
|
||||
)
|
||||
p.add_argument(
|
||||
"--in-place",
|
||||
action="store_true",
|
||||
help="Overwrite input file (creates .bak backup)",
|
||||
)
|
||||
args = p.parse_args()
|
||||
|
||||
text = read_text_input(args.path, allow_binary=args.force_text)
|
||||
cleaned, stats = clean_text(
|
||||
text,
|
||||
nfkc=args.nfkc,
|
||||
aggressive_homoglyphs=args.aggressive_homoglyphs,
|
||||
normalize_spaces=not args.no_normalize_spaces,
|
||||
strip_emoji_glue=args.strip_emoji_glue,
|
||||
)
|
||||
|
||||
out = args.output
|
||||
if args.in_place:
|
||||
if args.path in (None, "-"):
|
||||
eprint("--in-place requires a file path")
|
||||
return 2
|
||||
src = Path(args.path)
|
||||
bak = backup_path(src)
|
||||
out = str(src)
|
||||
elif out is None and args.path not in (None, "-"):
|
||||
out = str(cleaned_path(Path(args.path)))
|
||||
|
||||
write_text_output(cleaned, out)
|
||||
|
||||
if args.stats:
|
||||
eprint(json.dumps(stats, indent=2, ensure_ascii=False))
|
||||
else:
|
||||
eprint(
|
||||
f"removed={stats['removed_count']} replaced={stats['replaced_count']} "
|
||||
f"len {stats['input_length']}->{stats['output_length']}"
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,178 @@
|
||||
"""Small, dependency-free helpers for the text-only skill."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
MAX_INPUT_BYTES = int(os.environ.get("WATERMARKS_MAX_INPUT_BYTES", str(256 << 20)))
|
||||
MAX_STDIN_BYTES = int(os.environ.get("WATERMARKS_MAX_STDIN_BYTES", str(64 << 20)))
|
||||
BINARY_SNIFF_BYTES = 8192
|
||||
|
||||
BINARY_MAGIC: tuple[tuple[bytes, str], ...] = (
|
||||
(b"PK\x03\x04", "a ZIP container such as DOCX or ODT"),
|
||||
(b"%PDF-", "a PDF"),
|
||||
(b"\x89PNG\r\n\x1a\n", "a PNG image"),
|
||||
(b"\xff\xd8\xff", "a JPEG image"),
|
||||
(b"GIF87a", "a GIF image"),
|
||||
(b"GIF89a", "a GIF image"),
|
||||
(b"RIFF", "a RIFF container"),
|
||||
(b"\x1f\x8b", "a gzip archive"),
|
||||
(b"7z\xbc\xaf\x27\x1c", "a 7-Zip archive"),
|
||||
(b"Rar!\x1a\x07", "a RAR archive"),
|
||||
(b"\x7fELF", "an ELF binary"),
|
||||
)
|
||||
|
||||
_ALLOWED_CONTROLS = frozenset({0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x1B})
|
||||
|
||||
|
||||
def eprint(*args: object) -> None:
|
||||
print(*args, file=sys.stderr)
|
||||
|
||||
|
||||
def _configure_stdio() -> None:
|
||||
for stream, errors in (
|
||||
(sys.stdin, "surrogateescape"),
|
||||
(sys.stdout, "backslashreplace"),
|
||||
(sys.stderr, "backslashreplace"),
|
||||
):
|
||||
reconfigure = getattr(stream, "reconfigure", None)
|
||||
if reconfigure is not None:
|
||||
try:
|
||||
reconfigure(encoding="utf-8", errors=errors)
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
|
||||
|
||||
_configure_stdio()
|
||||
|
||||
|
||||
def looks_binary(data: bytes) -> str | None:
|
||||
if not data:
|
||||
return None
|
||||
for magic, label in BINARY_MAGIC:
|
||||
if data.startswith(magic):
|
||||
return label
|
||||
head = data[:BINARY_SNIFF_BYTES]
|
||||
if b"\x00" in head:
|
||||
return "binary data containing NUL bytes"
|
||||
controls = sum(1 for value in head if value < 0x20 and value not in _ALLOWED_CONTROLS)
|
||||
if controls / len(head) > 0.05:
|
||||
return "binary data dense in control bytes"
|
||||
return None
|
||||
|
||||
|
||||
def guard_binary(data: bytes, origin: str, *, allow_binary: bool = False) -> None:
|
||||
if allow_binary:
|
||||
return
|
||||
kind = looks_binary(data)
|
||||
if kind is None:
|
||||
return
|
||||
eprint(f"refusing to treat {origin} as text: it looks like {kind}.")
|
||||
eprint("This lightweight skill accepts text files only.")
|
||||
raise SystemExit(2)
|
||||
|
||||
|
||||
def _read_stdin_capped(*, allow_binary: bool = False) -> str:
|
||||
stream = getattr(sys.stdin, "buffer", None)
|
||||
if stream is None:
|
||||
text = sys.stdin.read()
|
||||
data = text.encode("utf-8", errors="surrogateescape")
|
||||
if len(data) > MAX_STDIN_BYTES:
|
||||
raise SystemExit(f"refusing stdin input larger than {MAX_STDIN_BYTES} bytes")
|
||||
guard_binary(data[:BINARY_SNIFF_BYTES], "stdin", allow_binary=allow_binary)
|
||||
return text
|
||||
|
||||
chunks: list[bytes] = []
|
||||
total = 0
|
||||
while True:
|
||||
chunk = stream.read(1 << 20)
|
||||
if not chunk:
|
||||
break
|
||||
if not chunks:
|
||||
guard_binary(
|
||||
chunk[:BINARY_SNIFF_BYTES],
|
||||
"stdin",
|
||||
allow_binary=allow_binary,
|
||||
)
|
||||
total += len(chunk)
|
||||
if total > MAX_STDIN_BYTES:
|
||||
raise SystemExit(f"refusing stdin input larger than {MAX_STDIN_BYTES} bytes")
|
||||
chunks.append(chunk)
|
||||
return b"".join(chunks).decode("utf-8", errors="surrogateescape")
|
||||
|
||||
|
||||
def read_text_input(path: str | None, *, allow_binary: bool = False) -> str:
|
||||
if path is None or path == "-":
|
||||
return _read_stdin_capped(allow_binary=allow_binary)
|
||||
source = Path(path)
|
||||
if source.stat().st_size > MAX_INPUT_BYTES:
|
||||
raise SystemExit(f"refusing input larger than {MAX_INPUT_BYTES} bytes: {path}")
|
||||
data = source.read_bytes()
|
||||
guard_binary(data, str(source), allow_binary=allow_binary)
|
||||
return data.decode("utf-8", errors="surrogateescape")
|
||||
|
||||
|
||||
def _default_file_mode() -> int:
|
||||
mask = os.umask(0)
|
||||
os.umask(mask)
|
||||
return 0o666 & ~mask
|
||||
|
||||
|
||||
def safe_write_bytes(path: str | Path, data: bytes) -> None:
|
||||
destination = Path(path)
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
if destination.is_symlink():
|
||||
raise OSError(f"refusing to write through symlink: {destination}")
|
||||
|
||||
fd, temporary = tempfile.mkstemp(
|
||||
prefix=f".{destination.name}.",
|
||||
suffix=".tmp",
|
||||
dir=str(destination.parent),
|
||||
)
|
||||
try:
|
||||
if hasattr(os, "fchmod"):
|
||||
os.fchmod(fd, _default_file_mode())
|
||||
with os.fdopen(fd, "wb") as handle:
|
||||
handle.write(data)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.replace(temporary, destination)
|
||||
except BaseException:
|
||||
try:
|
||||
os.unlink(temporary)
|
||||
except OSError:
|
||||
pass
|
||||
raise
|
||||
|
||||
|
||||
def write_text_output(text: str, path: str | None) -> None:
|
||||
if path is None or path == "-":
|
||||
sys.stdout.write(text)
|
||||
if text and not text.endswith("\n"):
|
||||
sys.stdout.write("\n")
|
||||
return
|
||||
safe_write_bytes(path, text.encode("utf-8", errors="surrogateescape"))
|
||||
|
||||
|
||||
def backup_path(source: Path) -> Path:
|
||||
backup = source.with_suffix(source.suffix + ".bak")
|
||||
try:
|
||||
safe_write_bytes(backup, source.read_bytes())
|
||||
except OSError as error:
|
||||
eprint(f"cannot create backup {backup}: {error}")
|
||||
raise SystemExit(2) from error
|
||||
return backup
|
||||
|
||||
|
||||
def emit_json(data: Any) -> None:
|
||||
json.dump(data, sys.stdout, indent=2, ensure_ascii=False)
|
||||
sys.stdout.write("\n")
|
||||
|
||||
|
||||
def cleaned_path(source: Path, suffix: str = ".cleaned") -> Path:
|
||||
return source.with_name(f"{source.stem}{suffix}{source.suffix}")
|
||||
+52
@@ -0,0 +1,52 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Inspect text for invisible Unicode / space homoglyphs (Layer A)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Allow running as script from any cwd
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
|
||||
from common import emit_json, read_text_input # noqa: E402
|
||||
from text_unicode import human_report, inspect_text # noqa: E402
|
||||
|
||||
|
||||
def main() -> int:
|
||||
p = argparse.ArgumentParser(description=__doc__)
|
||||
p.add_argument("path", nargs="?", default="-", help="Text file path, or - for stdin")
|
||||
p.add_argument("--json", action="store_true", help="JSON report")
|
||||
p.add_argument(
|
||||
"--aggressive",
|
||||
action="store_true",
|
||||
help="Also flag Latin confusable / fullwidth lookalikes",
|
||||
)
|
||||
p.add_argument(
|
||||
"--strip-emoji-glue",
|
||||
action="store_true",
|
||||
help="Paranoid: flag all load-bearing invisibles too (emoji glue, script joiners, flag tags, same-script fillers/selectors, orthographic Cf)",
|
||||
)
|
||||
p.add_argument(
|
||||
"--force-text",
|
||||
action="store_true",
|
||||
help="Scan even when the input looks like a binary container",
|
||||
)
|
||||
args = p.parse_args()
|
||||
|
||||
text = read_text_input(args.path, allow_binary=args.force_text)
|
||||
report = inspect_text(
|
||||
text,
|
||||
aggressive=args.aggressive,
|
||||
strip_emoji_glue=args.strip_emoji_glue,
|
||||
)
|
||||
if args.json:
|
||||
emit_json(report.to_dict())
|
||||
else:
|
||||
print(human_report(report))
|
||||
return 0 if report.suspicious_total == 0 else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
+521
@@ -0,0 +1,521 @@
|
||||
"""Layer A: invisible Unicode / homoglyph space detection and cleaning."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import unicodedata
|
||||
from collections import Counter
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
# Format / invisible controls commonly used for steganography or broken pastes.
|
||||
STRIP_CODEPOINTS: frozenset[int] = frozenset(
|
||||
{
|
||||
0x00AD, # soft hyphen
|
||||
0x034F, # combining grapheme joiner
|
||||
0x061C, # Arabic letter mark
|
||||
0x115F, # Hangul choseong filler
|
||||
0x1160, # Hangul jungseong filler
|
||||
0x17B4, # Khmer vowel inherent AQ
|
||||
0x17B5, # Khmer vowel inherent AA
|
||||
0x180B, # Mongolian free variation selector-1
|
||||
0x180C,
|
||||
0x180D,
|
||||
0x180E, # Mongolian vowel separator
|
||||
0x200B, # zero width space
|
||||
0x200C, # zero width non-joiner
|
||||
0x200D, # zero width joiner
|
||||
0x200E, # LRM
|
||||
0x200F, # RLM
|
||||
0x202A, # LRE
|
||||
0x202B, # RLE
|
||||
0x202C, # PDF
|
||||
0x202D, # LRO
|
||||
0x202E, # RLO
|
||||
0x2060, # word joiner
|
||||
0x2061, # function application
|
||||
0x2062, # invisible times
|
||||
0x2063, # invisible separator
|
||||
0x2064, # invisible plus
|
||||
0x2066, # LRI
|
||||
0x2067, # RLI
|
||||
0x2068, # FSI
|
||||
0x2069, # PDI
|
||||
0x206A, # inhibit symmetric swapping
|
||||
0x206B,
|
||||
0x206C,
|
||||
0x206D,
|
||||
0x206E,
|
||||
0x206F,
|
||||
0xFEFF, # BOM / ZWNBSP
|
||||
0xFE00, # variation selectors
|
||||
0xFE01,
|
||||
0xFE02,
|
||||
0xFE03,
|
||||
0xFE04,
|
||||
0xFE05,
|
||||
0xFE06,
|
||||
0xFE07,
|
||||
0xFE08,
|
||||
0xFE09,
|
||||
0xFE0A,
|
||||
0xFE0B,
|
||||
0xFE0C,
|
||||
0xFE0D,
|
||||
0xFE0E,
|
||||
0xFE0F,
|
||||
0xFFF9, # interlinear annotation
|
||||
0xFFFA,
|
||||
0xFFFB,
|
||||
}
|
||||
)
|
||||
|
||||
# Spaces that look like (or substitute for) U+0020.
|
||||
SPACE_HOMOGLYPHS: dict[int, str] = {
|
||||
0x00A0: " ", # no-break space
|
||||
0x1680: " ", # Ogham space mark
|
||||
0x2000: " ", # en quad
|
||||
0x2001: " ", # em quad
|
||||
0x2002: " ", # en space
|
||||
0x2003: " ", # em space
|
||||
0x2004: " ", # three-per-em space
|
||||
0x2005: " ", # four-per-em space
|
||||
0x2006: " ", # six-per-em space
|
||||
0x2007: " ", # figure space
|
||||
0x2008: " ", # punctuation space
|
||||
0x2009: " ", # thin space
|
||||
0x200A: " ", # hair space
|
||||
0x202F: " ", # narrow no-break space
|
||||
0x205F: " ", # medium mathematical space
|
||||
0x3000: " ", # ideographic space
|
||||
}
|
||||
|
||||
# Optional confusable Latin lookalikes (aggressive mode only).
|
||||
LATIN_CONFUSABLES: dict[int, str] = {
|
||||
0x0410: "A", # Cyrillic
|
||||
0x0412: "B",
|
||||
0x0415: "E",
|
||||
0x041A: "K",
|
||||
0x041C: "M",
|
||||
0x041D: "H",
|
||||
0x041E: "O",
|
||||
0x0420: "P",
|
||||
0x0421: "C",
|
||||
0x0422: "T",
|
||||
0x0425: "X",
|
||||
0x0430: "a",
|
||||
0x0435: "e",
|
||||
0x043E: "o",
|
||||
0x0440: "p",
|
||||
0x0441: "c",
|
||||
0x0443: "y",
|
||||
0x0445: "x",
|
||||
0x0456: "i",
|
||||
0xFF21: "A", # fullwidth
|
||||
0xFF22: "B",
|
||||
0xFF23: "C",
|
||||
0xFF24: "D",
|
||||
0xFF25: "E",
|
||||
0xFF26: "F",
|
||||
0xFF27: "G",
|
||||
0xFF28: "H",
|
||||
0xFF29: "I",
|
||||
0xFF2A: "J",
|
||||
0xFF2B: "K",
|
||||
0xFF2C: "L",
|
||||
0xFF2D: "M",
|
||||
0xFF2E: "N",
|
||||
0xFF2F: "O",
|
||||
0xFF30: "P",
|
||||
0xFF31: "Q",
|
||||
0xFF32: "R",
|
||||
0xFF33: "S",
|
||||
0xFF34: "T",
|
||||
0xFF35: "U",
|
||||
0xFF36: "V",
|
||||
0xFF37: "W",
|
||||
0xFF38: "X",
|
||||
0xFF39: "Y",
|
||||
0xFF3A: "Z",
|
||||
0xFF41: "a",
|
||||
0xFF42: "b",
|
||||
0xFF43: "c",
|
||||
0xFF44: "d",
|
||||
0xFF45: "e",
|
||||
0xFF46: "f",
|
||||
0xFF47: "g",
|
||||
0xFF48: "h",
|
||||
0xFF49: "i",
|
||||
0xFF4A: "j",
|
||||
0xFF4B: "k",
|
||||
0xFF4C: "l",
|
||||
0xFF4D: "m",
|
||||
0xFF4E: "n",
|
||||
0xFF4F: "o",
|
||||
0xFF50: "p",
|
||||
0xFF51: "q",
|
||||
0xFF52: "r",
|
||||
0xFF53: "s",
|
||||
0xFF54: "t",
|
||||
0xFF55: "u",
|
||||
0xFF56: "v",
|
||||
0xFF57: "w",
|
||||
0xFF58: "x",
|
||||
0xFF59: "y",
|
||||
0xFF5A: "z",
|
||||
}
|
||||
|
||||
# Variation selectors beyond FE0x (VS17–VS256 in Supplementary Special-purpose)
|
||||
_VS_SUPPLEMENT = range(0xE0100, 0xE01F0)
|
||||
|
||||
|
||||
# Bidi / directional format controls (subset of strip set, finer inspect labels)
|
||||
_BIDI_CPS: frozenset[int] = frozenset(
|
||||
{
|
||||
0x061C,
|
||||
0x200E,
|
||||
0x200F,
|
||||
0x202A,
|
||||
0x202B,
|
||||
0x202C,
|
||||
0x202D,
|
||||
0x202E,
|
||||
0x2066,
|
||||
0x2067,
|
||||
0x2068,
|
||||
0x2069,
|
||||
}
|
||||
)
|
||||
|
||||
# Zero-width family (common edit-based carriers)
|
||||
_ZW_FAMILY: frozenset[int] = frozenset(
|
||||
{0x200B, 0x200C, 0x200D, 0x2060, 0xFEFF, 0x180E}
|
||||
)
|
||||
|
||||
|
||||
def _is_private_use(cp: int) -> bool:
|
||||
"""BMP and supplementary private-use planes (Co: no portable meaning)."""
|
||||
return 0xE000 <= cp <= 0xF8FF or 0xF0000 <= cp <= 0xFFFFD or 0x100000 <= cp <= 0x10FFFD
|
||||
|
||||
|
||||
def _is_strip_cp(cp: int) -> bool:
|
||||
if cp in STRIP_CODEPOINTS:
|
||||
return True
|
||||
if cp in _VS_SUPPLEMENT:
|
||||
return True
|
||||
# Tag characters used in some stego schemes (U+E0001–U+E007F)
|
||||
if 0xE0001 <= cp <= 0xE007F:
|
||||
return True
|
||||
if _is_private_use(cp):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _strip_kind(cp: int) -> str:
|
||||
"""Finer-grained inspect kind for strip-class codepoints."""
|
||||
if 0xE0001 <= cp <= 0xE007F:
|
||||
return "tag_chars"
|
||||
if cp in _VS_SUPPLEMENT or 0xFE00 <= cp <= 0xFE0F or 0x180B <= cp <= 0x180D:
|
||||
return "variation_selector"
|
||||
if cp in _BIDI_CPS:
|
||||
return "bidi"
|
||||
if cp in _ZW_FAMILY:
|
||||
return "zwj_family"
|
||||
if _is_private_use(cp):
|
||||
return "private_use"
|
||||
return "strip"
|
||||
|
||||
|
||||
# Emoji presentation glue: zero-width joiner and text/emoji variation
|
||||
# selectors. These are invisible carriers when free-floating, but after an
|
||||
# emoji base they are part of the visible sequence (⚖️, 👨👩👧, ❤️🔥) and
|
||||
# stripping them visibly alters the text.
|
||||
EMOJI_GLUE_CODEPOINTS: frozenset[int] = frozenset({0x200D, 0xFE0E, 0xFE0F})
|
||||
|
||||
|
||||
def _is_emoji_glue(cp: int) -> bool:
|
||||
return cp in EMOJI_GLUE_CODEPOINTS
|
||||
|
||||
|
||||
def _is_emoji_base(cp: int) -> bool:
|
||||
"""Return True for characters that can start or continue an emoji sequence."""
|
||||
if 0x1F000 <= cp <= 0x1FAFF:
|
||||
return True
|
||||
if 0x2600 <= cp <= 0x27BF: # misc symbols / dingbats / arrows
|
||||
return True
|
||||
if 0x2B00 <= cp <= 0x2BFF: # misc symbols and arrows
|
||||
return True
|
||||
if cp in (0x00A9, 0x00AE, 0x2122, 0x3030, 0x303D, 0x3297, 0x3299):
|
||||
return True
|
||||
if cp in (0x0023, 0x002A) or 0x0030 <= cp <= 0x0039: # keycap bases
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
# ZWNJ/ZWJ are orthographic inside complex scripts (Persian میروم, Devanagari
|
||||
# क्ष); flag emoji are an emoji base followed by tag chars (🏴); and a
|
||||
# handful of Cf codepoints are normal Arabic/Syriac orthography, not carriers.
|
||||
# So are Mongolian free variation selectors (choose a glyph of the preceding
|
||||
# letter), Khmer inherent vowels (invisible but phonemic), and Hangul fillers
|
||||
# (hold a jamo slot in a partial syllable). Each is only meaningful directly
|
||||
# after a base from its own script; isolated instances are contraband.
|
||||
_SCRIPT_JOINERS: frozenset[int] = frozenset({0x200C, 0x200D})
|
||||
_TAG_RANGE = range(0xE0020, 0xE0080)
|
||||
_ORTHOGRAPHIC_CF: frozenset[int] = frozenset(
|
||||
{0x0600, 0x0601, 0x0602, 0x0603, 0x0604, 0x0605, 0x06DD, 0x070F, 0x08E2, 0x110BD, 0x110CD}
|
||||
)
|
||||
_MONGOLIAN_FVS: frozenset[int] = frozenset({0x180B, 0x180C, 0x180D})
|
||||
_KHMER_VOWELS: frozenset[int] = frozenset({0x17B4, 0x17B5})
|
||||
_HANGUL_FILLERS: frozenset[int] = frozenset({0x115F, 0x1160})
|
||||
_SCRIPT_GLUE: frozenset[int] = _MONGOLIAN_FVS | _KHMER_VOWELS | _HANGUL_FILLERS
|
||||
|
||||
|
||||
def _is_joining_letter(cp: int) -> bool:
|
||||
"""Non-ASCII letter/mark — the neighbour that makes a joiner orthographic."""
|
||||
return cp > 0x7F and unicodedata.category(chr(cp))[0] in ("L", "M")
|
||||
|
||||
|
||||
def _is_mongolian_letter(cp: int) -> bool:
|
||||
return 0x1800 <= cp <= 0x18AF and unicodedata.category(chr(cp))[0] == "L"
|
||||
|
||||
|
||||
def _is_khmer_letter(cp: int) -> bool:
|
||||
return 0x1780 <= cp <= 0x17FF and unicodedata.category(chr(cp))[0] == "L"
|
||||
|
||||
|
||||
def _is_hangul_jamo(cp: int) -> bool:
|
||||
return (
|
||||
0x1100 <= cp <= 0x11FF
|
||||
or 0xA960 <= cp <= 0xA97C # Hangul Jamo Extended-A
|
||||
or 0xD7B0 <= cp <= 0xD7C6 # Hangul Jamo Extended-B
|
||||
)
|
||||
|
||||
|
||||
def _is_glue(cp: int) -> bool:
|
||||
"""Load-bearing invisible char: emoji glue, script joiner, flag tag char,
|
||||
or same-script filler/selector (Mongolian FVS, Khmer vowel, Hangul filler)."""
|
||||
return (
|
||||
_is_emoji_glue(cp)
|
||||
or cp in _SCRIPT_JOINERS
|
||||
or cp in _TAG_RANGE
|
||||
or cp in _SCRIPT_GLUE
|
||||
)
|
||||
|
||||
|
||||
def _decide(
|
||||
ch: str,
|
||||
prev_kept: str | None,
|
||||
*,
|
||||
normalize_spaces: bool,
|
||||
treat_confusables: bool,
|
||||
strip_emoji_glue: bool,
|
||||
) -> tuple[str, str, str | None]:
|
||||
"""Classify one input char for both inspect and clean.
|
||||
|
||||
Returns ``(action, out_char, kind)`` where action is ``keep``, ``strip``
|
||||
or ``replace``; out_char is the surviving character for keep/replace; and
|
||||
kind is the inspect classification (None when not suspicious).
|
||||
"""
|
||||
cp = ord(ch)
|
||||
if _is_emoji_glue(cp) and not strip_emoji_glue:
|
||||
if prev_kept is not None and _is_emoji_base(ord(prev_kept)):
|
||||
return ("keep", ch, None)
|
||||
if not strip_emoji_glue:
|
||||
if cp in _SCRIPT_JOINERS and prev_kept is not None and _is_joining_letter(ord(prev_kept)):
|
||||
return ("keep", ch, None)
|
||||
if cp in _TAG_RANGE and prev_kept is not None and _is_emoji_base(ord(prev_kept)):
|
||||
return ("keep", ch, None)
|
||||
if cp in _MONGOLIAN_FVS and prev_kept is not None and _is_mongolian_letter(ord(prev_kept)):
|
||||
return ("keep", ch, None)
|
||||
if cp in _KHMER_VOWELS and prev_kept is not None and _is_khmer_letter(ord(prev_kept)):
|
||||
return ("keep", ch, None)
|
||||
if cp in _HANGUL_FILLERS and prev_kept is not None and _is_hangul_jamo(ord(prev_kept)):
|
||||
return ("keep", ch, None)
|
||||
if cp in _ORTHOGRAPHIC_CF:
|
||||
return ("keep", ch, None)
|
||||
if _is_strip_cp(cp):
|
||||
return ("strip", "", _strip_kind(cp))
|
||||
if normalize_spaces and cp in SPACE_HOMOGLYPHS:
|
||||
return ("replace", SPACE_HOMOGLYPHS[cp], "space")
|
||||
if treat_confusables and cp in LATIN_CONFUSABLES:
|
||||
return ("replace", LATIN_CONFUSABLES[cp], "confusable")
|
||||
if unicodedata.category(ch) == "Cf" and cp not in SPACE_HOMOGLYPHS:
|
||||
return ("strip", "", "other_cf")
|
||||
return ("keep", ch, None)
|
||||
|
||||
|
||||
def _char_label(ch: str) -> str:
|
||||
cp = ord(ch)
|
||||
name = unicodedata.name(ch, "UNKNOWN")
|
||||
cat = unicodedata.category(ch)
|
||||
return f"U+{cp:04X} {name} ({cat})"
|
||||
|
||||
|
||||
def _hit_confidence(kind: str) -> str:
|
||||
"""Layer A hits are edit-based carriers; space homoglyphs are weaker context."""
|
||||
return "informational" if kind == "space" else "probable"
|
||||
|
||||
|
||||
@dataclass
|
||||
class CharHit:
|
||||
codepoint: int
|
||||
char: str
|
||||
label: str
|
||||
count: int
|
||||
kind: str # strip | bidi | tag_chars | variation_selector | zwj_family | private_use | space | confusable | other_cf
|
||||
samples: list[int] = field(default_factory=list) # character offsets
|
||||
|
||||
|
||||
@dataclass
|
||||
class TextInspectReport:
|
||||
length: int
|
||||
suspicious_total: int
|
||||
hits: list[CharHit]
|
||||
notes: list[str] = field(default_factory=list)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"length": self.length,
|
||||
"suspicious_total": self.suspicious_total,
|
||||
"hits": [
|
||||
{
|
||||
"codepoint": f"U+{h.codepoint:04X}",
|
||||
"label": h.label,
|
||||
"count": h.count,
|
||||
"kind": h.kind,
|
||||
"confidence": _hit_confidence(h.kind),
|
||||
"sample_offsets": h.samples[:10],
|
||||
}
|
||||
for h in self.hits
|
||||
],
|
||||
"notes": self.notes,
|
||||
}
|
||||
|
||||
|
||||
def inspect_text(
|
||||
text: str,
|
||||
*,
|
||||
aggressive: bool = False,
|
||||
strip_emoji_glue: bool = False,
|
||||
) -> TextInspectReport:
|
||||
buckets: dict[tuple[int, str], list[int]] = {}
|
||||
prev_kept: str | None = None
|
||||
for i, ch in enumerate(text):
|
||||
action, out_char, kind = _decide(
|
||||
ch,
|
||||
prev_kept,
|
||||
normalize_spaces=True,
|
||||
treat_confusables=aggressive,
|
||||
strip_emoji_glue=strip_emoji_glue,
|
||||
)
|
||||
if kind is None:
|
||||
# Kept; glue (emoji/script joiner/tag) does not advance the
|
||||
# "previous kept" base so ZWJ chains and flag runs stay bound.
|
||||
if not _is_glue(ord(ch)):
|
||||
prev_kept = out_char
|
||||
continue
|
||||
key = (ord(ch), kind)
|
||||
buckets.setdefault(key, []).append(i)
|
||||
if action == "replace":
|
||||
prev_kept = out_char
|
||||
# strip: prev_kept unchanged
|
||||
|
||||
hits: list[CharHit] = []
|
||||
total = 0
|
||||
for (cp, kind), offsets in sorted(buckets.items(), key=lambda x: (-len(x[1]), x[0][0])):
|
||||
ch = chr(cp)
|
||||
hits.append(
|
||||
CharHit(
|
||||
codepoint=cp,
|
||||
char=ch,
|
||||
label=_char_label(ch),
|
||||
count=len(offsets),
|
||||
kind=kind,
|
||||
samples=offsets[:10],
|
||||
)
|
||||
)
|
||||
total += len(offsets)
|
||||
|
||||
notes = [
|
||||
"Layer A only: invisible/format Unicode and space homoglyphs (edit-based carriers).",
|
||||
"Statistical (token-sampling) watermarks are not detectable here; use Layer B rewrite.",
|
||||
"Inspect kinds: strip, bidi, tag_chars, variation_selector, zwj_family, private_use, space, confusable, other_cf.",
|
||||
"Load-bearing invisibles are preserved by default: emoji glue (ZWJ/VS after an emoji base), script joiners (ZWNJ/ZWJ inside complex scripts), flag tag chars, same-script fillers/selectors (Mongolian FVS, Khmer inherent vowels, Hangul jamo fillers), and orthographic Arabic/Syriac Cf marks. Use --strip-emoji-glue for paranoid mode (strips them all).",
|
||||
]
|
||||
if not hits:
|
||||
notes.append(
|
||||
"No deterministic Layer A (invisible Unicode/format) carriers detected; "
|
||||
"statistical and pixel-domain marks are out of scope here."
|
||||
)
|
||||
return TextInspectReport(length=len(text), suspicious_total=total, hits=hits, notes=notes)
|
||||
|
||||
|
||||
def clean_text(
|
||||
text: str,
|
||||
*,
|
||||
nfkc: bool = False,
|
||||
aggressive_homoglyphs: bool = False,
|
||||
normalize_spaces: bool = True,
|
||||
strip_emoji_glue: bool = False,
|
||||
) -> tuple[str, dict]:
|
||||
"""Return cleaned text and a stats dict."""
|
||||
removed: Counter[str] = Counter()
|
||||
replaced: Counter[str] = Counter()
|
||||
out_chars: list[str] = []
|
||||
prev_kept: str | None = None
|
||||
|
||||
for ch in text:
|
||||
action, out_char, _kind = _decide(
|
||||
ch,
|
||||
prev_kept,
|
||||
normalize_spaces=normalize_spaces,
|
||||
treat_confusables=aggressive_homoglyphs,
|
||||
strip_emoji_glue=strip_emoji_glue,
|
||||
)
|
||||
if action == "keep":
|
||||
out_chars.append(out_char)
|
||||
# Glue (emoji/script joiner/tag) does not advance the "previous
|
||||
# kept" base, so ZWJ chains (❤️🔥) and flag runs stay bound.
|
||||
if not _is_glue(ord(ch)):
|
||||
prev_kept = out_char
|
||||
elif action == "replace":
|
||||
out_chars.append(out_char)
|
||||
replaced[_char_label(ch)] += 1
|
||||
prev_kept = out_char
|
||||
else: # strip
|
||||
removed[_char_label(ch)] += 1
|
||||
# prev_kept unchanged
|
||||
|
||||
result = "".join(out_chars)
|
||||
if nfkc:
|
||||
before = result
|
||||
result = unicodedata.normalize("NFKC", result)
|
||||
if result != before:
|
||||
replaced["NFKC_normalize"] += abs(len(before) - len(result)) or 1
|
||||
|
||||
# Collapse runs of spaces only if we introduced space replacements? Keep conservative: no.
|
||||
|
||||
stats = {
|
||||
"input_length": len(text),
|
||||
"output_length": len(result),
|
||||
"removed": dict(removed),
|
||||
"replaced": dict(replaced),
|
||||
"removed_count": sum(removed.values()),
|
||||
"replaced_count": sum(v for k, v in replaced.items() if k != "NFKC_normalize"),
|
||||
}
|
||||
return result, stats
|
||||
|
||||
|
||||
def human_report(report: TextInspectReport) -> str:
|
||||
lines = [
|
||||
f"Length: {report.length} chars",
|
||||
f"Suspicious: {report.suspicious_total}",
|
||||
]
|
||||
if report.hits:
|
||||
lines.append("Hits:")
|
||||
for h in report.hits:
|
||||
lines.append(
|
||||
f" [{h.kind}/{_hit_confidence(h.kind)}] "
|
||||
f"{h.label} x{h.count} @ {h.samples[:5]}"
|
||||
)
|
||||
for n in report.notes:
|
||||
lines.append(f"Note: {n}")
|
||||
return "\n".join(lines)
|
||||
@@ -0,0 +1,76 @@
|
||||
"""Smoke tests for the standalone Cursor text skill."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SKILL = ROOT / "skills" / "clean-user-facing-text"
|
||||
|
||||
|
||||
def test_lightweight_clean_text_cli():
|
||||
result = subprocess.run(
|
||||
[sys.executable, str(SKILL / "scripts" / "clean_text.py"), "-", "--stats"],
|
||||
input="Hello\u200b world\u3000again",
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
capture_output=True,
|
||||
check=True,
|
||||
)
|
||||
|
||||
assert result.stdout.rstrip("\n") == "Hello world again"
|
||||
assert '"removed_count": 1' in result.stderr
|
||||
assert '"replaced_count": 1' in result.stderr
|
||||
|
||||
|
||||
def test_lightweight_skill_has_no_template_placeholders():
|
||||
skill_text = (SKILL / "SKILL.md").read_text(encoding="utf-8")
|
||||
|
||||
assert "TODO" not in skill_text
|
||||
assert (SKILL / "references" / "watermark-notes.md").is_file()
|
||||
|
||||
|
||||
def _run_installer(home: Path, *args: str, check: bool = True):
|
||||
env = os.environ.copy()
|
||||
env.pop("CURSOR_HOME", None)
|
||||
env.update({"HOME": str(home), "USERPROFILE": str(home)})
|
||||
return subprocess.run(
|
||||
[sys.executable, str(ROOT / "install_skill.py"), *args],
|
||||
env=env,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=check,
|
||||
)
|
||||
|
||||
|
||||
def test_installer_uses_cursor_skill_location(tmp_path):
|
||||
_run_installer(tmp_path)
|
||||
|
||||
assert (tmp_path / ".cursor" / "skills" / SKILL.name / "SKILL.md").is_file()
|
||||
|
||||
|
||||
def test_installer_preserves_existing_install_without_force(tmp_path):
|
||||
cursor_install = tmp_path / ".cursor" / "skills" / SKILL.name
|
||||
cursor_install.mkdir(parents=True)
|
||||
(cursor_install / "sentinel").write_text("keep", encoding="utf-8")
|
||||
|
||||
result = _run_installer(tmp_path, check=False)
|
||||
|
||||
assert result.returncode == 1
|
||||
assert (cursor_install / "sentinel").read_text(encoding="utf-8") == "keep"
|
||||
|
||||
|
||||
def test_installer_force_creates_backup_and_replaces(tmp_path):
|
||||
destination = tmp_path / ".cursor" / "skills" / SKILL.name
|
||||
destination.mkdir(parents=True)
|
||||
(destination / "old").write_text("old", encoding="utf-8")
|
||||
|
||||
_run_installer(tmp_path, "--force")
|
||||
|
||||
backups = list(destination.parent.glob(f"{SKILL.name}.backup.*"))
|
||||
assert len(backups) == 1
|
||||
assert (backups[0] / "old").read_text(encoding="utf-8") == "old"
|
||||
assert (destination / "SKILL.md").is_file()
|
||||
Reference in New Issue
Block a user