diff --git a/Makefile b/Makefile index d4be3019..5837faba 100644 --- a/Makefile +++ b/Makefile @@ -167,6 +167,16 @@ fix: @uv run ruff check --fix . @find . | grep -E "(__pycache__|\.pyc|\.pyo|\.pytest_cache|\.ruff_cache|\.mypy_cache)" | xargs rm -rf +# Reflow hard-wrapped markdown prose to one line per paragraph (docs style). +.PHONY: reflow-docs +reflow-docs: + uv run python scripts/reflow_md.py --apply + +# CI guard: fail if any in-scope doc has hard-wrapped prose. Wired into `quality`. +.PHONY: reflow-check +reflow-check: + uv run python scripts/reflow_md.py --check + # Find dead code with Vulture .PHONY: vulture vulture: @@ -240,6 +250,8 @@ quality: @uv run ruff format --check . @echo "==> ruff check" @uv run ruff check . + @echo "==> markdown prose (no hard-wrapping)" + @uv run python scripts/reflow_md.py --check @echo "==> mypy" @uv run mypy roboco/ tests/ @echo "==> pytest with coverage" diff --git a/scripts/reflow_md.py b/scripts/reflow_md.py new file mode 100644 index 00000000..b0d11f30 --- /dev/null +++ b/scripts/reflow_md.py @@ -0,0 +1,213 @@ +#!/usr/bin/env python3 +"""Reflow hard-wrapped markdown prose to one line per paragraph. + +Markdown and editors soft-wrap on their own, so manual mid-paragraph line +breaks (the ~75-char "wrap") add nothing. This joins wrapped lines within plain +paragraphs, list items, and (lazily) their continuations, and leaves code +fences, tables, headings, thematic breaks, HTML blocks, indented code, +blockquotes, and front matter byte-identical. + +Safety invariant: the non-whitespace token sequence of a file must be unchanged +(a reflow only edits whitespace). Any file that would fail the invariant is left +untouched and reported — the tool never risks losing or scrambling content. + +Usage: + python scripts/reflow_md.py # dry run: report what would change + python scripts/reflow_md.py --apply # rewrite files in place + python scripts/reflow_md.py --check # exit 1 if any in-scope file is wrapped +""" + +from __future__ import annotations + +import re +import sys +from pathlib import Path + +ROOT = Path() +SKIP_DIRS = { + ".OLD", + "node_modules", + ".venv", + ".next", + "dist", + ".git", + "__pycache__", + ".mypy_cache", + ".ruff_cache", + ".pytest_cache", +} +# Excluded subtrees / files: archive, generated (regenerated by `make lifecycle` +# — never hand-edit), gitignored internal notes, untracked working specs. +EXCLUDE_PREFIXES = ("docs/internal/", "agents/prompts/_generated/") +EXCLUDE_GLOBS = ("docs/SPEC_",) + +FENCE = re.compile(r"^(\s*)(`{3,}|~{3,})") +FENCE_CLOSE = re.compile(r"^\s*(`{3,}|~{3,})\s*$") +HEADING = re.compile(r"^\s*#{1,6}(\s|$)") +LIST_ITEM = re.compile(r"^(\s*)([-*+]|\d+[.)])\s+") +THEMATIC = re.compile(r"^\s*([-*_])(\s*\1){2,}\s*$") +SETEXT = re.compile(r"^\s*(=+|-+)\s*$") + + +def _is_indented_code(line: str) -> bool: + return line[:4] == " " and not LIST_ITEM.match(line) + + +def _is_passthrough(line: str) -> bool: + """A line copied verbatim and never joined into prose. + + Headings, code fences, tables/HTML/blockquotes (`|`, `<`, `>`), + thematic/setext rules, and indented code each stay on their own line. + """ + s = line.strip() + return bool( + HEADING.match(line) + or FENCE.match(line) + or s.startswith(("|", "<", ">")) + or THEMATIC.match(line) + or SETEXT.match(line) + or _is_indented_code(line) + ) + + +def _is_block_break(line: str) -> bool: + """A blank line or any verbatim block construct (ends a prose/list run).""" + return line.strip() == "" or _is_passthrough(line) + + +def _copy_fence(lines: list[str], i: int, out: list[str]) -> int: + """Copy a fenced code block verbatim through its closing fence.""" + out.append(lines[i]) + i += 1 + while i < len(lines): + out.append(lines[i]) + closed = FENCE_CLOSE.match(lines[i]) + i += 1 + if closed: + break + return i + + +def _join_list(lines: list[str], i: int, out: list[str]) -> int: + """Reflow a list block: one line per item, joining lazy continuations.""" + while ( + i < len(lines) and not _is_block_break(lines[i]) and LIST_ITEM.match(lines[i]) + ): + item = [lines[i]] + i += 1 + while ( + i < len(lines) + and lines[i].strip() != "" + and not LIST_ITEM.match(lines[i]) + and not _is_block_break(lines[i]) + ): + item.append(lines[i].strip()) + i += 1 + out.append( + " ".join(p.rstrip() if k == 0 else p.strip() for k, p in enumerate(item)) + ) + return i + + +def _join_paragraph(lines: list[str], i: int, out: list[str]) -> int: + """Join a plain paragraph's lines into one.""" + para: list[str] = [] + while ( + i < len(lines) + and not _is_block_break(lines[i]) + and not LIST_ITEM.match(lines[i]) + ): + para.append(lines[i].strip()) + i += 1 + out.append(" ".join(para)) + return i + + +def _skip_front_matter(lines: list[str], out: list[str]) -> int: + """Copy a leading `--- ... ---` front-matter block verbatim; return next i.""" + if not (lines and lines[0].strip() == "---"): + return 0 + out.append(lines[0]) + i = 1 + while i < len(lines) and lines[i].strip() != "---": + out.append(lines[i]) + i += 1 + if i < len(lines): + out.append(lines[i]) + i += 1 + return i + + +def reflow(text: str) -> str: + lines = text.split("\n") + out: list[str] = [] + i = _skip_front_matter(lines, out) + while i < len(lines): + line = lines[i] + if line.strip() == "": + out.append(line) + i += 1 + elif FENCE.match(line): + i = _copy_fence(lines, i, out) + elif _is_passthrough(line): + out.append(line) + i += 1 + elif LIST_ITEM.match(line): + i = _join_list(lines, i, out) + else: + i = _join_paragraph(lines, i, out) + return "\n".join(out) + + +def in_scope(p: Path) -> bool: + if any(part in SKIP_DIRS for part in p.parts): + return False + rel = p.as_posix() + if any(rel.startswith(pre) for pre in EXCLUDE_PREFIXES): + return False + return not any(g in rel for g in EXCLUDE_GLOBS) + + +def main() -> None: + args = sys.argv[1:] + apply = "--apply" in args + check = "--check" in args + + changed: list[Path] = [] + skipped: list[Path] = [] + for p in sorted(ROOT.rglob("*.md")): + if not in_scope(p): + continue + orig = p.read_text() + new = reflow(orig) + if new == orig: + continue + if orig.split() != new.split(): # token invariant — never touch + skipped.append(p) + continue + changed.append(p) + if apply: + p.write_text(new) + + if check: + for p in skipped: + print(f"warning: skipped (token-invariant): {p}") + if changed: + print(f"{len(changed)} file(s) have hard-wrapped prose. Fix with:") + print(" make reflow-docs") + for p in changed: + print(f" {p}") + sys.exit(1) + print("OK: no hard-wrapped markdown prose in scope.") + return + + verb = "reflowed" if apply else "would reflow" + print(f"{verb}: {len(changed)} skipped (token-invariant): {len(skipped)}") + for p in changed: + print(f" {p}") + for p in skipped: + print(f" !! skipped {p}") + + +if __name__ == "__main__": + main()