roboco/scripts/reflow_md.py

#!/usr/bin/env python3
"""Reflow hard-wrapped markdown prose to one line per paragraph.

Markdown and editors soft-wrap on their own, so manual mid-paragraph line
breaks (the ~75-char "wrap") add nothing. This joins wrapped lines within plain
paragraphs, list items, and (lazily) their continuations, and leaves code
fences, tables, headings, thematic breaks, HTML blocks, indented code,
blockquotes, and front matter byte-identical.

Safety invariant: the non-whitespace token sequence of a file must be unchanged
(a reflow only edits whitespace). Any file that would fail the invariant is left
untouched and reported — the tool never risks losing or scrambling content.

Usage:
    python scripts/reflow_md.py            # dry run: report what would change
    python scripts/reflow_md.py --apply    # rewrite files in place
    python scripts/reflow_md.py --check    # exit 1 if any in-scope file is wrapped
"""

from __future__ import annotations

import re
import sys
from pathlib import Path

ROOT = Path()
SKIP_DIRS = {
    ".OLD",
    "node_modules",
    ".venv",
    ".uv-cache",
    ".claude",
    ".next",
    "dist",
    ".git",
    "__pycache__",
    ".mypy_cache",
    ".ruff_cache",
    ".pytest_cache",
    ".superpowers",
}
# Excluded subtrees / files: archive, generated (regenerated by `make lifecycle`
# — never hand-edit), gitignored internal notes, untracked working specs.
EXCLUDE_PREFIXES = ("docs/internal/", "agents/prompts/_generated/")
EXCLUDE_GLOBS = ("docs/SPEC_",)

FENCE = re.compile(r"^(\s*)(`{3,}|~{3,})")
FENCE_CLOSE = re.compile(r"^\s*(`{3,}|~{3,})\s*$")
HEADING = re.compile(r"^\s*#{1,6}(\s|$)")
LIST_ITEM = re.compile(r"^(\s*)([-*+]|\d+[.)])\s+")
THEMATIC = re.compile(r"^\s*([-*_])(\s*\1){2,}\s*$")
SETEXT = re.compile(r"^\s*(=+|-+)\s*$")


def _is_indented_code(line: str) -> bool:
    return line[:4] == "    " and not LIST_ITEM.match(line)


def _is_passthrough(line: str) -> bool:
    """A line copied verbatim and never joined into prose.

    Headings, code fences, tables/HTML/blockquotes (`|`, `<`, `>`),
    thematic/setext rules, and indented code each stay on their own line.
    """
    s = line.strip()
    return bool(
        HEADING.match(line)
        or FENCE.match(line)
        or s.startswith(("|", "<", ">"))
        or THEMATIC.match(line)
        or SETEXT.match(line)
        or _is_indented_code(line)
    )


def _is_block_break(line: str) -> bool:
    """A blank line or any verbatim block construct (ends a prose/list run)."""
    return line.strip() == "" or _is_passthrough(line)


def _copy_fence(lines: list[str], i: int, out: list[str]) -> int:
    """Copy a fenced code block verbatim through its closing fence."""
    out.append(lines[i])
    i += 1
    while i < len(lines):
        out.append(lines[i])
        closed = FENCE_CLOSE.match(lines[i])
        i += 1
        if closed:
            break
    return i


def _join_list(lines: list[str], i: int, out: list[str]) -> int:
    """Reflow a list block: one line per item, joining lazy continuations."""
    while (
        i < len(lines) and not _is_block_break(lines[i]) and LIST_ITEM.match(lines[i])
    ):
        item = [lines[i]]
        i += 1
        while (
            i < len(lines)
            and lines[i].strip() != ""
            and not LIST_ITEM.match(lines[i])
            and not _is_block_break(lines[i])
        ):
            item.append(lines[i].strip())
            i += 1
        out.append(
            " ".join(p.rstrip() if k == 0 else p.strip() for k, p in enumerate(item))
        )
    return i


def _join_paragraph(lines: list[str], i: int, out: list[str]) -> int:
    """Join a plain paragraph's lines into one."""
    para: list[str] = []
    while (
        i < len(lines)
        and not _is_block_break(lines[i])
        and not LIST_ITEM.match(lines[i])
    ):
        para.append(lines[i].strip())
        i += 1
    out.append(" ".join(para))
    return i


def _skip_front_matter(lines: list[str], out: list[str]) -> int:
    """Copy a leading `--- ... ---` front-matter block verbatim; return next i."""
    if not (lines and lines[0].strip() == "---"):
        return 0
    out.append(lines[0])
    i = 1
    while i < len(lines) and lines[i].strip() != "---":
        out.append(lines[i])
        i += 1
    if i < len(lines):
        out.append(lines[i])
        i += 1
    return i


def reflow(text: str) -> str:
    lines = text.split("\n")
    out: list[str] = []
    i = _skip_front_matter(lines, out)
    while i < len(lines):
        line = lines[i]
        if line.strip() == "":
            out.append(line)
            i += 1
        elif FENCE.match(line):
            i = _copy_fence(lines, i, out)
        elif _is_passthrough(line):
            out.append(line)
            i += 1
        elif LIST_ITEM.match(line):
            i = _join_list(lines, i, out)
        else:
            i = _join_paragraph(lines, i, out)
    return "\n".join(out)


def in_scope(p: Path) -> bool:
    if any(part in SKIP_DIRS for part in p.parts):
        return False
    rel = p.as_posix()
    if any(rel.startswith(pre) for pre in EXCLUDE_PREFIXES):
        return False
    return not any(g in rel for g in EXCLUDE_GLOBS)


def main() -> None:
    args = sys.argv[1:]
    apply = "--apply" in args
    check = "--check" in args

    changed: list[Path] = []
    skipped: list[Path] = []
    for p in sorted(ROOT.rglob("*.md")):
        if not in_scope(p):
            continue
        orig = p.read_text()
        new = reflow(orig)
        if new == orig:
            continue
        if orig.split() != new.split():  # token invariant — never touch
            skipped.append(p)
            continue
        changed.append(p)
        if apply:
            p.write_text(new)

    if check:
        for p in skipped:
            print(f"warning: skipped (token-invariant): {p}")
        if changed:
            print(f"{len(changed)} file(s) have hard-wrapped prose. Fix with:")
            print("   make reflow-docs")
            for p in changed:
                print(f"   {p}")
            sys.exit(1)
        print("OK: no hard-wrapped markdown prose in scope.")
        return

    verb = "reflowed" if apply else "would reflow"
    print(f"{verb}: {len(changed)}   skipped (token-invariant): {len(skipped)}")
    for p in changed:
        print(f"   {p}")
    for p in skipped:
        print(f"   !! skipped {p}")


if __name__ == "__main__":
    main()