External/contextzip
1
0
Fork 0
mirror of https://github.com/akadeepesh/contextzip.git synced 2026-10-20 08:36:20 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat(redact): always redact secret files included via always_include

Browse Source 
.env files keep keys and comments but every value becomes [REDACTED];
other secret files get the pattern scan; binary ones are left out.
Applies regardless of limits.redact_secrets, including watch's debug ZIP.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
akadeepesh
2026-10-19 04:44:37 +05:30
co-authored by Claude Opus 5.5
parent 98d89eefeb
commit 842e689175
3 changed files with 213 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
contextzip/packager.py
+31 -2
View File
@@ -77,7 +77,7 @@ from rich.progress import (
)
from contextzip.filters import ResolveResult
from contextzip.redact import redact_secrets
from contextzip.redact import is_sensitive_path, redact_secrets, redact_sensitive_file
# Subdirectory of the workspace where generated ZIPs are written.
_OUTPUT_DIRNAME = "output"
@@ -161,7 +161,8 @@ class PackageResult:
skipped_in_zip: list[tuple[Path, str]] = field(default_factory=list)
# Files where at least one secret-shaped value was redacted before
# writing — (path relative to project root, pattern names matched).
# Always empty unless limits.redact_secrets is enabled.
# Empty unless limits.redact_secrets is enabled or a secret file such
# as .env was deliberately included (those are always redacted).
redacted: list[tuple[Path, list[str]]] = field(default_factory=list)
@property
@@ -184,6 +185,14 @@ class PackageResult:
# ---------------------------------------------------------------------------
class UnredactableSecretFile(OSError):
"""A deliberately included secret file that can't be redacted (not text).
An OSError so every caller's existing `except OSError` records it as a
skipped file with this message as the reason.
"""
def _write_member(
zf: zipfile.ZipFile,
abs_path: Path,
@@ -202,10 +211,30 @@ def _write_member(
that fails to decode as UTF-8, are written unmodified via the same
fast zf.write() path used when redaction is off entirely.
Secret files (rules/base.py's SECRET_PATTERNS — only here because a
project's `always_include` brought them back on purpose) are always
redacted, whatever *redact_enabled* says and whatever their size — see
redact.redact_sensitive_file. One that isn't UTF-8 text can't be
redacted, so it raises UnredactableSecretFile and is left out.
Returns the list of pattern names redacted (empty if none matched, or
if this file wasn't eligible for scanning). May raise OSError, exactly
as the old plain zf.write() call did — callers already handle that.
"""
if is_sensitive_path(rel):
data = abs_path.read_bytes()
try:
if abs_path in binary_paths or b"\x00" in data:
raise UnicodeDecodeError("utf-8", data, 0, 0, "binary")
text = data.decode("utf-8")
except UnicodeDecodeError:
raise UnredactableSecretFile(
"secret file that isn't text can't be redacted, so it was left out"
) from None
redacted_text, matched = redact_sensitive_file(rel, text)
zf.writestr(rel.as_posix(), redacted_text.encode("utf-8"))
return matched
if not redact_enabled or abs_path in binary_paths or abs_path in large_paths:
zf.write(abs_path, arcname=rel.as_posix())
return []
contextzip/redact.py
+174 -12
View File
@@ -7,14 +7,27 @@ enforced by the packaging logic — see the 0.4.0 changelog's "known
limitation."
This is deliberately a *second line of defense*, not a replacement for the
hard secrets/credentials exclusion list in packager.py's universal baseline.
That list keeps whole files that are almost certainly secrets (private keys,
credential files, .env) out of the archive entirely, unconditionally, and
that behavior is unchanged. This module instead handles the narrower case
those file-level rules can't catch: an ordinary source or config file that's
fine to include, but happens to have a secret-shaped value hardcoded
somewhere inside it (e.g. an AWS key pasted into a settings.py for local
testing).
secrets/credentials exclusion list (rules/base.py's SECRET_PATTERNS). That
list keeps whole files that are almost certainly secrets (private keys,
credential files, .env) out of the archive by default. This module instead
handles the narrower case those file-level rules can't catch: an ordinary
source or config file that's fine to include, but happens to have a
secret-shaped value hardcoded somewhere inside it (e.g. an AWS key pasted
into a settings.py for local testing).
Secret files the user deliberately includes
───────────────────────────────────────────
A project's `always_include` can bring a SECRET_PATTERNS file back on
purpose — most often a `.env`, so the AI tool can see which variables the
app expects. Those files are never written as-is (see redact_sensitive_file,
called unconditionally by packager.py, whatever `limits.redact_secrets`
says):
- `.env` / `.env.*` files keep every key, comment and blank line, but
every value becomes "[REDACTED]" — not just the secret-shaped ones,
because a dotenv value is a secret far too often to guess which aren't.
- Any other secret file gets the same pattern scan as above.
- A secret file that isn't UTF-8 text (a .p12, a .jks keystore) can't be
redacted, so packager.py skips it instead of zipping it raw.
Design choices, deliberately conservative to match the rest of the project:
- A curated list of high-precision patterns (known key/token formats,
@@ -28,10 +41,12 @@ Design choices, deliberately conservative to match the rest of the project:
left alone rather than redacted, so example/template files don't get
needlessly mangled.
Performance: this only ever runs on files the caller has already confirmed
are text and within `limits.max_file_size_mb` (see packager.py, which reuses
the binary/large-file classification `resolve_files()` already computes) —
never on binaries or oversized files. A handful of compiled regexes over a
Performance: the opt-in scan only ever runs on files the caller has already
confirmed are text and within `limits.max_file_size_mb` (see packager.py,
which reuses the binary/large-file classification `resolve_files()` already
computes) — never on binaries or oversized files. Deliberately included
secret files are the one exception: they're always scanned, whatever their
size. A handful of compiled regexes over a
small text file costs low-single-digit milliseconds; across an entire
project it's negligible next to the I/O of zipping in the first place.
"""
@@ -40,6 +55,13 @@ from __future__ import annotations
import re
from dataclasses import dataclass
from pathlib import PurePath
import pathspec
from contextzip.rules.base import SECRET_PATTERNS
REDACTED = "[REDACTED]"
@dataclass(frozen=True)
@@ -173,3 +195,143 @@ def redact_secrets(text: str) -> tuple[str, list[str]]:
text = pattern.regex.sub(_sub, text)
return text, matched_names
# ---------------------------------------------------------------------------
# Secret files the user deliberately included (see module docstring)
# ---------------------------------------------------------------------------
_SECRET_SPEC = pathspec.PathSpec.from_lines("gitwildmatch", SECRET_PATTERNS)
# KEY=value, optionally prefixed with `export` — the dotenv assignment form.
_ENV_ASSIGNMENT = re.compile(r"^(\s*(?:export\s+)?[A-Za-z_][A-Za-z0-9_.\-]*\s*=[ \t]*)(.*)$")
ENV_VALUE = ".env value"
def is_sensitive_path(rel: PurePath | str) -> bool:
"""True if *rel* (relative to the project root) matches SECRET_PATTERNS."""
rel_str = PurePath(rel).as_posix()
if _SECRET_SPEC.match_file(rel_str):
return True
parts = PurePath(rel_str).parts[:-1]
return any(
_SECRET_SPEC.match_file("/".join(parts[: i + 1]) + "/") for i in range(len(parts))
)
def is_env_file(rel: PurePath | str) -> bool:
"""True for `.env` and `.env.*` files (`.env.local`, `.env.production`, ...)."""
name = PurePath(rel).name
return name == ".env" or name.startswith(".env.")
def _split_line_ending(line: str) -> tuple[str, str]:
body = line.rstrip("\r\n")
return body, line[len(body):]
def _closing_quote(value: str, quote: str, start: int) -> int:
"""Index of the closing *quote* in *value* at or after *start*, or -1."""
i = start
while i < len(value):
if value[i] == "\\" and quote == '"':
i += 2
continue
if value[i] == quote:
return i
i += 1
return -1
def _safe_tail(tail: str) -> str:
"""Keep what follows a closing quote only if it's blank or a `#comment`."""
if not tail.strip() or re.match(r"\s+#", tail):
return tail
return " " + REDACTED
def redact_env(text: str) -> tuple[str, int]:
"""
Replace every value in a dotenv file with "[REDACTED]".
Keys, `export` prefixes, quotes, inline comments, blank lines and line
endings are kept, so the file still tells the reader which variables
exist. Quoted values that span several lines collapse to one redacted
line. Any other non-comment line (something that isn't KEY=value) is
redacted whole, since it might be a pasted secret.
Returns (redacted text, number of values redacted).
"""
lines = text.splitlines(keepends=True)
out: list[str] = []
count = 0
i = 0
while i < len(lines):
body, ending = _split_line_ending(lines[i])
i += 1
stripped = body.strip()
if not stripped or stripped.startswith("#"):
out.append(body + ending)
continue
m = _ENV_ASSIGNMENT.match(body)
if not m:
out.append(REDACTED + ending)
count += 1
continue
prefix, value = m.group(1), m.group(2)
if not value.strip():
out.append(body + ending)
continue
count += 1
quote = value[0]
if quote not in "\"'`":
# Unquoted: the value runs up to an inline ` #comment`.
comment = re.search(r"\s+#.*$", value)
tail = comment.group(0) if comment else ""
out.append(prefix + REDACTED + tail + ending)
continue
close = _closing_quote(value, quote, 1)
if close != -1:
out.append(prefix + quote + REDACTED + quote + _safe_tail(value[close + 1:]) + ending)
continue
# Multi-line quoted value: swallow lines until the closing quote.
tail = None
while i < len(lines):
next_body, ending = _split_line_ending(lines[i])
i += 1
close = _closing_quote(next_body, quote, 0)
if close != -1:
tail = next_body[close + 1:]
break
if tail is None:
# Never closed — everything after the opening quote is the value.
out.append(prefix + REDACTED + ending)
else:
out.append(prefix + quote + REDACTED + quote + _safe_tail(tail) + ending)
return "".join(out), count
def redact_sensitive_file(rel: PurePath | str, text: str) -> tuple[str, list[str]]:
"""
Redact a SECRET_PATTERNS file the user deliberately included.
`.env` files get every value redacted (see redact_env); any other secret
file gets the regular pattern scan. Returns the same shape as
redact_secrets(), with one ENV_VALUE entry per redacted dotenv value.
"""
names: list[str] = []
if is_env_file(rel):
text, count = redact_env(text)
names.extend([ENV_VALUE] * count)
text, matched = redact_secrets(text)
names.extend(matched)
return text, names
contextzip/watcher.py
+8 -2
View File
@@ -584,6 +584,7 @@ def _write_debug_zip(
from contextzip.packager import (
_workspace_dir,
_ensure_workspace_gitignore,
_write_member,
output_subdir_for_mode,
)
@@ -622,8 +623,13 @@ def _write_debug_zip(
) as inner_zf:
for abs_path in referenced_paths:
try:
rel = abs_path.relative_to(project_dir).as_posix()
inner_zf.write(abs_path, arcname=rel)
rel = abs_path.relative_to(project_dir)
# Same writer as a normal run, so a secret file
# (e.g. a .env named in a traceback) is redacted
# or left out, never zipped as-is.
_write_member(
inner_zf, abs_path, rel, False, frozenset(), frozenset()
)
except (ValueError, OSError):
pass # skip files we can't read or locate
zf.writestr("source-files.zip", inner_buf.getvalue())