mirror of
https://github.com/akadeepesh/contextzip.git
synced 2026-10-20 08:36:20 +02:00
feat: implement limits.redact_secrets enforcement
Previously persisted by config.py / the config UI but never actually
enforced (flagged as a known limitation as of 0.4.0). Text files
already going into the archive are now scanned for secret-shaped
values and matched values are replaced with [REDACTED] before
writing.
- New redact.py: curated high-precision patterns (AWS/Google/GitHub/
Slack/Stripe/Anthropic/OpenAI keys, JWTs, private-key blocks, and a
conservative generic key/secret/token/password assignment pattern)
with placeholder-value filtering (changeme, xxxxxxxx, <placeholder>,
${ENV_VAR} interpolation are left alone).
- packager.py: new _write_member() helper shared by create_zip() and
create_zip_silent(); only scans files already known (via
ResolveResult) to be text and within limits.max_file_size_mb —
binary and oversized files are never touched. PackageResult gains a
redacted field.
- cli.py: passes limits.redact_secrets into create_zip(), prints a
one-line redaction summary, forwards redacted entries to the report.
- cli_display.py: print_redaction_summary() — same 'surface it, don't
hide it' treatment as large-file warnings, silent when nothing was
redacted.
- report.py: itemizes every redacted file + which pattern matched.
- project_config.py: also touches the workspace-gitignore docstrings
as part of the same pass — see the next commit for the rest of that
change (config.py, webui/persist.py), which landed separately
because those files have no redaction-related content.
This commit is contained in:
@@ -55,6 +55,7 @@ from contextzip.cli_display import (
|
||||
print_apply_plan,
|
||||
print_apply_result,
|
||||
print_auto_cleanup,
|
||||
print_redaction_summary,
|
||||
)
|
||||
from contextzip.report import write_scan_report, write_apply_report
|
||||
from contextzip.applier import (
|
||||
@@ -673,6 +674,11 @@ def cmd_config(
|
||||
info(
|
||||
f"ai: enabled={ai.enabled} provider={ai.provider} max_files={ai.max_files}"
|
||||
)
|
||||
limits = project_cfg.limits
|
||||
info(
|
||||
f"limits: max_file_size_mb={limits.max_file_size_mb} "
|
||||
f"redact_secrets={limits.redact_secrets}"
|
||||
)
|
||||
cleanup_cfg = project_cfg.cleanup
|
||||
info(
|
||||
f"cleanup: enabled={cleanup_cfg.enabled} "
|
||||
@@ -984,6 +990,7 @@ def _run(
|
||||
console=console,
|
||||
mode=run_mode,
|
||||
prompt_txt=prompt_txt,
|
||||
redact_secrets_enabled=project_cfg.limits.redact_secrets,
|
||||
)
|
||||
except Exception as exc:
|
||||
err(f"Failed to create ZIP: {exc}")
|
||||
@@ -994,6 +1001,7 @@ def _run(
|
||||
resolved, project_dir, large_file_warn_bytes=large_file_warn_bytes
|
||||
)
|
||||
print_zip_write_warnings(result)
|
||||
print_redaction_summary(result)
|
||||
print_package_result(result)
|
||||
|
||||
# ── Report ───────────────────────────────────────────────────────────────
|
||||
@@ -1006,6 +1014,7 @@ def _run(
|
||||
ai_prompt=prompt,
|
||||
ai_selected=resolved.included if prompt else None,
|
||||
large_file_warn_bytes=large_file_warn_bytes,
|
||||
redacted=result.redacted,
|
||||
)
|
||||
print_report_hint(report_path)
|
||||
|
||||
|
||||
@@ -238,6 +238,23 @@ def print_zip_write_warnings(result, *, con: Console = console) -> None:
|
||||
)
|
||||
|
||||
|
||||
def print_redaction_summary(result, *, con: Console = console) -> None:
|
||||
"""
|
||||
A one-line heads-up when limits.redact_secrets caught something —
|
||||
same "surface it, don't hide it" treatment as large-file warnings.
|
||||
Silent when nothing was redacted, including when the setting is off.
|
||||
"""
|
||||
if not result.redacted:
|
||||
return
|
||||
n = len(result.redacted)
|
||||
total_hits = sum(len(names) for _, names in result.redacted)
|
||||
warn(
|
||||
f"Redacted {total_hits} secret-shaped value(s) in {n} file{'s' if n != 1 else ''} "
|
||||
"before zipping — see report",
|
||||
con=con,
|
||||
)
|
||||
|
||||
|
||||
def print_report_hint(report_path, *, con: Console = console) -> None:
|
||||
"""Dim pointer to the full report, printed once at the end of a run."""
|
||||
if report_path:
|
||||
|
||||
+120
-29
@@ -20,11 +20,12 @@ Phase 7 changes:
|
||||
instead of directly in .contextzip/, leaving room for config.json
|
||||
alongside them.
|
||||
- Ignoring is now handled by a self-contained .contextzip/.gitignore
|
||||
(ignore everything except config.json) instead of a blanket
|
||||
".contextzip/" entry in the project's top-level .gitignore — this keeps
|
||||
config.json trackable/shareable while output/ stays untracked, and it
|
||||
keeps working even when the workspace is relocated outside the default
|
||||
git-root anchor (see project_config.py / _resolve_workspace_location).
|
||||
(ignore everything, no exceptions) instead of a blanket ".contextzip/"
|
||||
entry in the project's top-level .gitignore — this keeps working even
|
||||
when the workspace is relocated outside the default git-root anchor
|
||||
(see project_config.py / _resolve_workspace_location). config.json is
|
||||
local by default like everything else in the workspace; sharing it
|
||||
with a team is a deliberate, manual `git add -f`, never automatic.
|
||||
|
||||
Phase 8 changes:
|
||||
- Every ZIP now gets a sidecar manifest written next to it (e.g.
|
||||
@@ -45,6 +46,15 @@ Phase 9 changes:
|
||||
to remember which run you just did. See `output_subdir_for_mode` /
|
||||
`zip_filename_for_mode`, which are the single source of truth other
|
||||
modules (watcher.py, cleanup.py) build on so the mapping never drifts.
|
||||
|
||||
Phase 10 changes:
|
||||
- Implements `limits.redact_secrets`, previously persisted by config.py /
|
||||
the config UI but never actually enforced (flagged as a known
|
||||
limitation as of 0.4.0). Text files that are already going into the
|
||||
archive get scanned for secret-shaped values (API keys, tokens,
|
||||
private-key blocks, etc. — see redact.py) and the matched values are
|
||||
replaced with "[REDACTED]" before writing. Binary and oversized files
|
||||
are never scanned — see `_should_scan_for_secrets`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -67,6 +77,7 @@ from rich.progress import (
|
||||
)
|
||||
|
||||
from contextzip.filters import ResolveResult
|
||||
from contextzip.redact import redact_secrets
|
||||
|
||||
# Subdirectory of the workspace where generated ZIPs are written.
|
||||
_OUTPUT_DIRNAME = "output"
|
||||
@@ -113,22 +124,25 @@ def zip_filename_for_mode(mode: str) -> str:
|
||||
"""The deterministic zip filename written for *mode* (see module docstring)."""
|
||||
return _MODE_ZIP_FILENAMES.get(mode, _MODE_ZIP_FILENAMES[_DEFAULT_MODE])
|
||||
|
||||
# Contents of .contextzip/.gitignore — ignore everything in the workspace
|
||||
# except the team-shareable project config and this file itself.
|
||||
# Contents of .contextzip/.gitignore — ignore absolutely everything in the
|
||||
# workspace, no exceptions. .contextzip/ is a local, per-machine scratch
|
||||
# space; nothing in it is ever pushed by default. Anyone who genuinely
|
||||
# wants to share something from it (e.g. config.json, for team-wide
|
||||
# settings) can still do so explicitly with `git add -f`, but contextzip
|
||||
# itself never carves out an exception — see project_config.py.
|
||||
_WORKSPACE_GITIGNORE_CONTENTS = (
|
||||
"# contextzip workspace\n"
|
||||
"# Everything here is a local, per-machine artifact except config.json,\n"
|
||||
"# which holds project-level contextzip preferences meant to be shared\n"
|
||||
"# with your team via Git.\n"
|
||||
"# Everything here is a local, per-machine artifact. Nothing in this\n"
|
||||
"# directory is ever pushed by default. To share something from it\n"
|
||||
"# (e.g. config.json) with your team anyway, use `git add -f`.\n"
|
||||
"*\n"
|
||||
"!.gitignore\n"
|
||||
"!config.json\n"
|
||||
)
|
||||
|
||||
# A previous version of contextzip added this block to the project's
|
||||
# top-level .gitignore. Now that .contextzip/.gitignore handles ignoring
|
||||
# on its own (and does so in a way that keeps config.json trackable), we
|
||||
# clean up that older entry the first time we touch a project — see
|
||||
# on its own (and works correctly regardless of where the workspace is
|
||||
# relocated), we clean up that older entry the first time we touch a
|
||||
# project — see
|
||||
# _migrate_legacy_root_gitignore_entry.
|
||||
_LEGACY_GITIGNORE_BLOCK = "# contextzip workspace\n.contextzip/\n"
|
||||
|
||||
@@ -145,6 +159,10 @@ class PackageResult:
|
||||
uncompressed_bytes: int
|
||||
compressed_bytes: int
|
||||
skipped_in_zip: list[tuple[Path, str]] = field(default_factory=list)
|
||||
# Files where at least one secret-shaped value was redacted before
|
||||
# writing — (path relative to project root, pattern names matched).
|
||||
# Always empty unless limits.redact_secrets is enabled.
|
||||
redacted: list[tuple[Path, list[str]]] = field(default_factory=list)
|
||||
|
||||
@property
|
||||
def compression_ratio(self) -> float:
|
||||
@@ -161,6 +179,49 @@ class PackageResult:
|
||||
return self.compressed_bytes > self.uncompressed_bytes
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Shared per-file write logic (redaction happens here, in exactly one place)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _write_member(
|
||||
zf: zipfile.ZipFile,
|
||||
abs_path: Path,
|
||||
rel: Path,
|
||||
redact_enabled: bool,
|
||||
binary_paths: frozenset[Path],
|
||||
large_paths: frozenset[Path],
|
||||
) -> list[str]:
|
||||
"""
|
||||
Write one file into *zf*.
|
||||
|
||||
When *redact_enabled* is True and *abs_path* isn't already known to be
|
||||
binary or oversized (per resolve_files()'s existing classification —
|
||||
never re-detected here), reads it as UTF-8 text and redacts any
|
||||
secret-shaped values before writing. Binary/oversized files, or text
|
||||
that fails to decode as UTF-8, are written unmodified via the same
|
||||
fast zf.write() path used when redaction is off entirely.
|
||||
|
||||
Returns the list of pattern names redacted (empty if none matched, or
|
||||
if this file wasn't eligible for scanning). May raise OSError, exactly
|
||||
as the old plain zf.write() call did — callers already handle that.
|
||||
"""
|
||||
if not redact_enabled or abs_path in binary_paths or abs_path in large_paths:
|
||||
zf.write(abs_path, arcname=rel.as_posix())
|
||||
return []
|
||||
|
||||
original_bytes = abs_path.read_bytes()
|
||||
try:
|
||||
text = original_bytes.decode("utf-8")
|
||||
except UnicodeDecodeError:
|
||||
zf.writestr(rel.as_posix(), original_bytes)
|
||||
return []
|
||||
|
||||
redacted_text, matched = redact_secrets(text)
|
||||
zf.writestr(rel.as_posix(), redacted_text.encode("utf-8") if matched else original_bytes)
|
||||
return matched
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public API
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -172,6 +233,7 @@ def create_zip_silent(
|
||||
output_path: Path | None,
|
||||
mode: str = _DEFAULT_MODE,
|
||||
prompt_txt: str | None = None,
|
||||
redact_secrets_enabled: bool = False,
|
||||
) -> PackageResult:
|
||||
"""
|
||||
Write the included files from *resolve_result* into a ZIP archive
|
||||
@@ -184,6 +246,9 @@ def create_zip_silent(
|
||||
*mode* selects which .contextzip/output/<mode>/ subfolder and
|
||||
deterministic filename to use — see `output_subdir_for_mode` /
|
||||
`zip_filename_for_mode`. Ignored when *output_path* is given.
|
||||
|
||||
*redact_secrets_enabled* mirrors the project's `limits.redact_secrets`
|
||||
setting — see redact.py.
|
||||
"""
|
||||
if output_path is not None:
|
||||
zip_path = output_path
|
||||
@@ -193,7 +258,10 @@ def create_zip_silent(
|
||||
zip_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
included: list[Path] = resolve_result.included
|
||||
binary_paths = frozenset(resolve_result.binary_files)
|
||||
large_paths = frozenset(p for p, _ in resolve_result.large_files)
|
||||
skipped_in_zip: list[tuple[Path, str]] = []
|
||||
redacted: list[tuple[Path, list[str]]] = []
|
||||
uncompressed = 0
|
||||
file_count = 0
|
||||
|
||||
@@ -223,7 +291,11 @@ def create_zip_silent(
|
||||
continue
|
||||
|
||||
try:
|
||||
zf.write(abs_path, arcname=rel.as_posix())
|
||||
matched = _write_member(
|
||||
zf, abs_path, rel, redact_secrets_enabled, binary_paths, large_paths
|
||||
)
|
||||
if matched:
|
||||
redacted.append((rel, matched))
|
||||
uncompressed += file_size
|
||||
file_count += 1
|
||||
except PermissionError:
|
||||
@@ -241,6 +313,7 @@ def create_zip_silent(
|
||||
uncompressed_bytes=uncompressed,
|
||||
compressed_bytes=compressed,
|
||||
skipped_in_zip=skipped_in_zip,
|
||||
redacted=redacted,
|
||||
)
|
||||
|
||||
|
||||
@@ -251,6 +324,7 @@ def create_zip(
|
||||
console: Console,
|
||||
mode: str = _DEFAULT_MODE,
|
||||
prompt_txt: str | None = None,
|
||||
redact_secrets_enabled: bool = False,
|
||||
) -> PackageResult:
|
||||
"""
|
||||
Write the included files from *resolve_result* into a ZIP archive.
|
||||
@@ -268,6 +342,11 @@ def create_zip(
|
||||
file is written as the first entry in the ZIP. Any AI tool that receives
|
||||
the ZIP will immediately see the task description and selected file list.
|
||||
|
||||
*redact_secrets_enabled* mirrors the project's `limits.redact_secrets`
|
||||
setting — see redact.py. Only ever scans files already known (via
|
||||
*resolve_result*) to be text and within `limits.max_file_size_mb`;
|
||||
binary/oversized files are always written unmodified.
|
||||
|
||||
Returns a :class:`PackageResult` with compression stats and any
|
||||
files that had to be skipped during writing (e.g. permission denied).
|
||||
"""
|
||||
@@ -280,7 +359,10 @@ def create_zip(
|
||||
zip_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
included: list[Path] = resolve_result.included
|
||||
binary_paths = frozenset(resolve_result.binary_files)
|
||||
large_paths = frozenset(p for p, _ in resolve_result.large_files)
|
||||
skipped_in_zip: list[tuple[Path, str]] = []
|
||||
redacted: list[tuple[Path, list[str]]] = []
|
||||
uncompressed = 0
|
||||
file_count = 0
|
||||
|
||||
@@ -325,7 +407,11 @@ def create_zip(
|
||||
continue
|
||||
|
||||
try:
|
||||
zf.write(abs_path, arcname=rel.as_posix())
|
||||
matched = _write_member(
|
||||
zf, abs_path, rel, redact_secrets_enabled, binary_paths, large_paths
|
||||
)
|
||||
if matched:
|
||||
redacted.append((rel, matched))
|
||||
uncompressed += file_size
|
||||
file_count += 1
|
||||
except PermissionError:
|
||||
@@ -345,6 +431,7 @@ def create_zip(
|
||||
uncompressed_bytes=uncompressed,
|
||||
compressed_bytes=compressed,
|
||||
skipped_in_zip=skipped_in_zip,
|
||||
redacted=redacted,
|
||||
)
|
||||
|
||||
|
||||
@@ -406,20 +493,20 @@ def _find_git_root(start: Path) -> Path | None:
|
||||
def _ensure_workspace_gitignore(workspace: Path) -> None:
|
||||
"""
|
||||
Ensure <workspace>/.gitignore exists with the standard contents that
|
||||
ignore everything in the workspace except config.json (and the
|
||||
.gitignore file itself).
|
||||
ignore the entire workspace, no exceptions — .contextzip/ is a local,
|
||||
per-machine scratch space and nothing in it is tracked by default,
|
||||
including config.json and this .gitignore file itself.
|
||||
|
||||
This is self-contained: it works no matter where *workspace* ends up
|
||||
living (git-root default, cwd, or a custom relocated path), since a
|
||||
nested .gitignore applies to its own directory regardless of where
|
||||
that directory sits in the tree — unlike a single blanket entry in a
|
||||
distant top-level .gitignore, which can't be relied on to reach a
|
||||
relocated workspace and (if it ignores the whole directory rather than
|
||||
its contents) would prevent config.json from ever being trackable.
|
||||
relocated workspace.
|
||||
|
||||
Idempotent — leaves an existing, already-correct file untouched, and
|
||||
only rewrites files that don't yet match (e.g. hand-edited or from an
|
||||
older contextzip version).
|
||||
older contextzip version that carved out an exception for config.json).
|
||||
"""
|
||||
gitignore_path = workspace / ".gitignore"
|
||||
|
||||
@@ -441,11 +528,14 @@ def _migrate_legacy_root_gitignore_entry(git_root: Path) -> None:
|
||||
.gitignore, if present.
|
||||
|
||||
That blanket directory-level ignore predates .contextzip/.gitignore and
|
||||
would otherwise stop config.json from ever becoming trackable (git does
|
||||
not descend into an ignored directory to apply nested un-ignore rules).
|
||||
Only ever removes the exact block contextzip itself wrote — never
|
||||
touches unrelated .gitignore content, and is a no-op if the block isn't
|
||||
present (e.g. it was already removed, or never added).
|
||||
is redundant now that the nested .gitignore ignores the whole workspace
|
||||
on its own — leaving the old root-level entry around is harmless but
|
||||
unnecessary clutter, and it would have stopped git from ever descending
|
||||
into .contextzip/ to notice a force-added file (`git add -f`) if the
|
||||
entry ignores the directory itself rather than its contents. Only ever
|
||||
removes the exact block contextzip itself wrote — never touches
|
||||
unrelated .gitignore content, and is a no-op if the block isn't present
|
||||
(e.g. it was already removed, or never added).
|
||||
"""
|
||||
gitignore_path = git_root / ".gitignore"
|
||||
if not gitignore_path.is_file():
|
||||
@@ -490,9 +580,10 @@ def _resolve_workspace_location(project_dir: Path) -> tuple[str, str]:
|
||||
|
||||
Precedence, highest wins:
|
||||
1. CONTEXTZIP_WORKSPACE_LOCATION env var
|
||||
2. Project config (.contextzip/config.json at git root — team-shared,
|
||||
committed; falls back to the deprecated .contextzip.json if that's
|
||||
all a project has)
|
||||
2. Project config (.contextzip/config.json at git root — local by
|
||||
default like the rest of the workspace, force-added with
|
||||
`git add -f` if a team wants to share it; falls back to the
|
||||
deprecated .contextzip.json if that's all a project has)
|
||||
3. Personal config (~/.config/contextzip/config.json — per-machine)
|
||||
4. Built-in default: "git-root"
|
||||
|
||||
|
||||
@@ -1,15 +1,17 @@
|
||||
"""
|
||||
project_config.py — Project-level, team-shareable contextzip settings.
|
||||
project_config.py — Project-level contextzip settings.
|
||||
|
||||
Loaded from `.contextzip/config.json`, anchored at the project's git root
|
||||
(or the project directory itself outside a git repo) — the same anchor
|
||||
`.contextzip/` itself resolves to by default. Unlike config.py's personal,
|
||||
per-machine store (~/.config/contextzip/config.json, holds API keys, never
|
||||
committed), this file is meant to be committed to the repo so every
|
||||
contributor gets the same project defaults automatically — no per-machine
|
||||
setup required. `.contextzip/` is gitignored by default (see packager.py's
|
||||
_ensure_workspace_gitignore), but config.json is deliberately carved out of
|
||||
that ignore rule so it stays trackable.
|
||||
committed), this file holds settings a whole team could plausibly want to
|
||||
share — but contextzip never presumes that on your behalf. `.contextzip/`
|
||||
is gitignored in full by default (see packager.py's
|
||||
_ensure_workspace_gitignore), config.json included; a team that wants to
|
||||
share these settings has to explicitly `git add -f .contextzip/config.json`
|
||||
to commit it. This is a deliberate design choice, not an oversight — see
|
||||
"Never presumes what belongs in git" below.
|
||||
|
||||
Currently supports:
|
||||
|
||||
@@ -78,11 +80,20 @@ built-in default for the newer preference fields.
|
||||
"large" before packaging (still included,
|
||||
just surfaced) instead of the fixed 1 MB
|
||||
default. Fractional values are allowed.
|
||||
redact_secrets — reserved for a future best-effort scrub of
|
||||
secret-shaped values (API keys, tokens) inside
|
||||
otherwise-included files, on top of the
|
||||
always-excluded credential file patterns.
|
||||
Currently persisted but not yet enforced.
|
||||
redact_secrets — when true, files that are already going into the
|
||||
archive are scanned for secret-shaped values
|
||||
(API keys, tokens, private-key blocks — see
|
||||
redact.py) and matched values are replaced
|
||||
with "[REDACTED]" before writing. This is a
|
||||
second line of defense on top of, not a
|
||||
replacement for, the always-excluded whole-file
|
||||
credential patterns (§5-equivalent baseline in
|
||||
packager.py) — those still keep obviously-secret
|
||||
files out entirely regardless of this setting.
|
||||
Only ever scans files already known to be text
|
||||
and within max_file_size_mb; binary and
|
||||
oversized files are never scanned. Off by
|
||||
default.
|
||||
|
||||
applied_zip_retention
|
||||
How many past `apply-zip` archives to keep in
|
||||
|
||||
@@ -0,0 +1,175 @@
|
||||
"""
|
||||
redact.py — Best-effort in-content secret redaction.
|
||||
|
||||
Implements the `limits.redact_secrets` project setting (see project_config.py),
|
||||
which until now was persisted and editable in the config UI but never actually
|
||||
enforced by the packaging logic — see the 0.4.0 changelog's "known
|
||||
limitation."
|
||||
|
||||
This is deliberately a *second line of defense*, not a replacement for the
|
||||
hard secrets/credentials exclusion list in packager.py's universal baseline.
|
||||
That list keeps whole files that are almost certainly secrets (private keys,
|
||||
credential files, .env) out of the archive entirely, unconditionally, and
|
||||
that behavior is unchanged. This module instead handles the narrower case
|
||||
those file-level rules can't catch: an ordinary source or config file that's
|
||||
fine to include, but happens to have a secret-shaped value hardcoded
|
||||
somewhere inside it (e.g. an AWS key pasted into a settings.py for local
|
||||
testing).
|
||||
|
||||
Design choices, deliberately conservative to match the rest of the project:
|
||||
- A curated list of high-precision patterns (known key/token formats,
|
||||
private-key blocks, JWTs, and a narrow generic key=value assignment
|
||||
form) rather than an entropy-based scan — same "curated list, not a
|
||||
heuristic" approach as the file-level secrets baseline in packager.py.
|
||||
- Only the matched *value* is replaced with "[REDACTED...]", not the
|
||||
whole line — a redacted file should stay useful context for whatever
|
||||
AI tool receives the archive.
|
||||
- Obvious placeholders ("changeme", "xxxxxxxx", "<your-key-here>") are
|
||||
left alone rather than redacted, so example/template files don't get
|
||||
needlessly mangled.
|
||||
|
||||
Performance: this only ever runs on files the caller has already confirmed
|
||||
are text and within `limits.max_file_size_mb` (see packager.py, which reuses
|
||||
the binary/large-file classification `resolve_files()` already computes) —
|
||||
never on binaries or oversized files. A handful of compiled regexes over a
|
||||
small text file costs low-single-digit milliseconds; across an entire
|
||||
project it's negligible next to the I/O of zipping in the first place.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _SecretPattern:
|
||||
name: str
|
||||
regex: re.Pattern[str]
|
||||
# Which regex group holds the actual secret value to redact.
|
||||
# 0 means "redact the whole match" (used for freestanding key formats
|
||||
# with no surrounding key=value context to preserve).
|
||||
value_group: int = 0
|
||||
|
||||
|
||||
_PLACEHOLDER_VALUES = {
|
||||
"changeme",
|
||||
"change_me",
|
||||
"your_api_key_here",
|
||||
"your-api-key-here",
|
||||
"your_key_here",
|
||||
"example",
|
||||
"placeholder",
|
||||
"insert_key_here",
|
||||
"todo",
|
||||
"fixme",
|
||||
"xxx",
|
||||
"n/a",
|
||||
"none",
|
||||
"null",
|
||||
"test",
|
||||
"fake",
|
||||
"dummy",
|
||||
}
|
||||
|
||||
|
||||
def _looks_like_placeholder(value: str) -> bool:
|
||||
v = value.strip().strip("'\"").lower()
|
||||
if not v:
|
||||
return True
|
||||
if v in _PLACEHOLDER_VALUES:
|
||||
return True
|
||||
if set(v) <= {"x"}: # xxxxxxxx, XXXXXXXX
|
||||
return True
|
||||
if v.startswith("<") and v.endswith(">"): # <your-key-here>
|
||||
return True
|
||||
if v.startswith("$") or v.startswith("{{") or v.startswith("%"):
|
||||
# env-var interpolation / templating placeholders, e.g. ${API_KEY},
|
||||
# {{ secrets.API_KEY }}, %API_KEY% — a reference, not a live secret.
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
# Freestanding, self-identifying formats — redact the whole match, no
|
||||
# surrounding key=value context needed.
|
||||
_PATTERNS: list[_SecretPattern] = [
|
||||
_SecretPattern("AWS Access Key ID", re.compile(r"\bAKIA[0-9A-Z]{16}\b")),
|
||||
_SecretPattern("Google API Key", re.compile(r"\bAIza[0-9A-Za-z\-_]{35}\b")),
|
||||
_SecretPattern("GitHub Token", re.compile(r"\bgh[pousr]_[A-Za-z0-9]{36,}\b")),
|
||||
_SecretPattern("Slack Token", re.compile(r"\bxox[baprs]-[0-9A-Za-z-]{10,48}\b")),
|
||||
_SecretPattern("Stripe Key", re.compile(r"\bsk_(?:live|test)_[0-9A-Za-z]{24,}\b")),
|
||||
_SecretPattern("Anthropic API Key", re.compile(r"\bsk-ant-[A-Za-z0-9\-_]{20,}\b")),
|
||||
_SecretPattern("OpenAI API Key", re.compile(r"\bsk-proj-[A-Za-z0-9\-_]{20,}\b")),
|
||||
_SecretPattern(
|
||||
"JWT",
|
||||
re.compile(r"\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b"),
|
||||
),
|
||||
_SecretPattern(
|
||||
"Private Key Block",
|
||||
re.compile(
|
||||
r"-----BEGIN (?:RSA |EC |OPENSSH |DSA |PGP )?PRIVATE KEY-----"
|
||||
r"[\s\S]+?"
|
||||
r"-----END (?:RSA |EC |OPENSSH |DSA |PGP )?PRIVATE KEY-----"
|
||||
),
|
||||
),
|
||||
# AWS secret access keys have no distinguishing prefix, so this one
|
||||
# needs the "aws_secret_access_key = ..." context to be high-precision.
|
||||
_SecretPattern(
|
||||
"AWS Secret Access Key",
|
||||
re.compile(
|
||||
r"(?i)aws_secret_access_key\s*[:=]\s*['\"]?([A-Za-z0-9/+=]{40})['\"]?"
|
||||
),
|
||||
value_group=1,
|
||||
),
|
||||
# Narrow generic assignment: fires when a key/secret/token/password-shaped
|
||||
# word appears in the assigned name (standalone or as part of a
|
||||
# compound identifier like DB_PASSWORD, STRIPE_SECRET_KEY), assigned a
|
||||
# quoted value of at least 10 chars — conservative on purpose to keep
|
||||
# false positives low, though a false positive here only means an
|
||||
# unnecessary "[REDACTED]" in the zip copy, not any change to the
|
||||
# user's actual file.
|
||||
_SecretPattern(
|
||||
"Generic Secret Assignment",
|
||||
re.compile(
|
||||
r"(?i)\b[A-Za-z0-9_]*?"
|
||||
r"(?:api[_-]?key|secret[_-]?key|access[_-]?token|"
|
||||
r"auth[_-]?token|client[_-]?secret|password|passwd)"
|
||||
r"[A-Za-z0-9_]*\s*[:=]\s*['\"]([^'\"\s]{10,})['\"]"
|
||||
),
|
||||
value_group=1,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def redact_secrets(text: str) -> tuple[str, list[str]]:
|
||||
"""
|
||||
Scan *text* for secret-shaped values and replace each one with
|
||||
"[REDACTED]", leaving everything else — including the surrounding
|
||||
key name, quotes, and punctuation — untouched.
|
||||
|
||||
Returns (possibly-modified text, list of pattern names that matched at
|
||||
least once — e.g. ["AWS Access Key ID", "Generic Secret Assignment"]).
|
||||
An empty list means nothing matched and *text* is returned unchanged
|
||||
(the same string object, so callers can cheaply check `if matched:`
|
||||
to skip rewriting a file that didn't need it).
|
||||
"""
|
||||
matched_names: list[str] = []
|
||||
|
||||
for pattern in _PATTERNS:
|
||||
|
||||
def _sub(m: re.Match[str], pattern: _SecretPattern = pattern) -> str:
|
||||
value = m.group(pattern.value_group)
|
||||
if _looks_like_placeholder(value):
|
||||
return m.group(0)
|
||||
matched_names.append(pattern.name)
|
||||
if pattern.value_group == 0:
|
||||
return "[REDACTED]"
|
||||
full = m.group(0)
|
||||
g_start, g_end = m.span(pattern.value_group)
|
||||
rel_start = g_start - m.start()
|
||||
rel_end = g_end - m.start()
|
||||
return full[:rel_start] + "[REDACTED]" + full[rel_end:]
|
||||
|
||||
text = pattern.regex.sub(_sub, text)
|
||||
|
||||
return text, matched_names
|
||||
@@ -33,6 +33,7 @@ def write_scan_report(
|
||||
ai_prompt: str | None = None,
|
||||
ai_selected: list[Path] | None = None,
|
||||
large_file_warn_bytes: int,
|
||||
redacted: list[tuple[Path, list[str]]] | None = None,
|
||||
) -> Path:
|
||||
"""
|
||||
Write the full scan/package report next to *zip_path*.
|
||||
@@ -111,6 +112,12 @@ def write_scan_report(
|
||||
w(f" {rel}")
|
||||
w("")
|
||||
|
||||
if redacted:
|
||||
w("Redacted secrets (limits.redact_secrets)")
|
||||
for rel, names in redacted:
|
||||
w(f" {rel.as_posix():<60} {', '.join(names)}")
|
||||
w("")
|
||||
|
||||
if resolved.skipped:
|
||||
w("Skipped files (unreadable or dangling symlink)")
|
||||
for p, reason in resolved.skipped:
|
||||
|
||||
Reference in New Issue
Block a user