External/securelens-backend
1
0
Fork 0
mirror of https://github.com/Rarebuffalo/securelens-backend.git synced 2026-06-19 07:00:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

optimize discover_files walking using os.walk directory pruning

Browse Source
This commit is contained in:
rarebuffalo
2026-06-15 00:58:40 +05:30
parent 6c1caa2f25
commit caba447de3
1 changed files with 35 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
cli/securelens/scanners/__init__.py
View File

@@ -99,6 +99,7 @@ def discover_files(root: Path, cfg: CLIConfig) -> list[Path]:
Respects .gitignore in the root and cfg.ignore_patterns. Respects .gitignore in the root and cfg.ignore_patterns.
Skips binaries and files larger than cfg.max_file_size_kb. Skips binaries and files larger than cfg.max_file_size_kb.
""" """
import os
# Build a combined spec from config ignore_patterns + .gitignore # Build a combined spec from config ignore_patterns + .gitignore
ignore_patterns = list(cfg.ignore_patterns) ignore_patterns = list(cfg.ignore_patterns)
gitignore_path = root / ".gitignore" gitignore_path = root / ".gitignore"
@@ -113,18 +114,41 @@ def discover_files(root: Path, cfg: CLIConfig) -> list[Path]:
spec = pathspec.PathSpec.from_lines("gitwildmatch", ignore_patterns) spec = pathspec.PathSpec.from_lines("gitwildmatch", ignore_patterns)
max_bytes = cfg.max_file_size_kb * 1024 max_bytes = cfg.max_file_size_kb * 1024
# Hardcoded directory blacklist to prune execution paths immediately
prune_dirs = {
".git", "node_modules", "venv", ".venv", "__pycache__",
"dist", "build", ".next", ".cache", ".npm", ".cargo",
".rustup", ".local", ".ssh", ".gnupg", ".docker", ".vscode",
".idea", "Library", "Pictures", "Music", "Videos", "Documents"
}
candidates: list[Path] = [] candidates: list[Path] = []
for p in root.rglob("*"): for dirpath, dirnames, filenames in os.walk(root):
if not p.is_file(): # 1. Prune standard blacklisted folders in-place
continue dirnames[:] = [d for d in dirnames if d not in prune_dirs]
rel = p.relative_to(root).as_posix()
if spec.match_file(rel): # 2. Prune directories matching the ignore spec
continue active_dirs = []
if p.suffix.lower() in BINARY_EXTENSIONS: for d in dirnames:
continue rel_path = os.path.relpath(os.path.join(dirpath, d), root)
if p.stat().st_size > max_bytes: if not spec.match_file(rel_path + "/"):
continue active_dirs.append(d)
candidates.append(p) dirnames[:] = active_dirs
# 3. Process files in the active directory
for f in filenames:
p = Path(dirpath) / f
rel = p.relative_to(root).as_posix()
if spec.match_file(rel):
continue
if p.suffix.lower() in BINARY_EXTENSIONS:
continue
try:
if p.stat().st_size > max_bytes:
continue
except OSError:
continue
candidates.append(p)
return sorted(candidates) return sorted(candidates)