#!/usr/bin/env python3 """Scan release files for private birth-data residues.""" from __future__ import annotations import argparse import ast import json import os import re import subprocess from pathlib import Path from typing import Iterable ROOT = Path(__file__).resolve().parents[1] SKIP_DIRS = { ".git", "__pycache__", ".pytest_cache", "node_modules", "scratch", "references/open_source_sources", } SKIP_FILES = { "scripts/public_release_privacy_scan.py", "tests/test_public_release_privacy_scan.py", } TEXT_SUFFIXES = { ".cfg", ".csv", ".html", ".js", ".json", ".md", ".py", ".txt", ".yaml", ".yml", } LOCAL_DENY_PATTERN_FILE = Path("scratch/local/public_release_deny_patterns.txt") ENV_DENY_PATTERNS = "PUBLIC_RELEASE_DENY_PATTERNS" def _literal_patterns(values: Iterable[str], prefix: str) -> list[tuple[str, re.Pattern[str]]]: patterns: list[tuple[str, re.Pattern[str]]] = [] for index, value in enumerate(values, start=1): value = value.strip() if not value or value.startswith("#"): continue patterns.append((f"{prefix}_{index:02d}", re.compile(re.escape(value), re.IGNORECASE))) return patterns def deny_patterns(root: Path = ROOT) -> list[tuple[str, re.Pattern[str]]]: patterns = _literal_patterns(os.environ.get(ENV_DENY_PATTERNS, "").splitlines(), "private_env_pattern") local_file = root / LOCAL_DENY_PATTERN_FILE if local_file.is_file(): patterns.extend( _literal_patterns( local_file.read_text(encoding="utf-8", errors="ignore").splitlines(), "private_local_pattern", ) ) return patterns def _is_skipped(path: Path, root: Path = ROOT) -> bool: try: rel = path.relative_to(root).as_posix() except ValueError: rel = path.as_posix() if rel in SKIP_FILES: return True return any(rel == item or rel.startswith(f"{item}/") for item in SKIP_DIRS) def iter_release_files(root: Path = ROOT) -> Iterable[Path]: completed = subprocess.run( ["git", "ls-files"], cwd=root, text=True, capture_output=True, check=False, ) if completed.returncode == 0: candidates = (root / line for line in completed.stdout.splitlines()) else: candidates = (path for path in root.rglob("*") if path.is_file()) for path in candidates: if not path.is_file() or _is_skipped(path, root): continue if path.suffix.lower() in TEXT_SUFFIXES: yield path def scan_text( path: Path, text: str, patterns: Iterable[tuple[str, re.Pattern[str]]] | None = None, ) -> list[dict[str, object]]: findings: list[dict[str, object]] = [] try: display_path = path.relative_to(ROOT).as_posix() except ValueError: display_path = path.as_posix() for rule_id, pattern in (patterns if patterns is not None else deny_patterns()): for match in pattern.finditer(text): line = text.count("\n", 0, match.start()) + 1 findings.append( { "rule_id": rule_id, "path": display_path, "line": line, } ) return findings def scan_executable_redaction_names(path: Path, text: str) -> list[dict[str, object]]: """Reject unquoted privacy placeholders that would raise at runtime.""" if path.suffix.lower() != ".py": return [] try: tree = ast.parse(text, filename=str(path)) except SyntaxError: return [] try: display_path = path.relative_to(ROOT).as_posix() except ValueError: display_path = path.as_posix() return [ {"rule_id": "executable_redaction_placeholder", "path": display_path, "line": node.lineno} for node in ast.walk(tree) if isinstance(node, ast.Name) and node.id.startswith("REDACTED_") ] def build_report(root: Path = ROOT) -> dict[str, object]: findings: list[dict[str, object]] = [] scanned = 0 patterns = deny_patterns(root) for path in iter_release_files(root): try: text = path.read_text(encoding="utf-8") except UnicodeDecodeError: text = path.read_text(encoding="utf-8", errors="ignore") scanned += 1 findings.extend(scan_text(path, text, patterns)) findings.extend(scan_executable_redaction_names(path, text)) return { "scope": "public_release_privacy_scan", "scanned_files": scanned, "finding_count": len(findings), "findings": findings, "status": "pass" if not findings else "fail", } def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--json", action="store_true", help="Emit JSON.") args = parser.parse_args() report = build_report() if args.json: print(json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True)) else: print(f"{report['status']}: {report['finding_count']} findings") for finding in report["findings"]: print(f"{finding['path']}:{finding['line']} {finding['rule_id']}") return 0 if report["status"] == "pass" else 1 if __name__ == "__main__": raise SystemExit(main())