`merge-work/leak-scan.py`

90 lines
#!/usr/bin/env python3
# leak-scan.py <jsonl> <banned_json>
# Word-boundary aware leak scan (mirrors build-vendor.sh). Scans KEYS + string
# VALUES recursively. Reports EVERY distinct real leak (not just first).
# Preserves legit superwords like "twill" (a "TWIL" ban won't flag it).
import sys, json, re
path = sys.argv[1]
banned = [b for b in json.loads(sys.argv[2]) if b.strip()]
wpats = [(b, re.compile(r'\b' + re.escape(b) + r'\b', re.IGNORECASE)) for b in banned]
toks  = [(b, re.sub(r'[^a-z0-9]', '', b.lower())) for b in banned]
toks  = [(b, t) for (b, t) in toks if t]

# ---- place-name FALSE-POSITIVE guard (mirrors dw-leak-scanner/leak-match.mjs) ----
# A bare "York" ban (the malibu/phillipe_romano/hollywood houses source from the
# real York/Brewster and legitimately ban "York") must NOT flag "New York" NYC
# place names ("Concrete New York"), the "York Weave" pattern, or the internal
# "york-scrape"/"york_scrape" provenance token. It DOES still flag a standalone
# "York" brand form or "yorkwall". Only the bare-"york" term is guarded; all
# other banned brands (Brewster, WallQuest, …) have no place-name FP risk.
_YORK_STRIP_TEXT = re.compile(r'new york|nueva york|york weave', re.IGNORECASE)
_YORK_STRIP_TOK  = re.compile(r'newyork|nuevayork|yorkweave|yorkscrape', re.IGNORECASE)
_YORK_WORD       = re.compile(r'\byork\b', re.IGNORECASE)
_YORK_TOK        = re.compile(r'york')

def _is_bare_york(b):
    return re.sub(r'[^a-z0-9]', '', b.lower()) == 'york'

def _york_survives_text(s):
    # strip the known place/pattern phrases, then re-test for a standalone York
    return bool(_YORK_WORD.search(_YORK_STRIP_TEXT.sub(' ', s)))

def _york_survives_tok(low):
    # low is already alnum-only lowercase; strip glued place forms then re-test
    return bool(_YORK_TOK.search(_YORK_STRIP_TOK.sub('', low)))

def check_text(s):
    for b, p in wpats:
        if p.search(s):
            if _is_bare_york(b) and not _york_survives_text(s): continue
            return b
    return None

def check_urlish(s):
    low = re.sub(r'[^a-z0-9]', '', s.lower())
    for b, t in toks:
        if t in low:
            if _is_bare_york(b) and not _york_survives_tok(low): continue
            return b
    return None

def is_slug_or_url(s):
    if re.search(r'(https?:)?//|[./\\]', s): return True
    if ' ' not in s and re.search(r'[-_]', s): return True
    return False

def scan(line):
    try: rec = json.loads(line)
    except Exception: rec = None
    if rec is None:
        return check_text(line)
    stack = [rec]
    while stack:
        o = stack.pop()
        if isinstance(o, dict):
            for k, v in o.items():
                kt = re.sub(r'[^a-z0-9]', '', str(k).lower())
                for b, t in toks:
                    if t and t in kt:
                        if _is_bare_york(b) and not _york_survives_tok(kt): continue
                        return b
                stack.append(v)
        elif isinstance(o, list):
            stack.extend(o)
        elif isinstance(o, str):
            hit = check_urlish(o) if is_slug_or_url(o) else check_text(o)
            if hit: return hit
    return None

leaks = {}
n = 0
with open(path) as f:
    for line in f:
        line = line.strip()
        if not line: continue
        n += 1
        hit = scan(line)
        if hit:
            leaks[hit] = leaks.get(hit, 0) + 1
print(json.dumps({"scanned": n, "leaks": leaks, "clean": len(leaks) == 0}))