`data/census/match-census.py`

186 lines
#!/usr/bin/env python3
"""
Cross-match enumerated businesses against LA city license dataset.
Two passes:
  Pass 1 — STRICT: same street_number + zip, fuzzy name match >=0.5
  Pass 2 — CHAIN: if business name matches a known chain pattern, accept any
           license at same street_number containing the parent-LLC pattern.
"""
import json, re
from collections import defaultdict
from difflib import SequenceMatcher

# ── Known chain patterns. yelp_name_token → list of parent-LLC fragments
CHAIN_PATTERNS = {
    'mcdonald': ['MCDONALD'],
    'starbucks': ['STARBUCKS'],
    'chevron': ['CHEVRON','SCI VENTURE','ASGHARI','SHAYESTEH'],  # franchise owners
    'chase': ['JPMORGAN','CHASE'],
    'wells fargo': ['WELLS FARGO','WACHOVIA'],
    'us bank': ['US BANK','U.S. BANCORP','U S BANK'],
    'bank of america': ['BANK OF AMERICA','BAC HOME LOANS'],
    'citi': ['CITIBANK'],
    'dominos': ["DOMINO'S","DOMINOS","TEAM WASHINGTON","DPC INC","DPC ENTERPRISES"],
    "domino's": ["DOMINO","DPC"],
    'subway': ['SUBWAY','DOCTOR\'S ASSOCIATES'],
    'jersey mike': ['JERSEY MIKE','JM SUBS'],
    '7-eleven': ['7-ELEVEN','7 ELEVEN','SEVEN-ELEVEN'],
    'cvs': ['CVS','GARFIELD BEACH'],
    'walgreens': ['WALGREEN'],
    'home depot': ['HOME DEPOT'],
    'home goods': ['HOMEGOODS','TJX','TJ MAXX','MARSHALLS'],
    'tj maxx': ['TJX','TJ MAXX','MARMAXX'],
    'marshalls': ['MARSHALLS','TJX','MARMAXX'],
    'big o tires': ['BIG O','TBC RETAIL','TBC CORPORATION'],
    'goodyear': ['GOODYEAR','SAM\'S CLUB','RUBBER COMPANY'],
    'jiffy lube': ['JIFFY LUBE'],
    'firestone': ['FIRESTONE','BRIDGESTONE'],
    'wingstop': ['WINGSTOP','WING STOP'],
    'chipotle': ['CHIPOTLE'],
    'taco bell': ['TACO BELL','YUM BRANDS','BORDER FOODS'],
    'kfc': ['KFC','KENTUCKY FRIED','TACALA','KBP'],
    'panda express': ['PANDA EXPRESS','PANDA RESTAURANT'],
    'el pollo loco': ['EL POLLO LOCO','EPL'],
    'in-n-out': ['IN-N-OUT','IN N OUT'],
    'cheesecake': ['CHEESECAKE'],
    'panera': ['PANERA'],
    'jack in the box': ['JACK IN THE BOX','JIB'],
    'del taco': ['DEL TACO'],
    "carl's jr": ['CARL\'S JR','CKE','HARDEE\'S','CARLS JR'],
    'shell': ['SHELL','EQUILON','MOTIVA'],
    'arco': ['ARCO','PB ENERGY','SOPUS','BP WEST COAST','TESORO','MARATHON','ATLANTIC RICHFIELD'],
    'mobil': ['MOBIL','EXXON','SOPUS'],
    'sinclair': ['SINCLAIR','HEP'],
    'jamba': ['JAMBA','FOCUS BRANDS'],
    'peet': ["PEET'S",'PEETS'],
    'coffee bean': ['COFFEE BEAN','CBTL','INTL COFFEE'],
    'pizza hut': ['PIZZA HUT','YUM BRANDS'],
    'wells fargo': ['WELLS FARGO','WACHOVIA'],
    'pet supplies plus': ['PET SUPPLIES'],
    'petco': ['PETCO'],
    'petsmart': ['PETSMART'],
    'westlake hardware': ['WESTLAKE'],
    "tj maxx": ["TJX","TJ MAXX","MARMAXX"],
    "trader joe": ["TRADER JOE"],
    "whole foods": ["WHOLE FOODS","MRS GOOCH","AMAZON GROCERY","WFM"],
    "out of the closet": ["OUT OF THE CLOSET","AHF","AIDS HEALTHCARE"],
    "anytime fitness": ["ANYTIME FITNESS","ANYTIME WELLNESS"],
    "club pilates": ["CLUB PILATES","PILATES FRANCHISE"],
    "european wax": ["EUROPEAN WAX"],
    "great clips": ["GREAT CLIPS"],
}

def norm(s):
    s = (s or '').lower()
    s = re.sub(r"['’]", '', s)  # strip apostrophes
    s = re.sub(r'[^a-z0-9]+', ' ', s)
    return s.strip()

def street_num(addr):
    m = re.match(r'^(\d+)', (addr or '').strip())
    return m.group(1) if m else None

def chain_keys(name):
    n = norm(name)
    matches = []
    for k in CHAIN_PATTERNS:
        if k in n:
            matches.append(k)
    return matches

# Load datasets
licenses = [json.loads(L) for L in open('la-licenses-ventura-corridor.jsonl') if L.strip()]
print(f"licenses loaded: {len(licenses):,}")

# Add the 34 Tarzana samples from earlier + 372 8-zip enumeration
yelp = json.load(open('tarzana-91356-yelp.json')) + [json.loads(L) for L in open('yelp-corridor-all.jsonl') if L.strip()]
# Dedupe by (name, address)
seen = set()
uniq = []
for r in yelp:
    k = (norm(r.get('name','')), norm(r.get('street_address','')))
    if k in seen: continue
    seen.add(k); uniq.append(r)
yelp = uniq
print(f"yelp candidates (deduped): {len(yelp)}")

# Index licenses by (zip, street_num)
idx = defaultdict(list)
for l in licenses:
    z = (l.get('zip_code') or '')[:5]
    n = street_num(l.get('street_address',''))
    if z and n: idx[(z,n)].append(l)
print(f"license index buckets: {len(idx):,}")

matched, flagged = [], []

for b in yelp:
    addr = b.get('street_address','')
    name = b.get('name','')
    z = (b.get('zip_code') or '').strip()[:5]
    if not z:
        # Extract trailing 5-digit zip (skip street numbers at the front).
        m = re.search(r'(?:CA|California)\s*,?\s*(\d{5})\b', addr) or re.search(r'(\d{5})\s*$', addr.strip())
        z = m.group(1) if m else ''
    n = street_num(addr)
    if not n or not z:
        flagged.append({**b, 'license_status':'no-match-found','reason':'no-street-num-or-zip'})
        continue

    cand = idx.get((z, n), [])
    # Pass-2 fallback: same zip, street_num ± 1..5 (corner stores often list adjacent)
    if not cand:
        for offset in (1,2,3,4,5,-1,-2,-3,-4,-5):
            try: alt = str(int(n)+offset)
            except: continue
            cand.extend(idx.get((z, alt), []))

    bn = norm(name)

    # Pass 1: direct name fuzzy match
    best = None; best_score = 0
    for l in cand:
        for cmp_name in (l.get('dba_name') or '', l.get('business_name') or ''):
            cn = norm(cmp_name)
            if not cn: continue
            sm = SequenceMatcher(None, bn, cn).ratio()
            bw = set(bn.split()); cw = set(cn.split())
            ov = len(bw & cw) / max(1, min(len(bw), len(cw))) if bw and cw else 0
            score = max(sm, ov)
            if score > best_score:
                best_score = score; best = (l, cmp_name)

    # Pass 2: chain pattern match
    chain_hit = None
    for ck in chain_keys(name):
        patterns = CHAIN_PATTERNS[ck]
        for l in cand:
            for cmp_name in (l.get('dba_name') or '', l.get('business_name') or ''):
                up = (cmp_name or '').upper()
                if any(p in up for p in patterns):
                    chain_hit = (l, cmp_name, ck); break
            if chain_hit: break
        if chain_hit: break

    if chain_hit:
        l, cmp_name, ck = chain_hit
        matched.append({**b, 'license_status':'matched-chain','license_name':cmp_name,'license_account':l.get('location_account'),'matched_via':f'chain:{ck}'})
    elif best and best_score >= 0.5:
        l, cmp_name = best
        matched.append({**b, 'license_status':'matched','license_name':cmp_name,'license_account':l.get('location_account'),'match_score':round(best_score,2)})
    else:
        flagged.append({**b, 'license_status':'no-match-found','best_score':round(best_score,2),'candidates_at_addr':len(cand)})

print(f"\nMATCHED: {len(matched)} ({len([m for m in matched if m['license_status']=='matched-chain'])} via chain pattern)")
print(f"FLAGGED (possibly no license): {len(flagged)}")

# Save
with open('census-corridor.jsonl','w') as f:
    for r in matched + flagged: f.write(json.dumps(r)+'\n')
print(f"→ census-corridor.jsonl ({len(matched)+len(flagged)} rows)")

# Top flagged sample
print("\nFlagged sample (top 15):")
for f in flagged[:15]:
    print(f"  ⚠️  {f.get('name','?')[:30]:<30} {f.get('street_address','?')[:35]:<35} z={f.get('zip_code','?')} cand={f.get('candidates_at_addr',0)}")