`scripts/analyze.py`

36 lines
import csv, collections, re
f="data/MasterLicenseData.csv"
states=collections.Counter(); classes=collections.Counter(); status=collections.Counter()
n=0; delim_samples=[]
name_kw=re.compile(r'WALLCOVER|WALLPAPER|PAPER ?HANG', re.I)
kw_hits=0
with open(f, newline='', encoding='latin-1') as fh:
    r=csv.DictReader(fh)
    cls_field='Classifications(s)'
    for row in r:
        n+=1
        states[row.get('State','')]+=1
        status[row.get('PrimaryStatus','')]+=1
        c=(row.get(cls_field) or '').strip()
        if len(delim_samples)<8 and (' ' in c or ',' in c or '|' in c) and c:
            delim_samples.append(c)
        # tokenize on whitespace/comma/pipe
        for tok in re.split(r'[\s,|/]+', c):
            if tok: classes[tok.upper()]+=1
        full=(row.get('FullBusinessName') or '')+' '+(row.get('BusinessName') or '')
        if name_kw.search(full): kw_hits+=1
print("TOTAL ROWS:", n)
print("\nSTATE (top8):", states.most_common(8))
print("\nPRIMARYSTATUS (top8):", status.most_common(8))
print("\nMULTI-CLASS DELIM SAMPLES:")
for s in delim_samples: print("   ["+s+"]")
print("\nTARGET CLASS COUNTS:")
for k in ["C52","C54","C33","D06","D-06","C61"]:
    print(f"   {k}: {classes.get(k,0)}")
print("\nALL D-CODES present:")
for k,v in sorted(classes.items()):
    if k.startswith('D'): print(f"   {k}: {v}")
print("\nTOP 25 CLASالسSES:")
for k,v in classes.most_common(25): print(f"   {k}: {v}")
print("\nNAME-KEYWORD (wallcover/wallpaper/paperhang) hits:", kw_hits)