`parse.py`

50 lines
#!/usr/bin/env python3
# Parse downloaded Goodrich Koroseal collection pages -> per-SKU codes + specs + image, JSONL.
import os, re, json, glob, html

COLL_DIR = os.path.expanduser("~/Projects/koroseal-goodrich-audit/coll")
OUT = os.path.expanduser("~/Projects/koroseal-goodrich-audit/goodrich_koroseal.jsonl")

# a Goodrich product code looks like FB21-01, AL21-05, G0167-XX, 9B21-35, etc.
CODE_RE = re.compile(r'^[A-Za-z0-9]{2,7}-[0-9A-Za-z]{1,4}$')
IMG_RE  = re.compile(r'uploads/[0-9]{4}/[0-9]{2}/([A-Za-z0-9_-]+)\.jpg', re.I)
TITLE_RE= re.compile(r'<title>(.*?)</title>', re.S|re.I)
OG_RE   = re.compile(r'property=["\']og:title["\']\s+content=["\'](.*?)["\']', re.I)
SPEC_RE = re.compile(r'(Width|Repeat|Match|Composition|Material|Fire Rating|Type|Usage|Durability|Weight|Roll Length|Backing)\s*[:\-]?\s*</[^>]+>\s*<[^>]+>\s*([^<]{1,60})', re.I)

def clean(s): return html.unescape(re.sub(r'\s+',' ',s or '')).strip()

rows=0
with open(OUT,'w') as out:
    for fp in sorted(glob.glob(os.path.join(COLL_DIR,'*.html'))):
        slug=os.path.basename(fp)[:-5]
        h=open(fp,encoding='utf-8',errors='ignore').read()
        name=''
        m=OG_RE.search(h) or TITLE_RE.search(h)
        if m: name=clean(m.group(1)).split('|')[0].strip()
        # codes from image filenames
        codes=set()
        for c in IMG_RE.findall(h):
            base=re.sub(r'-(150x150|300x300|scaled|\d+x\d+)$','',c,flags=re.I)
            if CODE_RE.match(base) and not re.match(r'^(logo|icon|banner)',base,re.I):
                codes.add(base.upper())
        # crude spec grab
        specs={}
        for k,v in SPEC_RE.findall(h):
            k=clean(k).title(); v=clean(v)
            if v and k not in specs: specs[k]=v
        series=sorted({re.match(r'^([A-Za-z0-9]{2,7})-',c).group(1) for c in codes if re.match(r'^([A-Za-z0-9]{2,7})-',c)})
        out.write(json.dumps({"slug":slug,"name":name,"series":series,"n_codes":len(codes),
                              "codes":sorted(codes),"specs":specs})+"\n")
        rows+=1
print(f"parsed {rows} collections -> {OUT}")
# quick summary
allcodes=set(); coll_no_codes=[]
for line in open(OUT):
    d=json.loads(line)
    allcodes|=set(d["codes"])
    if not d["codes"]: coll_no_codes.append(d["slug"])
print(f"total distinct Goodrich Koroseal SKU codes: {len(allcodes)}")
print(f"collections with 0 parsed codes: {len(coll_no_codes)} (may need JS/other img pattern): {coll_no_codes[:10]}")
open(os.path.expanduser("~/Projects/koroseal-goodrich-audit/all_codes.txt"),'w').write("\n".join(sorted(allcodes)))