`koroseal_spec_crawl.py`

40 lines
#!/usr/bin/env python3
# Crawl koroseal.com wallcovering product pages -> per-SKU specs + FB21-style code, JSONL.
import re,html,json,sys,urllib.request
from concurrent.futures import ThreadPoolExecutor
UA="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15"
OUT=open("/Users/macstudio3/Projects/koroseal-goodrich-audit/koroseal_specs.jsonl","w")
SM="/tmp/kor_sm.xml"

urls=sorted(set(re.findall(r'https://koroseal\.com/products/wallcoverings/[^\s<]+', open(SM,encoding="utf-8",errors="ignore").read())))
urls=[u for u in urls if not u.rstrip('/').endswith(('/in-stock','/archived-products','/brand','/wallcoverings'))]
print(f"product urls: {len(urls)}", file=sys.stderr)

SPEC_P=re.compile(r'<p>\s*([A-Za-z][A-Za-z /]+?)(?:<sup>\d+</sup>)?\s*:\s*<span>\s*([^<]+?)\s*</span>', re.S)
CODE=re.compile(r'/([A-Za-z0-9]{2,7})_(\d{2,3})_[A-Za-z0-9]+(?:_[A-Za-z0-9]+)?\.(?:jpg|png)', re.I)
H1=re.compile(r'<h1[^>]*>(.*?)</h1>', re.S|re.I)
def clean(s): return html.unescape(re.sub(r'\s+',' ',s or '')).strip()

def fetch(u):
    try:
        req=urllib.request.Request(u,headers={"User-Agent":UA})
        h=urllib.request.urlopen(req,timeout=15).read().decode("utf-8","ignore")
    except Exception as e:
        return {"url":u,"err":str(e)[:80]}
    specs={clean(k).title():clean(v) for k,v in SPEC_P.findall(h)}
    codes=sorted({f"{a.upper()}-{int(b):02d}" for a,b in CODE.findall(h)})
    parts=u.rstrip('/').split('/')
    series=parts[-2] if len(parts)>=2 and parts[-1] not in ('wallcoverings',) else parts[-1]
    color=parts[-1]
    title=clean(re.sub(r'<[^>]+>','',(H1.search(h).group(1) if H1.search(h) else '')))
    return {"url":u,"series":series,"color":color,"title":title,"codes":codes,"specs":specs}

n=0; hit=0
with ThreadPoolExecutor(max_workers=10) as ex:
    for r in ex.map(fetch, urls):
        OUT.write(json.dumps(r)+"\n"); n+=1
        if r.get("specs"): hit+=1
        if n%300==0: print(f"{n}/{len(urls)} ({hit} w/specs)", file=sys.stderr)
OUT.close()
print(f"DONE {n} pages, {hit} with specs", file=sys.stderr)