`onboard_gg_sg_wc.py`

70 lines
#!/usr/bin/env python3
"""Onboard the 242 NEW Singapore (goodrichglobal) wallcovering patterns into our staging.
Colorways from the Store API pa_full-collection terms; per-colorway images from variations;
distributor brand scraped from the page (.psg-single-brands-main, same Goodrich platform).
Parallel brand-scrape (headless Chrome). Appends to staging + brand map. $0 local."""
import json, os, re, subprocess, urllib.request, html as H
from concurrent.futures import ThreadPoolExecutor, as_completed

HERE = os.path.dirname(__file__)
SRC = os.path.join(HERE, 'staging', 'gg-sg-new-wc.jsonl')
STAGING = os.path.join(HERE, 'staging', 'sangetsu-staging.jsonl')
BRANDS = os.path.join(HERE, 'staging', 'sangetsu-brands.json')
CHROME = "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"
UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)'
BR = re.compile(r'psg-single-brands-main"?>(.*?)</div>', re.S | re.I); LN = re.compile(r'/brand/([a-z0-9-]+)/"[^>]*>([^<]+)<', re.I)

def api(u):
    return json.loads(urllib.request.urlopen(urllib.request.Request(u, headers={'User-Agent': UA}), timeout=30).read().decode('utf-8','ignore'))

def process(p):
    url = p['permalink']
    skus = []
    for at in (p.get('attributes') or []):
        if at.get('taxonomy') == 'pa_full-collection':
            skus = [t['name'] for t in at.get('terms', [])]
    imgs = {}
    try:
        base = url.split('/wallpaper-wallcovering/')[0].split('/singapore/')[0] + '/singapore/wp-json/wc/store/v1/products'
        for v in api(f"{base}?type=variation&per_page=100&parent={p['id']}"):
            val = None
            for va in (v.get('attributes') or []):
                if 'collection' in (va.get('name','').lower()): val = va.get('value')
            src = (v.get('images') or [{}])[0].get('src')
            if val and src: imgs[val] = src
    except Exception: pass
    if not imgs and p.get('images'):
        for s in skus: imgs[s] = p['images'][0]
    # brand
    dist, slug = None, None
    try:
        h = subprocess.run([CHROME,"--headless=new","--disable-gpu","--no-sandbox","--virtual-time-budget=12000","--dump-dom",url],
                           capture_output=True, timeout=45, text=True, errors='ignore').stdout
        m = BR.search(h or ''); a = LN.search(m.group(1)) if m else None
        if a: dist, slug = H.unescape(a.group(2).strip()), a.group(1)
    except Exception: pass
    rec = {"source":"sangetsu","source_url":url,"pattern":p['name'],"overview":None,
           "spec":{"width":None,"type":None,"weight":None,"backing":None,"is_grasscloth":False,"is_metallic":False},
           "colorway_skus":skus,"colorway_count":len(skus),"sku_images":imgs,
           "mfr_prefix":None,"status":"gg-sg-new","distributor":dist,"distributor_direct":False}
    return rec, (url, {"distributor": dist, "slug": slug})

def main():
    prods = [json.loads(l) for l in open(SRC) if l.strip()]
    print(f"onboarding {len(prods)} Singapore wallcovering patterns (parallel brand-scrape)")
    brands = json.load(open(BRANDS)) if os.path.exists(BRANDS) else {}
    done = 0
    with open(STAGING, 'a') as out, ThreadPoolExecutor(max_workers=6) as ex:
        for f in as_completed({ex.submit(process, p): p for p in prods}):
            rec, (url, bd) = f.result()
            out.write(json.dumps(rec, ensure_ascii=False) + '\n'); out.flush()
            brands[url] = bd; done += 1
            if done % 20 == 0: print(f"  {done}/{len(prods)} … last brand={bd['distributor']}")
    json.dump(brands, open(BRANDS,'w'), ensure_ascii=False, indent=0)
    from collections import Counter
    print("DONE. brand tally of the new:", dict(Counter(v['distributor'] for k,v in brands.items() if v.get('slug')).most_common()[:0] or []))
    print(f"appended {done} patterns")

if __name__ == '__main__':
    main()