`scrape_gg_sg_brands.py`

58 lines
#!/usr/bin/env python3
"""Scrape the distributor brand for the 242 Singapore (goodrichglobal) wallcovering patterns
that came in blank. Singapore markup differs from Thailand: class="psg-single-brands" +
/product_brand/<slug>/ (Thailand was ...-main + /brand/). Updates the distributor field on
those staging rows + the brand map. Parallel headless, $0 local, resumable-ish (idempotent)."""
import json, os, re, subprocess, html as H
from concurrent.futures import ThreadPoolExecutor, as_completed

HERE = os.path.dirname(__file__)
STAGING = os.path.join(HERE, 'staging', 'sangetsu-staging.jsonl')
BRANDS = os.path.join(HERE, 'staging', 'sangetsu-brands.json')
CHROME = "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"
# brand link: /singapore/product_brand/<slug>/ ... >Name</a>  (class psg-single-brands)
LINK = re.compile(r'/product_brand/([a-z0-9-]+)/"[^>]*class="psg-single-brands"[^>]*>([^<]+)<', re.I)
ALT = re.compile(r'class="psg-single-brands"[^>]*>([^<]+)</a>', re.I)

def brand(url):
    try:
        h = subprocess.run([CHROME,"--headless=new","--disable-gpu","--no-sandbox","--virtual-time-budget=12000","--dump-dom",url],
                           capture_output=True, timeout=45, text=True, errors='ignore').stdout
        m = LINK.search(h or '')
        if m: return url, H.unescape(m.group(2).strip()), m.group(1)
        a = ALT.search(h or '')
        if a: return url, H.unescape(a.group(1).strip()), None
        return url, None, None
    except Exception:
        return url, None, None

def main():
    rows = [json.loads(l) for l in open(STAGING) if l.strip()]
    targets = [r['source_url'] for r in rows if r.get('status') == 'gg-sg-new' and not r.get('distributor')]
    print(f"scraping brand for {len(targets)} Singapore patterns")
    got = {}
    with ThreadPoolExecutor(max_workers=6) as ex:
        done = 0
        for f in as_completed({ex.submit(brand, u): u for u in targets}):
            u, dist, slug = f.result()
            if dist: got[u] = {"distributor": dist, "slug": slug}
            done += 1
            if done % 25 == 0: print(f"  {done}/{len(targets)} … last={dist}")
    # rewrite staging distributor for those rows
    for r in rows:
        if r['source_url'] in got:
            r['distributor'] = got[r['source_url']]['distributor']
    with open(STAGING, 'w') as out:
        for r in rows: out.write(json.dumps(r, ensure_ascii=False) + '\n')
    # merge brand map
    bm = json.load(open(BRANDS)) if os.path.exists(BRANDS) else {}
    bm.update(got)
    json.dump(bm, open(BRANDS, 'w'), ensure_ascii=False, indent=0)
    from collections import Counter
    c = Counter(v['distributor'] for v in got.values())
    print(f"\nresolved {len(got)}/{len(targets)} distributors:")
    for k, n in c.most_common(): print(f"  {n:4d}  {k}")

if __name__ == '__main__':
    main()