`scrape_sangetsu_brands.py`

70 lines
#!/usr/bin/env python3
"""Scrape the REAL per-pattern distributor (the .psg-single-brands-main brand link:
Koroseal / Eijffinger / J.Josephson / Omexco / Versa / Cole & Son / ...) from each
Sangetsu pattern page on sangetsu-goodrich.co.th, into staging/sangetsu-brands.json
as {source_url: {distributor, slug}}. Headless Chrome (JS-rendered site), parallel,
RESUMABLE (skips URLs already in the output). $0 local."""
import json, os, re, subprocess, sys
from concurrent.futures import ThreadPoolExecutor, as_completed

HERE = os.path.dirname(__file__)
STAGING = os.path.join(HERE, 'staging', 'sangetsu-staging.jsonl')
OUT = os.path.join(HERE, 'staging', 'sangetsu-brands.json')
CHROME = "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"
WORKERS = int(os.environ.get('WORKERS', '6'))

def urls():
    s = set()
    for line in open(STAGING):
        line = line.strip()
        if line:
            u = json.loads(line).get('source_url')
            if u: s.add(u)
    return sorted(s)

BRAND_RE = re.compile(r'psg-single-brands-main"?>(.*?)</div>', re.S | re.I)
LINK_RE = re.compile(r'/brand/([a-z0-9-]+)/"[^>]*>([^<]+)<', re.I)

def scrape(u):
    try:
        html = subprocess.run(
            [CHROME, "--headless=new", "--disable-gpu", "--no-sandbox",
             "--virtual-time-budget=11000", "--dump-dom", u],
            capture_output=True, timeout=45, text=True, errors='ignore').stdout
        m = BRAND_RE.search(html or '')
        if not m: return u, None, None
        a = LINK_RE.search(m.group(1))
        if a: return u, a.group(2).strip(), a.group(1).strip()
        txt = re.sub(r'\s+', ' ', re.sub(r'<[^>]+>', ' ', m.group(1))).strip()
        return u, (txt.split('  ')[0] or None), None
    except Exception:
        return u, None, None

def main():
    out = {}
    if os.path.exists(OUT):
        try: out = json.load(open(OUT))
        except Exception: out = {}
    todo = [u for u in urls() if u not in out]
    print(f"total {len(out)+len(todo)} patterns | already {len(out)} | scraping {len(todo)} | {WORKERS} workers")
    done = 0
    with ThreadPoolExecutor(max_workers=WORKERS) as ex:
        futs = {ex.submit(scrape, u): u for u in todo}
        for f in as_completed(futs):
            u, dist, slug = f.result()
            out[u] = {"distributor": dist, "slug": slug}
            done += 1
            if done % 25 == 0 or done == len(todo):
                json.dump(out, open(OUT, 'w'), ensure_ascii=False, indent=0)
                print(f"  {done}/{len(todo)} … last: {dist!r}")
    json.dump(out, open(OUT, 'w'), ensure_ascii=False, indent=0)
    from collections import Counter
    c = Counter(v.get('distributor') for v in out.values())
    print("DONE. distributor tally:")
    for k, n in c.most_common(): print(f"  {n:4d}  {k!r}")
    miss = sum(1 for v in out.values() if not v.get('distributor'))
    print(f"missing/none: {miss}")

if __name__ == '__main__':
    main()