`crawl_vendor_corporate.py`

89 lines
#!/usr/bin/env python3
"""Overnight feed-first crawl of the 25 vendor CORPORATE sites for their FULL catalogs
(products NOT on Sangetsu/Goodrich). $0 — pure HTTP (Shopify products.json OR sitemap product
URLs + page meta). NO paid API, NO tokens. Per-vendor resumable. Enrichment (color via Pillow,
style via local Ollama qwen2.5vl) is a SEPARATE local pass — this file just harvests products."""
import json, os, re, time, gzip, urllib.request
from concurrent.futures import ThreadPoolExecutor, as_completed

HERE = os.path.dirname(__file__)
SITES = json.load(open(os.path.join(HERE, 'config', 'vendor-corporate.json')))
OUTDIR = os.path.join(HERE, 'staging', 'corporate')
UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)'

def get(u, timeout=25):
    r = urllib.request.urlopen(urllib.request.Request(u, headers={'User-Agent': UA}), timeout=timeout)
    data = r.read()
    if r.headers.get('Content-Encoding') == 'gzip': data = gzip.decompress(data)
    return data
def getjson(u): return json.loads(get(u).decode('utf-8', 'ignore'))
def gettext(u): return get(u).decode('utf-8', 'ignore')

def shopify(base):
    out, page = [], 1
    while page <= 60:
        try: d = getjson(f"{base}/products.json?limit=250&page={page}")
        except Exception: break
        ps = d.get('products', [])
        if not ps: break
        for p in ps:
            out.append({'title': p.get('title'), 'handle': p.get('handle'), 'type': p.get('product_type'),
                        'url': f"{base}/products/{p.get('handle')}", 'image': (p.get('images') or [{}])[0].get('src'),
                        'variants': len(p.get('variants') or []), 'src': 'shopify'})
        if len(ps) < 250: break
        page += 1; time.sleep(0.3)
    return out

def sitemap_urls(base):
    urls = set()
    for sm in ['/product-sitemap.xml', '/wp-sitemap-posts-product-1.xml', '/sitemap_index.xml', '/sitemap.xml']:
        try: xml = gettext(base + sm)
        except Exception: continue
        for s in re.findall(r'<loc>([^<]+sitemap[^<]*\.xml)</loc>', xml)[:15]:
            if any(k in s.lower() for k in ('product', 'wallcover', 'wallpaper', 'collection')):
                try: xml += gettext(s)
                except Exception: pass
        for loc in re.findall(r'<loc>([^<]+)</loc>', xml):
            if re.search(r'/(product|products|wallcovering|wallpaper|collection)s?/[^/?<]+/?$', loc, re.I):
                urls.add(loc.strip())
        if len(urls) > 5: break
    return sorted(urls)

def page_meta(u):
    try:
        h = gettext(u)
        t = re.search(r'<title>([^<]+)', h); og = re.search(r'og:image"[^>]*content="([^"]+)"', h)
        return {'url': u, 'title': (t.group(1).strip() if t else None), 'image': (og.group(1) if og else None), 'src': 'sitemap'}
    except Exception:
        return {'url': u, 'title': None, 'image': None, 'src': 'sitemap'}

def crawl(vendor, base):
    out_f = os.path.join(OUTDIR, re.sub(r'[^a-z0-9]+', '-', vendor.lower()).strip('-') + '.jsonl')
    if os.path.exists(out_f) and os.path.getsize(out_f) > 0: return 'skip(done)', 0
    prods = []
    try:
        if getjson(f"{base}/products.json?limit=1").get('products') is not None:
            prods = shopify(base)
    except Exception: pass
    if not prods:
        urls = sitemap_urls(base)
        with ThreadPoolExecutor(max_workers=6) as ex:
            prods = [f.result() for f in as_completed({ex.submit(page_meta, u): u for u in urls[:2500]})]
    with open(out_f, 'w') as o:
        for p in prods: o.write(json.dumps(p, ensure_ascii=False) + '\n')
    return 'ok', len(prods)

def main():
    os.makedirs(OUTDIR, exist_ok=True)
    total = 0
    for vendor, base in SITES.items():
        try:
            status, n = crawl(vendor, base); total += n
            print(f"[{time.strftime('%H:%M:%S')}] {vendor:22s} {status:12s} {n:5d} products", flush=True)
        except Exception as e:
            print(f"[{time.strftime('%H:%M:%S')}] {vendor:22s} ERROR {str(e)[:60]}", flush=True)
    print(f"ALL VENDORS DONE — {total} products harvested (offline, $0)", flush=True)

if __name__ == '__main__':
    main()