`scrape_sangetsu_storeapi.py`

53 lines
#!/usr/bin/env python3
"""Find MORE Sangetsu products via the WooCommerce Store API (feed-first, $0, no anti-bot).
The WP product sitemap listed 623; the Store API reports ~879 — so ~256 more patterns.
Paginates /wp-json/wc/store/v1/products, writes full product rows to
staging/sangetsu-storeapi.jsonl, and reports how many are NEW vs our current 623."""
import json, os, re, time, urllib.request

HERE = os.path.dirname(__file__)
BASE = "https://www.sangetsu-goodrich.co.th/wp-json/wc/store/v1/products"
OUT = os.path.join(HERE, 'staging', 'sangetsu-storeapi.jsonl')
STAGING = os.path.join(HERE, 'staging', 'sangetsu-staging.jsonl')
UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)'

def fetch(page):
    req = urllib.request.Request(f"{BASE}?per_page=100&page={page}", headers={'User-Agent': UA})
    return json.loads(urllib.request.urlopen(req, timeout=30).read().decode('utf-8', 'ignore'))

def slug_of(u): return (u or '').rstrip('/').split('/')[-1].lower()

def main():
    ours = set()
    for l in open(STAGING):
        l = l.strip()
        if l: ours.add(slug_of(json.loads(l).get('source_url', '')))
    all_prod, page = [], 1
    while True:
        try: batch = fetch(page)
        except Exception as e: print(f"page {page} stop: {e}"); break
        if not batch: break
        all_prod.extend(batch)
        print(f"  page {page}: +{len(batch)} (total {len(all_prod)})")
        if len(batch) < 100: break
        page += 1; time.sleep(0.4)
    with open(OUT, 'w') as f:
        for p in all_prod:
            f.write(json.dumps({
                'id': p.get('id'), 'name': p.get('name'), 'slug': p.get('slug'),
                'sku': p.get('sku'), 'type': p.get('type'), 'permalink': p.get('permalink'),
                'price': (p.get('prices') or {}).get('price'),
                'images': [im.get('src') for im in (p.get('images') or [])],
                'variations': p.get('variations') or [],
                'attributes': p.get('attributes') or [],
            }, ensure_ascii=False) + '\n')
    slugs = {slug_of(p.get('permalink')) or (p.get('slug') or '').lower() for p in all_prod}
    new = slugs - ours
    print(f"\nStore API products: {len(all_prod)} | distinct slugs: {len(slugs)}")
    print(f"already have (of 623): {len(slugs & ours)} | GENUINELY NEW: {len(new)}")
    for s in sorted(new)[:25]: print('  +', s)
    print(f"\n→ {OUT}")

if __name__ == '__main__':
    main()