`scrape_sangetsu_backfill.py`

128 lines
#!/usr/bin/env python3
"""
Sangetsu colorway/image BACKFILL — fills the 257 patterns whose original scrape
left colorway_skus:[] (they render as blank cards on japan.).

Source: sangetsu-goodrich.co.th is WooCommerce; its Store API is public
(the WP REST proper is locked by Kadence security, but /wc/store/v1 is open):
  GET /wc/store/v1/products?slug=<slug>              → parent (attributes = SKU list)
  GET /wc/store/v1/products?type=variation&parent=ID → variation images (filename = SKU)

A variable product's "Full Collection" attribute holds the clean colorway SKUs
(A222-10, A222-35, …); each variation's image filename embeds its SKU. We map
SKU→image by "basename starts with SKU" so the SKU's own trailing -NN isn't
confused with WordPress's -1 dedup suffix.

SAFE: only fills rows currently missing colorways; never touches populated rows.
Writes a SEPARATE merged file by default (staging/sangetsu-staging.backfilled.jsonl);
pass --apply to atomically replace staging/sangetsu-staging.jsonl (temp+rename).

Usage: scrape_sangetsu_backfill.py [--limit N] [--apply]
"""
import json, os, re, sys, time, urllib.request, urllib.error

ROOT = os.path.dirname(__file__)
STAGING = os.path.join(ROOT, "staging", "sangetsu-staging.jsonl")
OUT = os.path.join(ROOT, "staging", "sangetsu-staging.backfilled.jsonl")
API = "https://www.sangetsu-goodrich.co.th/wp-json/wc/store/v1"
UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) japan-enrich/sangetsu-backfill"

LIMIT = int(sys.argv[sys.argv.index("--limit") + 1]) if "--limit" in sys.argv else None
APPLY = "--apply" in sys.argv

DIM = re.compile(r"-\d+x\d+(?=\.\w+$)")           # WP thumbnail dimension suffix
def full_src(src):                                 # strip -150x150 → full-size upload
    return DIM.sub("", src or "")
def stem(src):
    return os.path.basename(full_src(src)).rsplit(".", 1)[0]
def slug_of(url):
    return (url or "").rstrip("/").rsplit("/", 1)[-1]


def get(path, tries=4):
    for i in range(tries):
        try:
            req = urllib.request.Request(API + path, headers={"User-Agent": UA, "Accept": "application/json"})
            with urllib.request.urlopen(req, timeout=45) as r:
                return json.loads(r.read().decode("utf-8"))
        except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as e:
            if i == tries - 1:
                return None
            time.sleep(1.2 * (i + 1))
    return None


def backfill_one(url):
    """Return (colorway_skus, sku_images, mfr_prefix) or (None,None,None)."""
    slug = slug_of(url)
    plist = get(f"/products?slug={slug}")
    if not (isinstance(plist, list) and plist):
        return None, None, None
    p = plist[0]
    # colorway SKUs: the attribute with the most term entries (the "Full Collection")
    skus = []
    for a in p.get("attributes") or []:
        terms = [t.get("name", "").strip() for t in (a.get("terms") or []) if t.get("name")]
        if len(terms) > len(skus):
            skus = terms
    # variation images: filename stem embeds the SKU
    var = get(f"/products?type=variation&parent={p['id']}&per_page=100") or []
    srcs = []
    for v in (var if isinstance(var, list) else []):
        for im in (v.get("images") or []):
            if im.get("src"):
                srcs.append(full_src(im["src"]))
    # parent gallery images too (some variations reuse the parent image)
    for im in (p.get("images") or []):
        if im.get("src"):
            srcs.append(full_src(im["src"]))
    srcs = list(dict.fromkeys(srcs))               # dedupe, keep order
    # fallback: if no attribute SKUs, derive from image stems that look like SKUs
    if not skus:
        cand = [stem(s) for s in srcs]
        skus = [c for c in dict.fromkeys(cand) if re.match(r"^[A-Z0-9]+(-[A-Z0-9]+)*$", c) and any(ch.isdigit() for ch in c)]
    imgs = {}
    for sku in skus:
        # prefer exact "<sku>.jpg", else a stem that starts with the sku (WP -1 dedup)
        exact = next((s for s in srcs if stem(s) == sku), None)
        pref = exact or next((s for s in srcs if stem(s).startswith(sku)), None)
        if pref:
            imgs[sku] = pref
    pm = re.match(r"^[A-Za-z]+", skus[0]) if skus else None
    prefix = pm.group(0) if pm else None
    return skus, imgs, prefix


def main():
    rows = [json.loads(l) for l in open(STAGING) if l.strip()]
    empties = [r for r in rows if not (r.get("colorway_skus"))]
    todo = empties[:LIMIT] if LIMIT else empties
    print(f"sangetsu backfill: {len(rows)} rows, {len(empties)} imageless, doing {len(todo)}", flush=True)
    filled = still = 0
    for i, r in enumerate(todo, 1):
        skus, imgs, prefix = backfill_one(r.get("source_url"))
        if skus and imgs:
            r["colorway_skus"] = skus
            r["colorway_count"] = len(skus)
            r["sku_images"] = imgs
            if prefix:
                r["mfr_prefix"] = prefix
            r["backfilled"] = True
            filled += 1
            print(f"  [{i}/{len(todo)}] {(r.get('pattern') or '?'):28} +{len(skus)} colorways / {len(imgs)} imgs", flush=True)
        else:
            still += 1
            print(f"  [{i}/{len(todo)}] {(r.get('pattern') or '?'):28} — no data ({slug_of(r.get('source_url'))})", flush=True)
        time.sleep(0.25)                            # polite
    dest = STAGING if APPLY else OUT
    tmp = dest + ".tmp"
    with open(tmp, "w") as f:
        for r in rows:
            f.write(json.dumps(r, ensure_ascii=False) + "\n")
    os.replace(tmp, dest)
    print(f"\nbackfilled {filled}, still-empty {still}  → {dest}{'  (APPLIED to staging)' if APPLY else '  (dry-run; --apply to merge)'}", flush=True)


if __name__ == "__main__":
    main()