`scripts/artmura-onboard/fill_specs.py`

63 lines
#!/usr/bin/env python3
"""Gently fill missing PDP spec fields in data/artmura.json (sequential, backoff-aware)."""
import json, os, re, time, urllib.request

HERE = os.path.dirname(os.path.abspath(__file__))
PKG = os.path.join(HERE, "data", "artmura.json")
UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"

def fetch(url):
    for i in range(5):
        try:
            req = urllib.request.Request(url, headers={"User-Agent": UA})
            with urllib.request.urlopen(req, timeout=30) as r:
                return r.read().decode("utf-8", "replace")
        except urllib.error.HTTPError as e:
            if e.code == 429:
                wait = 20 * (i + 1)
                print(f"    429 — backing off {wait}s")
                time.sleep(wait); continue
            raise
        except Exception:
            time.sleep(5)
    return None

def parse_specs(h):
    pairs = re.findall(r'product-info__label">([^<]+)</th>\s*<td class="product-info__value">(.*?)</td>', h, re.S)
    out, seen = {}, set()
    for lbl, val in pairs:
        lbl = lbl.strip().rstrip(":")
        if lbl in seen: continue
        seen.add(lbl)
        out[lbl] = re.sub(r"<[^>]+>", "", val).strip()
    return out

def width_inches(dim):
    m = re.search(r'([\d.]+)\s*"', dim or "")
    return float(m.group(1)) if m else None

d = json.load(open(PKG))
todo = [r for r in d["products"] if not r["dimensions"]]
print(f"filling {len(todo)} products with missing specs ...")
filled = 0
for n, r in enumerate(todo, 1):
    h = fetch(r["product_url"])
    if not h:
        print(f"  [{n}/{len(todo)}] {r['mfr_sku']} FAILED"); continue
    s = parse_specs(h)
    if s.get("Dimensions"):
        r["substrate"] = r["substrate"] or s.get("Substrate available")
        r["grade"] = s.get("Grade available")
        r["dimensions"] = s.get("Dimensions")
        r["width_inches"] = width_inches(s.get("Dimensions"))
        r["wall_coverage"] = s.get("Wall coverage")
        r["lead_time"] = s.get("Lead-time")
        r["origin"] = s.get("Origin")
        filled += 1
        print(f"  [{n}/{len(todo)}] {r['mfr_sku']} ✓ {r['dimensions']} · {r['origin']}")
    else:
        print(f"  [{n}/{len(todo)}] {r['mfr_sku']} no spec table")
    json.dump(d, open(PKG, "w"), indent=2)  # checkpoint each
    time.sleep(1.2)
print(f"done. filled {filled}/{len(todo)}. total w/dims now {sum(1 for r in d['products'] if r['dimensions'])}/161")