`recover_held.py`

143 lines
#!/usr/bin/env python3
"""
Image recovery for the image-held pile — SAFE (read-only external + local CSVs).

The 167 held products couldn't be matched by gallery filename (hash names,
numeric colorways). But rwltd.com is Shopify: {product_url}.js exposes an
authoritative variant.featured_image per colorway. Fetch each held pattern's .js
once, map colorway (from mfr_sku) -> the vendor's own featured_image, and promote
recovered rows into targets_ready.csv. Whatever still can't be matched stays held.

Writes only local CSVs. Idempotent. $0 (curl).
"""
import csv, json, os, re, time, subprocess, urllib.request, collections

HERE = os.path.dirname(os.path.abspath(__file__))

def slugify(s):
    return re.sub(r'-+', '-', re.sub(r'[^a-z0-9]+', '-', (s or '').lower())).strip('-')

def fetch_js(url):
    # Shopify's edge bot-protection fingerprints urllib (intermittent 403) but
    # lets curl through — shell to curl, with retry/backoff so a transient block
    # degrades gracefully instead of silently zeroing the whole recovery pass.
    for attempt in range(4):
        r = subprocess.run(
            ["curl", "-s", "--max-time", "25", "-A", "Mozilla/5.0", url + ".js"],
            capture_output=True, text=True)
        if r.returncode == 0 and r.stdout.strip():
            try:
                return json.loads(r.stdout)
            except Exception:
                pass
        time.sleep(1.5 * (attempt + 1))
    return None

def main():
    held = list(csv.DictReader(open(os.path.join(HERE, "targets_image_held.csv"))))
    ready = list(csv.DictReader(open(os.path.join(HERE, "targets_ready.csv"))))
    ready_cols = ready[0].keys() if ready else held[0].keys()

    # mfr_sku -> product_url from the catalog
    r = subprocess.run(["psql", "host=/tmp dbname=dw_unified", "-tA", "-F", "\t", "-c",
        "SELECT mfr_sku, product_url FROM rwltd_catalog WHERE product_url LIKE 'http%';"],
        capture_output=True, text=True)
    url_by_mfr = {}
    for line in r.stdout.strip().splitlines():
        if "\t" in line:
            m, u = line.split("\t", 1); url_by_mfr[m] = u

    # Verify against the authoritative vendor source (colorway == variant title):
    #  - held rows: recover an image if the vendor has one.
    #  - fuzzy "prefix" ready rows: CONFIRM via vendor-js (Cody gate) — the 4-char
    #    filename guess is not certainty; let the vendor's own data decide.
    #  - "exact" / already-"vendor-js" rows: trusted whole-token matches, kept as-is.
    trusted = [r for r in ready if r["image_confidence"] in ("exact", "vendor-js")]
    to_verify = held + [r for r in ready if r["image_confidence"] == "prefix"]

    js_cache = {}
    recovered, still_held = [], []
    for row in to_verify:
        mfr = row["mfr_sku"]
        url = url_by_mfr.get(mfr)
        colorway = mfr[len(slugify(row["pattern"]))+1:] if slugify(row["pattern"]) and \
                   slugify(mfr).startswith(slugify(row["pattern"])+"-") else slugify(mfr).rsplit("-",1)[-1]
        img = None
        if url:
            if url not in js_cache:
                js_cache[url] = fetch_js(url); time.sleep(0.4)
            d = js_cache[url]
            if d:
                cw = colorway.replace('-', '').lower()
                for v in d.get("variants", []):
                    vt = re.sub(r'[^a-z0-9]', '', (v.get("title") or "").lower())
                    fi = v.get("featured_image") or {}
                    if vt and cw and vt == cw and fi.get("src"):
                        img = fi["src"]; break
                if not img:  # looser: colorway token contained in variant title
                    for v in d.get("variants", []):
                        vt = re.sub(r'[^a-z0-9]', '', (v.get("title") or "").lower())
                        fi = v.get("featured_image") or {}
                        if vt and cw and (cw in vt or vt in cw) and fi.get("src"):
                            img = fi["src"]; break
        if img:
            row["image_url"] = img
            row["image_confidence"] = "vendor-js"
            recovered.append(row)
        else:
            # vendor can't confirm — hold for eyeball rather than ship a guess
            row["image_url"] = ""
            row["image_confidence"] = "unconfirmed-held"
            still_held.append(row)

    ready = trusted + recovered

    # collision post-pass: an image shared by >1 product is ambiguous (the vendor
    # reuses one swatch for two colorways) -> hold both rather than mis-ship.
    img_counts = collections.Counter(r["image_url"] for r in ready if r["image_url"])
    collided = {u for u, c in img_counts.items() if c > 1}
    if collided:
        keep = []
        for r in ready:
            if r["image_url"] in collided:
                r["image_confidence"] = "collision-held"
                r["image_url"] = ""
                still_held.append(r)
            else:
                keep.append(r)
        ready = keep

    def write_csv(path, recs, cols):
        with open(path, "w", newline="") as f:
            w = csv.DictWriter(f, fieldnames=list(cols)); w.writeheader()
            for r_ in recs:
                w.writerow({k: r_.get(k, "") for k in cols})

    write_csv(os.path.join(HERE, "targets_ready.csv"), ready, ready_cols)
    write_csv(os.path.join(HERE, "targets_image_held.csv"), still_held,
              held[0].keys() if held else ready_cols)

    conf_mix = dict(collections.Counter(r["image_confidence"] for r in ready))
    out = {
        "held_before": len(held),
        "verified_this_pass": len(to_verify),
        "recovered_or_confirmed_via_vendor_js": len(recovered),
        "still_held": len(still_held),
        "ready_now": len(ready),
        "ready_confidence_mix": conf_mix,
        "pages_fetched": len(js_cache),
    }
    print(json.dumps(out, indent=2))
    # keep summary.json authoritative (build_batch's is pre-recovery / stale otherwise)
    sp = os.path.join(HERE, "summary.json")
    try:
        s = json.load(open(sp))
    except Exception:
        s = {}
    s.update({"ready_to_onboard": len(ready), "image_held_for_eyeball": len(still_held),
              "ready_confidence_mix": conf_mix, "recovery": out})
    json.dump(s, open(sp, "w"), indent=2)

if __name__ == "__main__":
    main()