`build_mfr_index.py`

113 lines
#!/usr/bin/env python3
"""Build data/mfr_index.json — a normalized manufacturer_sku -> DW house-SKU index for
the whole Shopify catalog, so scanning a vendor's printed code (WDW2310, WHF3846.WT.0,
etc.) resolves to the DW product even though DW files it under a house SKU (CHC-/DWKK-/…)
with the vendor code tucked in a metafield Shopify can't search by value.

Uses a Bulk Operation (server-side full-catalog export -> one JSONL) rather than paging
~160k products by hand. Read-only. $0 (local + Shopify read)."""
import json, os, re, time, urllib.request, sys
from datetime import datetime, timezone

SHOP = "designer-laboratory-sandbox.myshopify.com"; API = "2024-10"
ROOT = os.path.dirname(os.path.abspath(__file__))
OUT = os.path.join(ROOT, "data", "mfr_index.json")
TOKEN = os.environ.get("SHOPIFY_ADMIN_TOKEN", "")
if not TOKEN:
    for line in open(os.path.expanduser("~/Projects/secrets-manager/.env")):
        if line.startswith("SHOPIFY_ADMIN_TOKEN="):
            TOKEN = line.strip().split("=", 1)[1]; break
assert TOKEN, "no SHOPIFY_ADMIN_TOKEN"

def gql(query):
    req = urllib.request.Request(f"https://{SHOP}/admin/api/{API}/graphql.json",
        data=json.dumps({"query": query}).encode(),
        headers={"X-Shopify-Access-Token": TOKEN, "Content-Type": "application/json"})
    return json.load(urllib.request.urlopen(req, timeout=90))

def norm(s):
    return re.sub(r'[^A-Z0-9]', '', (s or '').upper())

# A vendor code field can be compound: "WDW-2290; Diamond WMT5001" -> ["WDW-2290","Diamond WMT5001"]
def parts(val):
    return [p for p in re.split(r'[;,/|]+', val or '') if p.strip()]

# ── 1) kick off the bulk export ──────────────────────────────────────────────
BULK_Q = '''
{
  products {
    edges { node {
      id handle status title
      mf: metafield(namespace:"custom", key:"manufacturer_sku"){ value }
      mf2: metafield(namespace:"dwc", key:"manufacturer_sku"){ value }
      variants { edges { node { id sku } } }
    }}
  }
}'''
start = gql('mutation { bulkOperationRunQuery(query: """%s""") { bulkOperation{ id status } userErrors{ field message } } }' % BULK_Q)
ue = start.get("data", {}).get("bulkOperationRunQuery", {}).get("userErrors", [])
if ue:
    print("bulk start errors:", ue)
    # a stale running op blocks new ones — surface it
    cur = gql('{ currentBulkOperation { id status } }')["data"]["currentBulkOperation"]
    print("currentBulkOperation:", cur); sys.exit(1)
print("bulk op started, polling…")

# ── 2) poll to completion ────────────────────────────────────────────────────
url = None
for _ in range(180):  # up to ~15 min
    time.sleep(5)
    cur = gql('{ currentBulkOperation { id status errorCode objectCount url } }')["data"]["currentBulkOperation"]
    print(f"  status={cur['status']} objects={cur.get('objectCount')}")
    if cur["status"] == "COMPLETED":
        url = cur["url"]; break
    if cur["status"] in ("FAILED", "CANCELED"):
        print("bulk failed:", cur); sys.exit(1)
assert url, "bulk op did not complete in time"

# ── 3) download JSONL + parse (products + their variant child lines) ──────────
data = urllib.request.urlopen(url, timeout=300).read().decode()
prods = {}   # gid -> {handle,status,title,mfrs:[...]}
vskus = {}   # parentGid -> [sku,...]
for line in data.splitlines():
    if not line.strip(): continue
    o = json.loads(line)
    if "__parentId" in o and "sku" in o:           # variant child line
        vskus.setdefault(o["__parentId"], []).append(o.get("sku") or "")
    elif "handle" in o:                             # product line
        mfrs = []
        for k in ("mf", "mf2"):
            v = (o.get(k) or {}).get("value")
            if v: mfrs += parts(v)
        prods[o["id"]] = {"handle": o["handle"], "status": o.get("status"),
                          "title": o.get("title", ""), "mfrs": mfrs}

# ── 4) build normalized index: norm(vendor code) -> {sku,title,status,handle} ─
def house_sku(skus):
    real = [s for s in skus if s and not s.lower().endswith("-sample")]
    return (real or [s for s in skus if s] or [""])[0]

index = {}
collisions = 0
for gid, p in prods.items():
    sku = house_sku(vskus.get(gid, []))
    if not sku: continue
    keys = set()
    for m in p["mfrs"]:
        n = norm(m)
        if len(n) >= 4: keys.add(n)         # skip junk like "0" / "WT"
    # also index the house SKU itself (so scanning the house code resolves too)
    if len(norm(sku)) >= 4: keys.add(norm(sku))
    for n in keys:
        if n in index and index[n]["sku"] != sku: collisions += 1
        index.setdefault(n, {"sku": sku, "title": p["title"][:80],
                             "status": p["status"], "handle": p["handle"]})

meta = {"_meta": {"products": len(prods), "keys": len(index), "collisions": collisions,
                  "built_at": datetime.now(timezone.utc).isoformat()}}
json.dump({**meta, **index}, open(OUT, "w"))
print(f"\nindexed {len(prods)} products -> {len(index)} normalized keys ({collisions} collisions) -> {OUT}")
# spot-checks
for probe in ("WDW2290", "WDW2310", "CHC217000", "WHF3846WT0"):
    print(f"  {probe} -> {index.get(probe, 'NOT FOUND')}")