`refresh_mfr_index.py`

80 lines
#!/usr/bin/env python3
"""Incremental refresh of data/mfr_index.json — pulls ONLY products changed since the
last build/refresh (updated_at filter) and merges them in. Runs in seconds, $0, so it
can fire hourly during work hours to keep same-day-imported SKUs scannable. The nightly
full build_mfr_index.py is the backstop that also clears stale/deleted keys."""
import json, os, re, sys, time, urllib.request
from datetime import datetime, timezone, timedelta

SHOP = "designer-laboratory-sandbox.myshopify.com"; API = "2024-10"
ROOT = os.path.dirname(os.path.abspath(__file__))
OUT = os.path.join(ROOT, "data", "mfr_index.json")
TOKEN = os.environ.get("SHOPIFY_ADMIN_TOKEN", "")
if not TOKEN:
    for line in open(os.path.expanduser("~/Projects/secrets-manager/.env")):
        if line.startswith("SHOPIFY_ADMIN_TOKEN="):
            TOKEN = line.strip().split("=", 1)[1]; break
assert TOKEN, "no SHOPIFY_ADMIN_TOKEN"

def gql(query, variables=None):
    req = urllib.request.Request(f"https://{SHOP}/admin/api/{API}/graphql.json",
        data=json.dumps({"query": query, "variables": variables or {}}).encode(),
        headers={"X-Shopify-Access-Token": TOKEN, "Content-Type": "application/json"})
    return json.load(urllib.request.urlopen(req, timeout=90))

def norm(s):  return re.sub(r'[^A-Z0-9]', '', (s or '').upper())
def parts(v): return [p for p in re.split(r'[;,/|]+', v or '') if p.strip()]
def house_sku(skus):
    real = [s for s in skus if s and not s.lower().endswith("-sample")]
    return (real or [s for s in skus if s] or [""])[0]

if not os.path.exists(OUT):
    print("no index yet — run build_mfr_index.py first (full build)"); sys.exit(1)
idx = json.load(open(OUT))
meta = idx.get("_meta", {})
# baseline: last built_at, else the file's mtime, minus a 10-min safety overlap
since_iso = meta.get("built_at")
if not since_iso:
    since_iso = datetime.fromtimestamp(os.path.getmtime(OUT), tz=timezone.utc).isoformat()
since = datetime.fromisoformat(since_iso) - timedelta(minutes=10)
since_q = since.strftime("%Y-%m-%dT%H:%M:%SZ")
print(f"refreshing products updated since {since_q}")

Q = '''query($c:String,$q:String){ products(first:100, query:$q, after:$c){
  pageInfo{hasNextPage endCursor}
  edges{node{ id handle status title
    mf: metafield(namespace:"custom",key:"manufacturer_sku"){value}
    mf2: metafield(namespace:"dwc",key:"manufacturer_sku"){value}
    variants(first:10){edges{node{sku}}} }} } }'''
cur = None; changed = 0; pages = 0
while True:
    g = gql(Q, {"c": cur, "q": f"updated_at:>='{since_q}'"})
    d = g.get("data", {}).get("products")
    if d is None:
        print("GQL err:", json.dumps(g)[:300]); sys.exit(1)
    for e in d["edges"]:
        n = e["node"]
        sku = house_sku([v["node"]["sku"] for v in n["variants"]["edges"]])
        if not sku: continue
        keys = set()
        for k in ("mf", "mf2"):
            for m in parts((n.get(k) or {}).get("value")):
                if len(norm(m)) >= 4: keys.add(norm(m))
        if len(norm(sku)) >= 4: keys.add(norm(sku))
        rec = {"sku": sku, "title": (n.get("title") or "")[:80], "status": n.get("status"), "handle": n.get("handle")}
        for kk in keys: idx[kk] = rec
        changed += 1
    pages += 1
    if d["pageInfo"]["hasNextPage"] and pages < 80:
        cur = d["pageInfo"]["endCursor"]; time.sleep(0.3)
    else:
        if d["pageInfo"]["hasNextPage"]: print("⚠ stopped at 80 pages — nightly full build will reconcile")
        break

idx["_meta"] = {**meta, "keys": sum(1 for k in idx if k != "_meta"),
                "built_at": datetime.now(timezone.utc).isoformat(),
                "last_incremental": datetime.now(timezone.utc).isoformat(),
                "last_incremental_changed": changed}
json.dump(idx, open(OUT, "w"))
print(f"merged {changed} changed products across {pages} page(s) -> {idx['_meta']['keys']} keys")