← back to Dw Photo Capture
auto-save: 2026-06-25T17:36:07 (2 files) — build_mfr_index.py refresh_mfr_index.py
dcb3fdad5398800e8796e7e32e04c2d410f94ef1 · 2026-06-25 17:36:10 -0700 · Steve Abrams
Files touched
M build_mfr_index.pyA refresh_mfr_index.py
Diff
commit dcb3fdad5398800e8796e7e32e04c2d410f94ef1
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Thu Jun 25 17:36:10 2026 -0700
auto-save: 2026-06-25T17:36:07 (2 files) — build_mfr_index.py refresh_mfr_index.py
---
build_mfr_index.py | 4 ++-
refresh_mfr_index.py | 79 ++++++++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 82 insertions(+), 1 deletion(-)
diff --git a/build_mfr_index.py b/build_mfr_index.py
index 0a03f21..904d6bd 100644
--- a/build_mfr_index.py
+++ b/build_mfr_index.py
@@ -7,6 +7,7 @@ with the vendor code tucked in a metafield Shopify can't search by value.
Uses a Bulk Operation (server-side full-catalog export -> one JSONL) rather than paging
~160k products by hand. Read-only. $0 (local + Shopify read)."""
import json, os, re, time, urllib.request, sys
+from datetime import datetime, timezone
SHOP = "designer-laboratory-sandbox.myshopify.com"; API = "2024-10"
ROOT = os.path.dirname(os.path.abspath(__file__))
@@ -102,7 +103,8 @@ for gid, p in prods.items():
index.setdefault(n, {"sku": sku, "title": p["title"][:80],
"status": p["status"], "handle": p["handle"]})
-meta = {"_meta": {"products": len(prods), "keys": len(index), "collisions": collisions}}
+meta = {"_meta": {"products": len(prods), "keys": len(index), "collisions": collisions,
+ "built_at": datetime.now(timezone.utc).isoformat()}}
json.dump({**meta, **index}, open(OUT, "w"))
print(f"\nindexed {len(prods)} products -> {len(index)} normalized keys ({collisions} collisions) -> {OUT}")
# spot-checks
diff --git a/refresh_mfr_index.py b/refresh_mfr_index.py
new file mode 100644
index 0000000..10cb668
--- /dev/null
+++ b/refresh_mfr_index.py
@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+"""Incremental refresh of data/mfr_index.json — pulls ONLY products changed since the
+last build/refresh (updated_at filter) and merges them in. Runs in seconds, $0, so it
+can fire hourly during work hours to keep same-day-imported SKUs scannable. The nightly
+full build_mfr_index.py is the backstop that also clears stale/deleted keys."""
+import json, os, re, sys, time, urllib.request
+from datetime import datetime, timezone, timedelta
+
+SHOP = "designer-laboratory-sandbox.myshopify.com"; API = "2024-10"
+ROOT = os.path.dirname(os.path.abspath(__file__))
+OUT = os.path.join(ROOT, "data", "mfr_index.json")
+TOKEN = os.environ.get("SHOPIFY_ADMIN_TOKEN", "")
+if not TOKEN:
+ for line in open(os.path.expanduser("~/Projects/secrets-manager/.env")):
+ if line.startswith("SHOPIFY_ADMIN_TOKEN="):
+ TOKEN = line.strip().split("=", 1)[1]; break
+assert TOKEN, "no SHOPIFY_ADMIN_TOKEN"
+
+def gql(query, variables=None):
+ req = urllib.request.Request(f"https://{SHOP}/admin/api/{API}/graphql.json",
+ data=json.dumps({"query": query, "variables": variables or {}}).encode(),
+ headers={"X-Shopify-Access-Token": TOKEN, "Content-Type": "application/json"})
+ return json.load(urllib.request.urlopen(req, timeout=90))
+
+def norm(s): return re.sub(r'[^A-Z0-9]', '', (s or '').upper())
+def parts(v): return [p for p in re.split(r'[;,/|]+', v or '') if p.strip()]
+def house_sku(skus):
+ real = [s for s in skus if s and not s.lower().endswith("-sample")]
+ return (real or [s for s in skus if s] or [""])[0]
+
+if not os.path.exists(OUT):
+ print("no index yet — run build_mfr_index.py first (full build)"); sys.exit(1)
+idx = json.load(open(OUT))
+meta = idx.get("_meta", {})
+# baseline: last built_at, else the file's mtime, minus a 10-min safety overlap
+since_iso = meta.get("built_at")
+if not since_iso:
+ since_iso = datetime.fromtimestamp(os.path.getmtime(OUT), tz=timezone.utc).isoformat()
+since = datetime.fromisoformat(since_iso) - timedelta(minutes=10)
+since_q = since.strftime("%Y-%m-%dT%H:%M:%SZ")
+print(f"refreshing products updated since {since_q}")
+
+Q = '''query($c:String,$q:String){ products(first:100, query:$q, after:$c){
+ pageInfo{hasNextPage endCursor}
+ edges{node{ id handle status title
+ mf: metafield(namespace:"custom",key:"manufacturer_sku"){value}
+ mf2: metafield(namespace:"dwc",key:"manufacturer_sku"){value}
+ variants(first:10){edges{node{sku}}} }} } }'''
+cur = None; changed = 0; pages = 0
+while True:
+ g = gql(Q, {"c": cur, "q": f"updated_at:>='{since_q}'"})
+ d = g.get("data", {}).get("products")
+ if d is None:
+ print("GQL err:", json.dumps(g)[:300]); sys.exit(1)
+ for e in d["edges"]:
+ n = e["node"]
+ sku = house_sku([v["node"]["sku"] for v in n["variants"]["edges"]])
+ if not sku: continue
+ keys = set()
+ for k in ("mf", "mf2"):
+ for m in parts((n.get(k) or {}).get("value")):
+ if len(norm(m)) >= 4: keys.add(norm(m))
+ if len(norm(sku)) >= 4: keys.add(norm(sku))
+ rec = {"sku": sku, "title": (n.get("title") or "")[:80], "status": n.get("status"), "handle": n.get("handle")}
+ for kk in keys: idx[kk] = rec
+ changed += 1
+ pages += 1
+ if d["pageInfo"]["hasNextPage"] and pages < 80:
+ cur = d["pageInfo"]["endCursor"]; time.sleep(0.3)
+ else:
+ if d["pageInfo"]["hasNextPage"]: print("⚠ stopped at 80 pages — nightly full build will reconcile")
+ break
+
+idx["_meta"] = {**meta, "keys": sum(1 for k in idx if k != "_meta"),
+ "built_at": datetime.now(timezone.utc).isoformat(),
+ "last_incremental": datetime.now(timezone.utc).isoformat(),
+ "last_incremental_changed": changed}
+json.dump(idx, open(OUT, "w"))
+print(f"merged {changed} changed products across {pages} page(s) -> {idx['_meta']['keys']} keys")
← 47fbc5a Add manufacturer-SKU resolver: scanned vendor codes resolve
·
back to Dw Photo Capture
·
Auto-refresh mfr# index: hourly incremental (8-18) + nightly b974240 →