← back to Designer Wallcoverings

enrich: add resumable --all-published batch mode (cursor paginate, skip already-enriched, progress logging)

b8804ebf6ba2b07b9f055559c232248e48ab94ef · 2026-07-07 09:37:15 -0700 · Steve

Files touched

• M shopify/scripts/enrich-color-details-local.py

Diff

commit b8804ebf6ba2b07b9f055559c232248e48ab94ef
Author: Steve <steve@designerwallcoverings.com>
Date:   Tue Jul 7 09:37:15 2026 -0700

    enrich: add resumable --all-published batch mode (cursor paginate, skip already-enriched, progress logging)
---
 shopify/scripts/enrich-color-details-local.py | 67 ++++++++++++++++++++++-----
 1 file changed, 55 insertions(+), 12 deletions(-)

diff --git a/shopify/scripts/enrich-color-details-local.py b/shopify/scripts/enrich-color-details-local.py
index 840c93d0..897cbb0b 100644
--- a/shopify/scripts/enrich-color-details-local.py
+++ b/shopify/scripts/enrich-color-details-local.py
@@ -52,6 +52,27 @@ def get_products(ids, query_str, limit):
     return [e['node'] for e in gql(q, {'q': query_str, 'n': limit})['data']['products']['edges'] if e['node'].get('featuredImage')]
 
 
+def iter_published(skip_enriched=True):
+    """Cursor-paginate ALL published products, yielding those needing enrichment."""
+    q = ('query($c:String){ products(first:100, after:$c, query:"status:active published_status:published"){ '
+         'pageInfo{hasNextPage endCursor} edges{ node{ id title featuredImage{url} '
+         'cd: metafield(namespace:"custom", key:"color_details"){ value } } } } }')
+    cursor = None
+    while True:
+        d = gql(q, {'c': cursor})
+        conn = d['data']['products']
+        for e in conn['edges']:
+            n = e['node']
+            if not n.get('featuredImage'):
+                continue
+            if skip_enriched and n.get('cd') and n['cd'].get('value'):
+                continue
+            yield n
+        if not conn['pageInfo']['hasNextPage']:
+            break
+        cursor = conn['pageInfo']['endCursor']
+
+
 def dominant_colors(img_bytes, k=6):
     """Real dominant colors via median-cut. Returns [(hex,pct)] sorted desc, near-dupes merged."""
     im = Image.open(io.BytesIO(img_bytes)).convert('RGB')
@@ -111,6 +132,17 @@ def write_metafield(gid, colors):
         raise RuntimeError(errs)
 
 
+def enrich_one(p, dry):
+    img_bytes = urllib.request.urlopen(p['featuredImage']['url'], timeout=40).read()
+    hexpct = dominant_colors(img_bytes)
+    b64 = base64.b64encode(img_bytes).decode()
+    names = name_hexes(b64, [h for h, _ in hexpct])
+    colors = [{'name': names[i], 'hex': hexpct[i][0], 'pct': hexpct[i][1]} for i in range(len(hexpct))]
+    if not dry:
+        write_metafield(p['id'], colors)
+    return ', '.join(f"{c['name']} {c['pct']}%" for c in colors)
+
+
 def main():
     args = sys.argv[1:]
     dry = '--dry' in args
@@ -119,23 +151,34 @@ def main():
     if '--query' in args: query_str = args[args.index('--query') + 1]
     if '--limit' in args: limit = int(args[args.index('--limit') + 1])
 
+    if '--all-published' in args:
+        # Resumable full-catalog run: skips products that already have color_details.
+        print(f"FULL-CATALOG enrichment — hybrid (PIL pixels + {MODEL} naming), $0 local, resumable\n", flush=True)
+        ok = fail = 0; t_start = time.time()
+        for p in iter_published(skip_enriched=True):
+            t0 = time.time(); title = p['title'][:46]
+            try:
+                summary = enrich_one(p, dry)
+                ok += 1
+                print(f"[{ok}] {title} … ✓ {summary}  ({time.time()-t0:.0f}s)", flush=True)
+            except Exception as e:
+                fail += 1
+                print(f"[x] {title} … FAILED: {str(e)[:80]}", flush=True)
+            if (ok + fail) % 50 == 0:
+                el = time.time() - t_start
+                print(f"  — progress: {ok} ok / {fail} fail, {el/60:.0f}min elapsed, {el/max(1,ok+fail):.1f}s/ea —", flush=True)
+        print(f"\nDONE (full catalog): {ok} enriched, {fail} failed, {(time.time()-t_start)/60:.0f}min.", flush=True)
+        return
+
     products = get_products(ids or None, query_str, limit)
     print(f"Enriching {len(products)} product(s) — hybrid (PIL pixels + {MODEL} naming), $0 local\n")
     ok = fail = 0
     for p in products:
-        t0 = time.time()
-        title = p['title'][:48]
+        t0 = time.time(); title = p['title'][:48]
         try:
-            img_bytes = urllib.request.urlopen(p['featuredImage']['url'], timeout=40).read()
-            hexpct = dominant_colors(img_bytes)
-            b64 = base64.b64encode(img_bytes).decode()
-            names = name_hexes(b64, [h for h, _ in hexpct])
-            colors = [{'name': names[i], 'hex': hexpct[i][0], 'pct': hexpct[i][1]} for i in range(len(hexpct))]
-            summary = ', '.join(f"{c['name']} {c['pct']}%" for c in colors)
-            if dry:
-                print(f"• {title} … [dry] {summary}"); ok += 1; continue
-            write_metafield(p['id'], colors)
-            print(f"• {title} … ✓ {summary}  ({time.time()-t0:.0f}s)"); ok += 1
+            summary = enrich_one(p, dry)
+            tag = '[dry] ' if dry else ''
+            print(f"• {title} … ✓ {tag}{summary}  ({time.time()-t0:.0f}s)"); ok += 1
         except Exception as e:
             print(f"• {title} … FAILED: {e}"); fail += 1
     print(f"\nDone: {ok} enriched, {fail} failed.")

← 71defcf6 color-palette: fix double-json encode of data-colors (broke  ·  back to Designer Wallcoverings  ·  enrich: overwrite legacy 'percentage'-schema color_details ( cbd55ff1 →