sweep-to-100.js — drives search-by-color coverage to 100%: scans ALL active products live from Shopify, locally extracts palette for any still missing color_1_hex (catches products absent from the local mirror), writes canonical metafields. Re-runnable; runs after the DB backfill.

a96bd4e7d52f032dddc18d355ac19c9fb7ba5ac9 · 2026-06-13 22:57:03 -0700 · Steve Abrams
Files touched

A DW-Programming/sweep-to-100.js
Diff

commit a96bd4e7d52f032dddc18d355ac19c9fb7ba5ac9
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Sat Jun 13 22:57:03 2026 -0700

    sweep-to-100.js — drives search-by-color coverage to 100%: scans ALL active products live from Shopify, locally extracts palette for any still missing color_1_hex (catches products absent from the local mirror), writes canonical metafields. Re-runnable; runs after the DB backfill.
---
 DW-Programming/sweep-to-100.js | 85 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 85 insertions(+)

diff --git a/DW-Programming/sweep-to-100.js b/DW-Programming/sweep-to-100.js
new file mode 100644
index 00000000..a2c0d680
--- /dev/null
+++ b/DW-Programming/sweep-to-100.js
@@ -0,0 +1,85 @@
+#!/usr/bin/env node
+/**
+ * Drive search-by-color metafield coverage to 100%. Paginates ALL active products
+ * straight from Shopify; for any product MISSING custom.color_1_hex but having an
+ * image, extracts the palette LOCALLY (local-palette.py — $0) and writes the
+ * canonical color_{1,2,3} metafields + color_hex/color_details. Source of truth is
+ * Shopify itself, so it catches products absent from the local mirror too.
+ * Re-runnable: each pass only touches products still missing the metafield.
+ * Usage: node sweep-to-100.js [--conc 6] [--dry-run]
+ */
+const https = require('https');
+const { execFile } = require('child_process');
+const path = require('path');
+
+const SHOPIFY_DOMAIN = 'designer-laboratory-sandbox.myshopify.com';
+const API_VERSION = '2024-01';
+const TOKEN = process.env.SHOPIFY_ADMIN_TOKEN;
+const args = process.argv.slice(2);
+const DRY = args.includes('--dry-run');
+const CONC = parseInt(args.find((_, i, a) => a[i-1] === '--conc') || '6', 10) || 6;
+
+function palette(url){
+  return new Promise(res=>execFile('python3',[path.join(__dirname,'local-palette.py'),url],{timeout:30000,maxBuffer:1<<20},
+    (e,out)=>{ if(e) return res(null); try{res(JSON.parse(out));}catch{res(null);} }));
+}
+function gql(query,variables){
+  return new Promise((resolve,reject)=>{
+    const body=JSON.stringify({query,variables});
+    const req=https.request({hostname:SHOPIFY_DOMAIN,path:`/admin/api/${API_VERSION}/graphql.json`,method:'POST',
+      headers:{'Content-Type':'application/json','X-Shopify-Access-Token':TOKEN,'Content-Length':Buffer.byteLength(body)},timeout:30000},res=>{
+      let d='';res.on('data',c=>d+=c);res.on('end',()=>{try{resolve(JSON.parse(d));}catch(e){reject(e);}});
+    });
+    req.on('error',reject);req.on('timeout',()=>{req.destroy();reject(new Error('timeout'));});req.write(body);req.end();
+  });
+}
+async function writeMf(id,pal){
+  const owner=`gid://shopify/Product/${id}`, c=pal.ai_colors;
+  const mf=[
+    {ownerId:owner,namespace:'custom',key:'color_hex',type:'single_line_text_field',value:pal.color_hex},
+    {ownerId:owner,namespace:'custom',key:'color_details',type:'json',value:JSON.stringify(c)},
+  ];
+  for(let i=0;i<3;i++){ if(!c[i])break;
+    mf.push({ownerId:owner,namespace:'custom',key:`color_${i+1}_hex`,type:'single_line_text_field',value:c[i].hex});
+    mf.push({ownerId:owner,namespace:'custom',key:`color_${i+1}_percentage`,type:'number_decimal',value:String(c[i].percentage)});
+    mf.push({ownerId:owner,namespace:'custom',key:`color_${i+1}_name`,type:'single_line_text_field',value:c[i].name});
+  }
+  const r=await gql(`mutation($mf:[MetafieldsSetInput!]!){metafieldsSet(metafields:$mf){userErrors{message}}}`,{mf});
+  const ue=r?.data?.metafieldsSet?.userErrors; if(ue&&ue.length) throw new Error(ue[0].message);
+  if(r?.errors) throw new Error(JSON.stringify(r.errors).slice(0,100));
+}
+(async()=>{
+  if(!TOKEN){console.error('Missing SHOPIFY_ADMIN_TOKEN');process.exit(1);}
+  let cursor=null,page=0,scanned=0,have=0,missingNoImg=0; const todo=[];
+  console.log(`Sweep-to-100 — ${DRY?'DRY':'LIVE'} — scanning active products for missing color_1_hex…`);
+  while(true){
+    const after=cursor?`, after:"${cursor}"`:'';
+    const q=`{ products(first:100, query:"status:active"${after}){ pageInfo{hasNextPage endCursor} nodes{
+      id featuredImage{url} ch:metafield(namespace:"custom",key:"color_1_hex"){value} }}}`;
+    let r; try{r=await gql(q);}catch{ await new Promise(s=>setTimeout(s,3000)); continue; }
+    if(r.errors){ if(JSON.stringify(r.errors).includes('Throttled')){await new Promise(s=>setTimeout(s,4000));continue;} console.error(r.errors);break; }
+    const pg=r.data.products; page++;
+    for(const n of pg.nodes){ scanned++;
+      if(n.ch?.value){ have++; continue; }
+      if(!n.featuredImage?.url){ missingNoImg++; continue; }
+      todo.push({id:n.id.replace('gid://shopify/Product/',''),url:n.featuredImage.url});
+    }
+    if(page%20===0) console.log(`  scanned ${scanned} | have ${have} | to-fix ${todo.length} | no-image ${missingNoImg}`);
+    if(!pg.pageInfo.hasNextPage) break; cursor=pg.pageInfo.endCursor;
+    await new Promise(s=>setTimeout(s,200));
+  }
+  console.log(`\nScan done: ${scanned} active | ${have} already have color | ${todo.length} to extract | ${missingNoImg} no-image.`);
+  let done=0,failed=0,idx=0; const queue=[...todo];
+  async function worker(){
+    while(queue.length){
+      const p=queue.shift(); const n=++idx;
+      const pal=await palette(p.url);
+      if(!pal||!pal.ai_colors?.length){ failed++; continue; }
+      try{ if(!DRY) await writeMf(p.id,pal); done++; if(n%500===0) console.log(`  extracted ${n}/${todo.length}`); }
+      catch(e){ failed++; if(String(e.message).match(/Throttled|429/)) await new Promise(r=>setTimeout(r,3000)); }
+    }
+  }
+  await Promise.all(Array.from({length:CONC},worker));
+  console.log(`\nSweep done: ${done} newly enriched, ${failed} failed, ${missingNoImg} unreachable (no image).`);
+  console.log(`Coverage now: ${have+done}/${scanned} = ${((have+done)/scanned*100).toFixed(1)}% of active (minus ${missingNoImg} imageless).`);
+})().catch(e=>{console.error(e);process.exit(1);});
← 0e985f97 backfill-canonical-colors.js — push canonical color_1/2/3 me · back to Designer Wallcoverings · tag-designer-colors.js — FALLBACK for Boost tiers without me 1e5c69ba →