← back to Dw Yolo Loop
Kravet dedup sweep: bulk full-catalog true-dup detector (vendor+SKU / vendor+SKU+title keys); 18 real dup clusters found, 18 removable
e3ad9a182b7cc32365f19ff544b24deabb601b34 · 2026-06-15 10:46:05 -0700 · Steve Abrams
Files touched
A scripts/kravet-dedup-bulk.jsA scripts/kravet-dedup-sweep.js
Diff
commit e3ad9a182b7cc32365f19ff544b24deabb601b34
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Mon Jun 15 10:46:05 2026 -0700
Kravet dedup sweep: bulk full-catalog true-dup detector (vendor+SKU / vendor+SKU+title keys); 18 real dup clusters found, 18 removable
---
scripts/kravet-dedup-bulk.js | 68 +++++++++++++++++++++++++++++++++++++++++
scripts/kravet-dedup-sweep.js | 71 +++++++++++++++++++++++++++++++++++++++++++
2 files changed, 139 insertions(+)
diff --git a/scripts/kravet-dedup-bulk.js b/scripts/kravet-dedup-bulk.js
new file mode 100644
index 0000000..285bb6d
--- /dev/null
+++ b/scripts/kravet-dedup-bulk.js
@@ -0,0 +1,68 @@
+#!/usr/bin/env node
+/* READ-ONLY full-catalog dedup via Shopify BULK op (no per-page throttling). Scans ALL active
+ products, joins variants, filters Kravet family, flags TRUE duplicates:
+ - colorway-unique SKU (has . or /): dup = same VENDOR + same SKU (SKU reused across vendors,
+ so vendor is part of identity; within a vendor a full colorway SKU is unique).
+ - pattern-only SKU (no separator): dup = same VENDOR + SKU + TITLE (colorways share pattern SKU).
+ Writes /tmp/kravet_dedup.csv with created date+time so Steve picks the keeper. NO writes. */
+const https=require('https'), fs=require('fs');
+const T=process.env.T, S='designer-laboratory-sandbox.myshopify.com';
+const sleep=ms=>new Promise(r=>setTimeout(r,ms));
+function gql(q){return new Promise((res,rej)=>{const rq=https.request(`https://${S}/admin/api/2024-10/graphql.json`,{method:'POST',headers:{'X-Shopify-Access-Token':T,'Content-Type':'application/json'}},x=>{let b='';x.on('data',d=>b+=d);x.on('end',()=>{try{res(JSON.parse(b))}catch(e){rej(b.slice(0,200))}})});rq.on('error',rej);rq.write(JSON.stringify({query:q}));rq.end();});}
+function dl(u){return new Promise((res,rej)=>{https.get(u,r=>{let b='';r.on('data',d=>b+=d);r.on('end',()=>res(b));}).on('error',rej);});}
+const KFAM=new Set(['Kravet','Kravet Couture','Kravet Design','Lee Jofa','Lee Jofa Modern','Brunschwig & Fils','Cole & Son','GP & J Baker','Clarke And Clarke','Mulberry','Threads','Baker Lifestyle','Gaston Y Daniela','Gaston y Daniela','Andrew Martin']);
+const norm=s=>(s||'').toUpperCase().replace(/\s+/g,' ').trim();
+const titleKey=t=>norm(t).replace(/\s*[-#]\s*\d+$/,'');
+const isSample=o=>(o||[]).some(x=>String(x.value).toLowerCase()==='sample');
+const fmt=iso=>{ if(!iso) return ''; const d=new Date(iso); return d.toLocaleString('en-US',{year:'numeric',month:'short',day:'numeric',hour:'numeric',minute:'2-digit'}); };
+
+(async()=>{
+ const BULK=`mutation{ bulkOperationRunQuery(query:"""
+ { products(query:"status:active"){ edges{ node{ id title vendor createdAt
+ metafield(namespace:"custom",key:"manufacturer_sku"){ value }
+ variants{ edges{ node{ price selectedOptions{ value } } } } } } } }
+ """){ bulkOperation{ id status } userErrors{ field message } } }`;
+ const k=await gql(BULK);
+ const ue=k.data?.bulkOperationRunQuery?.userErrors||[];
+ if(ue.length){ console.log('cannot start bulk (another running?):',JSON.stringify(ue)); process.exit(1); }
+ console.log('bulk started; polling...');
+ let url=null;
+ for(let i=0;i<240;i++){ const s=await gql(`{ currentBulkOperation{ status objectCount url errorCode } }`); const op=s.data.currentBulkOperation;
+ if(i%5===0) console.log(` ${op.status} obj=${op.objectCount||0}`);
+ if(op.status==='COMPLETED'){ url=op.url; break; } if(op.status==='FAILED'){ console.log('FAILED',op.errorCode); process.exit(1); } await sleep(5000); }
+ if(!url){ console.log('no url'); process.exit(1); }
+ const data=await dl(url);
+ const lines=data.trim().split('\n').filter(Boolean).map(l=>JSON.parse(l));
+ const prod={}, vbyp={};
+ for(const n of lines){
+ if(n.id && n.id.includes('/Product/')) prod[n.id]={vendor:n.vendor,title:n.title,createdAt:n.createdAt,sku:norm(n.metafield?.value)};
+ else if(n.__parentId && n.price!==undefined) (vbyp[n.__parentId]=vbyp[n.__parentId]||[]).push(n);
+ }
+ const flat=[];
+ for(const [pid,p] of Object.entries(prod)){
+ if(!KFAM.has(p.vendor) || !p.sku) continue;
+ const reals=(vbyp[pid]||[]).map(v=>({p:parseFloat(v.price),s:isSample(v.selectedOptions)})).filter(v=>!v.s&&v.p>5);
+ flat.push({gid:pid.split('/').pop(),vendor:p.vendor,title:p.title,createdAt:p.createdAt,sku:p.sku,price:reals.length?Math.max(...reals.map(v=>v.p)):null});
+ }
+ const colorwayUnique=s=>/[.\/]/.test(s);
+ const groups={};
+ for(const p of flat){ const v=norm(p.vendor); const key=colorwayUnique(p.sku)?('CW:'+v+'|'+p.sku):('PT:'+v+'|'+p.sku+'||'+titleKey(p.title)); (groups[key]=groups[key]||[]).push(p); }
+ const dups=Object.entries(groups).filter(([k,a])=>a.length>1).map(([k,a])=>[k,a.sort((x,y)=>new Date(x.createdAt)-new Date(y.createdAt))]).sort((a,b)=>b[1].length-a[1].length);
+ const out=['dup_key,vendor,mfr_sku,product_id,price,created,keeper_hint,full_title'];
+ let dupProducts=0, priceDiff=0;
+ for(const [key,arr] of dups){ dupProducts+=arr.length;
+ const prices=arr.map(p=>p.price).filter(v=>v!=null); const diff=new Set(prices).size>1; if(diff) priceDiff++;
+ arr.forEach((p,idx)=>{ const keeper=idx===arr.length-1?'KEEP-newest':'dup-older';
+ out.push(`"${key}",${p.vendor},${p.sku},${p.gid},${p.price??''},"${fmt(p.createdAt)}",${keeper},"${(p.title||'').replace(/"/g,'')}"`); });
+ }
+ fs.writeFileSync('/tmp/kravet_dedup.csv',out.join('\n'));
+ console.log(`\n=== KRAVET-FAMILY TRUE DUP SWEEP (bulk, full catalog, READ-ONLY) ===`);
+ console.log(`KFAM active products scanned: ${flat.length}`);
+ console.log(`dup clusters: ${dups.length} | products involved: ${dupProducts} | EXTRA copies removable: ${dupProducts-dups.length}`);
+ console.log(`clusters with price conflict: ${priceDiff}`);
+ const perV={}; dups.forEach(([k,a])=>a.forEach(p=>perV[p.vendor]=(perV[p.vendor]||0)+1));
+ console.log('dup products per vendor:'); for(const [v,c] of Object.entries(perV).sort((a,b)=>b[1]-a[1])) console.log(` ${v.padEnd(20)} ${c}`);
+ console.log('\ntop dup clusters:');
+ for(const [key,arr] of dups.slice(0,20)) console.log(` ${key.replace(/^(CW|PT):/,'').slice(0,46).padEnd(46)} x${arr.length} prices=[${arr.map(p=>p.price??'?').join(', ')}]`);
+ console.log(`\nfull decision list -> /tmp/kravet_dedup.csv`);
+})().catch(e=>{console.error(e);process.exit(1);});
diff --git a/scripts/kravet-dedup-sweep.js b/scripts/kravet-dedup-sweep.js
new file mode 100644
index 0000000..1ca3a4b
--- /dev/null
+++ b/scripts/kravet-dedup-sweep.js
@@ -0,0 +1,71 @@
+#!/usr/bin/env node
+/* READ-ONLY dedup sweep across the Kravet family. Paginates all KFAM vendors, collects every ACTIVE
+ product's custom.manufacturer_sku -> {gid,title,status,createdAt,realPrice}. Flags any mfr_sku
+ carried by >1 active product (the dup-SKU trap, e.g. GDW5772.007.0 at $410 vs $500). Writes a
+ decision list with created date+time per dup so Steve can pick the keeper. NO writes/deletes. */
+const https=require('https'), fs=require('fs');
+const T=process.env.T, S='designer-laboratory-sandbox.myshopify.com';
+const sleep=ms=>new Promise(r=>setTimeout(r,ms));
+function gqlOnce(q){return new Promise((res,rej)=>{const rq=https.request(`https://${S}/admin/api/2024-10/graphql.json`,{method:'POST',headers:{'X-Shopify-Access-Token':T,'Content-Type':'application/json'}},x=>{let b='';x.on('data',d=>b+=d);x.on('end',()=>{try{res(JSON.parse(b))}catch(e){rej(b.slice(0,200))}})});rq.on('error',rej);rq.setTimeout(30000,()=>rq.destroy(new Error('timeout')));rq.write(JSON.stringify({query:q}));rq.end();});}
+async function gql(q){for(let i=0;i<4;i++){try{return await gqlOnce(q);}catch(e){if(i===3)throw e;await new Promise(r=>setTimeout(r,1500*(i+1)));}}}
+const KFAM=['Kravet','Kravet Couture','Kravet Design','Lee Jofa','Lee Jofa Modern','Brunschwig & Fils','Cole & Son','GP & J Baker','Clarke And Clarke','Mulberry','Threads','Baker Lifestyle','Gaston y Daniela','Andrew Martin'];
+const norm=s=>(s||'').toUpperCase().replace(/\s+/g,' ').trim();
+const isSample=o=>(o||[]).some(x=>String(x.value).toLowerCase()==='sample');
+const fmt=iso=>{ if(!iso) return ''; const d=new Date(iso); return d.toLocaleString('en-US',{year:'numeric',month:'short',day:'numeric',hour:'numeric',minute:'2-digit'}); };
+
+(async()=>{
+ const bySku={}; // sku -> [{gid,title,status,createdAt,price,vendor}]
+ for(const vendor of KFAM){
+ let cursor=null, n=0;
+ const vq=vendor.replace(/"/g,'\\"');
+ while(true){
+ const after=cursor?`, after:"${cursor}"`:'';
+ const q=`{ products(first:100, query:"vendor:'${vq}' status:active"${after}){
+ pageInfo{ hasNextPage endCursor }
+ edges{ node{ id title createdAt
+ metafield(namespace:"custom",key:"manufacturer_sku"){ value }
+ variants(first:10){ edges{ node{ price selectedOptions{ value } } } } } } } }`;
+ let s; try{ s=await gql(q); }catch(e){ console.log(` ${vendor} ERR ${e}`); break; }
+ const pr=s.data?.products; if(!pr) break;
+ for(const e of pr.edges){ const nd=e.node; n++;
+ const sku=norm(nd.metafield?.value); if(!sku) continue;
+ const reals=(nd.variants?.edges||[]).map(x=>({p:parseFloat(x.node.price),s:isSample(x.node.selectedOptions)})).filter(v=>!v.s && v.p>5);
+ const price=reals.length?Math.max(...reals.map(v=>v.p)):null;
+ (bySku[sku]=bySku[sku]||[]).push({gid:nd.id.split('/').pop(),title:nd.title,createdAt:nd.createdAt,price,vendor});
+ }
+ if(!pr.pageInfo.hasNextPage) break; cursor=pr.pageInfo.endCursor; await sleep(600);
+ }
+ console.log(` ${vendor.padEnd(20)} active=${n}`);
+ }
+ // TRUE dup = >1 active product sharing the SAME normalized title (color-level identical).
+ // Pattern-SKU colorway families (same manufacturer_sku, different color titles) are NOT dups.
+ const titleKey=t=>norm(t).replace(/\s*[-#]\s*\d+$/,''); // strip trailing "-1"/"#2"
+ const flat=[]; for(const [sku,arr] of Object.entries(bySku)) for(const p of arr) flat.push({...p,sku});
+ // FORMAT-AWARE dup key:
+ // - colorway-unique SKU (has '.' or '/', e.g. GDW5772.007.0 / 100/1004.CS.0 / 8024109.12): a 2nd
+ // product with the SAME full SKU is a dup by definition -> group by SKU alone.
+ // - pattern-only SKU (no separator, e.g. W4132 / 8024108 = shared by a whole colorway family):
+ // require SAME title too, so colorways aren't falsely merged.
+ // KEY by VENDOR too — the same full SKU (e.g. 30787.100.0) is reused across vendors (B&F vs Kravet)
+ // for totally different products, so vendor must be part of the identity.
+ const colorwayUnique=s=>/[.\/]/.test(s);
+ const byTitle={}; for(const p of flat){ const v=norm(p.vendor); const k=colorwayUnique(p.sku)?('CW:'+v+'|'+p.sku):('PT:'+v+'|'+p.sku+' || '+titleKey(p.title)); (byTitle[k]=byTitle[k]||[]).push(p); }
+ const dups=Object.entries(byTitle).filter(([k,a])=>a.length>1)
+ .map(([k,a])=>[k,a.sort((x,y)=>new Date(x.createdAt)-new Date(y.createdAt))]);
+ dups.sort((a,b)=>b[1].length-a[1].length);
+ const out=['title_key,vendor,mfr_sku,product_id,price,created,full_title'];
+ let dupProducts=0, priceDiffClusters=0;
+ for(const [k,arr] of dups){ dupProducts+=arr.length;
+ const prices=arr.map(p=>p.price).filter(v=>v!=null); if(new Set(prices).size>1) priceDiffClusters++;
+ for(const p of arr) out.push(`"${k}",${p.vendor},${p.sku},${p.gid},${p.price??''},"${fmt(p.createdAt)}","${(p.title||'').replace(/"/g,'')}"`);
+ }
+ fs.writeFileSync('/tmp/kravet_dedup.csv',out.join('\n'));
+ console.log(`\n=== KRAVET-FAMILY TRUE DUP SWEEP (same-title, READ-ONLY) ===`);
+ console.log(`dup clusters (>1 active product, identical title): ${dups.length} | total products: ${dupProducts} | extra copies removable: ${dupProducts-dups.length}`);
+ console.log(`clusters where copies have DIFFERENT prices (most urgent): ${priceDiffClusters}`);
+ const perV={}; dups.forEach(([k,a])=>a.forEach(p=>perV[p.vendor]=(perV[p.vendor]||0)+1));
+ console.log('dup products per vendor:'); for(const [v,c] of Object.entries(perV).sort((a,b)=>b[1]-a[1])) console.log(` ${v.padEnd(20)} ${c}`);
+ console.log('\nsample dup clusters (newest copy is usually the keeper):');
+ for(const [k,arr] of dups.slice(0,12)) console.log(` "${k.slice(0,40)}" x${arr.length} prices=[${arr.map(p=>p.price??'?').join(', ')}] ${arr[0].vendor}`);
+ console.log(`\nfull list -> /tmp/kravet_dedup.csv`);
+})().catch(e=>{console.error(e);process.exit(1);});
← 1193fa7 Add Schumacher roll-price pusher + propagate price-sheet cos
·
back to Dw Yolo Loop
·
Add Schumacher add-roll-variant builder (532 targets, invent 2cde42a →