`scripts/dedup-bulk-all.js`

69 lines
#!/usr/bin/env node
/* READ-ONLY full-catalog dedup via Shopify BULK op (no per-page throttling). Scans ALL active
   products, joins variants, filters ALL vendors, flags TRUE duplicates:
     - colorway-unique SKU (has . or /): dup = same VENDOR + same SKU  (SKU reused across vendors,
       so vendor is part of identity; within a vendor a full colorway SKU is unique).
     - pattern-only SKU (no separator): dup = same VENDOR + SKU + TITLE (colorways share pattern SKU).
   Writes /tmp/fleet_dedup.csv with created date+time so Steve picks the keeper. NO writes. */
const https=require('https'), fs=require('fs');
const T=process.env.T, S='designer-laboratory-sandbox.myshopify.com';
const sleep=ms=>new Promise(r=>setTimeout(r,ms));
function gql(q){return new Promise((res,rej)=>{const rq=https.request(`https://${S}/admin/api/2024-10/graphql.json`,{method:'POST',headers:{'X-Shopify-Access-Token':T,'Content-Type':'application/json'}},x=>{let b='';x.on('data',d=>b+=d);x.on('end',()=>{try{res(JSON.parse(b))}catch(e){rej(b.slice(0,200))}})});rq.on('error',rej);rq.write(JSON.stringify({query:q}));rq.end();});}
function dl(u){return new Promise((res,rej)=>{https.get(u,r=>{let b='';r.on('data',d=>b+=d);r.on('end',()=>res(b));}).on('error',rej);});}
const KFAM=new Set(['Kravet','Kravet Couture','Kravet Design','Lee Jofa','Lee Jofa Modern','Brunschwig & Fils','Cole & Son','GP & J Baker','Clarke And Clarke','Mulberry','Threads','Baker Lifestyle','Gaston Y Daniela','Gaston y Daniela','Andrew Martin']);
const norm=s=>(s||'').toUpperCase().replace(/\s+/g,' ').trim();
const titleKey=t=>norm(t).replace(/\s*[-#]\s*\d+$/,'');
const isSample=o=>(o||[]).some(x=>String(x.value).toLowerCase()==='sample');
const fmt=iso=>{ if(!iso) return ''; const d=new Date(iso); return d.toLocaleString('en-US',{year:'numeric',month:'short',day:'numeric',hour:'numeric',minute:'2-digit'}); };

(async()=>{
  const BULK=`mutation{ bulkOperationRunQuery(query:"""
    { products(query:"status:active"){ edges{ node{ id title vendor createdAt
      metafield(namespace:"custom",key:"manufacturer_sku"){ value }
      variants{ edges{ node{ price selectedOptions{ value } } } } } } } }
  """){ bulkOperation{ id status } userErrors{ field message } } }`;
  const k=await gql(BULK);
  const ue=k.data?.bulkOperationRunQuery?.userErrors||[];
  if(ue.length){ console.log('cannot start bulk (another running?):',JSON.stringify(ue)); process.exit(1); }
  console.log('bulk started; polling...');
  let url=null;
  for(let i=0;i<240;i++){ const s=await gql(`{ currentBulkOperation{ status objectCount url errorCode } }`); const op=s.data.currentBulkOperation;
    if(i%5===0) console.log(`  ${op.status} obj=${op.objectCount||0}`);
    if(op.status==='COMPLETED'){ url=op.url; break; } if(op.status==='FAILED'){ console.log('FAILED',op.errorCode); process.exit(1); } await sleep(5000); }
  if(!url){ console.log('no url'); process.exit(1); }
  const data=await dl(url);
  const lines=data.trim().split('\n').filter(Boolean).map(l=>JSON.parse(l));
  const prod={}, vbyp={};
  for(const n of lines){
    if(n.id && n.id.includes('/Product/')) prod[n.id]={vendor:n.vendor,title:n.title,createdAt:n.createdAt,sku:norm(n.metafield?.value)};
    else if(n.__parentId && n.price!==undefined) (vbyp[n.__parentId]=vbyp[n.__parentId]||[]).push(n);
  }
  const flat=[];
  for(const [pid,p] of Object.entries(prod)){
    if(!p.sku) continue;
    const reals=(vbyp[pid]||[]).map(v=>({p:parseFloat(v.price),s:isSample(v.selectedOptions)})).filter(v=>!v.s&&v.p>5);
    flat.push({gid:pid.split('/').pop(),vendor:p.vendor,title:p.title,createdAt:p.createdAt,sku:p.sku,price:reals.length?Math.max(...reals.map(v=>v.p)):null});
  }
  const colorwayUnique=s=>/[.\/]/.test(s);
  const groups={};
  for(const p of flat){ const v=norm(p.vendor); const key=colorwayUnique(p.sku)?('CW:'+v+'|'+p.sku):('PT:'+v+'|'+p.sku+'||'+titleKey(p.title)); (groups[key]=groups[key]||[]).push(p); }
  const dups=Object.entries(groups).filter(([k,a])=>a.length>1).map(([k,a])=>[k,a.sort((x,y)=>new Date(x.createdAt)-new Date(y.createdAt))]).sort((a,b)=>b[1].length-a[1].length);
  const out=['dup_key,vendor,mfr_sku,product_id,price,created,keeper_hint,full_title'];
  let dupProducts=0, priceDiff=0;
  for(const [key,arr] of dups){ dupProducts+=arr.length;
    const prices=arr.map(p=>p.price).filter(v=>v!=null); const diff=new Set(prices).size>1; if(diff) priceDiff++;
    arr.forEach((p,idx)=>{ const keeper=idx===arr.length-1?'KEEP-newest':'dup-older';
      out.push(`"${key}",${p.vendor},${p.sku},${p.gid},${p.price??''},"${fmt(p.createdAt)}",${keeper},"${(p.title||'').replace(/"/g,'')}"`); });
  }
  fs.writeFileSync('/tmp/fleet_dedup.csv',out.join('\n'));
  console.log(`\n=== KRAVET-FAMILY TRUE DUP SWEEP (bulk, full catalog, READ-ONLY) ===`);
  console.log(`ALL active products scanned: ${flat.length}`);
  console.log(`dup clusters: ${dups.length} | products involved: ${dupProducts} | EXTRA copies removable: ${dupProducts-dups.length}`);
  console.log(`clusters with price conflict: ${priceDiff}`);
  const perV={}; dups.forEach(([k,a])=>a.forEach(p=>perV[p.vendor]=(perV[p.vendor]||0)+1));
  console.log('dup products per vendor:'); for(const [v,c] of Object.entries(perV).sort((a,b)=>b[1]-a[1])) console.log(`  ${v.padEnd(20)} ${c}`);
  console.log('\ntop dup clusters:');
  for(const [key,arr] of dups.slice(0,20)) console.log(`  ${key.replace(/^(CW|PT):/,'').slice(0,46).padEnd(46)} x${arr.length} prices=[${arr.map(p=>p.price??'?').join(', ')}]`);
  console.log(`\nfull decision list -> /tmp/fleet_dedup.csv`);
})().catch(e=>{console.error(e);process.exit(1);});