`scripts/fleet-dedup-handle.js`

75 lines
#!/usr/bin/env node
/* READ-ONLY bulletproof fleet dup detector. Bulk-scans ALL active products and flags TRUE dups by
   two high-confidence signals only (no SKU-garbage false positives):
    (1) HANDLE-SUFFIX: handle `base-N` exists alongside active `base`, SAME vendor + SAME title-key.
        Shopify only mints `-N` handles on collision, so this is a definitive duplicate.
    (2) COLORWAY-SKU: same vendor + same manufacturer_sku + same title-key, where the SKU is a real
        article number (contains BOTH a separator . or / AND a digit) — excludes maker-name skus
        like SANCAR/TRUE/DONGHIA that pollute the naive grouping.
   Writes /tmp/fleet_dedup.csv (keeper = base / non-suffixed / cleaner handle). NO writes. */
const https=require('https'), fs=require('fs');
const T=process.env.T, S='designer-laboratory-sandbox.myshopify.com';
const sleep=ms=>new Promise(r=>setTimeout(r,ms));
function gql(q){return new Promise((res,rej)=>{const rq=https.request(`https://${S}/admin/api/2024-10/graphql.json`,{method:'POST',headers:{'X-Shopify-Access-Token':T,'Content-Type':'application/json'}},x=>{let b='';x.on('data',d=>b+=d);x.on('end',()=>{try{res(JSON.parse(b))}catch(e){rej(b.slice(0,200))}})});rq.on('error',rej);rq.write(JSON.stringify({query:q}));rq.end();});}
function dl(u){return new Promise((res,rej)=>{https.get(u,r=>{let b='';r.on('data',d=>b+=d);r.on('end',()=>res(b));}).on('error',rej);});}
const norm=s=>(s||'').toUpperCase().replace(/\s+/g,' ').trim();
const titleKey=t=>norm(t).replace(/\s*[-#]\s*\d+$/,'');
const isSample=o=>(o||[]).some(x=>String(x.value).toLowerCase()==='sample');
const fmt=iso=>{ if(!iso) return ''; return new Date(iso).toLocaleString('en-US',{year:'numeric',month:'short',day:'numeric',hour:'numeric',minute:'2-digit'}); };
const realSku=s=>/[.\/]/.test(s)&&/\d/.test(s);   // article-number-shaped, not a maker word

(async()=>{
  const BULK=`mutation{ bulkOperationRunQuery(query:"""
    { products(query:"status:active"){ edges{ node{ id handle title vendor createdAt
      metafield(namespace:"custom",key:"manufacturer_sku"){ value }
      variants{ edges{ node{ price selectedOptions{ value } } } } } } } }
  """){ bulkOperation{ id } userErrors{ field message } } }`;
  const k=await gql(BULK); const ue=k.data?.bulkOperationRunQuery?.userErrors||[];
  if(ue.length){ console.log('cannot start bulk:',JSON.stringify(ue)); process.exit(1); }
  console.log('bulk started; polling...'); let url=null;
  for(let i=0;i<240;i++){ const s=await gql(`{ currentBulkOperation{ status objectCount url errorCode } }`); const op=s.data.currentBulkOperation;
    if(i%6===0) console.log(`  ${op.status} obj=${op.objectCount||0}`);
    if(op.status==='COMPLETED'){ url=op.url; break; } if(op.status==='FAILED'){ console.log('FAILED',op.errorCode); process.exit(1); } await sleep(5000); }
  if(!url){ console.log('no url'); process.exit(1); }
  const data=await dl(url);
  const prod={}, vbyp={};
  for(const n of data.trim().split('\n').filter(Boolean).map(l=>JSON.parse(l))){
    if(n.id&&n.id.includes('/Product/')) prod[n.id]={gid:n.id.split('/').pop(),handle:n.handle,title:n.title,vendor:n.vendor,createdAt:n.createdAt,sku:norm(n.metafield?.value)};
    else if(n.__parentId&&n.price!==undefined) (vbyp[n.__parentId]=vbyp[n.__parentId]||[]).push(n);
  }
  const all=Object.entries(prod).map(([pid,p])=>{ const reals=(vbyp[pid]||[]).map(v=>({pr:parseFloat(v.price),sm:isSample(v.selectedOptions)})).filter(v=>!v.sm&&v.pr>5);
    return {...p, price:reals.length?Math.max(...reals.map(v=>v.pr)):null}; });
  const byHandle={}; all.forEach(p=>byHandle[p.handle]=p);
  const clusters={};   // keyed -> {keeper, dups:[]}
  const claimed=new Set();

  // (1) HANDLE-SUFFIX dups
  for(const p of all){ const m=p.handle.match(/^(.+)-(\d+)$/); if(!m) continue;
    const base=byHandle[m[1]]; if(!base) continue;
    if(norm(base.vendor)!==norm(p.vendor)) continue;
    if(titleKey(base.title)!==titleKey(p.title)) continue;
    const key='H:'+base.handle; (clusters[key]=clusters[key]||{keeper:base,dups:[]});
    clusters[key].dups.push(p); claimed.add(p.gid); claimed.add(base.gid);
  }
  // (2) COLORWAY-SKU dups (real article-number skus only), excluding anything already handle-claimed
  const bySku={}; for(const p of all){ if(!realSku(p.sku)) continue; const key=norm(p.vendor)+'|'+p.sku+'|'+titleKey(p.title); (bySku[key]=bySku[key]||[]).push(p); }
  for(const [key,arr] of Object.entries(bySku)){ if(arr.length<2) continue;
    if(arr.every(p=>claimed.has(p.gid))) continue;
    const ranked=arr.slice().sort((a,b)=> ( /-\d+$/.test(a.handle)?0:1)-(/-\d+$/.test(b.handle)?0:1) || (b.price||0)-(a.price||0) || new Date(b.createdAt)-new Date(a.createdAt));
    const keeper=ranked[0];
    clusters['S:'+key]={keeper,dups:ranked.slice(1).filter(p=>p.gid!==keeper.gid)};
  }

  const out=['signal,vendor,sku,dup_id,dup_handle,keep_id,keep_handle,keep_price,created'];
  let n=0; const perV={};
  for(const [key,c] of Object.entries(clusters)){ if(!c.dups.length) continue;
    for(const d of c.dups){ n++; perV[d.vendor]=(perV[d.vendor]||0)+1;
      out.push(`${key[0]},${d.vendor},${d.sku},${d.gid},${d.handle},${c.keeper.gid},${c.keeper.handle},${c.keeper.price??''},"${fmt(d.createdAt)}"`); }
  }
  fs.writeFileSync('/tmp/fleet_dedup.csv',out.join('\n'));
  console.log(`\n=== FLEET TRUE-DUP SWEEP (bulletproof: handle-suffix + real-sku, READ-ONLY) ===`);
  console.log(`active scanned: ${all.length} | dup products to remove: ${n}`);
  console.log('per vendor:'); for(const [v,c] of Object.entries(perV).sort((a,b)=>b[1]-a[1])) console.log(`  ${v.padEnd(24)} ${c}`);
  console.log(`\nlist -> /tmp/fleet_dedup.csv`);
})().catch(e=>{console.error(e);process.exit(1);});