`schu-gallery-dedup.js`

116 lines
#!/usr/bin/env node
/**
 * Dedup duplicate gallery images on Schumacher Shopify products (sandbox).
 * For each product: fetch all media bytes -> cluster by md5 (exact) + 16x16
 * average-hash (perceptual, hamming<=10) -> KEEP the first image of each unique
 * cluster (preserves swatch-first order), DETACH the rest via productDeleteMedia.
 *
 * DRY-RUN by default. Pass --apply to actually delete. Every detached media id
 * (+url) is snapshotted to a recovery JSON so it can be re-imported.
 *
 *   node schu-gallery-dedup.js              # dry-run, recent cohort
 *   node schu-gallery-dedup.js --apply      # execute
 *   node schu-gallery-dedup.js --since 2026-06-01 --apply
 *   node schu-gallery-dedup.js --all --apply # whole Schumacher catalog
 */
const fs = require('fs');
const crypto = require('crypto');
const sharp = require('sharp');
const env = fs.readFileSync(process.env.HOME + '/Projects/secrets-manager/.env', 'utf8');
const get = k => (env.match(new RegExp('^' + k + '=(.*)$', 'm')) || [])[1];
const STORE = get('SHOPIFY_STORE'), TOKEN = get('SHOPIFY_ADMIN_TOKEN');

const APPLY = process.argv.includes('--apply');
const ALL = process.argv.includes('--all');
const sinceIdx = process.argv.indexOf('--since');
const SINCE = sinceIdx > -1 ? process.argv[sinceIdx + 1] : '2026-06-01';
const VENDOR = 'Schumacher';
const SNAP = `/tmp/schu-gallery-dedup-${APPLY ? 'APPLIED' : 'DRYRUN'}.json`;
const LOG = `/tmp/schu-gallery-dedup.log`;
function log(s){ const line = s+'\n'; process.stdout.write(line); fs.appendFileSync(LOG, line); }

async function gql(query, variables) {
  for (let attempt=0; attempt<4; attempt++){
    const r = await fetch(`https://${STORE}/admin/api/2024-10/graphql.json`, {
      method:'POST', headers:{'Content-Type':'application/json','X-Shopify-Access-Token':TOKEN},
      body: JSON.stringify({ query, variables }) });
    const j = await r.json();
    if (j.errors && JSON.stringify(j.errors).includes('Throttled')) { await sleep(2000*(attempt+1)); continue; }
    if (j.errors) throw new Error(JSON.stringify(j.errors));
    return j.data;
  }
  throw new Error('throttled out');
}
const sleep = ms => new Promise(r=>setTimeout(r, ms));

const LIST = `query($q:String!,$c:String){products(first:40,query:$q,sortKey:CREATED_AT,reverse:true,after:$c){
  edges{cursor node{ id legacyResourceId title status createdAt
    media(first:60){edges{node{ ... on MediaImage { id image{url} } }}} }} pageInfo{hasNextPage}}}`;
const DEL = `mutation($pid:ID!,$ids:[ID!]!){productDeleteMedia(productId:$pid,mediaIds:$ids){deletedMediaIds mediaUserErrors{field message}}}`;

async function phash(buf){
  const px = await sharp(buf).greyscale().resize(16,16,{fit:'fill'}).raw().toBuffer();
  let s=0; for (const v of px) s+=v; const avg=s/px.length;
  let b=''; for (const v of px) b += v>avg?'1':'0'; return b;
}
const ham=(a,b)=>{let d=0;for(let i=0;i<a.length;i++)if(a[i]!==b[i])d++;return d;};

async function fetchMeta(m){
  try{ const r=await fetch(m.url); const buf=Buffer.from(await r.arrayBuffer());
    return {...m, ok:true, md5:crypto.createHash('md5').update(buf).digest('hex'), ph:await phash(buf), bytes:buf.length}; }
  catch(e){ return {...m, ok:false}; }
}

(async () => {
  fs.writeFileSync(LOG,'');
  const q = ALL ? `vendor:'${VENDOR}'` : `vendor:'${VENDOR}' AND created_at:>=${SINCE}`;
  log(`=== Schumacher gallery dedup — ${APPLY?'APPLY (writes)':'DRY-RUN (read-only)'} ===`);
  log(`scope: ${ALL?'ALL Schumacher':`created_at >= ${SINCE}`}`);

  let cursor=null, products=[];
  while(true){ const d=await gql(LIST,{q,c:cursor}); const e=d.products.edges;
    for(const x of e) products.push(x.node);
    if(!d.products.pageInfo.hasNextPage||!e.length)break; cursor=e[e.length-1].cursor; }
  log(`products in scope: ${products.length}`);

  const snapshot=[]; let pAffected=0, mDetached=0, mTotal=0, idx=0;
  let actAff=0, drAff=0;
  for (const p of products){
    idx++;
    const media=(p.media?.edges||[]).map(x=>x.node).filter(n=>n?.image?.url).map(n=>({id:n.id,url:n.image.url}));
    mTotal+=media.length;
    // fetch+hash with small concurrency
    const metas=[]; const CC=6;
    for(let i=0;i<media.length;i+=CC){ const chunk=await Promise.all(media.slice(i,i+CC).map(fetchMeta)); metas.push(...chunk); }
    const good=metas.filter(m=>m.ok);
    // cluster keep-first
    const kept=[]; const toDelete=[];
    for(const m of good){
      const dup=kept.find(k=>k.md5===m.md5 || ham(k.ph,m.ph)<=10);
      if(dup) toDelete.push(m); else kept.push(m);
    }
    if(toDelete.length){
      pAffected++; mDetached+=toDelete.length;
      if(p.status==='ACTIVE')actAff++; else if(p.status==='DRAFT')drAff++;
      snapshot.push({ productId:p.id, lid:p.legacyResourceId, title:p.title, status:p.status,
        before:media.length, kept:kept.length, deleted:toDelete.map(d=>({id:d.id,url:d.url})) });
      log(`  [${p.status}] ${p.title.replace(/ \| Wallcovering \| Schumacher/,'')}: ${media.length} -> ${kept.length} (detach ${toDelete.length})`);
      if(APPLY){
        const d=await gql(DEL,{pid:p.id, ids:toDelete.map(t=>t.id)});
        const errs=d.productDeleteMedia.mediaUserErrors;
        if(errs && errs.length) log(`     ⚠ userErrors: ${JSON.stringify(errs)}`);
        await sleep(350); // gentle on the API
      }
    }
    if(idx%25===0) log(`  …${idx}/${products.length} scanned`);
  }
  fs.writeFileSync(SNAP, JSON.stringify({apply:APPLY, scope:ALL?'ALL':`>=${SINCE}`, products:products.length, snapshot}, null, 2));
  log(`\n=== SUMMARY (${APPLY?'APPLIED':'DRY-RUN'}) ===`);
  log(`products in scope:      ${products.length}`);
  log(`products with dup imgs: ${pAffected}  (ACTIVE ${actAff} / DRAFT ${drAff})`);
  log(`total media:            ${mTotal}`);
  log(`media ${APPLY?'DETACHED':'to detach'}:    ${mDetached}  (${mTotal?((mDetached/mTotal)*100).toFixed(0):0}% of gallery slots)`);
  log(`recovery snapshot:      ${SNAP}`);
  if(!APPLY) log(`\nDry-run only. Re-run with --apply to detach.`);
})().catch(e=>{ log('ERR '+e.message); process.exit(1); });