`schu-visual-dup-audit.js`

90 lines
#!/usr/bin/env node
/**
 * Read-only VISUAL duplicate audit for latest Schumacher uploads (sandbox).
 * For each product: fetch every gallery image, compute (1) md5 of raw bytes
 * (exact-dup) and (2) a 16x16 grayscale average-hash (perceptual near-dup).
 * Reports images that are visually identical within the same product page.
 * No writes.
 */
const fs = require('fs');
const crypto = require('crypto');
const sharp = require('sharp');
const env = fs.readFileSync(process.env.HOME + '/Projects/secrets-manager/.env', 'utf8');
const get = k => (env.match(new RegExp('^' + k + '=(.*)$', 'm')) || [])[1];
const STORE = get('SHOPIFY_STORE'), TOKEN = get('SHOPIFY_ADMIN_TOKEN');
const VENDOR = process.argv[2] || 'Schumacher';
const LIMIT = parseInt(process.argv[3] || '40', 10);

async function gql(query, variables) {
  const r = await fetch(`https://${STORE}/admin/api/2024-10/graphql.json`, {
    method: 'POST', headers: { 'Content-Type': 'application/json', 'X-Shopify-Access-Token': TOKEN },
    body: JSON.stringify({ query, variables }) });
  const j = await r.json(); if (j.errors) throw new Error(JSON.stringify(j.errors)); return j.data;
}
const Q = `query($q:String!,$cursor:String){products(first:30,query:$q,sortKey:CREATED_AT,reverse:true,after:$cursor){
  edges{cursor node{title legacyResourceId status media(first:50){edges{node{...on MediaImage{image{url}}}}}}} pageInfo{hasNextPage}}}`;

// 16x16 grayscale -> avg hash (256-bit). hamming distance <= 10 ~ visual dup.
async function phash(buf) {
  const px = await sharp(buf).greyscale().resize(16,16,{fit:'fill'}).raw().toBuffer();
  let sum=0; for (const v of px) sum+=v; const avg=sum/px.length;
  let bits=''; for (const v of px) bits += v>avg?'1':'0';
  return bits;
}
const ham = (a,b) => { let d=0; for (let i=0;i<a.length;i++) if(a[i]!==b[i]) d++; return d; };

(async () => {
  const qstr = `vendor:'${VENDOR}'`;
  let cursor=null, prods=[];
  while (prods.length < LIMIT) {
    const d = await gql(Q,{q:qstr,cursor});
    for (const e of d.products.edges) prods.push(e.node);
    if (!d.products.pageInfo.hasNextPage || !d.products.edges.length) break;
    cursor = d.products.edges[d.products.edges.length-1].cursor;
  }
  prods = prods.slice(0, LIMIT);

  let flagged = [], totalImgs=0, fetchErr=0;
  for (const p of prods) {
    const urls = (p.media?.edges||[]).map(m=>m.node?.image?.url).filter(Boolean);
    const metas = [];
    for (const u of urls) {
      try {
        const r = await fetch(u); const ab = await r.arrayBuffer(); const buf = Buffer.from(ab);
        metas.push({ url:u, name:u.split('/').pop().split('?')[0], md5:crypto.createHash('md5').update(buf).digest('hex'), ph: await phash(buf) });
        totalImgs++;
      } catch(e){ fetchErr++; }
    }
    // exact byte dupes
    const byMd5 = {}; metas.forEach(m=>{(byMd5[m.md5]=byMd5[m.md5]||[]).push(m.name);});
    const exact = Object.values(byMd5).filter(a=>a.length>1);
    // perceptual dupes (cluster by hamming<=10), excluding exact pairs already counted
    const visual = [];
    for (let i=0;i<metas.length;i++) for (let j=i+1;j<metas.length;j++){
      if (metas[i].md5===metas[j].md5) continue;
      const d = ham(metas[i].ph, metas[j].ph);
      if (d<=10) visual.push([metas[i].name, metas[j].name, d]);
    }
    // cluster all images by visual identity (union exact + perceptual ham<=10) -> unique count
    const parent = metas.map((_,i)=>i);
    const find=x=>{while(parent[x]!==x){parent[x]=parent[parent[x]];x=parent[x];}return x;};
    const uni=(a,b)=>{parent[find(a)]=find(b);};
    for (let i=0;i<metas.length;i++) for (let j=i+1;j<metas.length;j++){
      if (metas[i].md5===metas[j].md5 || ham(metas[i].ph,metas[j].ph)<=10) uni(i,j);
    }
    const uniqueCount = new Set(metas.map((_,i)=>find(i))).size;
    p._n = metas.length; p._u = uniqueCount;
    if (exact.length || visual.length) flagged.push({ title:p.title, status:p.status, lid:p.legacyResourceId, n:metas.length, unique:uniqueCount, exact, visual });
  }
  const tImg = prods.reduce((s,p)=>s+(p._n||0),0), tUni = prods.reduce((s,p)=>s+(p._u||0),0);

  console.log(`\n=== ${VENDOR} VISUAL duplicate audit (sandbox, read-only) ===`);
  console.log(`Scanned ${prods.length} latest products · ${totalImgs} images fetched · ${fetchErr} fetch errors`);
  console.log(`Products with visual/exact dup images: ${flagged.length} / ${prods.length}`);
  console.log(`TOTAL images uploaded: ${tImg}  ·  TRULY UNIQUE: ${tUni}  ·  DUPLICATE WASTE: ${tImg-tUni} (${((tImg-tUni)/tImg*100).toFixed(0)}%)\n`);
  console.log('Per-product (images → unique):');
  for (const f of flagged) console.log(`   [${f.status}] ${f.title.replace(/ \| Wallcovering \| Schumacher/,'')}: ${f.n} → ${f.unique} unique`);
  fs.writeFileSync('/tmp/schu-visual-dup-audit.json', JSON.stringify(flagged,null,2));
  console.log(`\nDetail → /tmp/schu-visual-dup-audit.json`);
})().catch(e=>{console.error('ERR',e.message);process.exit(1);});