← back to Letsbegin
schumacher gallery dedup tooling: visual (md5+phash) audit + keep-first cleanup engine (cleaned 1498 dup images across 161 live products)
e1f7121b8098f126f1050b7b1cb484e1f48f4cf8 · 2026-06-12 07:57:05 -0700 · Steve Abrams
Files touched
A schu-dup-image-audit.jsA schu-gallery-dedup.jsA schu-visual-dup-audit.js
Diff
commit e1f7121b8098f126f1050b7b1cb484e1f48f4cf8
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Fri Jun 12 07:57:05 2026 -0700
schumacher gallery dedup tooling: visual (md5+phash) audit + keep-first cleanup engine (cleaned 1498 dup images across 161 live products)
---
schu-dup-image-audit.js | 116 +++++++++++++++++++++++++++++++++++++++++++++++
schu-gallery-dedup.js | 115 ++++++++++++++++++++++++++++++++++++++++++++++
schu-visual-dup-audit.js | 89 ++++++++++++++++++++++++++++++++++++
3 files changed, 320 insertions(+)
diff --git a/schu-dup-image-audit.js b/schu-dup-image-audit.js
new file mode 100644
index 0000000..8baa9b7
--- /dev/null
+++ b/schu-dup-image-audit.js
@@ -0,0 +1,116 @@
+#!/usr/bin/env node
+/**
+ * Read-only audit: latest Schumacher uploads on the sandbox Shopify store.
+ * Detects (a) duplicate images WITHIN a product (same image URL repeated in the
+ * gallery), and (b) the same image shared ACROSS multiple Schumacher products.
+ * No writes. Sandbox store only.
+ */
+const fs = require('fs');
+const env = fs.readFileSync(process.env.HOME + '/Projects/secrets-manager/.env', 'utf8');
+const get = k => (env.match(new RegExp('^' + k + '=(.*)$', 'm')) || [])[1];
+const STORE = get('SHOPIFY_STORE');
+const TOKEN = get('SHOPIFY_ADMIN_TOKEN');
+if (!STORE || !TOKEN) { console.error('missing store/token'); process.exit(1); }
+
+const VENDOR = process.argv[2] || 'Schumacher';
+const LIMIT = parseInt(process.argv[3] || '120', 10); // how many recent products to scan
+
+async function gql(query, variables) {
+ const r = await fetch(`https://${STORE}/admin/api/2024-10/graphql.json`, {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json', 'X-Shopify-Access-Token': TOKEN },
+ body: JSON.stringify({ query, variables }),
+ });
+ const j = await r.json();
+ if (j.errors) throw new Error(JSON.stringify(j.errors));
+ return j.data;
+}
+
+// strip the shopify cdn transform suffix (?v=, _ {width}x, .progressive) so we
+// compare the underlying asset, not a resized variant URL
+function canon(url) {
+ if (!url) return url;
+ let u = url.split('?')[0];
+ u = u.replace(/_\d+x\d+(?=\.)/,'').replace(/_\d+x(?=\.)/,'').replace(/_x\d+(?=\.)/,'');
+ return u;
+}
+
+const Q = `
+query($q:String!, $cursor:String){
+ products(first:50, query:$q, sortKey:CREATED_AT, reverse:true, after:$cursor){
+ edges{ cursor node{
+ id legacyResourceId title handle status vendor createdAt
+ media(first:50){ edges{ node{ ... on MediaImage { id image { url } } } } }
+ }}
+ pageInfo{ hasNextPage }
+ }
+}`;
+
+(async () => {
+ const qstr = `vendor:'${VENDOR}'`;
+ let cursor = null, scanned = [], pages = 0;
+ while (scanned.length < LIMIT) {
+ const d = await gql(Q, { q: qstr, cursor });
+ const edges = d.products.edges;
+ for (const e of edges) scanned.push(e.node);
+ pages++;
+ if (!d.products.pageInfo.hasNextPage || edges.length === 0) break;
+ cursor = edges[edges.length - 1].cursor;
+ if (scanned.length >= LIMIT) break;
+ }
+ scanned = scanned.slice(0, LIMIT);
+
+ const crossMap = new Map(); // canon url -> [{title, status, id}]
+ const withinDupes = []; // products with repeated image in own gallery
+ let totalImgs = 0, noImg = 0;
+
+ for (const p of scanned) {
+ const imgs = (p.media?.edges || []).map(m => m.node?.image?.url).filter(Boolean);
+ totalImgs += imgs.length;
+ if (imgs.length === 0) noImg++;
+ const seen = new Map();
+ for (const u of imgs) {
+ const c = canon(u);
+ seen.set(c, (seen.get(c) || 0) + 1);
+ if (!crossMap.has(c)) crossMap.set(c, []);
+ crossMap.get(c).push({ title: p.title, status: p.status, lid: p.legacyResourceId, handle: p.handle });
+ }
+ const repeats = [...seen.entries()].filter(([, n]) => n > 1);
+ if (repeats.length) {
+ withinDupes.push({ title: p.title, status: p.status, lid: p.legacyResourceId, handle: p.handle,
+ imgCount: imgs.length, repeats });
+ }
+ }
+
+ // cross-product: same canon url on >1 DISTINCT product
+ const crossDupes = [];
+ for (const [c, arr] of crossMap.entries()) {
+ const distinct = [...new Map(arr.map(a => [a.lid, a])).values()];
+ if (distinct.length > 1) crossDupes.push({ url: c, products: distinct });
+ }
+
+ console.log(`\n=== ${VENDOR} image-duplication audit (sandbox, read-only) ===`);
+ console.log(`Scanned ${scanned.length} most-recent ${VENDOR} products (${pages} page(s)).`);
+ console.log(`Created range: ${scanned[scanned.length-1]?.createdAt?.slice(0,10)} … ${scanned[0]?.createdAt?.slice(0,10)}`);
+ console.log(`Total images: ${totalImgs} · products w/ 0 images: ${noImg}`);
+
+ console.log(`\n--- (A) DUPLICATE IMAGE WITHIN A SINGLE PRODUCT (same photo repeated in gallery): ${withinDupes.length} products ---`);
+ for (const w of withinDupes.slice(0, 40)) {
+ console.log(` • [${w.status}] ${w.title} (gid ${w.lid}, ${w.imgCount} imgs)`);
+ for (const [u, n] of w.repeats) console.log(` ×${n} ${u.split('/').pop()}`);
+ }
+ if (withinDupes.length > 40) console.log(` …and ${withinDupes.length - 40} more`);
+
+ console.log(`\n--- (B) SAME IMAGE SHARED ACROSS MULTIPLE PRODUCTS: ${crossDupes.length} shared images ---`);
+ crossDupes.sort((a,b)=>b.products.length-a.products.length);
+ for (const c of crossDupes.slice(0, 30)) {
+ console.log(` • ${c.products.length} products share ${c.url.split('/').pop()}`);
+ for (const p of c.products.slice(0, 6)) console.log(` [${p.status}] ${p.title} (${p.lid})`);
+ if (c.products.length > 6) console.log(` …and ${c.products.length - 6} more`);
+ }
+ if (crossDupes.length > 30) console.log(` …and ${crossDupes.length - 30} more shared images`);
+
+ // machine-readable dump for follow-up
+ fs.writeFileSync('/tmp/schu-dup-image-audit.json', JSON.stringify({ vendor: VENDOR, scanned: scanned.length, withinDupes, crossDupes }, null, 2));
+ console.log(`\nFull detail → /tmp/schu-dup-image-audit.json`);
+})().catch(e => { console.error('ERR', e.message); process.exit(1); });
diff --git a/schu-gallery-dedup.js b/schu-gallery-dedup.js
new file mode 100644
index 0000000..114c5ec
--- /dev/null
+++ b/schu-gallery-dedup.js
@@ -0,0 +1,115 @@
+#!/usr/bin/env node
+/**
+ * Dedup duplicate gallery images on Schumacher Shopify products (sandbox).
+ * For each product: fetch all media bytes -> cluster by md5 (exact) + 16x16
+ * average-hash (perceptual, hamming<=10) -> KEEP the first image of each unique
+ * cluster (preserves swatch-first order), DETACH the rest via productDeleteMedia.
+ *
+ * DRY-RUN by default. Pass --apply to actually delete. Every detached media id
+ * (+url) is snapshotted to a recovery JSON so it can be re-imported.
+ *
+ * node schu-gallery-dedup.js # dry-run, recent cohort
+ * node schu-gallery-dedup.js --apply # execute
+ * node schu-gallery-dedup.js --since 2026-06-01 --apply
+ * node schu-gallery-dedup.js --all --apply # whole Schumacher catalog
+ */
+const fs = require('fs');
+const crypto = require('crypto');
+const sharp = require('sharp');
+const env = fs.readFileSync(process.env.HOME + '/Projects/secrets-manager/.env', 'utf8');
+const get = k => (env.match(new RegExp('^' + k + '=(.*)$', 'm')) || [])[1];
+const STORE = get('SHOPIFY_STORE'), TOKEN = get('SHOPIFY_ADMIN_TOKEN');
+
+const APPLY = process.argv.includes('--apply');
+const ALL = process.argv.includes('--all');
+const sinceIdx = process.argv.indexOf('--since');
+const SINCE = sinceIdx > -1 ? process.argv[sinceIdx + 1] : '2026-06-01';
+const VENDOR = 'Schumacher';
+const SNAP = `/tmp/schu-gallery-dedup-${APPLY ? 'APPLIED' : 'DRYRUN'}.json`;
+const LOG = `/tmp/schu-gallery-dedup.log`;
+function log(s){ const line = s+'\n'; process.stdout.write(line); fs.appendFileSync(LOG, line); }
+
+async function gql(query, variables) {
+ for (let attempt=0; attempt<4; attempt++){
+ const r = await fetch(`https://${STORE}/admin/api/2024-10/graphql.json`, {
+ method:'POST', headers:{'Content-Type':'application/json','X-Shopify-Access-Token':TOKEN},
+ body: JSON.stringify({ query, variables }) });
+ const j = await r.json();
+ if (j.errors && JSON.stringify(j.errors).includes('Throttled')) { await sleep(2000*(attempt+1)); continue; }
+ if (j.errors) throw new Error(JSON.stringify(j.errors));
+ return j.data;
+ }
+ throw new Error('throttled out');
+}
+const sleep = ms => new Promise(r=>setTimeout(r, ms));
+
+const LIST = `query($q:String!,$c:String){products(first:40,query:$q,sortKey:CREATED_AT,reverse:true,after:$c){
+ edges{cursor node{ id legacyResourceId title status createdAt
+ media(first:60){edges{node{ ... on MediaImage { id image{url} } }}} }} pageInfo{hasNextPage}}}`;
+const DEL = `mutation($pid:ID!,$ids:[ID!]!){productDeleteMedia(productId:$pid,mediaIds:$ids){deletedMediaIds mediaUserErrors{field message}}}`;
+
+async function phash(buf){
+ const px = await sharp(buf).greyscale().resize(16,16,{fit:'fill'}).raw().toBuffer();
+ let s=0; for (const v of px) s+=v; const avg=s/px.length;
+ let b=''; for (const v of px) b += v>avg?'1':'0'; return b;
+}
+const ham=(a,b)=>{let d=0;for(let i=0;i<a.length;i++)if(a[i]!==b[i])d++;return d;};
+
+async function fetchMeta(m){
+ try{ const r=await fetch(m.url); const buf=Buffer.from(await r.arrayBuffer());
+ return {...m, ok:true, md5:crypto.createHash('md5').update(buf).digest('hex'), ph:await phash(buf), bytes:buf.length}; }
+ catch(e){ return {...m, ok:false}; }
+}
+
+(async () => {
+ fs.writeFileSync(LOG,'');
+ const q = ALL ? `vendor:'${VENDOR}'` : `vendor:'${VENDOR}' AND created_at:>=${SINCE}`;
+ log(`=== Schumacher gallery dedup — ${APPLY?'APPLY (writes)':'DRY-RUN (read-only)'} ===`);
+ log(`scope: ${ALL?'ALL Schumacher':`created_at >= ${SINCE}`}`);
+
+ let cursor=null, products=[];
+ while(true){ const d=await gql(LIST,{q,c:cursor}); const e=d.products.edges;
+ for(const x of e) products.push(x.node);
+ if(!d.products.pageInfo.hasNextPage||!e.length)break; cursor=e[e.length-1].cursor; }
+ log(`products in scope: ${products.length}`);
+
+ const snapshot=[]; let pAffected=0, mDetached=0, mTotal=0, idx=0;
+ let actAff=0, drAff=0;
+ for (const p of products){
+ idx++;
+ const media=(p.media?.edges||[]).map(x=>x.node).filter(n=>n?.image?.url).map(n=>({id:n.id,url:n.image.url}));
+ mTotal+=media.length;
+ // fetch+hash with small concurrency
+ const metas=[]; const CC=6;
+ for(let i=0;i<media.length;i+=CC){ const chunk=await Promise.all(media.slice(i,i+CC).map(fetchMeta)); metas.push(...chunk); }
+ const good=metas.filter(m=>m.ok);
+ // cluster keep-first
+ const kept=[]; const toDelete=[];
+ for(const m of good){
+ const dup=kept.find(k=>k.md5===m.md5 || ham(k.ph,m.ph)<=10);
+ if(dup) toDelete.push(m); else kept.push(m);
+ }
+ if(toDelete.length){
+ pAffected++; mDetached+=toDelete.length;
+ if(p.status==='ACTIVE')actAff++; else if(p.status==='DRAFT')drAff++;
+ snapshot.push({ productId:p.id, lid:p.legacyResourceId, title:p.title, status:p.status,
+ before:media.length, kept:kept.length, deleted:toDelete.map(d=>({id:d.id,url:d.url})) });
+ log(` [${p.status}] ${p.title.replace(/ \| Wallcovering \| Schumacher/,'')}: ${media.length} -> ${kept.length} (detach ${toDelete.length})`);
+ if(APPLY){
+ const d=await gql(DEL,{pid:p.id, ids:toDelete.map(t=>t.id)});
+ const errs=d.productDeleteMedia.mediaUserErrors;
+ if(errs && errs.length) log(` ⚠ userErrors: ${JSON.stringify(errs)}`);
+ await sleep(350); // gentle on the API
+ }
+ }
+ if(idx%25===0) log(` …${idx}/${products.length} scanned`);
+ }
+ fs.writeFileSync(SNAP, JSON.stringify({apply:APPLY, scope:ALL?'ALL':`>=${SINCE}`, products:products.length, snapshot}, null, 2));
+ log(`\n=== SUMMARY (${APPLY?'APPLIED':'DRY-RUN'}) ===`);
+ log(`products in scope: ${products.length}`);
+ log(`products with dup imgs: ${pAffected} (ACTIVE ${actAff} / DRAFT ${drAff})`);
+ log(`total media: ${mTotal}`);
+ log(`media ${APPLY?'DETACHED':'to detach'}: ${mDetached} (${mTotal?((mDetached/mTotal)*100).toFixed(0):0}% of gallery slots)`);
+ log(`recovery snapshot: ${SNAP}`);
+ if(!APPLY) log(`\nDry-run only. Re-run with --apply to detach.`);
+})().catch(e=>{ log('ERR '+e.message); process.exit(1); });
diff --git a/schu-visual-dup-audit.js b/schu-visual-dup-audit.js
new file mode 100644
index 0000000..e56762b
--- /dev/null
+++ b/schu-visual-dup-audit.js
@@ -0,0 +1,89 @@
+#!/usr/bin/env node
+/**
+ * Read-only VISUAL duplicate audit for latest Schumacher uploads (sandbox).
+ * For each product: fetch every gallery image, compute (1) md5 of raw bytes
+ * (exact-dup) and (2) a 16x16 grayscale average-hash (perceptual near-dup).
+ * Reports images that are visually identical within the same product page.
+ * No writes.
+ */
+const fs = require('fs');
+const crypto = require('crypto');
+const sharp = require('sharp');
+const env = fs.readFileSync(process.env.HOME + '/Projects/secrets-manager/.env', 'utf8');
+const get = k => (env.match(new RegExp('^' + k + '=(.*)$', 'm')) || [])[1];
+const STORE = get('SHOPIFY_STORE'), TOKEN = get('SHOPIFY_ADMIN_TOKEN');
+const VENDOR = process.argv[2] || 'Schumacher';
+const LIMIT = parseInt(process.argv[3] || '40', 10);
+
+async function gql(query, variables) {
+ const r = await fetch(`https://${STORE}/admin/api/2024-10/graphql.json`, {
+ method: 'POST', headers: { 'Content-Type': 'application/json', 'X-Shopify-Access-Token': TOKEN },
+ body: JSON.stringify({ query, variables }) });
+ const j = await r.json(); if (j.errors) throw new Error(JSON.stringify(j.errors)); return j.data;
+}
+const Q = `query($q:String!,$cursor:String){products(first:30,query:$q,sortKey:CREATED_AT,reverse:true,after:$cursor){
+ edges{cursor node{title legacyResourceId status media(first:50){edges{node{...on MediaImage{image{url}}}}}}} pageInfo{hasNextPage}}}`;
+
+// 16x16 grayscale -> avg hash (256-bit). hamming distance <= 10 ~ visual dup.
+async function phash(buf) {
+ const px = await sharp(buf).greyscale().resize(16,16,{fit:'fill'}).raw().toBuffer();
+ let sum=0; for (const v of px) sum+=v; const avg=sum/px.length;
+ let bits=''; for (const v of px) bits += v>avg?'1':'0';
+ return bits;
+}
+const ham = (a,b) => { let d=0; for (let i=0;i<a.length;i++) if(a[i]!==b[i]) d++; return d; };
+
+(async () => {
+ const qstr = `vendor:'${VENDOR}'`;
+ let cursor=null, prods=[];
+ while (prods.length < LIMIT) {
+ const d = await gql(Q,{q:qstr,cursor});
+ for (const e of d.products.edges) prods.push(e.node);
+ if (!d.products.pageInfo.hasNextPage || !d.products.edges.length) break;
+ cursor = d.products.edges[d.products.edges.length-1].cursor;
+ }
+ prods = prods.slice(0, LIMIT);
+
+ let flagged = [], totalImgs=0, fetchErr=0;
+ for (const p of prods) {
+ const urls = (p.media?.edges||[]).map(m=>m.node?.image?.url).filter(Boolean);
+ const metas = [];
+ for (const u of urls) {
+ try {
+ const r = await fetch(u); const ab = await r.arrayBuffer(); const buf = Buffer.from(ab);
+ metas.push({ url:u, name:u.split('/').pop().split('?')[0], md5:crypto.createHash('md5').update(buf).digest('hex'), ph: await phash(buf) });
+ totalImgs++;
+ } catch(e){ fetchErr++; }
+ }
+ // exact byte dupes
+ const byMd5 = {}; metas.forEach(m=>{(byMd5[m.md5]=byMd5[m.md5]||[]).push(m.name);});
+ const exact = Object.values(byMd5).filter(a=>a.length>1);
+ // perceptual dupes (cluster by hamming<=10), excluding exact pairs already counted
+ const visual = [];
+ for (let i=0;i<metas.length;i++) for (let j=i+1;j<metas.length;j++){
+ if (metas[i].md5===metas[j].md5) continue;
+ const d = ham(metas[i].ph, metas[j].ph);
+ if (d<=10) visual.push([metas[i].name, metas[j].name, d]);
+ }
+ // cluster all images by visual identity (union exact + perceptual ham<=10) -> unique count
+ const parent = metas.map((_,i)=>i);
+ const find=x=>{while(parent[x]!==x){parent[x]=parent[parent[x]];x=parent[x];}return x;};
+ const uni=(a,b)=>{parent[find(a)]=find(b);};
+ for (let i=0;i<metas.length;i++) for (let j=i+1;j<metas.length;j++){
+ if (metas[i].md5===metas[j].md5 || ham(metas[i].ph,metas[j].ph)<=10) uni(i,j);
+ }
+ const uniqueCount = new Set(metas.map((_,i)=>find(i))).size;
+ p._n = metas.length; p._u = uniqueCount;
+ if (exact.length || visual.length) flagged.push({ title:p.title, status:p.status, lid:p.legacyResourceId, n:metas.length, unique:uniqueCount, exact, visual });
+ }
+ const tImg = prods.reduce((s,p)=>s+(p._n||0),0), tUni = prods.reduce((s,p)=>s+(p._u||0),0);
+
+ console.log(`\n=== ${VENDOR} VISUAL duplicate audit (sandbox, read-only) ===`);
+ console.log(`Scanned ${prods.length} latest products · ${totalImgs} images fetched · ${fetchErr} fetch errors`);
+ console.log(`Products with visual/exact dup images: ${flagged.length} / ${prods.length}`);
+ console.log(`TOTAL images uploaded: ${tImg} · TRULY UNIQUE: ${tUni} · DUPLICATE WASTE: ${tImg-tUni} (${((tImg-tUni)/tImg*100).toFixed(0)}%)\n`);
+ console.log('Per-product (images → unique):');
+ for (const f of flagged) console.log(` [${f.status}] ${f.title.replace(/ \| Wallcovering \| Schumacher/,'')}: ${f.n} → ${f.unique} unique`);
+ fs.writeFileSync('/tmp/schu-visual-dup-audit.json', JSON.stringify(flagged,null,2));
+ console.log(`\nDetail → /tmp/schu-visual-dup-audit.json`);
+})().catch(e=>{console.error('ERR',e.message);process.exit(1);});
← 19a2714 codehealth: add sandbox guards to 3 env-default Shopify writ
·
back to Letsbegin
·
auto-save: 2026-06-21T18:53:00 (9 files) — rw-fix-rollprice. 76bceba →