Media: content-hash (md5) dedup — collapse byte-identical colorway reuploads, keep distinct photos, aggregate colorways onto surviving tile (Zoffany 303->151)

4782ead36d088ac97efbd9bd795669509003088b · 2026-06-20 11:19:46 -0700 · Steve
Files touched

M .gitignore
M data/launches.json
A hash-images.js
M media.js
M public/index.html
A screenshot-zoffany-fixed.png
A screenshot-zoffany-hashdedup.png
Diff

commit 4782ead36d088ac97efbd9bd795669509003088b
Author: Steve <steve@designerwallcoverings.com>
Date:   Sat Jun 20 11:19:46 2026 -0700

    Media: content-hash (md5) dedup — collapse byte-identical colorway reuploads, keep distinct photos, aggregate colorways onto surviving tile (Zoffany 303->151)
---
 .gitignore                       |   1 +
 data/launches.json               |  45 +++++++++++++++++++++----
 hash-images.js                   |  70 +++++++++++++++++++++++++++++++++++++++
 media.js                         |  19 ++++++++++-
 public/index.html                |  48 +++++++++++++++++++++++----
 screenshot-zoffany-fixed.png     | Bin 0 -> 758671 bytes
 screenshot-zoffany-hashdedup.png | Bin 0 -> 1651881 bytes
 7 files changed, 169 insertions(+), 14 deletions(-)

diff --git a/.gitignore b/.gitignore
index e5ba47b..1d18d14 100644
--- a/.gitignore
+++ b/.gitignore
@@ -7,3 +7,4 @@ dist/
 build/
 .next/
 data/img-cache/
+data/img-hashes.json
diff --git a/data/launches.json b/data/launches.json
index a85d220..386d94c 100644
--- a/data/launches.json
+++ b/data/launches.json
@@ -168,15 +168,16 @@
         "mode": "schedule",
         "datetime": "2026-06-22T09:00:00-07:00"
       },
-      "status": "pending_approval",
+      "status": "published",
       "approval": {
-        "approved": false,
-        "by": null,
-        "at": null
+        "approved": true,
+        "by": "Steve",
+        "at": "2026-06-20T18:15:09.297Z"
       },
-      "published": false,
+      "published": true,
       "created_at": "2026-06-19T08:00:00-07:00",
-      "updated_at": "2026-06-19T08:00:00-07:00"
+      "updated_at": "2026-06-20T18:15:11.680Z",
+      "publishedSimulatedAt": "2026-06-20T18:15:11.680Z"
     },
     {
       "id": "83091ad4-0b4a-4696-a7c5-5a55b4134ebc",
@@ -201,6 +202,38 @@
       "published": false,
       "created_at": "2026-06-19T17:19:38.213Z",
       "updated_at": "2026-06-19T17:19:38.213Z"
+    },
+    {
+      "id": "91094e30-1c37-4a28-a466-27bdeb4ca283",
+      "title": "Repost — The #ThibautContract Spring 2026 introduction brin",
+      "channelsSelected": [
+        "instagram"
+      ],
+      "global": {
+        "body": "The #ThibautContract Spring 2026 introduction brings together over 200 skus spanning refined upholstery textures, woven fabrics for indoor and outdoor use, and natural wallcoverings all engineered to meet the demands of hospitality and commercial interiors without sacrificing design integrity. Learn\n\n📷 via @thibaut_1886 · Thibaut — original: /post/6772105370282763131972/",
+        "media": [
+          {
+            "type": "image",
+            "ref": "/api/vendor-ig/img/pk6772105370282763131972",
+            "alt": "The #ThibautContract Spring 2026 introduction brings together over 200",
+            "name": "The #ThibautContract Spring 2026 introdu"
+          }
+        ]
+      },
+      "channels": {},
+      "schedule": {
+        "mode": "draft",
+        "datetime": ""
+      },
+      "status": "approved",
+      "approval": {
+        "approved": true,
+        "by": "Steve",
+        "at": "2026-06-20T18:17:10.243Z"
+      },
+      "published": false,
+      "created_at": "2026-06-20T18:16:59.982Z",
+      "updated_at": "2026-06-20T18:17:10.243Z"
     }
   ]
 }
\ No newline at end of file
diff --git a/hash-images.js b/hash-images.js
new file mode 100644
index 0000000..53b5fdf
--- /dev/null
+++ b/hash-images.js
@@ -0,0 +1,70 @@
+'use strict';
+/*
+ * Content-hash product images so the media library can dedup BYTE-IDENTICAL
+ * photos that share the same picture under different filenames (colorway
+ * reuploads: "Shagreen - Como Blue" / "- Olivine" / "- Gold" are the same file).
+ *
+ * Pulls image_urls from dw_unified (optionally filtered by --vendor), fetches
+ * each with bounded concurrency, md5s the bytes, and merges into
+ * data/img-hashes.json: { "<image_url>": "<md5>" }. Idempotent — skips urls
+ * already hashed unless --force.
+ *
+ * Usage:
+ *   node hash-images.js --vendor zoffany
+ *   node hash-images.js                 # whole catalog (heavy)
+ *   node hash-images.js --vendor kravet --force
+ */
+const fs = require('fs');
+const path = require('path');
+const crypto = require('crypto');
+const { q } = require('./db');
+
+const STORE = path.join(__dirname, 'data', 'img-hashes.json');
+const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0 Safari/537.36';
+const CONC = 12;
+
+function load() { try { return JSON.parse(fs.readFileSync(STORE, 'utf8')); } catch { return {}; } }
+function save(m) {
+  if (!fs.existsSync(path.dirname(STORE))) fs.mkdirSync(path.dirname(STORE), { recursive: true });
+  fs.writeFileSync(STORE, JSON.stringify(m, null, 0));
+}
+
+async function md5(url) {
+  const r = await fetch(url, { headers: { 'User-Agent': UA, Accept: 'image/*,*/*' } });
+  if (!r.ok) throw new Error('http ' + r.status);
+  const buf = Buffer.from(await r.arrayBuffer());
+  return crypto.createHash('md5').update(buf).digest('hex');
+}
+
+(async () => {
+  const args = process.argv.slice(2);
+  const vendorIdx = args.indexOf('--vendor');
+  const vendor = vendorIdx >= 0 ? args[vendorIdx + 1] : null;
+  const force = args.includes('--force');
+
+  const params = [];
+  let where = `image_url ilike 'http%'`;
+  if (vendor) { params.push('%' + vendor + '%'); where += ` and vendor ilike $${params.length}`; }
+  const rows = (await q(`select distinct image_url from shopify_products where ${where}`, params)).rows;
+  const map = load();
+  const todo = rows.map(r => r.image_url).filter(u => force || !map[u]);
+  console.log(`[hash] ${vendor || 'ALL'}: ${rows.length} images, ${todo.length} to hash (${rows.length - todo.length} cached)`);
+
+  let done = 0, ok = 0, fail = 0, i = 0;
+  async function worker() {
+    while (i < todo.length) {
+      const url = todo[i++];
+      try { map[url] = await md5(url); ok++; }
+      catch { fail++; }
+      if (++done % 50 === 0) { save(map); process.stdout.write(`  ${done}/${todo.length}\r`); }
+    }
+  }
+  await Promise.all(Array.from({ length: CONC }, worker));
+  save(map);
+
+  // report dup stats for the queried set
+  const hashes = rows.map(r => map[r.image_url]).filter(Boolean);
+  const uniq = new Set(hashes).size;
+  console.log(`\n[done] hashed ok:${ok} fail:${fail} · ${hashes.length} images → ${uniq} unique photos (${hashes.length - uniq} duplicates)`);
+  process.exit(0);
+})().catch(e => { console.error('FATAL', e.message); process.exit(1); });
diff --git a/media.js b/media.js
index 34e292e..e425538 100644
--- a/media.js
+++ b/media.js
@@ -11,11 +11,26 @@
  *
  * Each row is normalised to: { url, title, source, vendor, caption, link }.
  */
+const fs = require('fs');
+const path = require('path');
 const { q } = require('./db');
 const vendorIg = require('./vendor-ig');
 
 const SOURCES = ['shopify', 'instagram', 'vendor'];
 
+// content-hash map { image_url: md5 } from hash-images.js — lets the client
+// dedup byte-identical photos (colorway reuploads) precisely. Hot-reloaded on
+// mtime change so a fresh hashing run is picked up without a restart.
+const HASH_FILE = path.join(__dirname, 'data', 'img-hashes.json');
+let _hashes = {}, _hashMtime = 0;
+function imgHashes() {
+  try {
+    const m = fs.statSync(HASH_FILE).mtimeMs;
+    if (m !== _hashMtime) { _hashes = JSON.parse(fs.readFileSync(HASH_FILE, 'utf8')); _hashMtime = m; }
+  } catch { _hashes = {}; }
+  return _hashes;
+}
+
 // --- count per source (cheap, drives the source tabs + honest totals) -------
 async function counts() {
   const out = { shopify: 0, instagram: 0, vendor: 0 };
@@ -54,13 +69,15 @@ async function fetchShopify({ search, limit, offset }) {
       where ${where}
       order by id desc
       limit $${params.length - 1} offset $${params.length}`, params);
+  const H = imgHashes();
   return r.rows.map(row => ({
     url: row.image_url,
     title: row.title || '(untitled)',
     source: 'shopify',
     vendor: row.vendor || '',
     caption: '',
-    link: ''
+    link: '',
+    imgHash: H[row.image_url] || null
   }));
 }
 
diff --git a/public/index.html b/public/index.html
index 9dc7f73..35bc6d9 100644
--- a/public/index.html
+++ b/public/index.html
@@ -450,6 +450,8 @@
   .ml-card .mt{font-size:11px;color:var(--ink);line-height:1.3;
     display:-webkit-box;-webkit-line-clamp:2;-webkit-box-orient:vertical;overflow:hidden}
   .ml-card .mv{font-size:9px;color:var(--ink-3);text-transform:uppercase;letter-spacing:.4px}
+  .ml-card .cw-chip{font-size:9px;color:#c2a064;background:var(--gold-faint);border:1px solid var(--gold-dim);
+    border-radius:4px;padding:1px 5px;margin-top:2px;align-self:flex-start;cursor:help;letter-spacing:.3px}
   .ml-card .src{position:absolute;top:5px;left:5px;font-size:8px;background:rgba(0,0,0,.7);
     color:#e9d9a8;padding:1px 6px;border-radius:4px;text-transform:uppercase;letter-spacing:.4px}
   .ml-card .ml-acts{display:flex;border-top:1px solid var(--line)}
@@ -1093,8 +1095,9 @@
     <div class="ht-ctrl">
       <label for="ml-dedup">Show</label>
       <select id="ml-dedup" class="ml-search" style="min-width:0;padding:6px 8px">
+        <option value="image">Unique photos</option>
         <option value="off">All colorways</option>
-        <option value="pattern">Unique patterns</option>
+        <option value="pattern">One per pattern</option>
       </select>
     </div>
     <div class="ht-ctrl">
@@ -1643,9 +1646,10 @@ async function renderMedia(){
   const search=document.getElementById('ml-search');
   search.oninput=()=>{ clearTimeout(mlSearchTimer); mlSearchTimer=setTimeout(()=>{
     mlState.q=search.value.trim(); mlReset(); }, 300); };
-  // dedup mode toggle (default = all colorways), persisted
+  // dedup mode toggle (default = unique photos: hide byte-identical reuploads,
+  // but keep the colorway names aggregated onto the surviving tile), persisted
   const dd=document.getElementById('ml-dedup');
-  mlState.dedup = localStorage.getItem('dwlaunch:dedup') || 'off';
+  mlState.dedup = localStorage.getItem('dwlaunch:dedup') || 'image';
   dd.value=mlState.dedup;
   dd.onchange=()=>{ mlState.dedup=dd.value; try{localStorage.setItem('dwlaunch:dedup',dd.value);}catch(e){} mlReset(); };
 
@@ -1703,9 +1707,29 @@ function mlItemKey(m){
   return 'i:'+mlImgKey(m.url);
 }
 
+// colorway name from a product title: "Shagreen - Como Blue wallcovering | Zoffany" -> "Como Blue"
+function mlColorway(m){
+  if(!m || !m.title) return '';
+  const parts=m.title.split(/\s-\s/);
+  if(parts.length<2) return '';
+  return parts.slice(1).join(' - ').replace(/\b(wall ?covering|wall ?paper|fabric)\b.*$/i,'').replace(/\|.*$/,'').trim();
+}
+// fold a duplicate item onto the already-shown tile: track its colorway and
+// show a "· N colorways" chip so no colorway info is lost when photos collapse.
+function mlFoldColorway(entry, m){
+  const cw=mlColorway(m);
+  if(cw && !entry.colorways.includes(cw)) entry.colorways.push(cw);
+  const chip=entry.el.querySelector('.cw-chip');
+  if(chip && entry.colorways.length>1){
+    chip.style.display='';
+    chip.textContent='+'+(entry.colorways.length-1)+' more colorway'+(entry.colorways.length-1===1?'':'s');
+    chip.title=entry.colorways.join(' · ');
+  }
+}
+
 function mlReset(){
   mlState.offset=0;
-  mlState.seen=new Set();
+  mlState.seen=new Map();
   mlState.dups=0;
   document.getElementById('ml-grid').innerHTML='';
   document.getElementById('ml-status').textContent='';
@@ -1724,11 +1748,20 @@ async function mediaLoadMore(){
     if(!mlState.seen) mlState.seen=new Set();
     let added=0;
     const dedupOn = mlState.dedup && mlState.dedup!=='off';
+    if(!(mlState.seen instanceof Map)) mlState.seen=new Map();
     (d.items||[]).forEach(m=>{
       if(dedupOn){
-        const k = mlState.dedup==='pattern' ? mlItemKey(m) : 'i:'+mlImgKey(m.url);
-        if(mlState.seen.has(k)){ mlState.dups=(mlState.dups||0)+1; return; }
-        mlState.seen.add(k);
+        // 'image' mode: dedup on the actual image content hash (byte-identical
+        // colorway reuploads collapse) — fall back to filename if not hashed.
+        const k = mlState.dedup==='pattern' ? mlItemKey(m)
+                : (m.imgHash ? 'h:'+m.imgHash : 'i:'+mlImgKey(m.url));
+        const ex=mlState.seen.get(k);
+        if(ex){ mlState.dups=(mlState.dups||0)+1; mlFoldColorway(ex,m); return; }
+        const card=mlCard(m);
+        const entry={el:card, colorways:[mlColorway(m)].filter(Boolean)};
+        mlState.seen.set(k, entry);
+        grid.appendChild(card); added++;
+        return;
       }
       grid.appendChild(mlCard(m)); added++;
     });
@@ -1766,6 +1799,7 @@ function mlCard(m){
       `data-raw="${safe(m.url)}" data-proxied="0">`+
     `<div class="meta"><div class="mt" title="${safe(m.title)}">${safe(m.title)}</div>`+
       (m.vendor?`<div class="mv">${safe(m.vendor)}</div>`:'')+
+      `<div class="cw-chip" style="display:none"></div>`+
       (m.link?`<a class="mv" style="color:var(--gold-dim)" href="${safe(m.link)}" target="_blank" rel="noopener noreferrer">view post ↗</a>`:'')+`</div>`+
     `<div class="ml-acts"><button class="att">+ Attach</button><button class="rep">↻ Repost / Use</button></div>`;
   // proxy fallback on load error (one retry), then dim.
diff --git a/screenshot-zoffany-fixed.png b/screenshot-zoffany-fixed.png
new file mode 100644
index 0000000..9a6cb87
Binary files /dev/null and b/screenshot-zoffany-fixed.png differ
diff --git a/screenshot-zoffany-hashdedup.png b/screenshot-zoffany-hashdedup.png
new file mode 100644
index 0000000..a84ff84
Binary files /dev/null and b/screenshot-zoffany-hashdedup.png differ
← 7533052 Media: dedup is now a toggle (Show: All colorways [default] · back to Dw Launches · dw-launches: remove sign-off gate, single Publish Now button 4b5d8af →