`hash-images.js`

71 lines
'use strict';
/*
 * Content-hash product images so the media library can dedup BYTE-IDENTICAL
 * photos that share the same picture under different filenames (colorway
 * reuploads: "Shagreen - Como Blue" / "- Olivine" / "- Gold" are the same file).
 *
 * Pulls image_urls from dw_unified (optionally filtered by --vendor), fetches
 * each with bounded concurrency, md5s the bytes, and merges into
 * data/img-hashes.json: { "<image_url>": "<md5>" }. Idempotent — skips urls
 * already hashed unless --force.
 *
 * Usage:
 *   node hash-images.js --vendor zoffany
 *   node hash-images.js                 # whole catalog (heavy)
 *   node hash-images.js --vendor kravet --force
 */
const fs = require('fs');
const path = require('path');
const crypto = require('crypto');
const { q } = require('./db');

const STORE = path.join(__dirname, 'data', 'img-hashes.json');
const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0 Safari/537.36';
const CONC = 12;

function load() { try { return JSON.parse(fs.readFileSync(STORE, 'utf8')); } catch { return {}; } }
function save(m) {
  if (!fs.existsSync(path.dirname(STORE))) fs.mkdirSync(path.dirname(STORE), { recursive: true });
  fs.writeFileSync(STORE, JSON.stringify(m, null, 0));
}

async function md5(url) {
  const r = await fetch(url, { headers: { 'User-Agent': UA, Accept: 'image/*,*/*' } });
  if (!r.ok) throw new Error('http ' + r.status);
  const buf = Buffer.from(await r.arrayBuffer());
  return crypto.createHash('md5').update(buf).digest('hex');
}

(async () => {
  const args = process.argv.slice(2);
  const vendorIdx = args.indexOf('--vendor');
  const vendor = vendorIdx >= 0 ? args[vendorIdx + 1] : null;
  const force = args.includes('--force');

  const params = [];
  let where = `image_url ilike 'http%'`;
  if (vendor) { params.push('%' + vendor + '%'); where += ` and vendor ilike $${params.length}`; }
  const rows = (await q(`select distinct image_url from shopify_products where ${where}`, params)).rows;
  const map = load();
  const todo = rows.map(r => r.image_url).filter(u => force || !map[u]);
  console.log(`[hash] ${vendor || 'ALL'}: ${rows.length} images, ${todo.length} to hash (${rows.length - todo.length} cached)`);

  let done = 0, ok = 0, fail = 0, i = 0;
  async function worker() {
    while (i < todo.length) {
      const url = todo[i++];
      try { map[url] = await md5(url); ok++; }
      catch { fail++; }
      if (++done % 50 === 0) { save(map); process.stdout.write(`  ${done}/${todo.length}\r`); }
    }
  }
  await Promise.all(Array.from({ length: CONC }, worker));
  save(map);

  // report dup stats for the queried set
  const hashes = rows.map(r => map[r.image_url]).filter(Boolean);
  const uniq = new Set(hashes).size;
  console.log(`\n[done] hashed ok:${ok} fail:${fail} · ${hashes.length} images → ${uniq} unique photos (${hashes.length - uniq} duplicates)`);
  process.exit(0);
})().catch(e => { console.error('FATAL', e.message); process.exit(1); });