← back to Dw Fleet Registry
resolve-heroes.mjs
196 lines
#!/usr/bin/env node
// Bake a DISTINCT, HIGH-RES hero image into every fleet site, with NO duplicates
// across cards and a branded fallback for the few sites that have nothing.
//
// Strategy (per Steve's rule: every site >=1 high-res image; no dups; else fallback):
// Phase 1 (parallel) — for each domain gather:
// - liveHero : best image off the live homepage (og/twitter/link/background/img),
// validated as a real image AND measured >= MINW px wide.
// - products : that site's OWN catalog images from /api/products (distinct, high-res).
// Phase 2 (sequential, deterministic) — greedy assignment that guarantees global
// uniqueness: a site keeps its own styled hero only if it OWNS it (url host == domain)
// or it's globally unique; template cousins that share one fall through to their own
// product image; blanks take a product image; last resort is /static/fallback-hero.jpg.
//
// node resolve-heroes.mjs # resolve all, write registry
// node resolve-heroes.mjs --domain x # debug one domain (no write)
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { execFileSync } from 'node:child_process';
const DIR = path.dirname(new URL(import.meta.url).pathname);
const REG = path.join(DIR, 'dw-fleet-registry.json');
const FALLBACK = '/static/fallback-hero.jpg';
const CONCURRENCY = 6;
const MINW = 480; // a hero must be at least this wide to count (rejects icon/badge junk; real product photos pass)
const JUNK = /(?:close-?icon|sprite|favicon|logo|badge|placeholder|loader|spinner|1x1|pixel|blank|transparent|data:image|\.svg(?:[?#]|$)|_(?:48|64|96|125|150|180)x)/i;
function absUrl(u, domain) {
if (!u) return null;
u = u.trim().replace(/&/g, '&');
if (u.startsWith('//')) return 'https:' + u;
if (u.startsWith('/')) return 'https://' + domain + u;
if (!/^https?:/i.test(u)) return 'https://' + domain + '/' + u.replace(/^\.?\//, '');
return u;
}
function hostOf(u) { try { return new URL(u).host.replace(/^www\./, ''); } catch { return ''; } }
// Ordered hero candidates from page HTML, best-first.
function pageCandidates(html, domain) {
const out = [];
const push = u => { const a = absUrl(u, domain); if (a && !out.includes(a)) out.push(a); };
for (const m of html.matchAll(/<meta[^>]+(?:property|name)=["'](?:og:image(?::secure_url)?|twitter:image(?::src)?)["'][^>]+content=["']([^"']+)["']/gi)) push(m[1]);
for (const m of html.matchAll(/<meta[^>]+content=["']([^"']+)["'][^>]+(?:property|name)=["'](?:og:image(?::secure_url)?|twitter:image(?::src)?)["']/gi)) push(m[1]);
for (const m of html.matchAll(/<link[^>]+rel=["'](?:image_src|preload)["'][^>]+href=["']([^"']+\.(?:jpg|jpeg|png|webp|avif)[^"']*)["']/gi)) push(m[1]);
for (const m of html.matchAll(/background(?:-image)?\s*:\s*[^;}"']*url\(\s*(["']?)([^"')]+)\1\s*\)/gi)) if (!JUNK.test(m[2])) push(m[2]);
for (const m of html.matchAll(/<img[^>]+(?:src|data-src|data-lazy-src)=["']([^"']+\.(?:jpg|jpeg|png|webp|avif)[^"']*)["']/gi)) if (!JUNK.test(m[1])) push(m[1]);
return out;
}
async function fetchBuf(url, ms, range) {
const ctrl = new AbortController();
const t = setTimeout(() => ctrl.abort(), ms);
try {
const headers = { 'User-Agent': 'Mozilla/5.0 (dw-fleet-hero/2.0)' };
if (range) headers.Range = range;
const r = await fetch(url, { signal: ctrl.signal, redirect: 'follow', headers });
const ct = r.headers.get('content-type') || '';
const buf = Buffer.from(await r.arrayBuffer());
return { ok: r.ok || r.status === 206, status: r.status, ct, buf };
} finally { clearTimeout(t); }
}
// Width of an image at url, or 0 if unreadable / not an image. Uses sips on a temp file.
async function imageWidth(url) {
let res;
try { res = await fetchBuf(url, 12000); } catch { return 0; }
if (!res.ok || !/^image\//i.test(res.ct) || res.buf.length < 100) return 0;
const tmp = path.join(os.tmpdir(), 'dwhero-' + Math.abs(hashStr(url)) + extOf(res.ct));
try {
fs.writeFileSync(tmp, res.buf);
const out = execFileSync('sips', ['-g', 'pixelWidth', tmp], { encoding: 'utf8' });
const m = out.match(/pixelWidth:\s*(\d+)/);
return m ? parseInt(m[1], 10) : 9999; // sips couldn't read (e.g. webp) -> trust it
} catch { return 9999; } finally { try { fs.unlinkSync(tmp); } catch {} }
}
function extOf(ct) { return /png/.test(ct) ? '.png' : /webp/.test(ct) ? '.webp' : /gif/.test(ct) ? '.gif' : '.jpg'; }
function hashStr(s) { let h = 0; for (let i = 0; i < s.length; i++) h = (h * 31 + s.charCodeAt(i)) | 0; return h; }
async function getPage(domain) {
for (let a = 0; a < 2; a++) {
try { const r = await fetchBuf('https://' + domain + '/', 12000); if (r.ok && /text\/html/i.test(r.ct)) return r.buf.toString('utf8'); }
catch {}
}
return '';
}
// Ask Shopify for a larger render where the master allows (no-op for non-Shopify / already-large).
function upsize(url, w = 1600) {
if (!url || !/cdn\.shopify\.com/i.test(url)) return url;
const [p, q] = url.split('?');
const params = new URLSearchParams(q || '');
params.set('width', String(w));
return p + '?' + params.toString();
}
function imgFromItem(x) {
const im = x.image_url || x.image || x.featuredImage || (Array.isArray(x.images) && x.images[0]);
if (!im) return null;
const u = typeof im === 'string' ? im : (im.src || im.url || null);
return u ? upsize(u) : null;
}
async function getProducts(domain) {
try {
const r = await fetchBuf('https://' + domain + '/api/products?limit=24&sort=newest', 10000);
if (!r.ok || !/json/i.test(r.ct)) return [];
const j = JSON.parse(r.buf.toString('utf8'));
const items = j.items || j.products || [];
return items.map(imgFromItem).filter(u => u && /^https?:/i.test(u) && !JUNK.test(u));
} catch { return []; }
}
// Fallback source: a local repo product json (for sites whose live /api is dead).
function localProducts(repoPath) {
if (!repoPath || !fs.existsSync(repoPath)) return [];
try {
const f = fs.readdirSync(repoPath).find(n => /products?.*\.json$/i.test(n) && !/backup/i.test(n));
if (!f) return [];
const j = JSON.parse(fs.readFileSync(path.join(repoPath, f), 'utf8'));
const items = Array.isArray(j) ? j : (j.items || j.products || []);
return items.map(imgFromItem).filter(u => u && /^https?:/i.test(u) && !JUNK.test(u));
} catch { return []; }
}
// Phase 1: gather raw material for one site.
async function gather(site) {
const domain = site.domain;
const [html, live] = await Promise.all([getPage(domain), getProducts(domain)]);
let products = live;
if (!products.length) products = localProducts(site.local?.repoPath);
let liveHero = null;
for (const c of pageCandidates(html.slice(0, 200000), domain)) {
const w = await imageWidth(c);
if (w >= MINW) { liveHero = c; break; }
}
return { domain, liveHero, products: [...new Set(products)] };
}
async function pool(items, n, fn) {
const results = new Array(items.length); let i = 0;
await Promise.all(Array.from({ length: n }, async () => { while (i < items.length) { const idx = i++; results[idx] = await fn(items[idx], idx); } }));
return results;
}
// ---- single-domain debug ----
const one = process.argv.includes('--domain') ? process.argv[process.argv.indexOf('--domain') + 1] : null;
if (one) { console.log(JSON.stringify(await gather({ domain: one }), null, 2)); process.exit(0); }
// ---- full run ----
const reg = JSON.parse(fs.readFileSync(REG, 'utf8'));
const sites = reg.sites || [];
const stamp = process.env.HERO_STAMP || '';
process.stderr.write('phase 1: gathering live heroes + product catalogs…\n');
const gathered = await pool(sites, CONCURRENCY, async (s) => { const g = await gather(s); process.stderr.write(g.liveHero || g.products.length ? '.' : 'x'); return g; });
process.stderr.write('\n');
const gmap = new Map(gathered.map(g => [g.domain, g]));
// Count how often each liveHero appears across sites (to detect shared template heroes).
const liveCount = {};
for (const g of gathered) if (g.liveHero) liveCount[g.liveHero] = (liveCount[g.liveHero] || 0) + 1;
// Phase 2: deterministic greedy unique assignment.
process.stderr.write('phase 2: assigning unique high-res heroes…\n');
const used = new Set();
const ordered = [...sites].sort((a, b) => a.domain.localeCompare(b.domain));
const tally = { live: 0, product: 0, fallback: 0 };
for (const s of ordered) {
const g = gmap.get(s.domain);
const owns = g.liveHero && (hostOf(g.liveHero) === s.domain.replace(/^www\./, '') || liveCount[g.liveHero] === 1);
let pick = null, src = null;
// 1) own styled hero (only if owned/unique and not already taken)
if (owns && !used.has(g.liveHero)) { pick = g.liveHero; src = 'live'; }
// 2) the site's own catalog — first product image that loads, is high-res, and is unused
if (!pick) {
for (const p of g.products) {
if (used.has(p)) continue;
const w = await imageWidth(p);
if (w >= MINW) { pick = p; src = 'product'; break; }
}
}
// 3) a shared live hero, only if still unused (better than fallback)
if (!pick && g.liveHero && !used.has(g.liveHero)) { pick = g.liveHero; src = 'live-shared'; }
// 4) branded fallback (may repeat across the truly-empty sites)
if (!pick) { pick = FALLBACK; src = 'fallback'; }
s.heroImage = pick; s.heroSource = src; if (stamp) s.heroAt = stamp;
if (pick !== FALLBACK) used.add(pick);
tally[src === 'product' ? 'product' : src === 'fallback' ? 'fallback' : 'live']++;
process.stderr.write(src === 'fallback' ? 'F' : src === 'product' ? 'p' : '.');
}
process.stderr.write('\n');
fs.writeFileSync(REG, JSON.stringify(reg, null, 1));
const uniqueUrls = new Set(sites.map(s => s.heroImage)).size;
console.log(`baked heroes: ${sites.length} sites — live:${tally.live} product:${tally.product} fallback:${tally.fallback}`);
console.log(`distinct hero URLs: ${uniqueUrls} (fallback shared by ${sites.filter(s => s.heroImage === FALLBACK).length})`);
console.log('fallback sites:', sites.filter(s => s.heroImage === FALLBACK).map(s => s.domain).join(', ') || '(none)');