← back to Dw Fleet Registry
viewer/server.mjs
369 lines
#!/usr/bin/env node
// Live DW fleet registry viewer — zero dependencies (node:http + built-in fetch).
// node viewer/server.mjs # serves on :9774
// Reads ../dw-fleet-registry.json fresh on each request. Proxies each site's live
// og:image as a card thumbnail (so the grid shows LIVE data, not snapshots), and
// can kick a re-probe.
import http from 'node:http';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { spawn } from 'node:child_process';
const DIR = path.dirname(new URL(import.meta.url).pathname);
const REG = path.join(DIR, '..', 'dw-fleet-registry.json');
const CATALOG = path.join(DIR, '..', 'catalog-hires.json');
const PORT = process.env.PORT || 9774;
// sharp is OPTIONAL — if it's not installed the proxy just serves full-res (graceful fallback).
let sharp = null;
try { sharp = (await import('sharp')).default; } catch { /* no sharp → /img?w= serves full-res */ }
let rescrapeRunning = false; // single-flight guard for /api/rescrape-heroes
const heroCache = new Map(); // domain -> {image, at}
const imgCache = new Map(); // url -> {ct, buf} (proxy cache; many fleet images send CORP same-origin and can't be hotlinked cross-origin)
const HERO_TTL = 10 * 60 * 1000; // re-resolve after 10 min so fixed sites recover
const JUNK = /(?:close-?icon|sprite|favicon|logo|badge|placeholder|loader|spinner|1x1|pixel|blank|transparent|data:image|\.svg(?:[?#]|$)|_(?:48|64|96|125|150|180)x)/i;
function send(res, code, type, body) {
res.writeHead(code, { 'Content-Type': type, 'Cache-Control': 'no-store' });
res.end(body);
}
const json = (res, code, obj) => send(res, code, 'application/json', JSON.stringify(obj));
function hashStr(s) { let h = 0; for (let i = 0; i < s.length; i++) h = (h * 31 + s.charCodeAt(i)) | 0; return h; }
// Read a request body into a Buffer with a hard cap (default 24 MB for hero uploads).
function readBody(req, cap = 24 * 1024 * 1024) {
return new Promise((resolve, reject) => {
const chunks = []; let n = 0;
req.on('data', c => { n += c.length; if (n > cap) { reject(new Error('payload too large')); req.destroy(); return; } chunks.push(c); });
req.on('end', () => resolve(Buffer.concat(chunks)));
req.on('error', reject);
});
}
// Persist a new hero for one domain into the registry (matches resolve-heroes.mjs's indent:1
// write format so diffs stay minimal). Returns the mutated site, or null if domain unknown.
function setHero(domain, image, source) {
const reg = JSON.parse(fs.readFileSync(REG, 'utf8'));
const site = (reg.sites || []).find(s => s.domain === domain);
if (!site) return null;
site.heroImage = image;
site.heroSource = source;
site.heroAt = new Date().toISOString();
fs.writeFileSync(REG, JSON.stringify(reg, null, 1));
heroCache.delete(domain);
return site;
}
// Pick a high-res catalog image that NO site currently uses (preserves the global
// no-duplicate-hero rule), assign it to `domain`, and persist. Returns the new URL or null.
function pickFreshHero(domain) {
const reg = JSON.parse(fs.readFileSync(REG, 'utf8'));
const sites = reg.sites || [];
const site = sites.find(s => s.domain === domain);
if (!site) return null;
const used = new Set(sites.map(s => s.heroImage).filter(Boolean));
let pool = [];
try { pool = (JSON.parse(fs.readFileSync(CATALOG, 'utf8')).pool || []); } catch { /* no catalog */ }
const candidates = pool.map(p => p && p.src).filter(u => u && !used.has(u));
if (!candidates.length) return null;
const next = candidates[Math.abs(hashStr(domain + (site.heroImage || '') + Date.now())) % candidates.length];
site.heroImage = next;
site.heroSource = 'catalog-refresh';
site.heroAt = new Date().toISOString();
fs.writeFileSync(REG, JSON.stringify(reg, null, 1));
heroCache.delete(domain);
return next;
}
// Normalize a possibly protocol-relative / root-relative URL to absolute https.
function absUrl(u, domain) {
if (!u) return null;
u = u.trim().replace(/&/g, '&');
if (u.startsWith('//')) return 'https:' + u;
if (u.startsWith('/')) return 'https://' + domain + u;
if (!/^https?:/i.test(u)) return 'https://' + domain + '/' + u.replace(/^\.?\//, '');
return u;
}
async function liveHero(domain) {
const hit = heroCache.get(domain);
if (hit && Date.now() - hit.at < HERO_TTL) return hit.image;
let image = null;
try {
const ctrl = new AbortController();
const t = setTimeout(() => ctrl.abort(), 8000);
const r = await fetch('https://' + domain + '/', { signal: ctrl.signal, headers: { 'User-Agent': 'dw-fleet-viewer/1.0' } });
clearTimeout(t);
const html = (await r.text()).slice(0, 120000);
// 1) social meta (og:image / twitter:image), either attribute order
const meta = html.match(/<meta[^>]+(?:property|name)=["'](?:og:image(?::secure_url)?|twitter:image(?::src)?)["'][^>]+content=["']([^"']+)["']/i)
|| html.match(/<meta[^>]+content=["']([^"']+)["'][^>]+(?:property|name)=["'](?:og:image(?::secure_url)?|twitter:image(?::src)?)["']/i);
if (meta) image = meta[1];
// 2) link rel=image_src / preload as=image
if (!image) {
const link = html.match(/<link[^>]+rel=["']image_src["'][^>]+href=["']([^"']+)["']/i)
|| html.match(/<link[^>]+rel=["']preload["'][^>]+as=["']image["'][^>]+href=["']([^"']+)["']/i);
if (link) image = link[1];
}
// 3) CSS background-image:url(...) — half the DW microsites put the hero here
if (!image) {
const bgs = [...html.matchAll(/background(?:-image)?\s*:\s*[^;}"']*url\(\s*(["']?)([^"')]+)\1\s*\)/gi)].map(m => m[2]);
const bg = bgs.find(u => !JUNK.test(u) && /\.(?:jpg|jpeg|png|webp|avif)/i.test(u)) || bgs.find(u => !JUNK.test(u) && !/^data:/i.test(u));
if (bg) image = bg;
}
// 4) first real content <img> (skip icons / logos / sprites / tiny thumbnails)
if (!image) {
const imgs = [...html.matchAll(/<img[^>]+(?:src|data-src|data-lazy-src)=["']([^"']+\.(?:jpg|jpeg|png|webp|avif)[^"']*)["']/gi)].map(m => m[1]);
const good = imgs.find(u => !JUNK.test(u));
if (good) image = good;
}
image = absUrl(image, domain);
} catch { /* unreachable / timeout */ }
heroCache.set(domain, { image, at: Date.now() });
return image;
}
const server = http.createServer(async (req, res) => {
const u = new URL(req.url, 'http://localhost');
if (u.pathname === '/') return send(res, 200, 'text/html; charset=utf-8', fs.readFileSync(path.join(DIR, 'index.html')));
if (u.pathname === '/img') {
const target = u.searchParams.get('u') || '';
if (!/^https?:\/\//i.test(target)) return send(res, 400, 'text/plain', 'bad url');
// Optional server-side downscale: /img?u=…&w=400 resizes non-resizable (non-Shopify)
// images to webp so a 3MB hero becomes ~40KB. Shopify URLs already arrive small via ?width=.
const w = Math.min(2000, Math.max(64, parseInt(u.searchParams.get('w') || '0', 10) || 0));
const key = target + '|' + w;
const hit = imgCache.get(key);
if (hit) { res.writeHead(200, { 'Content-Type': hit.ct, 'Cache-Control': 'public, max-age=86400' }); return res.end(hit.buf); }
try {
const ctrl = new AbortController(); const t = setTimeout(() => ctrl.abort(), 12000);
const r = await fetch(target, { signal: ctrl.signal, redirect: 'follow', headers: { 'User-Agent': 'Mozilla/5.0', 'Referer': target } });
clearTimeout(t);
if (!r.ok) return send(res, 502, 'text/plain', 'upstream ' + r.status);
let ct = r.headers.get('content-type') || 'image/jpeg';
let buf = Buffer.from(await r.arrayBuffer());
// Downscale to webp when a width is requested, sharp is available, and it's a raster image.
if (w && sharp && /image\/(jpe?g|png|webp|avif)/i.test(ct)) {
try { buf = await sharp(buf).rotate().resize({ width: w, withoutEnlargement: true }).webp({ quality: 80 }).toBuffer(); ct = 'image/webp'; }
catch { /* keep original buffer on any resize failure */ }
}
if (imgCache.size > 400) imgCache.clear(); // crude cap
imgCache.set(key, { ct, buf });
res.writeHead(200, { 'Content-Type': ct, 'Cache-Control': 'public, max-age=86400' });
return res.end(buf);
} catch { return send(res, 502, 'text/plain', 'proxy error'); }
}
if (u.pathname.startsWith('/static/')) {
const safe = u.pathname.replace(/\.\.+/g, '').replace(/^\/static\//, '');
const fp = path.join(DIR, 'static', safe);
try {
const ext = path.extname(fp).toLowerCase();
const type = ext === '.jpg' || ext === '.jpeg' ? 'image/jpeg' : ext === '.png' ? 'image/png' : ext === '.webp' ? 'image/webp' : ext === '.svg' ? 'image/svg+xml' : 'application/octet-stream';
const body = fs.readFileSync(fp);
res.writeHead(200, { 'Content-Type': type, 'Cache-Control': 'public, max-age=86400' });
return res.end(body);
} catch { return send(res, 404, 'text/plain', 'not found'); }
}
if (u.pathname === '/api/registry') {
try { return send(res, 200, 'application/json', fs.readFileSync(REG)); }
catch { return send(res, 500, 'application/json', '{"error":"registry not found — run build-registry.mjs"}'); }
}
if (u.pathname === '/api/hero') {
const d = u.searchParams.get('domain') || '';
if (!/^[a-z0-9.-]+$/i.test(d)) return send(res, 400, 'application/json', '{"image":null}');
// Serve the pre-resolved + validated hero baked into the registry (instant,
// no live fetch storm). Fall back to a live resolve only if none is baked.
try {
const reg = JSON.parse(fs.readFileSync(REG, 'utf8'));
const site = (reg.sites || []).find(s => s.domain === d);
if (site && site.heroImage) return send(res, 200, 'application/json', JSON.stringify({ image: site.heroImage, baked: true }));
} catch { /* fall through to live */ }
return send(res, 200, 'application/json', JSON.stringify({ image: await liveHero(d) }));
}
// Refresh one card to a NEW, globally-unused high-res catalog image.
if (u.pathname === '/api/hero/refresh' && req.method === 'POST') {
let d = '';
try { d = (JSON.parse((await readBody(req)).toString('utf8') || '{}').domain || ''); } catch { /* bad body */ }
if (!/^[a-z0-9.-]+$/i.test(d)) return json(res, 400, { error: 'bad domain' });
const next = pickFreshHero(d);
if (!next) return json(res, 409, { error: 'no fresh catalog image available' });
return json(res, 200, { ok: true, image: next });
}
// Set a card's hero to a dragged-in image URL.
if (u.pathname === '/api/hero/set' && req.method === 'POST') {
let body = {};
try { body = JSON.parse((await readBody(req)).toString('utf8') || '{}'); } catch { return json(res, 400, { error: 'bad json' }); }
const d = body.domain || ''; const url = (body.url || '').trim();
if (!/^[a-z0-9.-]+$/i.test(d)) return json(res, 400, { error: 'bad domain' });
if (!/^https?:\/\//i.test(url)) return json(res, 400, { error: 'url must be http(s)' });
const site = setHero(d, url, 'manual-url');
if (!site) return json(res, 404, { error: 'domain not in registry' });
return json(res, 200, { ok: true, image: url });
}
// Set a card's hero from a dropped local image file (raw body upload).
if (u.pathname === '/api/hero/upload' && req.method === 'POST') {
const d = u.searchParams.get('domain') || '';
const ext = (u.searchParams.get('ext') || 'jpg').toLowerCase().replace(/[^a-z0-9]/g, '').slice(0, 5) || 'jpg';
if (!/^[a-z0-9.-]+$/i.test(d)) return json(res, 400, { error: 'bad domain' });
try {
const buf = await readBody(req);
if (buf.length < 100) return json(res, 400, { error: 'empty file' });
const dir = path.join(DIR, 'static', 'heroes');
fs.mkdirSync(dir, { recursive: true });
const fname = d.replace(/[^a-z0-9.-]/gi, '_') + '-' + Date.now() + '.' + ext;
fs.writeFileSync(path.join(dir, fname), buf);
const image = '/static/heroes/' + fname;
const site = setHero(d, image, 'upload');
if (!site) return json(res, 404, { error: 'domain not in registry' });
return json(res, 200, { ok: true, image });
} catch (e) { return json(res, 500, { error: String(e && e.message || e) }); }
}
// Re-scrape EVERY site's live hero page and re-bake heroes into the registry.
// Runs resolve-heroes.mjs only (no live-status probe) — the "rescrape from hero
// pages for all" action. Single-flight: a second click while one is running 409s.
if (u.pathname === '/api/rescrape-heroes' && req.method === 'POST') {
if (rescrapeRunning) return json(res, 409, { error: 'a rescrape is already running' });
rescrapeRunning = true;
heroCache.clear();
const root = path.join(DIR, '..');
const stamp = new Date().toISOString();
let out = '', done = false;
// Reply at most once, and never throw if the client already disconnected (a
// short client timeout must not wedge the single-flight flag or crash the server).
const reply = (code, obj) => {
if (done) return; done = true; rescrapeRunning = false;
if (res.writableEnded || res.destroyed) return;
try { send(res, code, 'application/json', JSON.stringify(obj)); } catch { /* socket gone */ }
};
const h = spawn('node', ['resolve-heroes.mjs'], { cwd: root, env: { ...process.env, HERO_STAMP: stamp } });
h.stderr.on('data', d => out += d); h.stdout.on('data', d => out += d);
h.on('close', hc => reply(200, { ok: hc === 0, log: out.slice(-2000) }));
h.on('error', e => reply(500, { error: String(e && e.message || e) }));
// Safety net: even if the child never emits close/error, free the flag.
setTimeout(() => { if (!done) { done = true; rescrapeRunning = false; } }, 5 * 60 * 1000).unref();
return;
}
// High-res image library to choose hero images from (catalog-hires.json pool),
// sorted highest-res first, filtered to >= min width, optional ?q= title/meta search.
if (u.pathname === '/api/image-library') {
const q = (u.searchParams.get('q') || '').trim().toLowerCase();
const min = parseInt(u.searchParams.get('min') || '1400', 10) || 1400;
const kind = (u.searchParams.get('kind') || '').trim().toLowerCase(); // '', 'room', 'pattern', 'swatch'
let pool = [];
try { pool = (JSON.parse(fs.readFileSync(CATALOG, 'utf8')).pool || []); } catch { /* none */ }
const rank = { room: 0, pattern: 1, swatch: 2 };
let out = pool
.filter(p => p && p.src && Math.max(p.w || 0, p.h || 0) >= min)
.filter(p => !kind || (p.kind || 'pattern') === kind)
.filter(p => !q || ((p.title || '') + ' ' + (p.meta || '')).toLowerCase().includes(q))
.sort((a, b) => ((rank[a.kind] ?? 1) - (rank[b.kind] ?? 1)) || (Math.max(b.w, b.h) - Math.max(a.w, a.h)))
.map(p => ({ src: p.src, w: p.w || 0, h: p.h || 0, title: p.title || '', kind: p.kind || 'pattern', landscape: !!p.landscape }));
return json(res, 200, { count: out.length, images: out.slice(0, 600) });
}
// Read one site's hero rotation pool (heroImages[], highest-res first; primary = [0]).
if (u.pathname === '/api/hero-pool') {
const d = u.searchParams.get('domain') || '';
if (!/^[a-z0-9.-]+$/i.test(d)) return json(res, 400, { error: 'bad domain' });
try {
const reg = JSON.parse(fs.readFileSync(REG, 'utf8'));
const site = (reg.sites || []).find(s => s.domain === d);
if (!site) return json(res, 404, { error: 'domain not in registry' });
const pool = Array.isArray(site.heroImages) && site.heroImages.length ? site.heroImages
: (site.heroImage ? [site.heroImage] : []);
return json(res, 200, { domain: d, images: pool });
} catch { return json(res, 500, { error: 'registry read failed' }); }
}
// Set one site's full hero pool (ordered array). images[0] becomes the primary heroImage.
if (u.pathname === '/api/hero-pool/set' && req.method === 'POST') {
let body = {};
try { body = JSON.parse((await readBody(req)).toString('utf8') || '{}'); } catch { return json(res, 400, { error: 'bad json' }); }
const d = body.domain || '';
if (!/^[a-z0-9.-]+$/i.test(d)) return json(res, 400, { error: 'bad domain' });
let images = Array.isArray(body.images) ? body.images : [];
images = [...new Set(images.filter(u => typeof u === 'string' && /^(https?:\/\/|\/)/.test(u.trim())).map(u => u.trim()))];
try {
const reg = JSON.parse(fs.readFileSync(REG, 'utf8'));
const site = (reg.sites || []).find(s => s.domain === d);
if (!site) return json(res, 404, { error: 'domain not in registry' });
site.heroImages = images;
if (images.length) { site.heroImage = images[0]; site.heroSource = 'pool'; }
site.heroAt = new Date().toISOString();
fs.writeFileSync(REG, JSON.stringify(reg, null, 1));
heroCache.delete(d);
return json(res, 200, { ok: true, domain: d, images, primary: images[0] || null });
} catch (e) { return json(res, 500, { error: String(e && e.message || e) }); }
}
// Push one site's curated hero pool LIVE: write hero-pool.json into the repo +
// rsync it to the prod public/ dir (static — no reload needed). Reads heroImages[]
// from the registry and the slug->cwd->repo map in data/deploy-map.json.
if (u.pathname === '/api/hero-pool/deploy' && req.method === 'POST') {
let d = '';
try { d = (JSON.parse((await readBody(req)).toString('utf8') || '{}').domain || ''); } catch { /* bad */ }
if (!/^[a-z0-9.-]+$/i.test(d)) return json(res, 400, { error: 'bad domain' });
let reg, mapAll;
try { reg = JSON.parse(fs.readFileSync(REG, 'utf8')); } catch { return json(res, 500, { error: 'registry read failed' }); }
try { mapAll = JSON.parse(fs.readFileSync(path.join(DIR, '..', 'data', 'deploy-map.json'), 'utf8')); } catch { return json(res, 500, { error: 'deploy-map missing' }); }
const site = (reg.sites || []).find(s => s.domain === d);
const dep = mapAll[d];
if (!site) return json(res, 404, { error: 'domain not in registry' });
if (!dep || !dep.repo || !dep.cwd) return json(res, 409, { error: 'no deploy target for ' + d + ' (redirect-only or unmapped)' });
let images = Array.isArray(site.heroImages) ? site.heroImages : (site.heroImage ? [site.heroImage] : []);
const rsync = (src, dst) => new Promise((resolve) => { const p = spawn('rsync', ['-az', src, dst]); let e = ''; p.stderr.on('data', x => e += x); p.on('close', c => resolve({ ok: c === 0, err: e })); p.on('error', x => resolve({ ok: false, err: String(x) })); });
const sshMkdir = (dir) => new Promise((resolve) => { const p = spawn('ssh', ['my-server', `mkdir -p '${dir.replace(/'/g, "")}'`]); let e = ''; p.stderr.on('data', x => e += x); p.on('close', c => resolve({ ok: c === 0, err: e })); p.on('error', x => resolve({ ok: false, err: String(x) })); });
try {
fs.mkdirSync(path.join(dep.repo, 'public', 'rooms'), { recursive: true });
// Ensure the prod rooms/ dir exists (local openrsync has no --mkpath).
const hasRooms = images.some(img => /^\/static\/rooms\//.test(img));
if (hasRooms) { const mk = await sshMkdir(`${dep.cwd}/public/rooms`); if (!mk.ok) return json(res, 502, { error: 'mkdir prod rooms/ failed: ' + mk.err.slice(-200) }); }
// Any locally-staged room images (/static/rooms/<domain>/<file>) get pushed to
// prod public/rooms/ and rewritten to their live https URL in the pool.
const rewritten = [];
for (const img of images) {
const m = /^\/static\/rooms\/[^/]+\/(.+)$/.exec(img);
if (m) {
const file = m[1];
const localFile = path.join(DIR, 'static', 'rooms', d, file);
if (fs.existsSync(localFile)) {
const r = await rsync(localFile, `my-server:${dep.cwd}/public/rooms/${file}`);
if (!r.ok) return json(res, 502, { error: 'room rsync failed (' + file + '): ' + r.err.slice(-200) });
rewritten.push('https://' + d + '/rooms/' + file);
}
} else rewritten.push(img);
}
images = rewritten;
const poolPath = path.join(dep.repo, 'public', 'hero-pool.json');
fs.writeFileSync(poolPath, JSON.stringify(images, null, 0));
const pr = await rsync(poolPath, `my-server:${dep.cwd}/public/hero-pool.json`);
if (!pr.ok) return json(res, 502, { error: 'hero-pool.json rsync failed: ' + pr.err.slice(-200) });
// persist the rewritten (live) URLs back into the registry so the card shows them
site.heroImages = images; if (images.length) site.heroImage = images[0];
try { const fresh = JSON.parse(fs.readFileSync(REG, 'utf8')); const fs2 = fresh.sites.find(s => s.domain === d); if (fs2) { fs2.heroImages = images; if (images.length) fs2.heroImage = images[0]; fs.writeFileSync(REG, JSON.stringify(fresh, null, 1)); } } catch {}
heroCache.delete(d);
return json(res, 200, { ok: true, domain: d, count: images.length, url: 'https://' + d + '/hero-pool.json' });
} catch (e) { return json(res, 500, { error: String(e && e.message || e) }); }
}
if (u.pathname === '/api/reprobe' && req.method === 'POST') {
heroCache.clear(); // force fresh hero resolution after a re-probe
const root = path.join(DIR, '..');
const stamp = new Date().toISOString();
let out = '';
const p = spawn('node', ['probe.mjs'], { cwd: root });
p.stderr.on('data', d => out += d); p.stdout.on('data', d => out += d);
p.on('close', code => {
// After probing live status, re-bake validated hero images into the registry.
const h = spawn('node', ['resolve-heroes.mjs'], { cwd: root, env: { ...process.env, HERO_STAMP: stamp } });
h.stderr.on('data', d => out += d); h.stdout.on('data', d => out += d);
h.on('close', hc => send(res, 200, 'application/json', JSON.stringify({ ok: code === 0 && hc === 0, log: out.slice(-2000) })));
});
return;
}
send(res, 404, 'text/plain', 'not found');
});
server.listen(PORT, () => console.log(`DW fleet viewer → http://localhost:${PORT}`));