← back to Dw Fleet Registry
viewer: add 'Rescrape heroes (all)' button → POST /api/rescrape-heroes
4ab86830f1966bbe12169c7e05924d7013938d1a · 2026-06-01 15:05:49 -0700 · SteveStudio2
Re-scrapes every site's live hero page and re-bakes heroes via resolve-heroes.mjs
(no live-status probe, unlike Re-probe). Single-flight guarded — concurrent click 409s.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Files touched
M viewer/index.htmlM viewer/server.mjs
Diff
commit 4ab86830f1966bbe12169c7e05924d7013938d1a
Author: SteveStudio2 <stevestudio2@SteveStacStudio.lan>
Date: Mon Jun 1 15:05:49 2026 -0700
viewer: add 'Rescrape heroes (all)' button → POST /api/rescrape-heroes
Re-scrapes every site's live hero page and re-bakes heroes via resolve-heroes.mjs
(no live-status probe, unlike Re-probe). Single-flight guarded — concurrent click 409s.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
viewer/index.html | 72 +++++++++++++++++++++++++++++++++++++
viewer/server.mjs | 104 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 176 insertions(+)
diff --git a/viewer/index.html b/viewer/index.html
index ea9cca0..cf4c045 100644
--- a/viewer/index.html
+++ b/viewer/index.html
@@ -26,6 +26,14 @@
.thumb{aspect-ratio:16/9;background:#0a0b0d center/cover no-repeat;border-bottom:1px solid var(--rule);position:relative;display:flex;align-items:center;justify-content:center;overflow:hidden}
.thumb img.heroimg{position:absolute;inset:0;width:100%;height:100%;object-fit:cover;z-index:1}
.thumb .ph{font-family:'Cormorant Garamond',serif;font-size:22px;color:var(--muted);letter-spacing:.1em;text-align:center;padding:0 10px;z-index:0}
+ .thumb .heroref{position:absolute;top:6px;right:6px;z-index:4;width:26px;height:26px;padding:0;border-radius:50%;background:rgba(14,15,18,.72);color:var(--ink);border:1px solid var(--rule);font-size:13px;font-weight:400;line-height:1;display:flex;align-items:center;justify-content:center;opacity:0;transition:opacity .12s,background .12s}
+ .thumb:hover .heroref{opacity:1}
+ .thumb .heroref:hover{background:var(--accent);color:#1a1a1a;border-color:var(--accent)}
+ .thumb.dragover{outline:2px dashed var(--accent);outline-offset:-5px}
+ .thumb.dragover::after{content:'drop image to set hero';position:absolute;inset:0;z-index:3;display:flex;align-items:center;justify-content:center;background:rgba(14,15,18,.66);color:var(--ink);font-size:11px;letter-spacing:.06em;text-transform:uppercase}
+ .thumb.busy{opacity:.55}
+ #toast{position:fixed;left:50%;bottom:24px;transform:translateX(-50%);z-index:50;background:var(--panel);color:var(--ink);border:1px solid var(--rule);border-radius:8px;padding:9px 16px;font-size:12px;box-shadow:0 6px 24px rgba(0,0,0,.4);opacity:0;pointer-events:none;transition:opacity .2s}
+ #toast.show{opacity:1}
.body{padding:11px 12px 13px;display:flex;flex-direction:column;gap:6px;flex:1}
.dom{font-size:13px;font-weight:600;word-break:break-all;line-height:1.25}
.nm{font-size:11px;color:var(--muted)}
@@ -57,10 +65,12 @@
<div class="ctl">Density <input type="range" id="density" min="3" max="8" value="5"></div>
<div class="ctl"><input type="search" id="q" placeholder="filter domains…"></div>
<label class="ctl" style="cursor:pointer"><input type="checkbox" id="dedupe" checked> Hide redirect dupes</label>
+ <button id="rescrape" title="Re-scrape every site's live hero page and re-bake heroes (no live-status probe)">↻ Rescrape heroes (all)</button>
<button id="reprobe">↻ Re-probe live</button>
</div>
</header>
<div class="grid" id="grid"></div>
+<div id="toast"></div>
<script>
const $ = s => document.querySelector(s);
@@ -103,6 +113,7 @@ function render(){
'<div class="accentbar" style="background:'+s._accent+'"></div>'+
'<div class="thumb" data-domain="'+s.domain+'" style="background-color:'+s._accent+'22">'+
(s.heroImage?'<img class="heroimg" loading="lazy" src="'+heroSrc(s.heroImage)+'" alt="" onerror="this.remove()">':'')+
+ '<button class="heroref" type="button" title="Refresh to a new image — or drag an image here to set it" aria-label="refresh hero">↻</button>'+
'<div class="ph">'+(s.siteName||s.domain).slice(0,18)+'</div></div>'+
'<div class="body">'+
'<div class="dom">'+s.domain+'</div>'+
@@ -146,6 +157,67 @@ $('#density').oninput=e=>document.documentElement.style.setProperty('--cols', e.
// persist controls
for(const id of ['sort','density']){ const el=$('#'+id); const k='dwfleet-'+id; if(localStorage[k]){ el.value=localStorage[k]; if(id==='density')document.documentElement.style.setProperty('--cols',el.value);} el.addEventListener('change',()=>localStorage[k]=el.value); el.addEventListener('input',()=>localStorage[k]=el.value); }
$('#reprobe').onclick=async()=>{ const b=$('#reprobe'); b.textContent='probing…'; b.disabled=true; try{ await fetch('/api/reprobe',{method:'POST'}); await load(); }finally{ b.textContent='↻ Re-probe live'; b.disabled=false; } };
+$('#rescrape').onclick=async()=>{ const b=$('#rescrape'); b.textContent='rescraping…'; b.disabled=true; try{ const r=await fetch('/api/rescrape-heroes',{method:'POST'}); const j=await r.json().catch(()=>({})); if(j&&j.ok){ toast('Heroes rescraped — registry rebaked'); } else { toast('Rescrape failed'+(j&&j.error?': '+j.error:'')); } await load(); }catch{ toast('Rescrape failed'); }finally{ b.textContent='↻ Rescrape heroes (all)'; b.disabled=false; } };
+
+// ---- per-card hero refresh + drag-to-set ----
+let toastT;
+function toast(msg){ const t=$('#toast'); t.textContent=msg; t.classList.add('show'); clearTimeout(toastT); toastT=setTimeout(()=>t.classList.remove('show'),2600); }
+// Paint a freshly-assigned hero into a thumb and keep SITES in sync so re-render persists it.
+function paintHero(thumb, url){
+ const src=heroSrc(url);
+ let img=thumb.querySelector('img.heroimg');
+ if(!img){ img=document.createElement('img'); img.className='heroimg'; img.loading='lazy'; img.alt=''; img.onerror=function(){this.remove();}; thumb.appendChild(img); }
+ img.src=src; thumb.style.backgroundImage='';
+ const ph=thumb.querySelector('.ph'); if(ph) ph.style.display='none';
+ const s=SITES.find(x=>x.domain===thumb.dataset.domain); if(s) s.heroImage=url;
+}
+async function postJSON(path, body){ const r=await fetch(path,{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify(body)}); return r.json().catch(()=>({error:'bad response'})); }
+
+// Refresh button (delegated; cards are <a>, so swallow the click).
+$('#grid').addEventListener('click', async (e)=>{
+ const btn=e.target.closest('.heroref'); if(!btn) return;
+ e.preventDefault(); e.stopPropagation();
+ const thumb=btn.closest('.thumb'); const d=thumb.dataset.domain;
+ thumb.classList.add('busy'); btn.disabled=true;
+ try{
+ const j=await postJSON('/api/hero/refresh',{domain:d});
+ if(j.ok){ paintHero(thumb, j.image); toast('New hero for '+d); }
+ else toast('Refresh failed: '+(j.error||'unknown'));
+ }catch{ toast('Refresh failed'); }
+ finally{ thumb.classList.remove('busy'); btn.disabled=false; }
+});
+
+// Drag-and-drop onto a chip: a local file uploads; a dragged web image sets by URL.
+function extractUrl(dt){
+ const uri=(dt.getData('text/uri-list')||'').split('\n').map(s=>s.trim()).find(s=>/^https?:\/\//i.test(s));
+ if(uri) return uri;
+ const html=dt.getData('text/html'); if(html){ const m=html.match(/<img[^>]+src=["']([^"']+)["']/i); if(m&&/^https?:\/\//i.test(m[1])) return m[1]; }
+ const txt=(dt.getData('text/plain')||'').trim(); if(/^https?:\/\/.+\.(jpg|jpeg|png|webp|avif|gif)/i.test(txt)) return txt;
+ return null;
+}
+$('#grid').addEventListener('dragover', (e)=>{ const t=e.target.closest('.thumb'); if(!t) return; e.preventDefault(); e.dataTransfer.dropEffect='copy'; t.classList.add('dragover'); });
+$('#grid').addEventListener('dragleave', (e)=>{ const t=e.target.closest('.thumb'); if(t && !t.contains(e.relatedTarget)) t.classList.remove('dragover'); });
+$('#grid').addEventListener('drop', async (e)=>{
+ const thumb=e.target.closest('.thumb'); if(!thumb) return;
+ e.preventDefault(); thumb.classList.remove('dragover');
+ const d=thumb.dataset.domain; const dt=e.dataTransfer;
+ thumb.classList.add('busy');
+ try{
+ const file=dt.files && dt.files[0];
+ if(file && /^image\//i.test(file.type)){
+ const ext=(file.name.split('.').pop()||(file.type.split('/')[1])||'jpg').toLowerCase();
+ const buf=await file.arrayBuffer();
+ const r=await fetch('/api/hero/upload?domain='+encodeURIComponent(d)+'&ext='+encodeURIComponent(ext),{method:'POST',headers:{'Content-Type':file.type||'application/octet-stream'},body:buf});
+ const j=await r.json().catch(()=>({error:'bad response'}));
+ if(j.ok){ paintHero(thumb, j.image); toast('Hero updated from '+file.name); } else toast('Upload failed: '+(j.error||'unknown'));
+ } else {
+ const url=extractUrl(dt);
+ if(!url){ toast('No image found in drop'); }
+ else { const j=await postJSON('/api/hero/set',{domain:d,url}); if(j.ok){ paintHero(thumb, j.image); toast('Hero set for '+d); } else toast('Set failed: '+(j.error||'unknown')); }
+ }
+ }catch(err){ toast('Drop failed'); }
+ finally{ thumb.classList.remove('busy'); }
+});
load();
</script>
</body>
diff --git a/viewer/server.mjs b/viewer/server.mjs
index 9bfcdb9..c49a7aa 100644
--- a/viewer/server.mjs
+++ b/viewer/server.mjs
@@ -12,7 +12,9 @@ import { spawn } from 'node:child_process';
const DIR = path.dirname(new URL(import.meta.url).pathname);
const REG = path.join(DIR, '..', 'dw-fleet-registry.json');
+const CATALOG = path.join(DIR, '..', 'catalog-hires.json');
const PORT = process.env.PORT || 9774;
+let rescrapeRunning = false; // single-flight guard for /api/rescrape-heroes
const heroCache = new Map(); // domain -> {image, at}
const imgCache = new Map(); // url -> {ct, buf} (proxy cache; many fleet images send CORP same-origin and can't be hotlinked cross-origin)
const HERO_TTL = 10 * 60 * 1000; // re-resolve after 10 min so fixed sites recover
@@ -22,6 +24,54 @@ function send(res, code, type, body) {
res.writeHead(code, { 'Content-Type': type, 'Cache-Control': 'no-store' });
res.end(body);
}
+const json = (res, code, obj) => send(res, code, 'application/json', JSON.stringify(obj));
+
+function hashStr(s) { let h = 0; for (let i = 0; i < s.length; i++) h = (h * 31 + s.charCodeAt(i)) | 0; return h; }
+
+// Read a request body into a Buffer with a hard cap (default 24 MB for hero uploads).
+function readBody(req, cap = 24 * 1024 * 1024) {
+ return new Promise((resolve, reject) => {
+ const chunks = []; let n = 0;
+ req.on('data', c => { n += c.length; if (n > cap) { reject(new Error('payload too large')); req.destroy(); return; } chunks.push(c); });
+ req.on('end', () => resolve(Buffer.concat(chunks)));
+ req.on('error', reject);
+ });
+}
+
+// Persist a new hero for one domain into the registry (matches resolve-heroes.mjs's indent:1
+// write format so diffs stay minimal). Returns the mutated site, or null if domain unknown.
+function setHero(domain, image, source) {
+ const reg = JSON.parse(fs.readFileSync(REG, 'utf8'));
+ const site = (reg.sites || []).find(s => s.domain === domain);
+ if (!site) return null;
+ site.heroImage = image;
+ site.heroSource = source;
+ site.heroAt = new Date().toISOString();
+ fs.writeFileSync(REG, JSON.stringify(reg, null, 1));
+ heroCache.delete(domain);
+ return site;
+}
+
+// Pick a high-res catalog image that NO site currently uses (preserves the global
+// no-duplicate-hero rule), assign it to `domain`, and persist. Returns the new URL or null.
+function pickFreshHero(domain) {
+ const reg = JSON.parse(fs.readFileSync(REG, 'utf8'));
+ const sites = reg.sites || [];
+ const site = sites.find(s => s.domain === domain);
+ if (!site) return null;
+ const used = new Set(sites.map(s => s.heroImage).filter(Boolean));
+ let pool = [];
+ try { pool = (JSON.parse(fs.readFileSync(CATALOG, 'utf8')).pool || []); } catch { /* no catalog */ }
+ const candidates = pool.map(p => p && p.src).filter(u => u && !used.has(u));
+ if (!candidates.length) return null;
+ const next = candidates[Math.abs(hashStr(domain + (site.heroImage || '') + Date.now())) % candidates.length];
+ site.heroImage = next;
+ site.heroSource = 'catalog-refresh';
+ site.heroAt = new Date().toISOString();
+ fs.writeFileSync(REG, JSON.stringify(reg, null, 1));
+ heroCache.delete(domain);
+ return next;
+}
// Normalize a possibly protocol-relative / root-relative URL to absolute https.
function absUrl(u, domain) {
@@ -124,6 +174,60 @@ const server = http.createServer(async (req, res) => {
} catch { /* fall through to live */ }
return send(res, 200, 'application/json', JSON.stringify({ image: await liveHero(d) }));
}
+ // Refresh one card to a NEW, globally-unused high-res catalog image.
+ if (u.pathname === '/api/hero/refresh' && req.method === 'POST') {
+ let d = '';
+ try { d = (JSON.parse((await readBody(req)).toString('utf8') || '{}').domain || ''); } catch { /* bad body */ }
+ if (!/^[a-z0-9.-]+$/i.test(d)) return json(res, 400, { error: 'bad domain' });
+ const next = pickFreshHero(d);
+ if (!next) return json(res, 409, { error: 'no fresh catalog image available' });
+ return json(res, 200, { ok: true, image: next });
+ }
+ // Set a card's hero to a dragged-in image URL.
+ if (u.pathname === '/api/hero/set' && req.method === 'POST') {
+ let body = {};
+ try { body = JSON.parse((await readBody(req)).toString('utf8') || '{}'); } catch { return json(res, 400, { error: 'bad json' }); }
+ const d = body.domain || ''; const url = (body.url || '').trim();
+ if (!/^[a-z0-9.-]+$/i.test(d)) return json(res, 400, { error: 'bad domain' });
+ if (!/^https?:\/\//i.test(url)) return json(res, 400, { error: 'url must be http(s)' });
+ const site = setHero(d, url, 'manual-url');
+ if (!site) return json(res, 404, { error: 'domain not in registry' });
+ return json(res, 200, { ok: true, image: url });
+ }
+ // Set a card's hero from a dropped local image file (raw body upload).
+ if (u.pathname === '/api/hero/upload' && req.method === 'POST') {
+ const d = u.searchParams.get('domain') || '';
+ const ext = (u.searchParams.get('ext') || 'jpg').toLowerCase().replace(/[^a-z0-9]/g, '').slice(0, 5) || 'jpg';
+ if (!/^[a-z0-9.-]+$/i.test(d)) return json(res, 400, { error: 'bad domain' });
+ try {
+ const buf = await readBody(req);
+ if (buf.length < 100) return json(res, 400, { error: 'empty file' });
+ const dir = path.join(DIR, 'static', 'heroes');
+ fs.mkdirSync(dir, { recursive: true });
+ const fname = d.replace(/[^a-z0-9.-]/gi, '_') + '-' + Date.now() + '.' + ext;
+ fs.writeFileSync(path.join(dir, fname), buf);
+ const image = '/static/heroes/' + fname;
+ const site = setHero(d, image, 'upload');
+ if (!site) return json(res, 404, { error: 'domain not in registry' });
+ return json(res, 200, { ok: true, image });
+ } catch (e) { return json(res, 500, { error: String(e && e.message || e) }); }
+ }
+ // Re-scrape EVERY site's live hero page and re-bake heroes into the registry.
+ // Runs resolve-heroes.mjs only (no live-status probe) — the "rescrape from hero
+ // pages for all" action. Single-flight: a second click while one is running 409s.
+ if (u.pathname === '/api/rescrape-heroes' && req.method === 'POST') {
+ if (rescrapeRunning) return json(res, 409, { error: 'a rescrape is already running' });
+ rescrapeRunning = true;
+ heroCache.clear();
+ const root = path.join(DIR, '..');
+ const stamp = new Date().toISOString();
+ let out = '';
+ const h = spawn('node', ['resolve-heroes.mjs'], { cwd: root, env: { ...process.env, HERO_STAMP: stamp } });
+ h.stderr.on('data', d => out += d); h.stdout.on('data', d => out += d);
+ h.on('close', hc => { rescrapeRunning = false; send(res, 200, 'application/json', JSON.stringify({ ok: hc === 0, log: out.slice(-2000) })); });
+ h.on('error', e => { rescrapeRunning = false; json(res, 500, { error: String(e && e.message || e) }); });
+ return;
+ }
if (u.pathname === '/api/reprobe' && req.method === 'POST') {
heroCache.clear(); // force fresh hero resolution after a re-probe
const root = path.join(DIR, '..');
← 30cf448 fleet: 5 parked domains live + unique heroes; rebake clears
·
back to Dw Fleet Registry
·
viewer: harden rescrape single-flight against client disconn 14428b6 →