← back to Stack Map Viewer
viewer: Scrapers ⏱ view — sortable staleness table (all 50 scrapers ranked by days-since-last-run, color-coded, row→5W card)
175dbc0b05499f85efcd55641db3fe49e6ad8153 · 2026-09-22 12:55:49 -0700 · Steve Abrams
Files touched
Diff
commit 175dbc0b05499f85efcd55641db3fe49e6ad8153
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Tue Sep 22 12:55:49 2026 -0700
viewer: Scrapers ⏱ view — sortable staleness table (all 50 scrapers ranked by days-since-last-run, color-coded, row→5W card)
---
index.html | 65 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++--
server.js | 40 ++++++++++++++++++++++++++++++++++++++
2 files changed, 103 insertions(+), 2 deletions(-)
diff --git a/index.html b/index.html
index 7e1a4ed..c0f9ae3 100644
--- a/index.html
+++ b/index.html
@@ -43,6 +43,21 @@
#treemapbox svg text{pointer-events:none;font-family:inherit}
#treemapbox rect{cursor:pointer;stroke:#0f1117;stroke-width:1.5}
#treemapbox rect:hover{filter:brightness(1.3)}
+ /* scrapers staleness table */
+ #scrapersbox{padding:10px 20px 40px;display:none}
+ #scrapersbox .sumbar{color:var(--dim);font-size:12px;margin:2px 0 12px}
+ #scrapersbox .sumbar b{color:#eef1f7}
+ table.stale{border-collapse:collapse;width:100%;font-size:13px}
+ table.stale th{text-align:left;color:#9fc4ff;font-weight:700;font-size:11px;letter-spacing:.4px;padding:6px 10px;border-bottom:1px solid #2a3247;cursor:pointer;user-select:none;white-space:nowrap}
+ table.stale th:hover{color:#eef1f7}
+ table.stale th .ar{color:#5f7bb0;font-size:10px}
+ table.stale td{padding:6px 10px;border-bottom:1px solid #181c28}
+ table.stale tr{cursor:pointer}
+ table.stale tbody tr:hover td{background:#181c28}
+ table.stale td.num{color:#6f7d97;text-align:right;font-variant-numeric:tabular-nums}
+ table.stale td.age{font-weight:700;font-variant-numeric:tabular-nums;white-space:nowrap}
+ table.stale td.tbl{color:#6f7d97;font-size:11px}
+ .agepill{display:inline-block;min-width:8px}
/* drawer */
#drawer{position:fixed;top:0;right:-460px;width:440px;height:100%;background:var(--panel);border-left:1px solid #2a3247;
transition:right .18s;overflow:auto;padding:18px;box-shadow:-8px 0 30px rgba(0,0,0,.5);z-index:50}
@@ -90,10 +105,11 @@
<button class="vbtn" data-view="map">Map</button>
<button class="vbtn" data-view="tree">Tree</button>
<button class="vbtn" data-view="treemap">Treemap</button>
+ <button class="vbtn" data-view="scrapers">Scrapers ⏱</button>
<button class="tbtn" id="resetLayout" title="Clear saved node positions">Reset layout</button>
<span class="tspace" id="viewhint"></span>
</div>
-<div id="wrap"><div id="svgbox"></div><div id="treebox"></div><div id="treemapbox"></div></div>
+<div id="wrap"><div id="svgbox"></div><div id="treebox"></div><div id="treemapbox"></div><div id="scrapersbox"></div></div>
<div id="folders"></div>
<div class="foot" id="foot"></div>
<div id="drawer"><span id="close">✕</span>
@@ -377,6 +393,49 @@ async function renderTreemap(d){
.text(x=>((x.x1-x.x0)>40&&(x.y1-x.y0)>34)?x.data.count:'');
}
+// ── Scrapers staleness view (sortable, stalest-first, click row → 5W card) ───
+let SCR=null, SCRSORT={key:'days',dir:-1};
+function ageCell(d){
+ if(d==null)return '<span class="agepill" style="color:#6f7d97">— no data</span>';
+ const c=d>30?'#f3b6b6':d>14?'#ffd27a':'#5fd0a8';
+ return '<span class="agepill" style="color:'+c+'">'+d+'d</span>';
+}
+function sortScr(rows){
+ const {key,dir}=SCRSORT;
+ return rows.slice().sort((a,b)=>{
+ let av=a[key],bv=b[key];
+ if(key==='days'){ // nulls always last regardless of dir
+ if(av==null&&bv==null)return a.skill.localeCompare(b.skill);
+ if(av==null)return 1; if(bv==null)return -1; return (av-bv)*dir;
+ }
+ if(key==='rows'){av=av==null?-1:av;bv=bv==null?-1:bv;return (av-bv)*dir;}
+ return String(av||'').localeCompare(String(bv||''))*dir;
+ });
+}
+function renderScrapersTable(){
+ if(!SCR)return;
+ const box=document.getElementById('scrapersbox');
+ const cols=[['skill','Vendor scraper'],['last','Last run'],['days','Age'],['rows','Rows'],['tbl','Staging table']];
+ const arrow=k=>SCRSORT.key===k?(' <span class="ar">'+(SCRSORT.dir<0?'▼':'▲')+'</span>'):'';
+ const head='<tr><th style="cursor:default;color:#6f7d97">#</th>'+cols.map(c=>'<th data-k="'+c[0]+'">'+c[1]+arrow(c[0])+'</th>').join('')+'</tr>';
+ const rows=sortScr(SCR.rows).map((r,i)=>'<tr data-skill="'+r.skill+'">'
+ +'<td class="num">'+(i+1)+'</td>'
+ +'<td>'+r.skill.replace(/-scraper-manager$|-scraper$/,'')+'</td>'
+ +'<td class="age">'+(r.last||'<span style="color:#6f7d97">—</span>')+'</td>'
+ +'<td class="age">'+ageCell(r.days)+'</td>'
+ +'<td class="num">'+(r.rows==null?'—':r.rows.toLocaleString())+'</td>'
+ +'<td class="tbl">'+(r.tbl||'<span style="color:#8a4a4a">unmapped</span>')+'</td></tr>').join('');
+ box.innerHTML='<div class="sumbar"><b>'+SCR.count+'</b> scrapers · <b style="color:#f3b6b6">'+SCR.stale30+'</b> stale >30d · <b style="color:#ffd27a">'+SCR.nodata+'</b> no data · newest timestamp in each staging table · click a row for its 5W card · click a header to re-sort</div>'
+ +'<table class="stale"><thead>'+head+'</thead><tbody>'+rows+'</tbody></table>';
+ box.querySelectorAll('th[data-k]').forEach(th=>th.addEventListener('click',()=>{const k=th.dataset.k;if(SCRSORT.key===k)SCRSORT.dir*=-1;else SCRSORT={key:k,dir:k==='skill'||k==='tbl'?1:-1};renderScrapersTable();}));
+ box.querySelectorAll('tr[data-skill]').forEach(tr=>tr.addEventListener('click',()=>startDrill('skill',tr.dataset.skill,tr.dataset.skill)));
+}
+async function renderScrapers(){
+ const box=document.getElementById('scrapersbox');
+ box.innerHTML='<div class="sumbar">Loading scraper run dates…</div>';
+ try{const r=await fetch('/api/scrapers');SCR=await r.json();}catch(e){box.innerHTML='<div class="sumbar">Could not load /api/scrapers.</div>';return;}
+ renderScrapersTable();
+}
// ── View switching ───────────────────────────────────────────────────────────
function setView(v){
VIEW=v;lsSet('smv_view',v);
@@ -384,9 +443,11 @@ function setView(v){
document.getElementById('svgbox').style.display=v==='map'?'block':'none';
document.getElementById('treebox').style.display=v==='tree'?'block':'none';
document.getElementById('treemapbox').style.display=v==='treemap'?'block':'none';
+ document.getElementById('scrapersbox').style.display=v==='scrapers'?'block':'none';
document.getElementById('resetLayout').style.display=v==='map'?'inline-block':'none';
- const hint={map:'drag nodes to rearrange · click to drill',tree:'click any row to expand deeper',treemap:'rectangle area = member count · click a leaf to drill'};
+ const hint={map:'drag nodes to rearrange · click to drill',tree:'click any row to expand deeper',treemap:'rectangle area = member count · click a leaf to drill',scrapers:'sorted stalest-first · click a header to re-sort · click a row for the 5W card'};
document.getElementById('viewhint').textContent=hint[v]||'';
+ if(v==='scrapers'){renderScrapers();return;}
if(!DATA)return;
if(v==='map')renderMap(DATA);
else if(v==='tree')renderTree(DATA);
diff --git a/server.js b/server.js
index 79f2c7e..d5c1180 100644
--- a/server.js
+++ b/server.js
@@ -420,6 +420,42 @@ const DRILLERS = {
kv(id) { return { title: id, breadcrumb: id, children: [] }; },
};
+// ── Scraper staleness ranking: every scraper → newest timestamp in its staging table ──
+// Skill→table resolved from SKILL.md (no DB); then just 2 batched psql calls for all tables.
+function scraperStaleness() {
+ return dcache('scraper-staleness', () => {
+ const scrapers = bucket(lsdirs(SKILLS), /scraper-manager$|-scraper$/);
+ const map = scrapers.map(s => {
+ const { body, desc } = parseSkillMd(s);
+ const m = (desc + ' ' + body.slice(0, 4000)).match(/\b([a-z][a-z0-9_]*_(?:catalog|colorways))\b/);
+ return { skill: s, tbl: m ? m[1] : null };
+ });
+ const tbls = [...new Set(map.map(x => x.tbl).filter(Boolean))];
+ const stats = {};
+ if (tbls.length) {
+ const inClause = tbls.map(t => `'${t}'`).join(',');
+ const q1 = `select table_name, string_agg(column_name, ',') from information_schema.columns where table_name in (${inClause}) and column_name in (${TS_COLS.map(c => `'${c}'`).join(',')}) group by table_name`;
+ const colsByTbl = {};
+ for (const r of lines(sh(`psql -h /tmp -d dw_unified -tAF'|' -c ${shq(q1)} 2>/dev/null`))) { const [t, cs] = r.split('|'); if (t && cs) colsByTbl[t] = cs.split(','); }
+ const parts = [];
+ for (const t of tbls) {
+ const cs = colsByTbl[t]; if (!cs || !cs.length) continue;
+ const g = cs.map(c => c + '::timestamptz').join(',');
+ const expr = cs.length > 1 ? 'greatest(' + g + ')' : g;
+ parts.push(`select '${t}' t, coalesce(to_char(max(${expr}),'YYYY-MM-DD HH24:MI'),'') last, count(*) n, coalesce(floor(extract(epoch from now()-max(${expr}))/86400)::int::text,'') days from ${t}`);
+ }
+ if (parts.length) for (const r of lines(sh(`psql -h /tmp -d dw_unified -tAF'|' -c ${shq(parts.join(' union all '))} 2>/dev/null`))) {
+ const [t, last, n, days] = r.split('|'); stats[t] = { last, rows: +n, days: days === '' ? null : +days };
+ }
+ }
+ const rows = map.map(x => { const st = x.tbl ? stats[x.tbl] : null; return { skill: x.skill, tbl: x.tbl || '', last: st ? st.last : '', rows: st ? st.rows : null, days: st && st.days != null ? st.days : null }; });
+ rows.sort((a, b) => a.days == null && b.days == null ? a.skill.localeCompare(b.skill) : a.days == null ? 1 : b.days == null ? -1 : b.days - a.days);
+ const stale30 = rows.filter(r => r.days != null && r.days > 30).length;
+ const nodata = rows.filter(r => r.days == null).length;
+ return { ts: new Date().toISOString(), count: rows.length, stale30, nodata, rows };
+ });
+}
+
function drill(type, id) {
const fn = DRILLERS[type];
if (!fn) return { title: type + ' · ' + id, breadcrumb: type + '/' + id, children: [], error: 'unknown type ' + type };
@@ -443,6 +479,10 @@ const server = http.createServer((req, res) => {
res.writeHead(200, { 'content-type': 'application/json' });
return res.end(JSON.stringify({ kind, items: (s._lists[kind] || []).slice().sort() }));
}
+ if (u.pathname === '/api/scrapers') {
+ res.writeHead(200, { 'content-type': 'application/json' });
+ return res.end(JSON.stringify(scraperStaleness()));
+ }
if (u.pathname === '/api/drill') {
const type = u.searchParams.get('type') || '';
const id = u.searchParams.get('id') || '';
← 4c7440f viewer: add LAST RUN to scraper cards (newest scrape timesta
·
back to Stack Map Viewer
·
scraper staleness: measure TRUE last-scrape (prefer last_scr db6fb5e →