← back to Stack Map Viewer

Scrapers view: expand from 50 skill-backed to ALL 168 vendors (vendor_registry + every *_catalog table), exclude backup/temp tables, mark ◆ has-skill vs registry-only; big lines (Kravet/Schumacher/Thibaut/Scalamandre) now covered

71dc5249f3d0ef6ec77bfcbf19ebf87fba02a0bf · 2026-09-22 13:33:43 -0700 · Steve Abrams

Files touched

• M index.html
• M server.js

Diff

commit 71dc5249f3d0ef6ec77bfcbf19ebf87fba02a0bf
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Tue Sep 22 13:33:43 2026 -0700

    Scrapers view: expand from 50 skill-backed to ALL 168 vendors (vendor_registry + every *_catalog table), exclude backup/temp tables, mark ◆ has-skill vs registry-only; big lines (Kravet/Schumacher/Thibaut/Scalamandre) now covered
---
 index.html | 23 +++++++++++++---------
 server.js  | 66 +++++++++++++++++++++++++++++++++++++++-----------------------
 2 files changed, 56 insertions(+), 33 deletions(-)

diff --git a/index.html b/index.html
index 71e84d1..732b583 100644
--- a/index.html
+++ b/index.html
@@ -330,7 +330,12 @@ function renderCrumbs(){
   document.getElementById('crumbs').innerHTML=CRUMBS.map((c,i)=>'<a data-i="'+i+'">'+c.label+'</a>').join('<span class="sep">›</span>');
   document.querySelectorAll('#crumbs a').forEach(a=>a.addEventListener('click',()=>{CRUMBS=CRUMBS.slice(0,+a.dataset.i+1);renderDrill();}));
 }
-document.getElementById('close').addEventListener('click',()=>{document.getElementById('drawer').classList.remove('open');CRUMBS=[];});
+function closeDrawer(){document.getElementById('drawer').classList.remove('open');CRUMBS=[];}
+document.getElementById('close').addEventListener('click',closeDrawer);
+// dismiss the drill card via Esc or a click outside it — an open fixed drawer otherwise
+// buries the right-column category chips with no affordance to reach them (only the ✕ closed it).
+document.addEventListener('keydown',e=>{if(e.key==='Escape'&&document.getElementById('drawer').classList.contains('open'))closeDrawer();});
+document.addEventListener('mousedown',e=>{const dw=document.getElementById('drawer');if(dw.classList.contains('open')&&!e.target.closest('#drawer'))closeDrawer();},true);
 
 // ── Tree view (recursive collapsible, same /api/drill) ───────────────────────
 function treeRoots(d){return [
@@ -411,7 +416,7 @@ function sortScr(rows){
   return rows.slice().sort((a,b)=>{
     let av=a[key],bv=b[key];
     if(key==='days'){ // nulls always last regardless of dir
-      if(av==null&&bv==null)return a.skill.localeCompare(b.skill);
+      if(av==null&&bv==null)return (a.name||'').localeCompare(b.name||'');
       if(av==null)return 1; if(bv==null)return -1; return (av-bv)*dir;
     }
     if(key==='cadence'){av=a.scrapeDates||0;bv=b.scrapeDates||0;return (av-bv)*dir;}
@@ -422,20 +427,20 @@ function sortScr(rows){
 function renderScrapersTable(){
   if(!SCR)return;
   const box=document.getElementById('scrapersbox');
-  const cols=[['skill','Vendor scraper'],['last','Last scrape'],['days','Age'],['cadence','History'],['rows','Rows'],['tbl','Staging table']];
+  const cols=[['name','Vendor'],['last','Last scrape'],['days','Age'],['cadence','History'],['rows','Rows'],['tbl','Staging table']];
   const arrow=k=>SCRSORT.key===k?(' <span class="ar">'+(SCRSORT.dir<0?'▼':'▲')+'</span>'):'';
   const head='<tr><th style="cursor:default;color:#6f7d97">#</th>'+cols.map(c=>'<th data-k="'+c[0]+'">'+c[1]+arrow(c[0])+'</th>').join('')+'</tr>';
-  const rows=sortScr(SCR.rows).map((r,i)=>'<tr data-skill="'+r.skill+'">'
+  const rows=sortScr(SCR.rows).map((r,i)=>'<tr'+(r.skill?' data-skill="'+r.skill+'"':' style="cursor:default"')+'>'
     +'<td class="num">'+(i+1)+'</td>'
-    +'<td>'+r.skill.replace(/-scraper-manager$|-scraper$/,'')+'</td>'
-    +'<td class="age">'+(r.last?esc(r.last)+(r.source==='db-touch'?' <span style="color:#c79a4a;font-weight:400;font-size:10px" title="no true scrape timestamp; date is from created_at/updated_at (a DB touch)">⚠ db-touch</span>':''):'<span style="color:#6f7d97">—</span>')+'</td>'
+    +'<td>'+esc(r.name)+(r.skill?' <span style="color:#5f7bb0;font-size:10px" title="has a scraper-manager skill — click for its 5W card">◆</span>':' <span style="color:#6f7d97;font-size:10px" title="registry vendor — no dedicated skill">·reg</span>')+'</td>'
+    +'<td class="age">'+(r.last?esc(r.last)+(r.source==='db-touch'?' <span style="color:#c79a4a;font-weight:400;font-size:10px" title="no true scrape timestamp; date is from updated_at (a DB touch)">⚠ db-touch</span>':''):'<span style="color:#6f7d97">—</span>')+'</td>'
     +'<td class="age">'+ageCell(r.days)+'</td>'
     +'<td>'+cadenceCell(r)+'</td>'
     +'<td class="num">'+(r.rows==null?'—':r.rows.toLocaleString())+'</td>'
-    +'<td class="tbl">'+(r.tbl||'<span style="color:#8a4a4a">unmapped</span>')+'</td></tr>').join('');
-  box.innerHTML='<div class="sumbar"><b>'+SCR.count+'</b> scrapers · <b style="color:#f3b6b6">'+SCR.stale30+'</b> stale &gt;30d · <b style="color:#ffd27a">'+SCR.onceOnly+'</b> onboarded-once · <b style="color:#f3b6b6">'+SCR.neverScraped+'</b> never scraped · <b>'+SCR.nodata+'</b> no data · <b>Last scrape</b> = true scrape ts (not a DB touch) · click a row for its 5W card · headers re-sort</div>'
+    +'<td class="tbl">'+(r.tbl||'<span style="color:#8a4a4a">no table</span>')+'</td></tr>').join('');
+  box.innerHTML='<div class="sumbar"><b>'+SCR.count+'</b> vendors · <b style="color:#f3b6b6">'+SCR.stale30+'</b> stale &gt;30d · <b style="color:#ffd27a">'+SCR.onceOnly+'</b> onboarded-once · <b style="color:#f3b6b6">'+SCR.neverScraped+'</b> never scraped · <b>'+SCR.nodata+'</b> no data · <b>'+SCR.withSkill+'</b> ◆ have a skill · <b>Last scrape</b> = true scrape ts (not a DB touch) · click a ◆ row for its 5W card · headers re-sort</div>'
     +'<table class="stale"><thead>'+head+'</thead><tbody>'+rows+'</tbody></table>';
-  box.querySelectorAll('th[data-k]').forEach(th=>th.addEventListener('click',()=>{const k=th.dataset.k;if(SCRSORT.key===k)SCRSORT.dir*=-1;else SCRSORT={key:k,dir:k==='skill'||k==='tbl'?1:-1};renderScrapersTable();}));
+  box.querySelectorAll('th[data-k]').forEach(th=>th.addEventListener('click',()=>{const k=th.dataset.k;if(SCRSORT.key===k)SCRSORT.dir*=-1;else SCRSORT={key:k,dir:k==='name'||k==='tbl'?1:-1};renderScrapersTable();}));
   box.querySelectorAll('tr[data-skill]').forEach(tr=>tr.addEventListener('click',()=>startDrill('skill',tr.dataset.skill,tr.dataset.skill)));
 }
 async function renderScrapers(){
diff --git a/server.js b/server.js
index bd5f875..ff24ee3 100644
--- a/server.js
+++ b/server.js
@@ -222,7 +222,7 @@ const qtbl = (t) => '"' + String(t).replace(/"/g, '') + '"'; // quote table name
 let _existTbls = null, _existTs = 0;
 function existingCatalogTables() {
   if (_existTbls && Date.now() - _existTs < 60000) return _existTbls;
-  const rows = lines(sh(`psql -h /tmp -d dw_unified -tAc "select table_name from information_schema.tables where table_schema='public' and table_name ~ 'catalog|colorway'" 2>/dev/null`));
+  const rows = lines(sh(`psql -h /tmp -d dw_unified -tAc "select table_name from information_schema.tables where table_schema='public' and table_name ~ '_(catalog|colorways?)$' and table_name !~ '(bak|backup|prebackfill|_old|_tmp|_temp|_copy)'" 2>/dev/null`));
   _existTbls = new Set(rows); _existTs = Date.now();
   return _existTbls;
 }
@@ -533,38 +533,56 @@ const DRILLERS = {
 
 // ── Scraper staleness ranking: every scraper → newest timestamp in its staging table ──
 // Skill→table resolved from SKILL.md (no DB); then just 2 batched psql calls for all tables.
+const prettyTbl = (t) => t.replace(/_catalog$|_colorways?$/, '').replace(/_/g, ' ');
+const prettySkill = (s) => s.replace(/-scraper-manager$|-scraper$/, '');
+// freshness stats for a set of tables, CHUNKED so 200+ tables stay under the 15s shell timeout
+function statsForTables(tbls) {
+  const stats = {}; const CH = 60;
+  for (let i = 0; i < tbls.length; i += CH) {
+    const chunk = tbls.slice(i, i + CH);
+    const q1 = `select table_name, string_agg(column_name, ',') from information_schema.columns where table_name in (${chunk.map(t => `'${t}'`).join(',')}) and column_name in (${TS_COLS.map(c => `'${c}'`).join(',')}) group by table_name`;
+    const colsByTbl = {};
+    for (const r of lines(sh(`psql -h /tmp -d dw_unified -tAF'|' -c ${shq(q1)} 2>/dev/null`))) { const [t, cs] = r.split('|'); if (t && cs) colsByTbl[t] = cs.split(','); }
+    const parts = [];
+    for (const t of chunk) {
+      const cs = colsByTbl[t]; if (!cs || !cs.length) continue;
+      const sc = cs.filter(c => SCRAPE_COLS.includes(c)), fc = cs.filter(c => FALLBACK_COLS.includes(c));
+      parts.push(`select '${t}' t, coalesce(to_char(max(${gexpr(sc)}),'YYYY-MM-DD HH24:MI'),'') s, coalesce(to_char(max(${gexpr(fc)}),'YYYY-MM-DD HH24:MI'),'') f, count(*) n, count(distinct (${gexpr(sc)})::date) sd, coalesce(to_char(min(${gexpr(sc)}),'YYYY-MM-DD'),'') smin from ${qtbl(t)}`);
+    }
+    if (parts.length) for (const r of lines(sh(`psql -h /tmp -d dw_unified -tAF'|' -c ${shq(parts.join(' union all '))} 2>/dev/null`))) {
+      const [t, s, f, n, sd, smin] = r.split('|'); stats[t] = lastRunFromParts(s, f, n, sd, smin);
+    }
+  }
+  return stats;
+}
 function scraperStaleness() {
   return dcache('scraper-staleness', () => {
-    const scrapers = bucket(lsdirs(SKILLS), /scraper-manager$|-scraper$/);
-    const map = scrapers.map(s => {
+    const exist = existingCatalogTables();                 // every catalog/colorway table (the full vendor universe)
+    const nameByTbl = {};                                   // authoritative vendor name per table
+    for (const l of lines(sh(`psql -h /tmp -d dw_unified -tAF'|' -c "select catalog_table, coalesce(nullif(vendor_name,''),vendor_code) from vendor_registry where catalog_table is not null" 2>/dev/null`))) { const [t, nm] = l.split('|'); if (t && nm && !nameByTbl[t]) nameByTbl[t] = nm; }
+    const entries = {};                                     // keyed by table
+    for (const t of exist) entries[t] = { tbl: t, name: nameByTbl[t] || prettyTbl(t), skill: null };
+    // attach a dedicated skill (for click-to-card) where one maps to the table
+    const skillOnly = [];
+    for (const s of bucket(lsdirs(SKILLS), /scraper-manager$|-scraper$/)) {
       const { body, desc } = parseSkillMd(s);
       const m = (desc + ' ' + body.slice(0, 4000)).match(/\b([a-z0-9][a-z0-9_]*_(?:catalog|colorways))\b/);
-      return { skill: s, tbl: resolveVendorTable(s, m ? m[1] : null) };
-    });
-    const tbls = [...new Set(map.map(x => x.tbl).filter(Boolean))];
-    const stats = {};
-    if (tbls.length) {
-      const inClause = tbls.map(t => `'${t}'`).join(',');
-      const q1 = `select table_name, string_agg(column_name, ',') from information_schema.columns where table_name in (${inClause}) and column_name in (${TS_COLS.map(c => `'${c}'`).join(',')}) group by table_name`;
-      const colsByTbl = {};
-      for (const r of lines(sh(`psql -h /tmp -d dw_unified -tAF'|' -c ${shq(q1)} 2>/dev/null`))) { const [t, cs] = r.split('|'); if (t && cs) colsByTbl[t] = cs.split(','); }
-      const parts = [];
-      for (const t of tbls) {
-        const cs = colsByTbl[t]; if (!cs || !cs.length) continue;
-        const sc = cs.filter(c => SCRAPE_COLS.includes(c)), fc = cs.filter(c => FALLBACK_COLS.includes(c));
-        parts.push(`select '${t}' t, coalesce(to_char(max(${gexpr(sc)}),'YYYY-MM-DD HH24:MI'),'') s, coalesce(to_char(max(${gexpr(fc)}),'YYYY-MM-DD HH24:MI'),'') f, count(*) n, count(distinct (${gexpr(sc)})::date) sd, coalesce(to_char(min(${gexpr(sc)}),'YYYY-MM-DD'),'') smin from ${qtbl(t)}`);
-      }
-      if (parts.length) for (const r of lines(sh(`psql -h /tmp -d dw_unified -tAF'|' -c ${shq(parts.join(' union all '))} 2>/dev/null`))) {
-        const [t, s, f, n, sd, smin] = r.split('|'); stats[t] = lastRunFromParts(s, f, n, sd, smin);
-      }
+      const tbl = resolveVendorTable(s, m ? m[1] : null);
+      if (tbl && entries[tbl]) entries[tbl].skill = entries[tbl].skill || s;
+      else if (tbl) entries[tbl] = { tbl, name: nameByTbl[tbl] || prettyTbl(tbl), skill: s };
+      else skillOnly.push({ name: prettySkill(s), skill: s });      // non-vendor scraper skill (feed-first, fineartamerica…)
     }
-    const rows = map.map(x => { const st = x.tbl ? stats[x.tbl] : null; return { skill: x.skill, tbl: x.tbl || '', last: st ? st.last : '', rows: st ? st.rows : null, days: st && st.days != null ? st.days : null, source: st ? st.source : '', cadence: st ? st.cadence : '', scrapeDates: st ? st.scrapeDates : 0, firstScrape: st ? st.firstScrape : '' }; });
-    rows.sort((a, b) => a.days == null && b.days == null ? a.skill.localeCompare(b.skill) : a.days == null ? 1 : b.days == null ? -1 : b.days - a.days);
+    const stats = statsForTables(Object.keys(entries));
+    const mk = (e, st) => ({ name: e.name, skill: e.skill || '', tbl: e.tbl || '', last: st ? st.last : '', rows: st ? st.rows : null, days: st && st.days != null ? st.days : null, source: st ? st.source : '', cadence: st ? st.cadence : '', scrapeDates: st ? st.scrapeDates : 0, firstScrape: st ? st.firstScrape : '' });
+    const rows = Object.values(entries).map(e => mk(e, stats[e.tbl]));
+    for (const so of skillOnly) rows.push(mk(so, null));
+    rows.sort((a, b) => a.days == null && b.days == null ? a.name.localeCompare(b.name) : a.days == null ? 1 : b.days == null ? -1 : b.days - a.days);
     const stale30 = rows.filter(r => r.days != null && r.days > 30).length;
     const nodata = rows.filter(r => r.days == null).length;
     const onceOnly = rows.filter(r => r.cadence === 'once').length;
     const neverScraped = rows.filter(r => r.cadence === 'never').length;
-    return { ts: new Date().toISOString(), count: rows.length, stale30, nodata, onceOnly, neverScraped, rows };
+    const withSkill = rows.filter(r => r.skill).length;
+    return { ts: new Date().toISOString(), count: rows.length, stale30, nodata, onceOnly, neverScraped, withSkill, rows };
   });
 }
 

← 278a01e scraper view: authoritative vendor_registry.catalog_table re  ·  back to Stack Map Viewer  ·  Scrapers view: add Health column from live scraper-status.js dc60ca8 →