viewer: add LAST RUN to scraper cards (newest scrape timestamp + age + row count from dw_unified staging table)

4c7440fca8c2cbfbf868c89bf977731abec12272 · 2026-09-22 12:51:34 -0700 · Steve Abrams

Files touched

M index.html
M server.js

Diff

commit 4c7440fca8c2cbfbf868c89bf977731abec12272
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Tue Sep 22 12:51:34 2026 -0700

    viewer: add LAST RUN to scraper cards (newest scrape timestamp + age + row count from dw_unified staging table)
---
 index.html |  1 +
 server.js  | 24 +++++++++++++++++++++++-
 2 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/index.html b/index.html
index e8cbabd..7e1a4ed 100644
--- a/index.html
+++ b/index.html
@@ -293,6 +293,7 @@ function renderInfo(info){
     +row('WHERE',info.where,'#9fc4ff')
     +row('WHO',info.who,'#d6b8ff')
     +row('WHEN',info.when,'#f3b6b6')
+    +row('LAST RUN',info.lastRun,'#5fd0a8')
     +'</div>';
 }
 async function renderDrill(){
diff --git a/server.js b/server.js
index 15f3c3e..79f2c7e 100644
--- a/server.js
+++ b/server.js
@@ -202,6 +202,24 @@ function parseSkillMd(id) {
   return { dir, name, desc, body };
 }
 
+// last time a vendor scraper actually ran = newest scrape-ish timestamp in its dw_unified staging table
+const TS_COLS = ['last_scraped', 'scraped_at', 'specs_scraped_at', 'imported_at', 'updated_at', 'price_updated_at', 'created_at'];
+function tableLastRun(tbl) {
+  if (!/^[a-z0-9_]+$/.test(tbl || '')) return null;
+  const q1 = `select column_name from information_schema.columns where table_name='${tbl}' and column_name in (${TS_COLS.map(c => `'${c}'`).join(',')})`;
+  const cols = lines(sh(`psql -h /tmp -d dw_unified -tAc ${shq(q1)} 2>/dev/null`));
+  if (!cols.length) return null;
+  const g = cols.map(c => c + '::timestamptz').join(',');
+  const expr = cols.length > 1 ? 'greatest(' + g + ')' : g;
+  const q2 = `select coalesce(to_char(max(${expr}),'YYYY-MM-DD HH24:MI'),'') || '|' || count(*) || '|' || coalesce(floor(extract(epoch from now()-max(${expr}))/86400)::text,'') from ${tbl}`;
+  const out = sh(`psql -h /tmp -d dw_unified -tAc ${shq(q2)} 2>/dev/null`).trim();
+  if (!out) return null;
+  const [last, rows, days] = out.split('|');
+  if (!last) return null;
+  const age = days === '' ? '' : (+days === 0 ? 'today' : +days === 1 ? '1 day ago' : days + ' days ago');
+  return { last, rows, age, tbl };
+}
+
 function skillInfo(id) {
   const { dir, name, desc, body } = parseSkillMd(id);
   const hay = desc + ' \n' + body.slice(0, 4000);
@@ -243,12 +261,16 @@ function skillInfo(id) {
     || sentences(desc).slice(1, 3).find(s => s.length > 25) || firstSentence(desc) || '—';
   // WHAT — one-line summary
   const what = firstSentence(desc) || desc || '—';
+  // LAST RUN — newest scrape timestamp in the vendor's staging table (scrapers only)
+  const vendorTbl = tables.find(t => /_catalog$/.test(t)) || tables[0];
+  const lr = vendorTbl ? tableLastRun(vendorTbl) : null;
+  const lastRun = lr ? (lr.last + (lr.age ? ' (' + lr.age + ')' : '') + ' · ' + Number(lr.rows).toLocaleString() + ' rows · ' + lr.tbl) : null;
   return {
     description: desc || '(no description found in SKILL.md)',
     what, why,
     who: whoBits.length ? whoBits.join(' · ') : 'nothing references it (pure on-demand)',
     where: whereBits.join('  ·  '),
-    when, name,
+    when, lastRun, name,
   };
 }

← 18c85c0 viewer: click any skill/canary → description + who/what/wher · back to Stack Map Viewer · viewer: Scrapers ⏱ view — sortable staleness table (all 50 s 175dbc0 →