scraper staleness: add crawled_at + promote created_at (new-row insert = real scrape activity); only updated_at stays db-touch — fixes york/elitis reading Mar instead of Jun, anna-french now correctly 22d

fa2db1093c20b480dd0cbaf9e2897ac77f2d618b · 2026-09-22 13:17:37 -0700 · Steve Abrams

Files touched

M server.js

Diff

commit fa2db1093c20b480dd0cbaf9e2897ac77f2d618b
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Tue Sep 22 13:17:37 2026 -0700

    scraper staleness: add crawled_at + promote created_at (new-row insert = real scrape activity); only updated_at stays db-touch — fixes york/elitis reading Mar instead of Jun, anna-french now correctly 22d
---
 server.js | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/server.js b/server.js
index 598b510..209a9d2 100644
--- a/server.js
+++ b/server.js
@@ -203,12 +203,14 @@ function parseSkillMd(id) {
   return { dir, name, desc, body };
 }
 
-// last time a vendor scraper actually RAN. Prefer true-scrape columns; fall back to
-// created/updated only when no scrape column exists — because updated_at moves on ANY
-// bulk DB touch (e.g. the Apr-1 gemini-tagger migration hit 152 catalog tables), which
-// would falsely read as a fresh "run". Source is labelled so a db-touch can't pose as a scrape.
-const SCRAPE_COLS = ['last_scraped', 'scraped_at', 'specs_scraped_at', 'imported_at', 'price_updated_at'];
-const FALLBACK_COLS = ['created_at', 'updated_at'];
+// last time a vendor scraper actually RAN. Scrape signals = a crawl/insert/scrape ts.
+// crawled_at + created_at COUNT: a new row's created_at means a scraper genuinely found a
+// new SKU (the overnight new-products refresh inserts rows without bumping last_scraped,
+// e.g. anna-french +22 rows Aug-31), so treating created_at as activity is correct.
+// ONLY updated_at is demoted — it moves on ANY bulk edit of EXISTING rows (the Apr-1
+// gemini-tagger migration touched 152 tables), so it alone is the "db-touch" fallback.
+const SCRAPE_COLS = ['last_scraped', 'scraped_at', 'crawled_at', 'specs_scraped_at', 'imported_at', 'price_updated_at', 'created_at'];
+const FALLBACK_COLS = ['updated_at'];
 const TS_COLS = [...SCRAPE_COLS, ...FALLBACK_COLS];
 const gexpr = (cols) => cols.length ? (cols.length > 1 ? 'greatest(' + cols.map(c => c + '::timestamptz').join(',') + ')' : cols[0] + '::timestamptz') : 'null::timestamptz';
 function lastRunFromParts(sLast, fLast, rows, sd, smin) {

← 3dcba1b Add 'date last run' to all drill cards (officers, agents, sk · back to Stack Map Viewer · scraper view: authoritative vendor_registry.catalog_table re 278a01e →