fix(TK-11357): versa scraper counts failed page-scrapes so a degraded run can't read clean (finding #3)

87cf6ccc17e3c4d549ab67ec37144110709660ed · 2026-09-14 16:31:49 -0700 · Steve

The per-product catch logged+skipped a failed page but the RESULTS banner counted
only DB errors, so N of 25 pages silently vanishing (network / selector drift /
Versa redirect) printed "Errors: 0" and looked healthy — the TK-11431 false-green
class. Now: a scrapeFailures counter + failedSkus list, a rewritten banner
(pages attempted / scraped OK / FAILED / rows / DB errors), and a `degraded`
verdict (any failed page, DB error, or zero-row/zero-page run) that prints a loud
DEGRADED RUN and sets process.exitCode=1 so a cron/caller never mistakes a partial
refresh for success. Its dep lib/versa-colorway-extraction.js is already committed,
so this is not a broken-import commit. Closes review finding #3.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Files touched

M DW-Programming/versa-evo-pvcfree-scraper.js

Diff

commit 87cf6ccc17e3c4d549ab67ec37144110709660ed
Author: Steve <steve@designerwallcoverings.com>
Date:   Mon Sep 14 16:31:49 2026 -0700

    fix(TK-11357): versa scraper counts failed page-scrapes so a degraded run can't read clean (finding #3)
    
    The per-product catch logged+skipped a failed page but the RESULTS banner counted
    only DB errors, so N of 25 pages silently vanishing (network / selector drift /
    Versa redirect) printed "Errors: 0" and looked healthy — the TK-11431 false-green
    class. Now: a scrapeFailures counter + failedSkus list, a rewritten banner
    (pages attempted / scraped OK / FAILED / rows / DB errors), and a `degraded`
    verdict (any failed page, DB error, or zero-row/zero-page run) that prints a loud
    DEGRADED RUN and sets process.exitCode=1 so a cron/caller never mistakes a partial
    refresh for success. Its dep lib/versa-colorway-extraction.js is already committed,
    so this is not a broken-import commit. Closes review finding #3.
    
    Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 DW-Programming/versa-evo-pvcfree-scraper.js | 41 ++++++++++++++++++++++++-----
 1 file changed, 35 insertions(+), 6 deletions(-)

diff --git a/DW-Programming/versa-evo-pvcfree-scraper.js b/DW-Programming/versa-evo-pvcfree-scraper.js
index b1f1d272..06f455a8 100644
--- a/DW-Programming/versa-evo-pvcfree-scraper.js
+++ b/DW-Programming/versa-evo-pvcfree-scraper.js
@@ -42,6 +42,13 @@ async function scrapeCollection() {
   await db.connect();
 
   const allProducts = [];
+  // TK-11357 (vp-engineering review 2026-09-14): COUNT failed page scrapes so a
+  // degraded run can't read as clean. Before this, a page that threw was logged
+  // and skipped, but the RESULTS banner counted only DB errors — so N of 25
+  // pages silently vanishing (network / selector drift / Versa redirect) reported
+  // "Errors: 0" and looked healthy (the TK-11431 false-green class).
+  let scrapeFailures = 0;
+  const failedSkus = [];
 
   console.log(`Scraping ${KNOWN_PRODUCTS.length} products from Versa Evo PVC-free collection...\n`);
 
@@ -104,8 +111,10 @@ async function scrapeCollection() {
 
     } catch (err) {
       console.error(`  ERROR: ${err.message}`);
-      // Failed pages are skipped; do not replace verified rows with guesses.
-
+      // Failed pages are skipped; do not replace verified rows with guesses —
+      // but they are COUNTED so the banner can't report a degraded run as clean.
+      scrapeFailures++;
+      failedSkus.push(prod.sku);
     }
 
     // Rate limit
@@ -135,14 +144,34 @@ async function scrapeCollection() {
     }
   }
 
+  const pagesOK = KNOWN_PRODUCTS.length - scrapeFailures;
   console.log(`\n========== RESULTS ==========`);
-  console.log(`Total processed: ${allProducts.length}`);
-  console.log(`Inserted (new): ${inserted}`);
-  console.log(`Updated (existing): ${updated}`);
-  console.log(`Errors: ${errors}`);
+  console.log(`Pages attempted:   ${KNOWN_PRODUCTS.length}`);
+  console.log(`Pages scraped OK:  ${pagesOK}`);
+  console.log(`Pages FAILED:      ${scrapeFailures}${scrapeFailures ? ' -> ' + failedSkus.join(', ') : ''}`);
+  console.log(`Rows produced:     ${allProducts.length}`);
+  console.log(`Inserted (new):    ${inserted}`);
+  console.log(`Updated (existing):${updated}`);
+  console.log(`DB errors:         ${errors}`);
   console.log(`Collection: ${COLLECTION_NAME}`);
   console.log(`==============================`);
 
+  // A run is only HEALTHY when every page scraped, no DB error, and it actually
+  // produced rows. Any of: a failed page, a DB error, or a zero-row/zero-page
+  // run (the "0 of 0 looks clean" NOT-MEASURED trap) is a DEGRADED run — say so
+  // loudly and exit non-zero so a caller / cron never mistakes it for success.
+  const degraded = scrapeFailures > 0 || errors > 0 || allProducts.length === 0 || pagesOK === 0;
+  if (degraded) {
+    console.error(
+      `\n⚠️  DEGRADED RUN — ${scrapeFailures} page-scrape failure(s), ${errors} DB error(s), ` +
+      `${allProducts.length} row(s) from ${pagesOK}/${KNOWN_PRODUCTS.length} pages. ` +
+      `NOT a clean run; do not treat versa_catalog as fully refreshed.`
+    );
+    process.exitCode = 1;
+  } else {
+    console.log(`\n✅ CLEAN RUN — all ${KNOWN_PRODUCTS.length} pages scraped, ${allProducts.length} rows upserted, 0 errors.`);
+  }
+
   await db.end();
 }

← 674994ec fix(TK-11403): price-integrity gate fails CLOSED on empty/mi · back to Designer Wallcoverings · auto-data-snapshot: 2026-09-14T16:46:18 (4 data files) — DW- 4aed5a55 →