recrawl-schumacher: dry-run skips DB write + self-heal pinterest/facebook rows

127175f045c34678ab45f66118c43d31e54eb59e · 2026-06-10 17:40:05 -0700 · Steve

Files touched

M recrawl-schumacher.js

Diff

commit 127175f045c34678ab45f66118c43d31e54eb59e
Author: Steve <steve@designerwallcoverings.com>
Date:   Wed Jun 10 17:40:05 2026 -0700

    recrawl-schumacher: dry-run skips DB write + self-heal pinterest/facebook rows
---
 recrawl-schumacher.js | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/recrawl-schumacher.js b/recrawl-schumacher.js
index ea28a17..9e9313f 100644
--- a/recrawl-schumacher.js
+++ b/recrawl-schumacher.js
@@ -42,6 +42,9 @@
 //   --canary N    process only the first N rows (default 5) — run FIRST on
 //                 Kamatera to confirm the deployed extractors still match live
 //                 HTML / CDN before the full pass.
+//   --dry-run     READ-ONLY: extract + print images/specs but SKIP the DB
+//                 UPDATE entirely. Mandatory for the validation canary so no
+//                 bad write can touch prod before the extraction is eyeballed.
 //   --all         process every eligible row (full ~4,474-row pass).
 //   --net-new     restrict to rows whose mfr_sku is NOT already in
 //                 shopify_products (skip the already-live cohort).
@@ -72,6 +75,7 @@ const { discoverCdnImages, fetchProductSpecs, extractImagesFromHtml, httpGet } =
 const argv = process.argv.slice(2);
 const FULL = argv.includes('--all');
 const NET_NEW_ONLY = argv.includes('--net-new');
+const DRY_RUN = argv.includes('--dry-run');
 const canaryIdx = argv.indexOf('--canary');
 const CANARY_N = canaryIdx >= 0 ? (parseInt(argv[canaryIdx + 1], 10) || 5) : (FULL ? 0 : 5);
 
@@ -92,7 +96,8 @@ function isImg(u) {
            split_part(product_url,'?',1) AS url,
            image_url
     FROM ${TABLE}
-    WHERE (all_images IS NULL OR all_images = '' OR all_images = '[]' OR all_images = '""')
+    WHERE (all_images IS NULL OR all_images = '' OR all_images = '[]' OR all_images = '""'
+           OR all_images ILIKE '%pinterest%' OR all_images ILIKE '%facebook%')
       AND mfr_sku IS NOT NULL AND mfr_sku <> ''
   `;
   if (NET_NEW_ONLY) {
@@ -105,7 +110,8 @@ function isImg(u) {
   const { rows } = await pool.query(sql);
   console.log(`Schumacher additive all_images re-crawl — ${rows.length} rows`
     + `${CANARY_N > 0 ? ` (CANARY/limit ${CANARY_N})` : ' (FULL)'}`
-    + `${NET_NEW_ONLY ? ' [net-new only]' : ''}`);
+    + `${NET_NEW_ONLY ? ' [net-new only]' : ''}`
+    + `${DRY_RUN ? ' [DRY-RUN — NO DB WRITES]' : ''}`);
 
   let n = 0, updated = 0, withImgs = 0, withSpecs = 0, noImg = 0, errs = 0;
   for (const r of rows) {
@@ -156,6 +162,16 @@ function isImg(u) {
         continue;
       }
 
+      if (DRY_RUN) {
+        // READ-ONLY canary: extract + print, but NEVER touch the DB so a bad
+        // extraction can't write to prod before it is eyeballed.
+        withImgs++;
+        if (Object.keys(specs).length) withSpecs++;
+        console.log(`  [dry-run] ${sku}: ${images.length} imgs, ${Object.keys(specs).length} specs (NOT written)`);
+        console.log(`    ${images.slice(0, 5).join('\n    ')}${images.length > 5 ? '\n    …' : ''}`);
+        continue;
+      }
+
       await pool.query(
         `UPDATE ${TABLE}
             SET all_images = $1,

← f432b38 coordonne recrawl: drop shared Coordonne_Qualities material · back to Dw Vendor Scrapers Local · Add flag-discontinued.js: HEAD-probe net-new image-less rows ba1e429 →