← back to Designer Wallcoverings

auto-save: 2026-07-13T12:24:56 (6 files) — DW-Programming/ImportNewSkufromURL/data/scraper-audit/reconcile/brewster-import-queue.jsonl DW-Programming/ImportNewSkufromURL/scripts/run-all-vendor-scrapers.ts DW-Programming/ImportNewSkufromURL/scripts/scraper-healthcheck.ts pending-approval/boost-filter-consolidation-2026-06-25 scripts/pr-contact-for-price

a56af202e865007244b2569e67d8d5addce7ecc0 · 2026-07-13 12:25:02 -0700 · Steve Abrams

Files touched

• M DW-Programming/ImportNewSkufromURL/data/scraper-audit/reconcile/brewster-import-queue.jsonl
• M DW-Programming/ImportNewSkufromURL/scripts/run-all-vendor-scrapers.ts
• M DW-Programming/ImportNewSkufromURL/scripts/scraper-healthcheck.ts

Diff

commit a56af202e865007244b2569e67d8d5addce7ecc0
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Mon Jul 13 12:25:02 2026 -0700

    auto-save: 2026-07-13T12:24:56 (6 files) — DW-Programming/ImportNewSkufromURL/data/scraper-audit/reconcile/brewster-import-queue.jsonl DW-Programming/ImportNewSkufromURL/scripts/run-all-vendor-scrapers.ts DW-Programming/ImportNewSkufromURL/scripts/scraper-healthcheck.ts pending-approval/boost-filter-consolidation-2026-06-25 scripts/pr-contact-for-price
---
 .../reconcile/brewster-import-queue.jsonl          | 28 +++++++-------
 .../scripts/run-all-vendor-scrapers.ts             | 45 ++++++++++++++++++++--
 .../scripts/scraper-healthcheck.ts                 |  5 +++
 3 files changed, 61 insertions(+), 17 deletions(-)

diff --git a/DW-Programming/ImportNewSkufromURL/data/scraper-audit/reconcile/brewster-import-queue.jsonl b/DW-Programming/ImportNewSkufromURL/data/scraper-audit/reconcile/brewster-import-queue.jsonl
index 2fb28c38..d29d5c61 100644
--- a/DW-Programming/ImportNewSkufromURL/data/scraper-audit/reconcile/brewster-import-queue.jsonl
+++ b/DW-Programming/ImportNewSkufromURL/data/scraper-audit/reconcile/brewster-import-queue.jsonl
@@ -206,20 +206,20 @@
 {"vendor":"brewster","sku":"FAB10030","url":"https://www.brewsterwallcovering.com/fab10030-white-glossy-adhesive-film","status":"done","brand":"Jeffrey Stevens","name":"FAB10030 - White Glossy Adhesive Film - by Fablon","color":"by Fablon","image":"","specs":{},"attempts":1,"note":"https://admin.shopify.com/store/designer-laboratory-sandbox/products/7883946950707","doneAt":"2026-07-13T18:52:11.999Z"}
 {"vendor":"brewster","sku":"FAB10031","url":"https://www.brewsterwallcovering.com/fab10031-glossy-white-adhesive-film","status":"done","brand":"Jeffrey Stevens","name":"FAB10031 - Glossy White Adhesive Film - by Fablon","color":"by Fablon","image":"","specs":{},"attempts":1,"note":"https://admin.shopify.com/store/designer-laboratory-sandbox/products/7883947212851","doneAt":"2026-07-13T18:53:28.132Z"}
 {"vendor":"brewster","sku":"FAB10036","url":"https://www.brewsterwallcovering.com/fab10036-red-adhesive-film","status":"done","brand":"Jeffrey Stevens","name":"FAB10036 - Red Adhesive Film - by Fablon","color":"by Fablon","image":"","specs":{},"attempts":1,"note":"https://admin.shopify.com/store/designer-laboratory-sandbox/products/7883947507763","doneAt":"2026-07-13T18:54:43.657Z"}
-{"vendor":"brewster","sku":"FAB10044","url":"https://www.brewsterwallcovering.com/fab10044-black-adhesive-film","status":"pending","brand":"Jeffrey Stevens","name":"FAB10044 - Black Adhesive Film - by Fablon","color":"by Fablon","image":"","specs":{},"attempts":0,"note":"resurrected 2026-07-13 (was infra-failed)"}
-{"vendor":"brewster","sku":"FAB10047","url":"https://www.brewsterwallcovering.com/fab10047-transparent-glossy-adhesive-film","status":"pending","brand":"Jeffrey Stevens","name":"FAB10047 - Transparent Glossy Adhesive Film - by Fablon","color":"by Fablon","image":"","specs":{},"attempts":0,"note":"resurrected 2026-07-13 (was infra-failed)"}
-{"vendor":"brewster","sku":"FAB10048","url":"https://www.brewsterwallcovering.com/fab10048-white-matte-adhesive-film","status":"pending","brand":"Jeffrey Stevens","name":"FAB10048 - White Matte Adhesive Film - by Fablon","color":"by Fablon","image":"","specs":{},"attempts":0,"note":"resurrected 2026-07-13 (was infra-failed)"}
-{"vendor":"brewster","sku":"FAB10056","url":"https://www.brewsterwallcovering.com/fab10056-black-matte-adhesive-film","status":"pending","brand":"Jeffrey Stevens","name":"FAB10056 - Black Matte Adhesive Film - by Fablon","color":"by Fablon","image":"","specs":{},"attempts":0,"note":"resurrected 2026-07-13 (was infra-failed)"}
-{"vendor":"brewster","sku":"FAB10064","url":"https://www.brewsterwallcovering.com/fab10064-mahogany-adhesive-film","status":"pending","brand":"Jeffrey Stevens","name":"FAB10064 - Mahogany Adhesive Film - by Fablon","color":"by Fablon","image":"","specs":{},"attempts":0,"note":"resurrected 2026-07-13 (was infra-failed)"}
-{"vendor":"brewster","sku":"FAB10070","url":"https://www.brewsterwallcovering.com/fab10070-oak-pale-adhesive-film","status":"pending","brand":"Jeffrey Stevens","name":"FAB10070 - Oak Pale Adhesive Film - by Fablon","color":"by Fablon","image":"","specs":{},"attempts":0,"note":"resurrected 2026-07-13 (was infra-failed)"}
-{"vendor":"brewster","sku":"FAB10100","url":"https://www.brewsterwallcovering.com/fab10100-carrara-black-adhesive-film","status":"pending","brand":"Jeffrey Stevens","name":"FAB10100 - Carrara Black Adhesive Film - by Fablon","color":"by Fablon","image":"","specs":{},"attempts":0,"note":"resurrected 2026-07-13 (was infra-failed)"}
-{"vendor":"brewster","sku":"FAB10128","url":"https://www.brewsterwallcovering.com/fab10128-carrara-grey-adhesive-film","status":"pending","brand":"Jeffrey Stevens","name":"FAB10128 - Carrara Grey Adhesive Film - by Fablon","color":"by Fablon","image":"","specs":{},"attempts":0,"note":"resurrected 2026-07-13 (was infra-failed)"}
-{"vendor":"brewster","sku":"FAB10132","url":"https://www.brewsterwallcovering.com/fab10132-zebra-adhesive-film","status":"pending","brand":"Jeffrey Stevens","name":"Zebra Adhesive Film - Fablon Sticky Back Plastic","color":"","image":"","specs":{},"attempts":0,"note":"resurrected 2026-07-13 (was infra-failed)"}
-{"vendor":"brewster","sku":"FAB10134","url":"https://www.brewsterwallcovering.com/fab10134-heritage-silver-adhesive-film","status":"pending","brand":"Jeffrey Stevens","name":"FAB10134 - Heritage Silver Adhesive Film - by Fablon","color":"by Fablon","image":"","specs":{},"attempts":0,"note":"resurrected 2026-07-13 (was infra-failed)"}
-{"vendor":"brewster","sku":"FAB10158","url":"https://www.brewsterwallcovering.com/fab10158-hessian-adhesive-film","status":"pending","brand":"Jeffrey Stevens","name":"FAB10158 - Hessian Adhesive Film - by Fablon","color":"by Fablon","image":"","specs":{},"attempts":0,"note":"resurrected 2026-07-13 (was infra-failed)"}
-{"vendor":"brewster","sku":"FAB10192","url":"https://www.brewsterwallcovering.com/fab10192-pebbles-adhesive-film","status":"pending","brand":"Jeffrey Stevens","name":"Pebbles Adhesive Film - Fablon Sticky Back Plastic","color":"","image":"","specs":{},"attempts":0,"note":"resurrected 2026-07-13 (was infra-failed)"}
-{"vendor":"brewster","sku":"FAB10222","url":"https://www.brewsterwallcovering.com/fab10222-brick-wall-adhesive-film","status":"pending","brand":"Jeffrey Stevens","name":"Brick Wall Adhesive Film - Fablon Sticky Back Plastic","color":"","image":"","specs":{},"attempts":0,"note":"resurrected 2026-07-13 (was infra-failed)"}
-{"vendor":"brewster","sku":"FAB10224","url":"https://www.brewsterwallcovering.com/fab10224-romance-floral-adhesive-film","status":"pending","brand":"Jeffrey Stevens","name":"Romance Floral Adhesive Film - Fablon Sticky Back Plastic","color":"","image":"","specs":{},"attempts":0,"note":"resurrected 2026-07-13 (was infra-failed)"}
+{"vendor":"brewster","sku":"FAB10044","url":"https://www.brewsterwallcovering.com/fab10044-black-adhesive-film","status":"done","brand":"Jeffrey Stevens","name":"FAB10044 - Black Adhesive Film - by Fablon","color":"by Fablon","image":"","specs":{},"attempts":1,"note":"https://admin.shopify.com/store/designer-laboratory-sandbox/products/7883947835443","doneAt":"2026-07-13T18:56:10.428Z"}
+{"vendor":"brewster","sku":"FAB10047","url":"https://www.brewsterwallcovering.com/fab10047-transparent-glossy-adhesive-film","status":"done","brand":"Jeffrey Stevens","name":"FAB10047 - Transparent Glossy Adhesive Film - by Fablon","color":"by Fablon","image":"","specs":{},"attempts":1,"note":"https://admin.shopify.com/store/designer-laboratory-sandbox/products/7883947999283","doneAt":"2026-07-13T18:57:16.380Z"}
+{"vendor":"brewster","sku":"FAB10048","url":"https://www.brewsterwallcovering.com/fab10048-white-matte-adhesive-film","status":"done","brand":"Jeffrey Stevens","name":"FAB10048 - White Matte Adhesive Film - by Fablon","color":"by Fablon","image":"","specs":{},"attempts":1,"note":"https://admin.shopify.com/store/designer-laboratory-sandbox/products/7883948228659","doneAt":"2026-07-13T18:58:38.334Z"}
+{"vendor":"brewster","sku":"FAB10056","url":"https://www.brewsterwallcovering.com/fab10056-black-matte-adhesive-film","status":"done","brand":"Jeffrey Stevens","name":"FAB10056 - Black Matte Adhesive Film - by Fablon","color":"by Fablon","image":"","specs":{},"attempts":1,"note":"https://admin.shopify.com/store/designer-laboratory-sandbox/products/7883949703219","doneAt":"2026-07-13T18:59:52.508Z"}
+{"vendor":"brewster","sku":"FAB10064","url":"https://www.brewsterwallcovering.com/fab10064-mahogany-adhesive-film","status":"done","brand":"Jeffrey Stevens","name":"FAB10064 - Mahogany Adhesive Film - by Fablon","color":"by Fablon","image":"","specs":{},"attempts":1,"note":"https://admin.shopify.com/store/designer-laboratory-sandbox/products/7883949834291","doneAt":"2026-07-13T19:01:02.872Z"}
+{"vendor":"brewster","sku":"FAB10070","url":"https://www.brewsterwallcovering.com/fab10070-oak-pale-adhesive-film","status":"done","brand":"Jeffrey Stevens","name":"FAB10070 - Oak Pale Adhesive Film - by Fablon","color":"by Fablon","image":"","specs":{},"attempts":1,"note":"https://admin.shopify.com/store/designer-laboratory-sandbox/products/7883950555187","doneAt":"2026-07-13T19:02:10.679Z"}
+{"vendor":"brewster","sku":"FAB10100","url":"https://www.brewsterwallcovering.com/fab10100-carrara-black-adhesive-film","status":"done","brand":"Jeffrey Stevens","name":"FAB10100 - Carrara Black Adhesive Film - by Fablon","color":"by Fablon","image":"","specs":{},"attempts":1,"note":"https://admin.shopify.com/store/designer-laboratory-sandbox/products/7883950882867","doneAt":"2026-07-13T19:03:42.546Z"}
+{"vendor":"brewster","sku":"FAB10128","url":"https://www.brewsterwallcovering.com/fab10128-carrara-grey-adhesive-film","status":"done","brand":"Jeffrey Stevens","name":"FAB10128 - Carrara Grey Adhesive Film - by Fablon","color":"by Fablon","image":"","specs":{},"attempts":1,"note":"https://admin.shopify.com/store/designer-laboratory-sandbox/products/7883951243315","doneAt":"2026-07-13T19:05:01.087Z"}
+{"vendor":"brewster","sku":"FAB10132","url":"https://www.brewsterwallcovering.com/fab10132-zebra-adhesive-film","status":"done","brand":"Jeffrey Stevens","name":"Zebra Adhesive Film - Fablon Sticky Back Plastic","color":"","image":"","specs":{},"attempts":1,"note":"https://admin.shopify.com/store/designer-laboratory-sandbox/products/7883951472691","doneAt":"2026-07-13T19:06:17.815Z"}
+{"vendor":"brewster","sku":"FAB10134","url":"https://www.brewsterwallcovering.com/fab10134-heritage-silver-adhesive-film","status":"done","brand":"Jeffrey Stevens","name":"FAB10134 - Heritage Silver Adhesive Film - by Fablon","color":"by Fablon","image":"","specs":{},"attempts":1,"note":"https://admin.shopify.com/store/designer-laboratory-sandbox/products/7883951767603","doneAt":"2026-07-13T19:07:34.617Z"}
+{"vendor":"brewster","sku":"FAB10158","url":"https://www.brewsterwallcovering.com/fab10158-hessian-adhesive-film","status":"done","brand":"Jeffrey Stevens","name":"FAB10158 - Hessian Adhesive Film - by Fablon","color":"by Fablon","image":"","specs":{},"attempts":1,"note":"https://admin.shopify.com/store/designer-laboratory-sandbox/products/7883952357427","doneAt":"2026-07-13T19:08:58.819Z"}
+{"vendor":"brewster","sku":"FAB10192","url":"https://www.brewsterwallcovering.com/fab10192-pebbles-adhesive-film","status":"done","brand":"Jeffrey Stevens","name":"Pebbles Adhesive Film - Fablon Sticky Back Plastic","color":"","image":"","specs":{},"attempts":1,"note":"https://admin.shopify.com/store/designer-laboratory-sandbox/products/7883956846643","doneAt":"2026-07-13T19:22:44.266Z"}
+{"vendor":"brewster","sku":"FAB10222","url":"https://www.brewsterwallcovering.com/fab10222-brick-wall-adhesive-film","status":"done","brand":"Jeffrey Stevens","name":"Brick Wall Adhesive Film - Fablon Sticky Back Plastic","color":"","image":"","specs":{},"attempts":1,"note":"https://admin.shopify.com/store/designer-laboratory-sandbox/products/7883957600307","doneAt":"2026-07-13T19:24:08.431Z"}
+{"vendor":"brewster","sku":"FAB10224","url":"https://www.brewsterwallcovering.com/fab10224-romance-floral-adhesive-film","status":"pending","brand":"Jeffrey Stevens","name":"Romance Floral Adhesive Film - Fablon Sticky Back Plastic","color":"","image":"","specs":{},"attempts":0,"note":"socket hang up (env failure — attempt refunded)"}
 {"vendor":"brewster","sku":"FAB10226","url":"https://www.brewsterwallcovering.com/fab10226-stone-wall-adhesive-film","status":"pending","brand":"Jeffrey Stevens","name":"Stone Wall Adhesive Film - Fablon Sticky Back Plastic","color":"","image":"","specs":{},"attempts":0,"note":"resurrected 2026-07-13 (was infra-failed)"}
 {"vendor":"brewster","sku":"FAB10247","url":"https://www.brewsterwallcovering.com/fab10247-black-and-white-damask-adhesive-film","status":"pending","brand":"Jeffrey Stevens","name":"Black &amp; White Damask Adhesive Film - Fablon Sticky Back Plastic","color":"","image":"","specs":{},"attempts":0,"note":"resurrected 2026-07-13 (was infra-failed)"}
 {"vendor":"brewster","sku":"FAB10276","url":"https://www.brewsterwallcovering.com/fab10276-clouds-window-film","status":"pending","brand":"Jeffrey Stevens","name":"FAB10276 - Clouds Window Film - by Fablon","color":"by Fablon","image":"","specs":{},"attempts":0,"note":"resurrected 2026-07-13 (was infra-failed)"}
diff --git a/DW-Programming/ImportNewSkufromURL/scripts/run-all-vendor-scrapers.ts b/DW-Programming/ImportNewSkufromURL/scripts/run-all-vendor-scrapers.ts
index 3afe0df7..65571379 100644
--- a/DW-Programming/ImportNewSkufromURL/scripts/run-all-vendor-scrapers.ts
+++ b/DW-Programming/ImportNewSkufromURL/scripts/run-all-vendor-scrapers.ts
@@ -44,6 +44,38 @@ const PRIORITY_VENDORS = [
   "zimmer-rohde",
 ];
 
+// ── Monthly staggered refresh mode (2026-07-13) ──────────────────────────────
+// REFRESH_MODE=monthly → scrape ONLY the vendors whose stable day-of-month == today,
+// drawn from the healthcheck's VERIFIED set (not the hardcoded PRIORITY list). Each
+// vendor still runs through its OWN <vendor>-new-products-scraper.ts (per Steve's
+// every-vendor-own-scraper rule). Spreads the fleet across the month so no single day
+// hammers the machine. Set MONTHLY_DAY=N to force a day; MONTHLY_ALL=1 to run all
+// verified regardless of day (for a manual full pass). Default (unset) = unchanged
+// PRIORITY_VENDORS behavior.
+function hashDay(id: string): number {
+  let h = 0;
+  for (const c of id) h = (h * 31 + c.charCodeAt(0)) >>> 0;
+  return (h % 28) + 1; // 1..28 (every month has these days)
+}
+function resolveVendorList(): string[] {
+  if (process.env.REFRESH_MODE !== "monthly") return PRIORITY_VENDORS;
+  const statusPath = path.join(__dirname, "..", "data", "scraper-audit", "scraper-status.json");
+  let verified: string[] = [];
+  try {
+    const j = JSON.parse(fs.readFileSync(statusPath, "utf8"));
+    verified = Object.entries(j.scrapers || {})
+      .filter(([, v]: any) => v && (v.status === "verified" || v.status === "created-verified"))
+      .map(([id]) => id);
+  } catch (e: any) {
+    console.error("monthly refresh: could not read scraper-status.json —", e.message);
+    return [];
+  }
+  if (process.env.MONTHLY_ALL === "1") return verified;
+  const today = process.env.MONTHLY_DAY ? parseInt(process.env.MONTHLY_DAY, 10) : new Date().getDate();
+  const day = today > 28 ? 28 : today; // fold 29-31 into 28 so those vendors still run
+  return verified.filter((id) => hashDay(id) === day);
+}
+
 async function downloadImage(url: string, referer: string): Promise<string | null> {
   if (!url || url.startsWith("data:")) return null;
   try {
@@ -191,9 +223,16 @@ async function run() {
   let totalProducts = 0;
   let successfulVendors = 0;
 
-  for (let i = 0; i < PRIORITY_VENDORS.length; i++) {
-    const vendorId = PRIORITY_VENDORS[i];
-    console.log(`\n[${i + 1}/${PRIORITY_VENDORS.length}] 🔍 ${vendorId.toUpperCase()}`);
+  const VENDORS = resolveVendorList();
+  if (process.env.REFRESH_MODE === "monthly") {
+    console.log(`🗓️  Monthly refresh — day ${process.env.MONTHLY_DAY || new Date().getDate()}: ${VENDORS.length} vendor(s) due today${process.env.MONTHLY_ALL === "1" ? " (ALL verified)" : ""}`);
+    if (VENDORS.length === 0) { console.log("   nothing scheduled today — done."); await pool.end(); return; }
+    if (process.env.DRY_RUN === "1") { VENDORS.forEach((v) => console.log(`   would refresh: ${v}`)); await pool.end(); return; }
+  }
+
+  for (let i = 0; i < VENDORS.length; i++) {
+    const vendorId = VENDORS[i];
+    console.log(`\n[${i + 1}/${VENDORS.length}] 🔍 ${vendorId.toUpperCase()}`);
 
     const result = await runVendorScraper(vendorId);
     if (result.success) {
diff --git a/DW-Programming/ImportNewSkufromURL/scripts/scraper-healthcheck.ts b/DW-Programming/ImportNewSkufromURL/scripts/scraper-healthcheck.ts
index a8d819b7..86c77d51 100644
--- a/DW-Programming/ImportNewSkufromURL/scripts/scraper-healthcheck.ts
+++ b/DW-Programming/ImportNewSkufromURL/scripts/scraper-healthcheck.ts
@@ -28,6 +28,10 @@ const TIMEOUT_MS = parseInt(process.env.HC_TIMEOUT_MS || '180000', 10);
 function loadStatus(): any {
   try { return JSON.parse(fs.readFileSync(STATUS, 'utf8')); } catch { return { scrapers: {} }; }
 }
+// Not real external vendors — the DW store itself (returns 0 products by design).
+// Excluded so they don't pollute the broken count every run (2026-07-13).
+const NON_VENDOR = new Set(['designer-wallcoverings']);
+
 function loadVendorUrls(): Array<{ id: string; url: string }> {
   const out: Array<{ id: string; url: string }> = [];
   try {
@@ -35,6 +39,7 @@ function loadVendorUrls(): Array<{ id: string; url: string }> {
       const i = l.indexOf('|');
       const id = i > 0 ? l.slice(0, i) : l;
       const url = i > 0 ? l.slice(i + 1).trim() : '';
+      if (NON_VENDOR.has(id)) return; // DW store itself, not a scrapeable vendor
       // Only vendors that have a real scraper to exercise.
       if (fs.existsSync(path.join(SCRAPER_DIR, `${id}-new-products-scraper.ts`))) out.push({ id, url });
     });

← 5f5d633d auto-save: 2026-07-13T11:54:48 (5 files) — DW-Programming/Im  ·  back to Designer Wallcoverings  ·  banned-word backstop extended to TAGS at the drain chokepoin 314eb729 →