probe shared vendor pairs without downloading duplicate feeds

30003d2cc40bb5b13daf38b387da5467e3561846 · 2026-08-28 09:43:43 -0700 · Steve Abrams

Files touched

M scripts/crawl-microsites.js

Diff

commit 30003d2cc40bb5b13daf38b387da5467e3561846
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Fri Aug 28 09:43:43 2026 -0700

    probe shared vendor pairs without downloading duplicate feeds
---
 scripts/crawl-microsites.js | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/scripts/crawl-microsites.js b/scripts/crawl-microsites.js
index 07fe9cd..d4fe048 100644
--- a/scripts/crawl-microsites.js
+++ b/scripts/crawl-microsites.js
@@ -259,14 +259,14 @@ function isAstekHost(url) {
   try { return new URL(url).hostname.toLowerCase() === 'astek.designerwallcoverings.com'; }
   catch { return false; }
 }
-async function getText(url, ms) {
+async function getText(url, ms, options = {}) {
   const headers = { 'user-agent': 'all-dw-crawler/1.0 (+all.designerwallcoverings.com)' };
   if (MICROSITE_AUTH && isInternalHost(url)) headers.Authorization = MICROSITE_AUTH;
   if (CF_ACCESS_ASTEK_ID && CF_ACCESS_ASTEK_SECRET && isAstekHost(url)) {
     headers['CF-Access-Client-Id'] = CF_ACCESS_ASTEK_ID;
     headers['CF-Access-Client-Secret'] = CF_ACCESS_ASTEK_SECRET;
   }
-  const r = await fetch(url, { redirect: 'follow', signal: AbortSignal.timeout(ms), headers });
+  const r = await fetch(url, { ...options, redirect: 'follow', signal: AbortSignal.timeout(ms), headers });
   return { status: r.status, ok: r.ok, finalUrl: r.url, body: await r.text() };
 }
 
@@ -401,7 +401,10 @@ async function crawlOne(seed) {
     handles: [], handleCount: 0,
   };
   try {
-    const root = await getText(rec.url, ROOT_TIMEOUT_MS);
+    // Unified pairs only need endpoint reachability here. Their catalog membership/count
+    // already comes from dw_unified; downloading the same large shared feed 1,022 times
+    // would waste bandwidth and delay every audit.
+    const root = await getText(rec.url, ROOT_TIMEOUT_MS, seed.variant ? { method: 'HEAD' } : {});
     rec.status = root.status; rec.finalUrl = root.finalUrl;
     // 401 = healthy-but-gated (Steve's convention); still "up".
     rec.up = root.ok || root.status === 401;
@@ -416,7 +419,10 @@ async function crawlOne(seed) {
 
   // A co-located microsite (feedBase = localhost) is readable even if its PUBLIC host
   // has no DNS yet (rec.up=false) — so always try the localhost feed when feedBase is set.
-  if (feedBase || (rec.up && rec.status !== 401)) {
+  if (seed.variant && rec.up) {
+    rec.productCount = seed.activeProductsDB || 0;
+    rec.feed = rec.productCount > 0;
+  } else if (feedBase || (rec.up && rec.status !== 401)) {
     const f = await feedProducts(host, feedBase);
     if (f.has) {
       rec.feed = true;

← 6032ba8 serve every unified vendor through shared host endpoints · back to All Designerwallcoverings · keep internal vendor hostnames within DNS limits 9cd80ae →