Auto-discover all co-located pm2 vendor viewers + read /api/skus feed → 49→156 vendors live in all.dw (fulfills 'every vendor viewer must be live'); private-label sources stay BANNED

80c128c7299f6137431a4b3629488ad5213b4b01 · 2026-07-14 13:11:20 -0700 · Steve Abrams
Files touched

M scripts/crawl-microsites.js
Diff

commit 80c128c7299f6137431a4b3629488ad5213b4b01
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Tue Jul 14 13:11:20 2026 -0700

    Auto-discover all co-located pm2 vendor viewers + read /api/skus feed → 49→156 vendors live in all.dw (fulfills 'every vendor viewer must be live'); private-label sources stay BANNED
---
 scripts/crawl-microsites.js | 69 +++++++++++++++++++++++++++++++++++++++------
 1 file changed, 61 insertions(+), 8 deletions(-)

diff --git a/scripts/crawl-microsites.js b/scripts/crawl-microsites.js
index 88f62dd..45cd86d 100644
--- a/scripts/crawl-microsites.js
+++ b/scripts/crawl-microsites.js
@@ -48,10 +48,42 @@ function seedFromRegistry() {
   const reg = JSON.parse(fs.readFileSync(SEED, 'utf8'));
   const out = [];
   for (const b of reg.brands || []) out.push({ slug: b.slug, vendor: b.vendor || null, type: 'brand', note: b.note || null, internalViewer: b.internalViewer || null });
-  for (const i of reg.internal || []) out.push({ slug: i.slug, vendor: null, type: 'internal', note: i.label || null });
+  for (const i of reg.internal || []) out.push({ slug: i.slug, vendor: null, type: 'internal', note: i.label || null, internalViewer: i.internalViewer || null });
   return out;
 }
 
+// Auto-discover EVERY co-located vendor viewer running on this host and seed its
+// localhost feed — so any *.DW / internal.*.DW viewer that is up is crawled with
+// ZERO registry upkeep (Steve's standing rule: all vendor viewers must be live in all.dw).
+// The norm-fold in mergeSeeds collapses these onto their public CF twin (no duplicates).
+function seedFromPm2() {
+  try {
+    const cp = require('child_process');
+    const procs = JSON.parse(cp.execSync('pm2 jlist', { encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 }));
+    const lsof = cp.execSync('lsof -nP -iTCP -sTCP:LISTEN 2>/dev/null || true', { encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 });
+    const pidPort = new Map();
+    for (const line of lsof.split('\n')) {
+      const m = line.match(/^\S+\s+(\d+)\s.*?:(\d+)\s+\(LISTEN\)/);
+      if (m && !pidPort.has(m[1])) pidPort.set(m[1], m[2]);
+    }
+    const SUFFIX = /-(viewer|internal|landing|showroom|house|site)$/;
+    const out = [];
+    for (const p of procs) {
+      const name = p.name || '';
+      const cwd = (p.pm2_env && p.pm2_env.pm_cwd) || '';
+      const isViewer = SUFFIX.test(name) || /dw-vendor-microsites\/vendors\//.test(cwd);
+      if (!isViewer) continue;
+      const port = pidPort.get(String(p.pid));
+      if (!port) continue;
+      const slug = name.replace(SUFFIX, '').replace(/_/g, '-').toLowerCase();
+      if (!slug) continue;
+      out.push({ slug, vendor: null, type: 'pm2', internalViewer: `http://127.0.0.1:${port} (pm2 ${name})` });
+    }
+    console.log(`seedFromPm2: ${out.length} co-located vendor viewers discovered`);
+    return out;
+  } catch (e) { console.warn('seedFromPm2 skipped:', e.message); return []; }
+}
+
 async function seedFromDb() {
   // Best-effort: candidate <slug>.designerwallcoverings.com for every vendor with
   // active products. Unreachable candidates simply drop out of the crawl.
@@ -145,21 +177,32 @@ async function seedFromCfZone() {
   return out;
 }
 
-function mergeSeeds(reg, db, cf) {
+function mergeSeeds(reg, db, cf, pm2) {
   const bySlug = new Map();
   // A registry brand can pin a canonical slug for a vendor whose auto-slug differs
   // (brand slug "pierre-frey" vs slugify("Pierre Frey")="pierrefrey"). Fold the
   // DB-derived twin into the registry entry by vendor so we don't emit a dark duplicate.
   const vendorToSlug = new Map();
   for (const s of reg) if (s.vendor) vendorToSlug.set(String(s.vendor).toLowerCase(), s.slug);
+  // Also fold hyphen/underscore twins (pm2 "andrew-martin" vs CF "andrewmartin") onto one
+  // canonical key so a co-located pm2 feed ENRICHES the public twin instead of duplicating it.
+  const norm = (v) => String(v || '').toLowerCase().replace(/[^a-z0-9]/g, '');
+  const normToKey = new Map();
   // Order matters: registry first (richest metadata + canonical slugs), then DB
-  // (vendor twins), then CF-zone (authoritative fill for everything else). Earlier
-  // sources win on key collisions via the `...s, ...prev` merge below.
-  for (const s of [...reg, ...db, ...(cf || [])]) {
+  // (vendor twins), then CF-zone (authoritative fill), then pm2 (localhost feeds).
+  // Earlier sources win on identity fields; internalViewer is FILLED from whoever has it.
+  for (const s of [...reg, ...db, ...(cf || []), ...(pm2 || [])]) {
     if (!s.slug || BANNED.test(s.slug) || BANNED.test(s.vendor || '')) continue;
-    const key = (s.vendor && vendorToSlug.get(String(s.vendor).toLowerCase())) || s.slug;
+    const n = norm(s.slug);
+    const key = (s.vendor && vendorToSlug.get(String(s.vendor).toLowerCase())) || normToKey.get(n) || s.slug;
+    if (!normToKey.has(n)) normToKey.set(n, key);
     const prev = bySlug.get(key);
-    bySlug.set(key, prev ? { ...s, ...prev, activeProductsDB: prev.activeProductsDB ?? s.activeProductsDB, dbSampleImage: prev.dbSampleImage ?? s.dbSampleImage } : s);
+    bySlug.set(key, prev
+      ? { ...s, ...prev,
+          internalViewer: prev.internalViewer || s.internalViewer || null,
+          activeProductsDB: prev.activeProductsDB ?? s.activeProductsDB,
+          dbSampleImage: prev.dbSampleImage ?? s.dbSampleImage }
+      : s);
   }
   return [...bySlug.values()];
 }
@@ -263,6 +306,16 @@ async function feedProducts(host, feedBase) {
       const arr = Array.isArray(j.rows) ? j.rows : Array.isArray(j.products) ? j.products : null;
       if (arr && arr.length) return buildFeedResult(arr, host, j.total ?? j.count ?? null, false);
     }
+  } catch { /* fall through */ }
+  // 1b. dw-vendor-microsites line-viewer family serves its catalog at /api/skus
+  //     → shape { total, rows:[{sku, mfr_sku, pattern_name, …}] }.
+  try {
+    const { status, body } = await getText(`${dwBase}/api/skus?limit=100000&all=1`, FULL_FEED_TIMEOUT_MS);
+    if (status === 200) {
+      const j = JSON.parse(body);
+      const arr = Array.isArray(j.rows) ? j.rows : Array.isArray(j.skus) ? j.skus : null;
+      if (arr && arr.length) return buildFeedResult(arr, host, j.total ?? j.all ?? j.count ?? null, false);
+    }
   } catch { /* fall through to Shopify */ }
   // 2. Shopify storefront — follow /products.json to completion.
   try {
@@ -345,7 +398,7 @@ async function pool(items, worker, n) {
 async function crawlMicrosites() {
   const t0 = Date.now();
   const [db, cf] = await Promise.all([seedFromDb(), seedFromCfZone()]);
-  const seeds = mergeSeeds(seedFromRegistry(), db, cf);
+  const seeds = mergeSeeds(seedFromRegistry(), db, cf, seedFromPm2());
   const sites = await pool(seeds, crawlOne, CONCURRENCY);
 
   const up = sites.filter((s) => s.up);
← 2a6cffc chore: v1.4.1 (session close) — vendor facet alpha-sort · back to All Designerwallcoverings · chore: v1.5.0 (session close — pm2 viewer auto-discovery + / 9317a40 →