[object Object]

← back to All Designerwallcoverings

microsite crawler: seed from Cloudflare zone (authoritative, self-completing) + register -internal viewers

9d3836efe58d2a79d123a2804d4d96500dd8d20a · 2026-07-09 16:38:57 -0700 · Steve

- add cfZoneSubdomains() 3rd seed source: fetch all A/CNAME for zone e4b88c70,
  derive <sub>.designerwallcoverings.com, merge with known+DB seeds (dedup)
- guarded: missing token / CF failure logs a warning + falls back to existing seeds
- denylist _-prefixed / _domainconnect / _dmarc / apex; www kept out (apex dup)
- token via env (CLOUDFLARE_API_TOKEN) + MICROSITE_CF_ZONE default = the zone
- register fromental/crezana/gracie/zuber -internal viewers in internal[]
- fix stale internalViewer ports 10071-10074 -> real Kamatera 10103-10106

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Files touched

Diff

commit 9d3836efe58d2a79d123a2804d4d96500dd8d20a
Author: Steve <steve@designerwallcoverings.com>
Date:   Thu Jul 9 16:38:57 2026 -0700

    microsite crawler: seed from Cloudflare zone (authoritative, self-completing) + register -internal viewers
    
    - add cfZoneSubdomains() 3rd seed source: fetch all A/CNAME for zone e4b88c70,
      derive <sub>.designerwallcoverings.com, merge with known+DB seeds (dedup)
    - guarded: missing token / CF failure logs a warning + falls back to existing seeds
    - denylist _-prefixed / _domainconnect / _dmarc / apex; www kept out (apex dup)
    - token via env (CLOUDFLARE_API_TOKEN) + MICROSITE_CF_ZONE default = the zone
    - register fromental/crezana/gracie/zuber -internal viewers in internal[]
    - fix stale internalViewer ports 10071-10074 -> real Kamatera 10103-10106
    
    Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 config/known-subdomains.json | 28 +++++++++++++---
 scripts/crawl-microsites.js  | 77 +++++++++++++++++++++++++++++++++++++++++---
 2 files changed, 97 insertions(+), 8 deletions(-)

diff --git a/config/known-subdomains.json b/config/known-subdomains.json
index 935902c..23ec431 100644
--- a/config/known-subdomains.json
+++ b/config/known-subdomains.json
@@ -54,7 +54,7 @@
    "knownProject": "~/Projects/dw-vendor-microsites/vendors/fromental",
    "note": "ONBOARDING/STAGING 2026-07-09 — ultra-luxury hand-painted & embroidered scenic wallpaper + fabric + artwork. 735 SKUs in fromental_catalog (feed-first Shopify products.json, $0). QUOTE-ONLY (179 priced artwork/shop, 556 bespoke). Customer-facing brand (not private-label). Shopify DRAFT import gated (DWFR-, settlement-gate scenic/bird imagery first). Microsite deploy + DNS gated.",
    "hasMicrosite": true,
-   "internalViewer": "http://127.0.0.1:10071 (~/Projects/fromental-internal, pm2 fromental-internal)"
+   "internalViewer": "http://127.0.0.1:10103 (~/Projects/fromental-internal, pm2 fromental-internal)"
   },
   {
    "slug": "gracie",
@@ -62,7 +62,7 @@
    "knownProject": "~/Projects/dw-vendor-microsites/vendors/gracie",
    "note": "ONBOARDING/STAGING 2026-07-09 — ultra-luxury hand-painted scenic & chinoiserie wallpaper panels (Gracie Studio, est. 1898). 150 patterns in gracie_catalog (feed-first Next.js/Sanity page-data, $0). QUOTE-ONLY bespoke. Shopify DRAFT import gated (DWGA-, MANDATORY settlement-gate — chinoiserie is bird/butterfly/botanical heavy). Microsite deploy + DNS gated.",
    "hasMicrosite": true,
-   "internalViewer": "http://127.0.0.1:10073 (~/Projects/gracie-internal, pm2 gracie-internal)"
+   "internalViewer": "http://127.0.0.1:10105 (~/Projects/gracie-internal, pm2 gracie-internal)"
   },
   {
    "slug": "zuber",
@@ -70,7 +70,7 @@
    "knownProject": "~/Projects/dw-vendor-microsites/vendors/zuber",
    "note": "ONBOARDING/STAGING 2026-07-09 — historic French house of hand-block-printed antique scenic/panoramic wallpaper (Zuber & Cie). 192 SKUs in zuber_catalog (feed-first sitemap.xml → plain-fetch pages, $0). QUOTE-ONLY heritage made-to-order. Shopify DRAFT import gated (DWZB-, MANDATORY settlement-gate — scenics are botanical/floral/bird heavy). Microsite deploy + DNS gated.",
    "hasMicrosite": true,
-   "internalViewer": "http://127.0.0.1:10074 (~/Projects/zuber-internal, pm2 zuber-internal)"
+   "internalViewer": "http://127.0.0.1:10106 (~/Projects/zuber-internal, pm2 zuber-internal)"
   },
   {
    "slug": "crezana",
@@ -78,7 +78,7 @@
    "knownProject": "~/Projects/dw-vendor-microsites/vendors/crezana",
    "note": "ONBOARDING/STAGING 2026-07-09 — Crezana Design textile/fabric line (product_type Fabric). 443 gallery images in crezana_catalog (feed-first GoDaddy gpub bundles, $0). NOTE: source is an editorial GoDaddy gallery with NO SKUs/prices/specs — synthetic names, QUOTE-ONLY; real activation BLOCKED on a Crezana line sheet. Shopify DRAFT import + microsite deploy + DNS gated.",
    "hasMicrosite": true,
-   "internalViewer": "http://127.0.0.1:10072 (~/Projects/crezana-internal, pm2 crezana-internal)"
+   "internalViewer": "http://127.0.0.1:10104 (~/Projects/crezana-internal, pm2 crezana-internal)"
   }
  ],
  "internal": [
@@ -102,6 +102,26 @@
   {
    "slug": "all",
    "label": "This directory site (all-dw-landing)"
+  },
+  {
+   "slug": "fromental-internal",
+   "label": "Fromental — internal curation viewer",
+   "note": "dw-vendor-microsites fromental internal viewer; Kamatera pm2 fromental-internal on :10103"
+  },
+  {
+   "slug": "crezana-internal",
+   "label": "Crezana Design — internal curation viewer",
+   "note": "dw-vendor-microsites crezana internal viewer; Kamatera pm2 crezana-internal on :10104"
+  },
+  {
+   "slug": "gracie-internal",
+   "label": "Gracie — internal curation viewer",
+   "note": "dw-vendor-microsites gracie internal viewer; Kamatera pm2 gracie-internal on :10105"
+  },
+  {
+   "slug": "zuber-internal",
+   "label": "Zuber — internal curation viewer",
+   "note": "dw-vendor-microsites zuber internal viewer; Kamatera pm2 zuber-internal on :10106"
   }
  ]
 }
\ No newline at end of file
diff --git a/scripts/crawl-microsites.js b/scripts/crawl-microsites.js
index 9b038e4..4f67b92 100644
--- a/scripts/crawl-microsites.js
+++ b/scripts/crawl-microsites.js
@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 // crawl-microsites.js — the live-crawl half of all.designerwallcoverings.com.
 //
-// Polls every KNOWN + DB-derived *.designerwallcoverings.com subdomain, records
+// Polls every KNOWN + DB-derived + Cloudflare-zone *.designerwallcoverings.com subdomain, records
 // which are up + their identity (title / hero / product feed), and writes a single
 // snapshot the all- server serves at /api/microsites (directory + merged products).
 //
@@ -22,6 +22,11 @@ const OUT = path.join(__dirname, '..', 'data', 'microsites.json');
 const SEED = path.join(__dirname, '..', 'config', 'known-subdomains.json');
 const BASE = 'designerwallcoverings.com';
 const CONCURRENCY = 8;
+// Cloudflare zone that owns designerwallcoverings.com — the authoritative subdomain
+// list. Seeding from it makes the directory complete now AND self-completing for any
+// future subdomain (no code change needed when a new site's DNS lands).
+const CF_ZONE = process.env.MICROSITE_CF_ZONE || 'e4b88c70e4c949f4556cc693133c2f42';
+const CF_TIMEOUT_MS = 10000;
 const ROOT_TIMEOUT_MS = 8000;
 const FEED_TIMEOUT_MS = 6000;        // per Shopify page / initial feed probe
 const FULL_FEED_TIMEOUT_MS = 15000;  // DW-family full-set fetch (returns everything in one shot)
@@ -80,14 +85,77 @@ async function seedFromDb() {
   } finally { try { await pool.end(); } catch {} }
 }
 
-function mergeSeeds(reg, db) {
+// Resolve the Cloudflare API token: prefer env (how prod gets it), else fall back to
+// the local secrets-manager master .env (so a local `node scripts/crawl-microsites.js`
+// works without exporting anything). Same pattern seedFromDb uses for the DB password.
+function cfToken() {
+  if (process.env.CLOUDFLARE_API_TOKEN) return process.env.CLOUDFLARE_API_TOKEN;
+  try {
+    const env = fs.readFileSync(os.homedir() + '/Projects/secrets-manager/.env', 'utf8');
+    const m = env.match(/^CLOUDFLARE_API_TOKEN=(.*)$/m);
+    if (m) return m[1].replace(/^["']|["']$/g, '').trim();
+  } catch {}
+  return null;
+}
+
+// ── seed source #3: the Cloudflare zone (authoritative subdomain list) ────────────
+// Fetch every A/CNAME record for the zone and derive its <sub>.designerwallcoverings.com
+// slug. This is what makes the directory complete + self-completing. GUARDED: any
+// failure (missing token, network, non-2xx, unexpected shape) logs a warning and
+// returns [] so the crawl NEVER breaks — the known-registry + DB seeds still run.
+async function seedFromCfZone() {
+  const token = cfToken();
+  if (!token) { console.warn('seedFromCfZone: no CLOUDFLARE_API_TOKEN — falling back to known + DB seeds only'); return []; }
+  const skip = new Set(['www', BASE, '']);   // apex dup + bare apex
+  const slugs = new Set();
+  try {
+    for (const rtype of ['A', 'CNAME']) {
+      let page = 1;
+      // per_page=500 pulls the whole zone in one page (~166 records), but paginate defensively.
+      for (;;) {
+        const url = `https://api.cloudflare.com/client/v4/zones/${CF_ZONE}/dns_records?type=${rtype}&per_page=500&page=${page}`;
+        const r = await fetch(url, {
+          signal: AbortSignal.timeout(CF_TIMEOUT_MS),
+          headers: { Authorization: `Bearer ${token}`, 'content-type': 'application/json' },
+        });
+        if (!r.ok) throw new Error(`CF API ${rtype} p${page} HTTP ${r.status}`);
+        const j = await r.json();
+        if (!j || j.success !== true || !Array.isArray(j.result)) throw new Error(`CF API ${rtype} p${page} unexpected shape`);
+        for (const rec of j.result) {
+          const name = String(rec.name || '').toLowerCase();
+          // Extract the label(s) left of the base domain.
+          if (name === BASE) continue;                 // apex
+          if (!name.endsWith('.' + BASE)) continue;    // foreign record
+          const sub = name.slice(0, -('.' + BASE).length);
+          // Denylist: DNS control/verification records, anything starting with "_", apex dup.
+          if (sub.startsWith('_') || sub === '_domainconnect' || sub === '_dmarc' || skip.has(sub)) continue;
+          slugs.add(sub);
+        }
+        const ti = j.result_info && j.result_info.total_pages;
+        if (!ti || page >= ti) break;
+        page++;
+      }
+    }
+  } catch (e) {
+    console.warn('seedFromCfZone failed (using known + DB seeds only):', e.message);
+    return [];
+  }
+  const out = [...slugs].map((slug) => ({ slug, vendor: null, type: 'cf-zone', note: null }));
+  console.log(`seedFromCfZone: ${out.length} subdomains from Cloudflare zone ${CF_ZONE.slice(0, 8)}…`);
+  return out;
+}
+
+function mergeSeeds(reg, db, cf) {
   const bySlug = new Map();
   // A registry brand can pin a canonical slug for a vendor whose auto-slug differs
   // (brand slug "pierre-frey" vs slugify("Pierre Frey")="pierrefrey"). Fold the
   // DB-derived twin into the registry entry by vendor so we don't emit a dark duplicate.
   const vendorToSlug = new Map();
   for (const s of reg) if (s.vendor) vendorToSlug.set(String(s.vendor).toLowerCase(), s.slug);
-  for (const s of [...reg, ...db]) {              // registry first, DB fills gaps
+  // Order matters: registry first (richest metadata + canonical slugs), then DB
+  // (vendor twins), then CF-zone (authoritative fill for everything else). Earlier
+  // sources win on key collisions via the `...s, ...prev` merge below.
+  for (const s of [...reg, ...db, ...(cf || [])]) {
     if (!s.slug || BANNED.test(s.slug) || BANNED.test(s.vendor || '')) continue;
     const key = (s.vendor && vendorToSlug.get(String(s.vendor).toLowerCase())) || s.slug;
     const prev = bySlug.get(key);
@@ -254,7 +322,8 @@ async function pool(items, worker, n) {
 
 async function crawlMicrosites() {
   const t0 = Date.now();
-  const seeds = mergeSeeds(seedFromRegistry(), await seedFromDb());
+  const [db, cf] = await Promise.all([seedFromDb(), seedFromCfZone()]);
+  const seeds = mergeSeeds(seedFromRegistry(), db, cf);
   const sites = await pool(seeds, crawlOne, CONCURRENCY);
 
   const up = sites.filter((s) => s.up);

← c22ea62 all.dw: make microsite feed ports env-overridable (MICROSITE  ·  back to All Designerwallcoverings  ·  auto-save: 2026-07-09T22:12:24 (1 files) — server.js f37876c →