← back to All Designerwallcoverings
scripts/crawl-microsites.js
454 lines
#!/usr/bin/env node
// crawl-microsites.js — the live-crawl half of all.designerwallcoverings.com.
//
// Polls every KNOWN + DB-derived + Cloudflare-zone *.designerwallcoverings.com subdomain, records
// which are up + their identity (title / hero / product feed), and writes a single
// snapshot the all- server serves at /api/microsites (directory + merged products).
//
// Data path is deliberately DIFFERENT from the main product grid: that reads
// dw_unified directly; THIS reads what each microsite actually publishes over HTTP.
// A live crawl only ever sees public pages, so cost/net/wholesale can't leak — but
// we still run the standing BANNED private-label scrub on every display string.
//
// Runs standalone (`node scripts/crawl-microsites.js`) OR in-process from server.js
// (require + crawlMicrosites()). $0 — local HTTP only, no paid APIs.
'use strict';
const fs = require('fs');
const path = require('path');
const os = require('os');
try { require('dotenv').config(); } catch {}
const OUT = path.join(__dirname, '..', 'data', 'microsites.json');
const SEED = path.join(__dirname, '..', 'config', 'known-subdomains.json');
const BASE = 'designerwallcoverings.com';
const CONCURRENCY = 8;
// Cloudflare zone that owns designerwallcoverings.com — the authoritative subdomain
// list. Seeding from it makes the directory complete now AND self-completing for any
// future subdomain (no code change needed when a new site's DNS lands).
const CF_ZONE = process.env.MICROSITE_CF_ZONE || 'e4b88c70e4c949f4556cc693133c2f42';
const CF_TIMEOUT_MS = 10000;
const ROOT_TIMEOUT_MS = 8000;
const FEED_TIMEOUT_MS = 6000; // per Shopify page / initial feed probe
const FULL_FEED_TIMEOUT_MS = 15000; // DW-family full-set fetch (returns everything in one shot)
const SHOPIFY_PAGE_CAP = 40; // /products.json pages (≤10k products) — politeness ceiling
const SAMPLE_PER_SITE = 24; // products carried per site into the merged feed (directory UI)
const MERGED_CAP = 4000; // hard ceiling on the merged products array
// Standing rule: banned upstream/private-label names never ship on a public surface.
const BANNED = /brewster|york|wallquest|wall\s*quest|newwall|new\s*wall\b|command\s*54|justin\s*david|nicolette\s*mayer|seabrook|chesapeake|nextwall|desima|carlsten/i;
// Display scrub: never surface the word "Wallpaper" where the house uses "Wallcovering".
const clean = (s) => (s == null ? s : String(s)
.replace(/\bWallpapers\b/gi, 'Wallcoverings').replace(/\bWallpaper\b/gi, 'Wallcovering'));
const slugify = (v) => String(v || '').toLowerCase().normalize('NFD')
.replace(/[̀-ͯ]/g, '').replace(/&/g, 'and').replace(/[^a-z0-9]/g, '');
// ── seed slug list: hand registry ∪ DB vendor candidates ─────────────────────────
function seedFromRegistry() {
const reg = JSON.parse(fs.readFileSync(SEED, 'utf8'));
const out = [];
for (const b of reg.brands || []) out.push({ slug: b.slug, vendor: b.vendor || null, type: 'brand', note: b.note || null, internalViewer: b.internalViewer || null });
for (const i of reg.internal || []) out.push({ slug: i.slug, vendor: null, type: 'internal', note: i.label || null, internalViewer: i.internalViewer || null });
return out;
}
// Auto-discover EVERY co-located vendor viewer running on this host and seed its
// localhost feed — so any *.DW / internal.*.DW viewer that is up is crawled with
// ZERO registry upkeep (Steve's standing rule: all vendor viewers must be live in all.dw).
// The norm-fold in mergeSeeds collapses these onto their public CF twin (no duplicates).
function seedFromPm2() {
try {
const cp = require('child_process');
const procs = JSON.parse(cp.execSync('pm2 jlist', { encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 }));
const lsof = cp.execSync('lsof -nP -iTCP -sTCP:LISTEN 2>/dev/null || true', { encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 });
const pidPort = new Map();
for (const line of lsof.split('\n')) {
const m = line.match(/^\S+\s+(\d+)\s.*?:(\d+)\s+\(LISTEN\)/);
if (m && !pidPort.has(m[1])) pidPort.set(m[1], m[2]);
}
const SUFFIX = /-(viewer|internal|landing|showroom|house|site)$/;
const out = [];
for (const p of procs) {
const name = p.name || '';
const cwd = (p.pm2_env && p.pm2_env.pm_cwd) || '';
const isViewer = SUFFIX.test(name) || /dw-vendor-microsites\/vendors\//.test(cwd);
if (!isViewer) continue;
const port = pidPort.get(String(p.pid));
if (!port) continue;
const slug = name.replace(SUFFIX, '').replace(/_/g, '-').toLowerCase();
if (!slug) continue;
out.push({ slug, vendor: null, type: 'pm2', internalViewer: `http://127.0.0.1:${port} (pm2 ${name})` });
}
console.log(`seedFromPm2: ${out.length} co-located vendor viewers discovered`);
return out;
} catch (e) { console.warn('seedFromPm2 skipped:', e.message); return []; }
}
async function seedFromDb() {
// Best-effort: candidate <slug>.designerwallcoverings.com for every vendor with
// active products. Unreachable candidates simply drop out of the crawl.
let Pool;
try { ({ Pool } = require('pg')); } catch { return []; }
const PW = (() => {
try {
const env = fs.readFileSync(os.homedir() + '/Projects/secrets-manager/.env', 'utf8');
const m = env.match(/^DW_ADMIN_DB_PASSWORD=(.*)$/m);
if (m) return m[1].replace(/^["']|["']$/g, '').trim();
} catch {}
return process.env.PGPASSWORD || '';
})();
const pool = process.env.DATABASE_URL
? new Pool({ connectionString: process.env.DATABASE_URL, max: 1 })
: new Pool({ connectionString: 'postgresql:///dw_unified?host=/tmp', max: 1, password: PW || undefined });
try {
const { rows } = await pool.query(`
SELECT vendor, COUNT(*)::int AS active_products,
(ARRAY_REMOVE(ARRAY_AGG(image_url ORDER BY created_at_shopify DESC NULLS LAST), NULL))[1] AS sample_image
FROM shopify_products
WHERE UPPER(COALESCE(status,'')) = 'ACTIVE' AND vendor IS NOT NULL AND vendor <> ''
GROUP BY vendor`);
return rows
.filter((r) => !BANNED.test(r.vendor || ''))
.map((r) => ({ slug: slugify(r.vendor), vendor: r.vendor, type: 'vendor', activeProductsDB: r.active_products, dbSampleImage: r.sample_image }))
.filter((r) => r.slug);
} catch (e) {
console.error('seedFromDb skipped:', e.message);
return [];
} finally { try { await pool.end(); } catch {} }
}
// Resolve the Cloudflare API token: prefer env (how prod gets it), else fall back to
// the local secrets-manager master .env (so a local `node scripts/crawl-microsites.js`
// works without exporting anything). Same pattern seedFromDb uses for the DB password.
function cfToken() {
if (process.env.CLOUDFLARE_API_TOKEN) return process.env.CLOUDFLARE_API_TOKEN;
try {
const env = fs.readFileSync(os.homedir() + '/Projects/secrets-manager/.env', 'utf8');
const m = env.match(/^CLOUDFLARE_API_TOKEN=(.*)$/m);
if (m) return m[1].replace(/^["']|["']$/g, '').trim();
} catch {}
return null;
}
// ── seed source #3: the Cloudflare zone (authoritative subdomain list) ────────────
// Fetch every A/CNAME record for the zone and derive its <sub>.designerwallcoverings.com
// slug. This is what makes the directory complete + self-completing. GUARDED: any
// failure (missing token, network, non-2xx, unexpected shape) logs a warning and
// returns [] so the crawl NEVER breaks — the known-registry + DB seeds still run.
async function seedFromCfZone() {
const token = cfToken();
if (!token) { console.warn('seedFromCfZone: no CLOUDFLARE_API_TOKEN — falling back to known + DB seeds only'); return []; }
const skip = new Set(['www', BASE, '']); // apex dup + bare apex
const slugs = new Set();
try {
for (const rtype of ['A', 'CNAME']) {
let page = 1;
// per_page=500 pulls the whole zone in one page (~166 records), but paginate defensively.
for (;;) {
const url = `https://api.cloudflare.com/client/v4/zones/${CF_ZONE}/dns_records?type=${rtype}&per_page=500&page=${page}`;
const r = await fetch(url, {
signal: AbortSignal.timeout(CF_TIMEOUT_MS),
headers: { Authorization: `Bearer ${token}`, 'content-type': 'application/json' },
});
if (!r.ok) throw new Error(`CF API ${rtype} p${page} HTTP ${r.status}`);
const j = await r.json();
if (!j || j.success !== true || !Array.isArray(j.result)) throw new Error(`CF API ${rtype} p${page} unexpected shape`);
for (const rec of j.result) {
const name = String(rec.name || '').toLowerCase();
// Extract the label(s) left of the base domain.
if (name === BASE) continue; // apex
if (!name.endsWith('.' + BASE)) continue; // foreign record
const sub = name.slice(0, -('.' + BASE).length);
// Denylist: DNS control/verification records, anything starting with "_", apex dup.
if (sub.startsWith('_') || sub === '_domainconnect' || sub === '_dmarc' || skip.has(sub)) continue;
slugs.add(sub);
}
const ti = j.result_info && j.result_info.total_pages;
if (!ti || page >= ti) break;
page++;
}
}
} catch (e) {
console.warn('seedFromCfZone failed (using known + DB seeds only):', e.message);
return [];
}
const out = [...slugs].map((slug) => ({ slug, vendor: null, type: 'cf-zone', note: null }));
console.log(`seedFromCfZone: ${out.length} subdomains from Cloudflare zone ${CF_ZONE.slice(0, 8)}…`);
return out;
}
function mergeSeeds(reg, db, cf, pm2) {
const bySlug = new Map();
// A registry brand can pin a canonical slug for a vendor whose auto-slug differs
// (brand slug "pierre-frey" vs slugify("Pierre Frey")="pierrefrey"). Fold the
// DB-derived twin into the registry entry by vendor so we don't emit a dark duplicate.
const vendorToSlug = new Map();
for (const s of reg) if (s.vendor) vendorToSlug.set(String(s.vendor).toLowerCase(), s.slug);
// Also fold hyphen/underscore twins (pm2 "andrew-martin" vs CF "andrewmartin") onto one
// canonical key so a co-located pm2 feed ENRICHES the public twin instead of duplicating it.
const norm = (v) => String(v || '').toLowerCase().replace(/[^a-z0-9]/g, '');
const normToKey = new Map();
// Order matters: registry first (richest metadata + canonical slugs), then DB
// (vendor twins), then CF-zone (authoritative fill), then pm2 (localhost feeds).
// Earlier sources win on identity fields; internalViewer is FILLED from whoever has it.
for (const s of [...reg, ...db, ...(cf || []), ...(pm2 || [])]) {
if (!s.slug || BANNED.test(s.slug) || BANNED.test(s.vendor || '')) continue;
const n = norm(s.slug);
const key = (s.vendor && vendorToSlug.get(String(s.vendor).toLowerCase())) || normToKey.get(n) || s.slug;
if (!normToKey.has(n)) normToKey.set(n, key);
const prev = bySlug.get(key);
bySlug.set(key, prev
? { ...s, ...prev,
internalViewer: prev.internalViewer || s.internalViewer || null,
activeProductsDB: prev.activeProductsDB ?? s.activeProductsDB,
dbSampleImage: prev.dbSampleImage ?? s.dbSampleImage }
: s);
}
return [...bySlug.values()];
}
// ── per-site fetch ───────────────────────────────────────────────────────────────
// Many DW microsites are Basic-Auth gated (the 401 IS the healthy gate). Send the
// shared internal cred (env MICROSITE_BASIC_AUTH="user:pass") so the crawler can read
// their /api/products feed. External Shopify storefronts simply ignore the header.
const MICROSITE_AUTH = process.env.MICROSITE_BASIC_AUTH
? 'Basic ' + Buffer.from(process.env.MICROSITE_BASIC_AUTH).toString('base64')
: null;
// Only attach the internal cred to OUR OWN surfaces — localhost co-located viewers
// and *.designerwallcoverings.com — never broadcast it to a third-party host.
function isInternalHost(url) {
try {
const h = new URL(url).hostname;
return h === '127.0.0.1' || h === 'localhost' || /(^|\.)designerwallcoverings\.com$/i.test(h);
} catch { return false; }
}
// CF Access service-token headers for astek (Zero-Trust pilot — memo 260727A / TK-11).
// Once astek moves behind Cloudflare Access and its nginx auth_basic is dropped, the
// shared Basic-Auth cred no longer authenticates the crawler — the CF Access service
// token does. Scoped to the astek host ONLY (a CF Access service token is per-application).
// Inert + byte-identical (Basic-Auth-only, exactly as today) whenever CF_ACCESS_ASTEK_ID /
// CF_ACCESS_ASTEK_SECRET are unset. During the transition window (Access enforced but the
// origin nginx auth_basic still present) BOTH headers ride — satisfying the CF edge AND
// the origin gate — and once auth_basic is dropped the leftover Basic header is harmless.
const CF_ACCESS_ASTEK_ID = process.env.CF_ACCESS_ASTEK_ID || '';
const CF_ACCESS_ASTEK_SECRET = process.env.CF_ACCESS_ASTEK_SECRET || '';
function isAstekHost(url) {
try { return new URL(url).hostname.toLowerCase() === 'astek.designerwallcoverings.com'; }
catch { return false; }
}
async function getText(url, ms) {
const headers = { 'user-agent': 'all-dw-crawler/1.0 (+all.designerwallcoverings.com)' };
if (MICROSITE_AUTH && isInternalHost(url)) headers.Authorization = MICROSITE_AUTH;
if (CF_ACCESS_ASTEK_ID && CF_ACCESS_ASTEK_SECRET && isAstekHost(url)) {
headers['CF-Access-Client-Id'] = CF_ACCESS_ASTEK_ID;
headers['CF-Access-Client-Secret'] = CF_ACCESS_ASTEK_SECRET;
}
const r = await fetch(url, { redirect: 'follow', signal: AbortSignal.timeout(ms), headers });
return { status: r.status, ok: r.ok, finalUrl: r.url, body: await r.text() };
}
function metaTitle(html) {
const t = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
return t ? clean(t[1].replace(/\s+/g, ' ').trim()).slice(0, 120) : null;
}
function ogImage(html) {
const m = html.match(/<meta[^>]+property=["']og:image["'][^>]+content=["']([^"']+)["']/i)
|| html.match(/<meta[^>]+content=["']([^"']+)["'][^>]+property=["']og:image["']/i);
return m ? m[1] : null;
}
const num = (v) => { const n = parseFloat(v); return Number.isFinite(n) && n > 0 ? n : null; };
const firstImage = (v) => {
if (!v) return null;
const x = Array.isArray(v) ? v[0] : v;
if (!x) return null;
return typeof x === 'string' ? x : (x.src || x.url || x.image || null);
};
// Normalize the two feed shapes the fleet uses into one product record:
// • DW-family landing /api/products → { count, products:[{title, handle, store_url, price, images, ...}] }
// • Shopify storefront /products.json → { products:[{title, handle, variants[].price, images[].src, vendor}] }
// Resolve a possibly-relative asset/link against the microsite's own origin so it
// renders correctly when surfaced on all.designerwallcoverings.com.
const abs = (u, host) => {
if (!u) return null;
if (/^https?:\/\//i.test(u)) return u;
return `https://${host}${u.startsWith('/') ? '' : '/'}${u}`;
};
// Derive an internal membership handle even when a feed omits one — internal line
// viewers (gracie/zuber/crezana family) emit mfr_sku/pattern_name but no handle.
// Never displayed; membership key only.
const rowHandle = (p) => p.handle || (p.dw_sku || p.mfr_sku || p.sku ? slugify(p.dw_sku || p.mfr_sku || p.sku) : null);
function normalizeRow(p, host) {
const shopify = Array.isArray(p.variants);
const handle = rowHandle(p);
return {
title: clean(p.title || p.display_name || p.pattern_name || ''),
handle: handle,
vendor: clean(p.vendor || null),
type: p.product_type || p.series || null,
image: abs(firstImage(p.images) || firstImage(p.image) || p.swatch || null, host),
price: shopify ? (p.variants[0] ? num(p.variants[0].price) : null) : num(p.price),
// Prefer the row's own store link; else the landing's product page.
url: abs(p.store_url || (handle ? `/products/${handle}` : null), host),
};
}
// Turn a raw feed array into a feed result: the COMPLETE handle set for the site
// (the membership source the all- server resolves "This Item" against — internal keys,
// never displayed) PLUS a scrubbed ≤24-product sample for the directory UI.
function buildFeedResult(arr, host, reportedTotal, truncated) {
const handles = [...new Set(arr.map((p) => rowHandle(p)).filter(Boolean).map(String))];
const products = arr
.filter((p) => !BANNED.test(p.title || '') && !BANNED.test(p.vendor || '') && !BANNED.test((p.tags || []).join(' ')))
.slice(0, SAMPLE_PER_SITE)
.map((p) => normalizeRow(p, host));
return { has: true, count: reportedTotal != null ? reportedTotal : arr.length, truncated: !!truncated, handles, products };
}
async function feedProducts(host, feedBase) {
// Capture the FULL handle set per microsite (not a 24-sample) so membership is exact.
// • DW-family /api/products → returns the WHOLE catalog in one shot (ignores limit);
// shape { count|total, rows|products:[…] }.
// • Shopify /products.json → 250/page, paginate to completion.
// feedBase lets a co-located, Basic-Auth-gated microsite be read on localhost
// (http://127.0.0.1:<port>) — bypassing its public 401 with no credential.
const dwBase = feedBase || `https://${host}`;
// 1. DW-family family endpoint first.
try {
const { status, body } = await getText(`${dwBase}/api/products?limit=100000`, FULL_FEED_TIMEOUT_MS);
if (status === 200) {
const j = JSON.parse(body);
const arr = Array.isArray(j.rows) ? j.rows : Array.isArray(j.products) ? j.products : null;
if (arr && arr.length) return buildFeedResult(arr, host, j.total ?? j.count ?? null, false);
}
} catch { /* fall through */ }
// 1b. dw-vendor-microsites line-viewer family serves its catalog at /api/skus
// → shape { total, rows:[{sku, mfr_sku, pattern_name, …}] }.
try {
const { status, body } = await getText(`${dwBase}/api/skus?limit=100000&all=1`, FULL_FEED_TIMEOUT_MS);
if (status === 200) {
const j = JSON.parse(body);
const arr = Array.isArray(j.rows) ? j.rows : Array.isArray(j.skus) ? j.skus : null;
if (arr && arr.length) return buildFeedResult(arr, host, j.total ?? j.all ?? j.count ?? null, false);
}
} catch { /* fall through to Shopify */ }
// 2. Shopify storefront — follow /products.json to completion.
try {
const all = [];
let truncated = false;
for (let page = 1; page <= SHOPIFY_PAGE_CAP; page++) {
const { status, body } = await getText(`https://${host}/products.json?limit=250&page=${page}`, FEED_TIMEOUT_MS);
if (status !== 200) break;
const j = JSON.parse(body);
const arr = Array.isArray(j.products) ? j.products : [];
if (!arr.length) break;
all.push(...arr);
if (arr.length < 250) break;
if (page === SHOPIFY_PAGE_CAP) truncated = true; // hit the politeness ceiling
}
if (all.length) return buildFeedResult(all, host, all.length, truncated);
} catch { /* no feed */ }
return { has: false };
}
async function crawlOne(seed) {
const host = `${seed.slug}.${BASE}`;
// Co-located Basic-Auth-gated microsite → read its feed on localhost, no credential.
const lv = (seed.internalViewer || '').match(/https?:\/\/(?:127\.0\.0\.1|localhost):\d+/);
const feedBase = lv ? lv[0] : null;
const rec = {
slug: seed.slug, host, url: `https://${host}/`, type: seed.type,
vendor: clean(seed.vendor), note: seed.note || null,
activeProductsDB: seed.activeProductsDB ?? null,
up: false, status: 0, finalUrl: null, title: null, hero: seed.dbSampleImage || null,
productCount: 0, feed: false, products: [],
// Full handle set for this microsite — the membership source the server resolves
// "This Item" deep-links against. Empty for gated (401) or broken/empty (up-200-but-no-feed,
// e.g. daisybennett serving the 1890swallpaper vhost) sites → server suppresses cleanly.
handles: [], handleCount: 0,
};
try {
const root = await getText(rec.url, ROOT_TIMEOUT_MS);
rec.status = root.status; rec.finalUrl = root.finalUrl;
// 401 = healthy-but-gated (Steve's convention); still "up".
rec.up = root.ok || root.status === 401;
if (root.body) {
// A gated/undeployed public host returns a 401/403 page — don't let its
// "401 Authorization Required" <title> become the directory card label.
const t = metaTitle(root.body);
if (t && !/^\s*(401|403)\b|authorization required|forbidden|access denied/i.test(t)) rec.title = t;
rec.hero = ogImage(root.body) || rec.hero;
}
} catch (e) { rec.error = e.name === 'TimeoutError' ? 'timeout' : e.message; }
// A co-located microsite (feedBase = localhost) is readable even if its PUBLIC host
// has no DNS yet (rec.up=false) — so always try the localhost feed when feedBase is set.
if (feedBase || (rec.up && rec.status !== 401)) {
const f = await feedProducts(host, feedBase);
if (f.has) {
rec.feed = true;
if (feedBase && !rec.up) rec.up = true; // serving locally = available (pre-DNS)
rec.productCount = f.count;
rec.feedTruncated = !!f.truncated;
rec.handles = f.handles; // COMPLETE handle set (membership source)
rec.handleCount = f.handles.length;
rec.products = f.products; // already scrubbed + ≤SAMPLE_PER_SITE
if (!rec.hero && rec.products[0]) rec.hero = rec.products[0].image;
}
}
// Fall back to the vendor name when no usable page <title> was found (gated/undeployed sites).
if (!rec.title) rec.title = clean(seed.vendor) || seed.slug;
return rec;
}
async function pool(items, worker, n) {
const out = new Array(items.length);
let i = 0;
await Promise.all(Array.from({ length: Math.min(n, items.length) }, async () => {
while (i < items.length) { const idx = i++; out[idx] = await worker(items[idx], idx); }
}));
return out;
}
async function crawlMicrosites() {
const t0 = Date.now();
const [db, cf] = await Promise.all([seedFromDb(), seedFromCfZone()]);
const seeds = mergeSeeds(seedFromRegistry(), db, cf, seedFromPm2());
const sites = await pool(seeds, crawlOne, CONCURRENCY);
const up = sites.filter((s) => s.up);
// Merged product feed across every reachable microsite (bounded).
const merged = [];
for (const s of up) {
for (const p of s.products) {
if (merged.length >= MERGED_CAP) break;
merged.push({ ...p, site: s.slug, siteUrl: s.url });
}
}
const snapshot = {
crawled_at: new Date().toISOString(),
took_ms: Date.now() - t0,
total_candidates: sites.length,
up: up.length,
with_feed: sites.filter((s) => s.feed).length,
with_handles: sites.filter((s) => s.handleCount > 0).length,
total_handles: sites.reduce((n, s) => n + (s.handleCount || 0), 0),
products: merged.length,
sites: sites.sort((a, b) => (b.up - a.up) || (b.productCount - a.productCount) || a.slug.localeCompare(b.slug)),
merged,
};
fs.mkdirSync(path.dirname(OUT), { recursive: true });
fs.writeFileSync(OUT, JSON.stringify(snapshot));
console.log(`microsite crawl: ${up.length}/${sites.length} up, ${snapshot.with_feed} with feeds (${snapshot.with_handles} handle-sets, ${snapshot.total_handles.toLocaleString()} handles), ${merged.length} sample products in ${snapshot.took_ms}ms [$0 local]`);
return snapshot;
}
module.exports = { crawlMicrosites, OUT };
if (require.main === module) {
crawlMicrosites().then(() => process.exit(0)).catch((e) => { console.error('crawl failed:', e.message); process.exit(1); });
}