← back to All Designerwallcoverings
all.: add live *.designerwallcoverings.com microsite crawl (directory + merged products), served at /api/microsites, polled every 10 min in-process + launchd
260257fcb8f63de052122aead660d4e20404d174 · 2026-07-06 15:18:45 -0700 · Steve Abrams
Files touched
M .gitignoreA config/known-subdomains.jsonA scripts/crawl-microsites.jsM server.js
Diff
commit 260257fcb8f63de052122aead660d4e20404d174
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Mon Jul 6 15:18:45 2026 -0700
all.: add live *.designerwallcoverings.com microsite crawl (directory + merged products), served at /api/microsites, polled every 10 min in-process + launchd
---
.gitignore | 4 +
config/known-subdomains.json | 59 +++++++++++
scripts/crawl-microsites.js | 237 +++++++++++++++++++++++++++++++++++++++++++
server.js | 30 ++++++
4 files changed, 330 insertions(+)
diff --git a/.gitignore b/.gitignore
index 1924158..fd573d0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,3 +6,7 @@ tmp/
dist/
build/
.next/
+
+# live crawl snapshot (regenerated every 10 min by scripts/crawl-microsites.js)
+data/microsites.json
+logs/
diff --git a/config/known-subdomains.json b/config/known-subdomains.json
new file mode 100644
index 0000000..24449d2
--- /dev/null
+++ b/config/known-subdomains.json
@@ -0,0 +1,59 @@
+{
+ "_comment": "Hand-maintained registry of *.designerwallcoverings.com subdomains that are KNOWN to exist (or have a local project). Brand microsites match DB vendors by slug; internal sites are non-brand tools. Edit this file, then re-run scripts/build-sites-json.js.",
+ "brands": [
+ {
+ "slug": "artmura",
+ "vendor": "Artmura",
+ "note": "LIVE reference build for the dw-vendor-landing template",
+ "hasMicrosite": true,
+ "internalViewer": "http://127.0.0.1:9948 (~/Projects/artmura-internal, pm2 artmura-internal)"
+ },
+ {
+ "slug": "astek",
+ "vendor": "Astek",
+ "knownProject": "~/Projects/astek-landing",
+ "localUrl": "http://127.0.0.1:9944",
+ "note": "Internal-only editorial landing (Basic-Auth), running on Mac2 :9944 \u2014 vendor-request system live (memo/stock/price REQ emails)",
+ "hasMicrosite": true
+ },
+ {
+ "slug": "reidwitlin",
+ "vendor": "Reid Witlin",
+ "knownProject": "~/Projects/reidwitlin-landing",
+ "note": "Local project built; deploy status tracked by probe",
+ "hasMicrosite": true,
+ "internalViewer": "http://127.0.0.1:9945 (pm2 reidwitlin-landing)"
+ },
+ {
+ "slug": "quadrille",
+ "vendor": "Quadrille",
+ "knownProject": "~/Projects/quadrille-house-site",
+ "note": "Local project built; deploy status tracked by probe",
+ "hasMicrosite": true,
+ "internalViewer": "http://127.0.0.1:9947 (~/Projects/quadrille-internal, pm2 quadrille-internal)"
+ }
+ ],
+ "internal": [
+ {
+ "slug": "new",
+ "label": "New-import catalog viewer",
+ "note": "DO NOT TOUCH the new-import-viewer project (live Shopify write machinery)"
+ },
+ {
+ "slug": "pairs",
+ "label": "Pairs-well-with viewer"
+ },
+ {
+ "slug": "chat",
+ "label": "Chat agent"
+ },
+ {
+ "slug": "appointments",
+ "label": "Appointments / smart scheduling"
+ },
+ {
+ "slug": "all",
+ "label": "This directory site (all-dw-landing)"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/scripts/crawl-microsites.js b/scripts/crawl-microsites.js
new file mode 100644
index 0000000..3b5b507
--- /dev/null
+++ b/scripts/crawl-microsites.js
@@ -0,0 +1,237 @@
+#!/usr/bin/env node
+// crawl-microsites.js — the live-crawl half of all.designerwallcoverings.com.
+//
+// Polls every KNOWN + DB-derived *.designerwallcoverings.com subdomain, records
+// which are up + their identity (title / hero / product feed), and writes a single
+// snapshot the all- server serves at /api/microsites (directory + merged products).
+//
+// Data path is deliberately DIFFERENT from the main product grid: that reads
+// dw_unified directly; THIS reads what each microsite actually publishes over HTTP.
+// A live crawl only ever sees public pages, so cost/net/wholesale can't leak — but
+// we still run the standing BANNED private-label scrub on every display string.
+//
+// Runs standalone (`node scripts/crawl-microsites.js`) OR in-process from server.js
+// (require + crawlMicrosites()). $0 — local HTTP only, no paid APIs.
+'use strict';
+const fs = require('fs');
+const path = require('path');
+const os = require('os');
+try { require('dotenv').config(); } catch {}
+
+const OUT = path.join(__dirname, '..', 'data', 'microsites.json');
+const SEED = path.join(__dirname, '..', 'config', 'known-subdomains.json');
+const BASE = 'designerwallcoverings.com';
+const CONCURRENCY = 8;
+const ROOT_TIMEOUT_MS = 8000;
+const FEED_TIMEOUT_MS = 6000;
+const SAMPLE_PER_SITE = 24; // products carried per site into the merged feed
+const MERGED_CAP = 4000; // hard ceiling on the merged products array
+
+// Standing rule: banned upstream/private-label names never ship on a public surface.
+const BANNED = /brewster|york|wallquest|wall\s*quest|newwall|new\s*wall\b|command\s*54|justin\s*david|nicolette\s*mayer|seabrook|chesapeake|nextwall|desima|carlsten/i;
+// Display scrub: never surface the word "Wallpaper" where the house uses "Wallcovering".
+const clean = (s) => (s == null ? s : String(s)
+ .replace(/\bWallpapers\b/gi, 'Wallcoverings').replace(/\bWallpaper\b/gi, 'Wallcovering'));
+
+const slugify = (v) => String(v || '').toLowerCase().normalize('NFD')
+ .replace(/[̀-ͯ]/g, '').replace(/&/g, 'and').replace(/[^a-z0-9]/g, '');
+
+// ── seed slug list: hand registry ∪ DB vendor candidates ─────────────────────────
+function seedFromRegistry() {
+ const reg = JSON.parse(fs.readFileSync(SEED, 'utf8'));
+ const out = [];
+ for (const b of reg.brands || []) out.push({ slug: b.slug, vendor: b.vendor || null, type: 'brand', note: b.note || null });
+ for (const i of reg.internal || []) out.push({ slug: i.slug, vendor: null, type: 'internal', note: i.label || null });
+ return out;
+}
+
+async function seedFromDb() {
+ // Best-effort: candidate <slug>.designerwallcoverings.com for every vendor with
+ // active products. Unreachable candidates simply drop out of the crawl.
+ let Pool;
+ try { ({ Pool } = require('pg')); } catch { return []; }
+ const PW = (() => {
+ try {
+ const env = fs.readFileSync(os.homedir() + '/Projects/secrets-manager/.env', 'utf8');
+ const m = env.match(/^DW_ADMIN_DB_PASSWORD=(.*)$/m);
+ if (m) return m[1].replace(/^["']|["']$/g, '').trim();
+ } catch {}
+ return process.env.PGPASSWORD || '';
+ })();
+ const pool = process.env.DATABASE_URL
+ ? new Pool({ connectionString: process.env.DATABASE_URL, max: 1 })
+ : new Pool({ connectionString: 'postgresql:///dw_unified?host=/tmp', max: 1, password: PW || undefined });
+ try {
+ const { rows } = await pool.query(`
+ SELECT vendor, COUNT(*)::int AS active_products,
+ (ARRAY_REMOVE(ARRAY_AGG(image_url ORDER BY created_at_shopify DESC NULLS LAST), NULL))[1] AS sample_image
+ FROM shopify_products
+ WHERE UPPER(COALESCE(status,'')) = 'ACTIVE' AND vendor IS NOT NULL AND vendor <> ''
+ GROUP BY vendor`);
+ return rows
+ .filter((r) => !BANNED.test(r.vendor || ''))
+ .map((r) => ({ slug: slugify(r.vendor), vendor: r.vendor, type: 'vendor', activeProductsDB: r.active_products, dbSampleImage: r.sample_image }))
+ .filter((r) => r.slug);
+ } catch (e) {
+ console.error('seedFromDb skipped:', e.message);
+ return [];
+ } finally { try { await pool.end(); } catch {} }
+}
+
+function mergeSeeds(reg, db) {
+ const bySlug = new Map();
+ for (const s of [...reg, ...db]) { // registry first, DB fills gaps
+ if (!s.slug || BANNED.test(s.slug) || BANNED.test(s.vendor || '')) continue;
+ const prev = bySlug.get(s.slug);
+ bySlug.set(s.slug, prev ? { ...s, ...prev, activeProductsDB: prev.activeProductsDB ?? s.activeProductsDB, dbSampleImage: prev.dbSampleImage ?? s.dbSampleImage } : s);
+ }
+ return [...bySlug.values()];
+}
+
+// ── per-site fetch ───────────────────────────────────────────────────────────────
+async function getText(url, ms) {
+ const r = await fetch(url, { redirect: 'follow', signal: AbortSignal.timeout(ms), headers: { 'user-agent': 'all-dw-crawler/1.0 (+all.designerwallcoverings.com)' } });
+ return { status: r.status, ok: r.ok, finalUrl: r.url, body: await r.text() };
+}
+
+function metaTitle(html) {
+ const t = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
+ return t ? clean(t[1].replace(/\s+/g, ' ').trim()).slice(0, 120) : null;
+}
+function ogImage(html) {
+ const m = html.match(/<meta[^>]+property=["']og:image["'][^>]+content=["']([^"']+)["']/i)
+ || html.match(/<meta[^>]+content=["']([^"']+)["'][^>]+property=["']og:image["']/i);
+ return m ? m[1] : null;
+}
+
+const num = (v) => { const n = parseFloat(v); return Number.isFinite(n) && n > 0 ? n : null; };
+const firstImage = (v) => {
+ if (!v) return null;
+ const x = Array.isArray(v) ? v[0] : v;
+ if (!x) return null;
+ return typeof x === 'string' ? x : (x.src || x.url || x.image || null);
+};
+
+// Normalize the two feed shapes the fleet uses into one product record:
+// • DW-family landing /api/products → { count, products:[{title, handle, store_url, price, images, ...}] }
+// • Shopify storefront /products.json → { products:[{title, handle, variants[].price, images[].src, vendor}] }
+// Resolve a possibly-relative asset/link against the microsite's own origin so it
+// renders correctly when surfaced on all.designerwallcoverings.com.
+const abs = (u, host) => {
+ if (!u) return null;
+ if (/^https?:\/\//i.test(u)) return u;
+ return `https://${host}${u.startsWith('/') ? '' : '/'}${u}`;
+};
+
+function normalizeRow(p, host) {
+ const shopify = Array.isArray(p.variants);
+ return {
+ title: clean(p.title || p.display_name || ''),
+ handle: p.handle || null,
+ vendor: clean(p.vendor || null),
+ type: p.product_type || p.series || null,
+ image: abs(firstImage(p.images) || firstImage(p.image) || p.swatch || null, host),
+ price: shopify ? (p.variants[0] ? num(p.variants[0].price) : null) : num(p.price),
+ // Prefer the row's own store link; else the landing's product page.
+ url: abs(p.store_url || (p.handle ? `/products/${p.handle}` : null), host),
+ };
+}
+
+async function feedProducts(host) {
+ // DW-family landings serve /api/products; Shopify sites serve /products.json.
+ // Try the family endpoint first, then the Shopify one. ?limit=250 gives a count signal.
+ const endpoints = [
+ { url: `https://${host}/api/products?limit=250`, key: (j) => (Array.isArray(j.rows) ? j.rows : Array.isArray(j.products) ? j.products : []), total: (j) => j.total ?? j.count ?? null },
+ { url: `https://${host}/products.json?limit=250`, key: (j) => (Array.isArray(j.products) ? j.products : []), total: () => null },
+ ];
+ for (const ep of endpoints) {
+ try {
+ const { status, body } = await getText(ep.url, FEED_TIMEOUT_MS);
+ if (status !== 200) continue;
+ const j = JSON.parse(body);
+ const arr = ep.key(j);
+ if (!arr.length) continue;
+ const products = arr
+ .filter((p) => !BANNED.test(p.title || '') && !BANNED.test(p.vendor || '') && !BANNED.test((p.tags || []).join(' ')))
+ .map((p) => normalizeRow(p, host));
+ const reported = ep.total(j);
+ return { has: true, count: reported != null ? reported : products.length, sawMax: arr.length >= 250, products };
+ } catch { /* try next endpoint */ }
+ }
+ return { has: false };
+}
+
+async function crawlOne(seed) {
+ const host = `${seed.slug}.${BASE}`;
+ const rec = {
+ slug: seed.slug, host, url: `https://${host}/`, type: seed.type,
+ vendor: clean(seed.vendor), note: seed.note || null,
+ activeProductsDB: seed.activeProductsDB ?? null,
+ up: false, status: 0, finalUrl: null, title: null, hero: seed.dbSampleImage || null,
+ productCount: 0, feed: false, products: [],
+ };
+ try {
+ const root = await getText(rec.url, ROOT_TIMEOUT_MS);
+ rec.status = root.status; rec.finalUrl = root.finalUrl;
+ // 401 = healthy-but-gated (Steve's convention); still "up".
+ rec.up = root.ok || root.status === 401;
+ if (root.body) { rec.title = metaTitle(root.body) || rec.title; rec.hero = ogImage(root.body) || rec.hero; }
+ } catch (e) { rec.error = e.name === 'TimeoutError' ? 'timeout' : e.message; }
+
+ if (rec.up && rec.status !== 401) {
+ const f = await feedProducts(host);
+ if (f.has) {
+ rec.feed = true;
+ rec.productCount = f.sawMax ? f.count : f.count; // 250 cap noted via sawMax
+ rec.feedTruncated = !!f.sawMax;
+ rec.products = f.products.slice(0, SAMPLE_PER_SITE);
+ if (!rec.hero && rec.products[0]) rec.hero = rec.products[0].image;
+ }
+ }
+ return rec;
+}
+
+async function pool(items, worker, n) {
+ const out = new Array(items.length);
+ let i = 0;
+ await Promise.all(Array.from({ length: Math.min(n, items.length) }, async () => {
+ while (i < items.length) { const idx = i++; out[idx] = await worker(items[idx], idx); }
+ }));
+ return out;
+}
+
+async function crawlMicrosites() {
+ const t0 = Date.now();
+ const seeds = mergeSeeds(seedFromRegistry(), await seedFromDb());
+ const sites = await pool(seeds, crawlOne, CONCURRENCY);
+
+ const up = sites.filter((s) => s.up);
+ // Merged product feed across every reachable microsite (bounded).
+ const merged = [];
+ for (const s of up) {
+ for (const p of s.products) {
+ if (merged.length >= MERGED_CAP) break;
+ merged.push({ ...p, site: s.slug, siteUrl: s.url });
+ }
+ }
+ const snapshot = {
+ crawled_at: new Date().toISOString(),
+ took_ms: Date.now() - t0,
+ total_candidates: sites.length,
+ up: up.length,
+ with_feed: sites.filter((s) => s.feed).length,
+ products: merged.length,
+ sites: sites.sort((a, b) => (b.up - a.up) || (b.productCount - a.productCount) || a.slug.localeCompare(b.slug)),
+ merged,
+ };
+ fs.mkdirSync(path.dirname(OUT), { recursive: true });
+ fs.writeFileSync(OUT, JSON.stringify(snapshot));
+ console.log(`microsite crawl: ${up.length}/${sites.length} up, ${snapshot.with_feed} with feeds, ${merged.length} products in ${snapshot.took_ms}ms [$0 local]`);
+ return snapshot;
+}
+
+module.exports = { crawlMicrosites, OUT };
+
+if (require.main === module) {
+ crawlMicrosites().then(() => process.exit(0)).catch((e) => { console.error('crawl failed:', e.message); process.exit(1); });
+}
diff --git a/server.js b/server.js
index 3739b8f..38d99e7 100644
--- a/server.js
+++ b/server.js
@@ -10,6 +10,10 @@ const fs = require('fs');
const path = require('path');
const { Pool } = require('pg');
try { require('dotenv').config(); } catch {}
+// Live-crawl half: polls *.designerwallcoverings.com microsites every REFRESH_MS
+// and serves the snapshot at /api/microsites. Same 10-min cadence as the PG
+// product snapshot, so the LIVE site stays fresh with no external cron.
+const { crawlMicrosites, OUT: MICROSITES } = require('./scripts/crawl-microsites');
const PORT = process.env.PORT || 9958;
const PUB = path.join(__dirname, 'public');
@@ -190,6 +194,15 @@ function loadVendors() {
return vendorsCache;
}
+// Microsite crawl snapshot (written by scripts/crawl-microsites.js on the REFRESH_MS
+// tick). Served from disk, mtime-cached, so the request path never blocks on a crawl.
+let micrositesCache = null, micrositesMtime = 0;
+function loadMicrosites() {
+ const st = fs.statSync(MICROSITES);
+ if (!micrositesCache || st.mtimeMs !== micrositesMtime) { micrositesCache = fs.readFileSync(MICROSITES); micrositesMtime = st.mtimeMs; }
+ return micrositesCache;
+}
+
// Basic Auth gate (Steve 2026-07-02: all. is published but un/pw-gated).
// Override with BASIC_AUTH=user:pass in .env; /healthz stays open for probes.
const BASIC_AUTH = process.env.BASIC_AUTH || 'admin:DW2024!';
@@ -257,6 +270,13 @@ const server = http.createServer((req, res) => {
} catch { res.writeHead(500, { 'Content-Type': 'application/json' }); return res.end('{"error":"data unavailable"}'); }
}
+ if (u.pathname === '/api/microsites') { // live-crawl directory + merged products
+ try {
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8', 'Cache-Control': 'public, max-age=60' });
+ return res.end(loadMicrosites());
+ } catch { res.writeHead(503, { 'Content-Type': 'application/json' }); return res.end('{"error":"crawl snapshot not ready"}'); }
+ }
+
if (u.pathname === '/healthz') {
res.writeHead(200, { 'Content-Type': 'application/json' });
return res.end(JSON.stringify({ ok: ROWS.length > 0, products: ROWS.length, loaded_at: LOADED_AT }));
@@ -270,9 +290,19 @@ const server = http.createServer((req, res) => {
res.end(fs.readFileSync(file));
});
+// Microsite crawl runs on the SAME 10-min cadence but is non-fatal — a failed or
+// slow crawl must never take the product grid down. Kicked off after listen so the
+// server is answering immediately; re-runs every REFRESH_MS staggered 30s off the
+// PG snapshot so the two refreshes don't contend.
+function pollMicrosites() {
+ crawlMicrosites().catch((e) => console.error('microsite crawl failed:', e.message));
+}
+
loadSnapshot()
.then(() => {
server.listen(PORT, () => console.log(`all. catalog → http://127.0.0.1:${PORT} (${ROWS.length.toLocaleString()} products)`));
setInterval(() => loadSnapshot().catch((e) => console.error('snapshot refresh failed:', e.message)), REFRESH_MS);
+ pollMicrosites();
+ setTimeout(() => setInterval(pollMicrosites, REFRESH_MS), 30 * 1000);
})
.catch((e) => { console.error('FATAL: initial snapshot load failed:', e.message); process.exit(1); });
← 96f3c15 Astek-viewer alignment: accordion filter sections + CARD FIE
·
back to All Designerwallcoverings
·
reconcile local to deployed prod (5bc3e1a) + re-layer micros 228a80f →