← back to All Designerwallcoverings
load every unified vendor as a public and internal site pair
88bd22ec74dabf19bc8fbfd23b73a82a4b4fafb9 · 2026-08-28 08:45:52 -0700 · Steve Abrams
Files touched
A lib/vendor-pairs.jsM package.jsonM public/microsites.htmlM scripts/build-data.pyM scripts/crawl-microsites.jsM server.jsA test/vendor-pairs.test.jsA verification/e2e-proof.json
Diff
commit 88bd22ec74dabf19bc8fbfd23b73a82a4b4fafb9
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Fri Aug 28 08:45:52 2026 -0700
load every unified vendor as a public and internal site pair
---
lib/vendor-pairs.js | 48 +++++++++++++++++++++++++++++++++++++++++++++
package.json | 3 ++-
public/microsites.html | 11 +++++++++++
scripts/build-data.py | 3 ++-
scripts/crawl-microsites.js | 21 +++++++++++++++++++-
server.js | 6 +++++-
test/vendor-pairs.test.js | 27 +++++++++++++++++++++++++
verification/e2e-proof.json | 34 ++++++++++++++++++++++++++++++++
8 files changed, 149 insertions(+), 4 deletions(-)
diff --git a/lib/vendor-pairs.js b/lib/vendor-pairs.js
new file mode 100644
index 0000000..d692ef3
--- /dev/null
+++ b/lib/vendor-pairs.js
@@ -0,0 +1,48 @@
+'use strict';
+
+const hostSlugify = (value) => String(value || '').toLowerCase().normalize('NFD')
+ .replace(/[\u0300-\u036f]/g, '')
+ .replace(/&/g, ' and ')
+ .replace(/[^a-z0-9]+/g, '-')
+ .replace(/^-+|-+$/g, '');
+
+function buildVendorPairSeeds(vendors) {
+ const seen = new Set();
+ const out = [];
+ for (const row of vendors || []) {
+ const vendor = String(row.name || row.vendor || '').trim();
+ const slug = hostSlugify(row.site_slug || vendor);
+ if (!vendor || !slug || seen.has(slug)) continue;
+ seen.add(slug);
+ const common = {
+ vendor,
+ pairSlug: slug,
+ source: 'dw_unified',
+ activeProductsDB: Number(row.active_count || row.activeProducts || 0),
+ dbSampleImage: Array.isArray(row.images) ? (row.images[0] || null) : (row.sampleImage || null),
+ created_at: row.created_at || null,
+ };
+ out.push({ ...common, slug, type: 'vendor-public', variant: 'public' });
+ out.push({ ...common, slug: `${slug}-internal`, type: 'vendor-internal', variant: 'internal' });
+ }
+ return out;
+}
+
+function summarizePairs(sites) {
+ const pairs = new Map();
+ for (const site of sites || []) {
+ if (!site.pairSlug || !site.variant) continue;
+ const pair = pairs.get(site.pairSlug) || { public: false, internal: false };
+ pair[site.variant] = !!site.up;
+ pairs.set(site.pairSlug, pair);
+ }
+ const values = [...pairs.values()];
+ return {
+ vendors: values.length,
+ complete: values.filter((p) => p.public && p.internal).length,
+ public_up: values.filter((p) => p.public).length,
+ internal_up: values.filter((p) => p.internal).length,
+ };
+}
+
+module.exports = { hostSlugify, buildVendorPairSeeds, summarizePairs };
diff --git a/package.json b/package.json
index 26f6396..9f3af1b 100644
--- a/package.json
+++ b/package.json
@@ -5,7 +5,8 @@
"main": "server.js",
"scripts": {
"start": "node server.js",
- "build-data": "python3 scripts/build-data.py"
+ "build-data": "python3 scripts/build-data.py",
+ "test": "node --test"
},
"dependencies": {
"@browserbasehq/sdk": "^2.15.0",
diff --git a/public/microsites.html b/public/microsites.html
index 1f44a0b..4e0be8e 100644
--- a/public/microsites.html
+++ b/public/microsites.html
@@ -79,6 +79,7 @@
body.hf-slug [data-f="slug"] { display:none; }
body.hf-count [data-f="count"] { display:none; }
body.hf-status [data-f="status"] { display:none; }
+ body.hf-created [data-f="created"] { display:none; }
body.hf-cta [data-f="cta"] { display:none; }
body.hf-inquiry [data-f="inquiry"] { display:none; }
.card { background:#fff; border:1px solid var(--line); overflow:hidden; display:flex; flex-direction:column;
@@ -152,6 +153,8 @@
<span><b id="statUp">—</b>Sites Live</span>
<span><b id="statFeeds">—</b>With Catalogs</span>
<span><b id="statProducts">—</b>Products Indexed</span>
+ <span><b id="statPairs">—</b>Vendor Pairs</span>
+ <span><b id="statComplete">—</b>Pairs Complete</span>
</div>
<div class="fresh" id="fresh">refreshed —</div>
</section>
@@ -221,6 +224,7 @@ const FIELDS = [
['slug', 'Slug', false],
['count', 'Products', true],
['status', 'Status', false],
+ ['created', 'Created', true],
['cta', 'Visit link', true],
['inquiry', 'Inquiry', true],
];
@@ -295,6 +299,8 @@ fetch('/api/microsites').then(r=>r.json()).then(d=>{
statUp.textContent = d.up != null ? d.up.toLocaleString() : state.all.length.toLocaleString();
statFeeds.textContent = (d.with_feed!=null ? d.with_feed : state.all.filter(s=>s.feed).length).toLocaleString();
statProducts.textContent = (d.products!=null ? d.products : 0).toLocaleString();
+ statPairs.textContent = ((d.vendor_pairs && d.vendor_pairs.vendors) || 0).toLocaleString();
+ statComplete.textContent = ((d.vendor_pairs && d.vendor_pairs.complete) || 0).toLocaleString();
fresh.textContent = 'refreshed ' + ago(d.crawled_at);
apply();
}).catch(()=>{ fresh.textContent='crawl snapshot not ready — check back shortly'; });
@@ -354,6 +360,10 @@ function card(s){
: countTxt)
: 'Editorial lookbook');
const clean = v => String(v == null ? '' : v).replace(/[<>]/g,'');
+ const created = s.created_at ? new Date(s.created_at) : null;
+ const createdText = created && Number.isFinite(created.getTime())
+ ? created.toLocaleString(undefined, { year:'numeric', month:'short', day:'numeric', hour:'numeric', minute:'2-digit' })
+ : '—';
el.innerHTML = `
${badge}
<div class="collage${pool.length<=1?' single':''}" data-f="image">${cells}</div>
@@ -363,6 +373,7 @@ function card(s){
<div class="fline" data-f="type"><b>Type</b> ${clean(s.type) || '—'}</div>
<div class="fline" data-f="slug"><b>Slug</b> ${clean(s.slug) || '—'}</div>
<div class="fline" data-f="status"><b>Status</b> ${gated ? 'Trade-Gated' : 'Live'}</div>
+ <div class="fline" data-f="created" title="${created ? created.toISOString() : ''}"><b>Created</b> 🕓 ${createdText}</div>
<div class="cta" data-f="cta">Visit Microsite →</div>
<button type="button" class="mailv" data-f="inquiry" title="Compose a stock & price inquiry to this vendor — saves a Gmail DRAFT, never auto-sends">✉ Stock & Price</button>
</div>`;
diff --git a/scripts/build-data.py b/scripts/build-data.py
index d2c0b5b..98c1b9e 100644
--- a/scripts/build-data.py
+++ b/scripts/build-data.py
@@ -51,7 +51,7 @@ SELECT COALESCE(json_agg(t), '[]') FROM (
# 2. Registry: every vendor id
reg = json.loads(q("""
SELECT COALESCE(json_agg(t), '[]') FROM (
- SELECT id, vendor_name, vendor_code, is_private_label, private_label_name,
+ SELECT id, vendor_name, vendor_code, is_private_label, private_label_name, created_at,
is_active, shopify_count, total_products
FROM vendor_registry
) t;"""))
@@ -109,6 +109,7 @@ for row in reg:
registry_ids=[row["id"]],
active_count=(lv or {}).get("active_count", 0),
images=(lv or {}).get("images"),
+ created_at=row.get("created_at"),
is_private_label=bool(row.get("is_private_label")))
# Live store vendors with no registry row still belong in a front-facing directory
diff --git a/scripts/crawl-microsites.js b/scripts/crawl-microsites.js
index 05d1d9a..97834e2 100644
--- a/scripts/crawl-microsites.js
+++ b/scripts/crawl-microsites.js
@@ -17,9 +17,11 @@ const fs = require('fs');
const path = require('path');
const os = require('os');
try { require('dotenv').config(); } catch {}
+const { buildVendorPairSeeds, summarizePairs } = require('../lib/vendor-pairs');
const OUT = path.join(__dirname, '..', 'data', 'microsites.json');
const SEED = path.join(__dirname, '..', 'config', 'known-subdomains.json');
+const VENDORS = path.join(__dirname, '..', 'data', 'vendors.json');
const BASE = 'designerwallcoverings.com';
const CONCURRENCY = 8;
// Cloudflare zone that owns designerwallcoverings.com — the authoritative subdomain
@@ -52,6 +54,21 @@ function seedFromRegistry() {
return out;
}
+// data/vendors.json is generated exclusively from dw_unified by build-data.py. Expand
+// every canonical display vendor into the required hostname pair. These candidates stay
+// in the snapshot even when one side is missing, making rollout gaps visible in all.dw.
+function seedVendorPairsFromUnified() {
+ try {
+ const snapshot = JSON.parse(fs.readFileSync(VENDORS, 'utf8'));
+ const pairs = buildVendorPairSeeds(snapshot.vendors || []);
+ console.log(`seedVendorPairsFromUnified: ${pairs.length / 2} vendor pairs from dw_unified snapshot`);
+ return pairs;
+ } catch (e) {
+ console.warn('seedVendorPairsFromUnified skipped:', e.message);
+ return [];
+ }
+}
+
// Auto-discover EVERY co-located vendor viewer running on this host and seed its
// localhost feed — so any *.DW / internal.*.DW viewer that is up is crawled with
// ZERO registry upkeep (Steve's standing rule: all vendor viewers must be live in all.dw).
@@ -421,7 +438,8 @@ async function pool(items, worker, n) {
async function crawlMicrosites() {
const t0 = Date.now();
const [db, cf] = await Promise.all([seedFromDb(), seedFromCfZone()]);
- const seeds = mergeSeeds(seedFromRegistry(), db, cf, seedFromPm2());
+ const paired = seedVendorPairsFromUnified();
+ const seeds = mergeSeeds([...seedFromRegistry(), ...paired], db, cf, seedFromPm2());
const sites = await pool(seeds, crawlOne, CONCURRENCY);
const up = sites.filter((s) => s.up);
@@ -445,6 +463,7 @@ async function crawlMicrosites() {
sites: sites.sort((a, b) => (b.up - a.up) || (b.productCount - a.productCount) || a.slug.localeCompare(b.slug)),
merged,
};
+ snapshot.vendor_pairs = summarizePairs(snapshot.sites);
fs.mkdirSync(path.dirname(OUT), { recursive: true });
fs.writeFileSync(OUT, JSON.stringify(snapshot));
console.log(`microsite crawl: ${up.length}/${sites.length} up, ${snapshot.with_feed} with feeds (${snapshot.with_handles} handle-sets, ${snapshot.total_handles.toLocaleString()} handles), ${merged.length} sample products in ${snapshot.took_ms}ms [$0 local]`);
diff --git a/server.js b/server.js
index 663c758..b4eeeae 100644
--- a/server.js
+++ b/server.js
@@ -386,7 +386,10 @@ function buildVendorViewer() {
const item = new Map(); // slug → { url, handles:Set } for per-PRODUCT deep-link membership
try {
const d = JSON.parse(fs.readFileSync(MICROSITES, 'utf8'));
- for (const s of (d.sites || [])) {
+ // Public sites are useful fallbacks, but the explicit vendorname-internal twin is
+ // the canonical line viewer and therefore wins in the second pass.
+ const sites = d.sites || [];
+ for (const s of [...sites.filter((x) => x.type !== 'vendor-internal'), ...sites.filter((x) => x.type === 'vendor-internal')]) {
if (!s.up || !s.host || BANNED.test(s.host)) continue;
const url = `https://${s.host}/`;
// "This Item" deep-links to /product/<handle>. Since microsite handle == dw_unified.handle,
@@ -401,6 +404,7 @@ function buildVendorViewer() {
const setBoth = (k) => { map.set(k, url); if (entry) item.set(k, entry); };
if (s.vendor && !BANNED.test(s.vendor)) setBoth(slugify(s.vendor));
if (s.slug) setBoth(String(s.slug).toLowerCase());
+ if (s.pairSlug) setBoth(String(s.pairSlug).toLowerCase());
}
} catch { /* crawl artifact not ready yet — viewers resolve after the first crawl lands */ }
// Crawl-resistant overrides win over anything the artifact produced (dead/stale subdomains).
diff --git a/test/vendor-pairs.test.js b/test/vendor-pairs.test.js
new file mode 100644
index 0000000..611c401
--- /dev/null
+++ b/test/vendor-pairs.test.js
@@ -0,0 +1,27 @@
+'use strict';
+
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const { hostSlugify, buildVendorPairSeeds, summarizePairs } = require('../lib/vendor-pairs');
+
+test('hostname slug keeps words readable and stable', () => {
+ assert.equal(hostSlugify('Farrow & Ball'), 'farrow-and-ball');
+ assert.equal(hostSlugify('Coordonn\u00e9 Europe'), 'coordonne-europe');
+});
+
+test('every unified vendor expands to public and internal records', () => {
+ const rows = buildVendorPairSeeds([{ name: 'Astek', active_count: 12, images: ['hero.jpg'], created_at: '2026-01-02T03:04:00Z' }]);
+ assert.deepEqual(rows.map((r) => r.slug), ['astek', 'astek-internal']);
+ assert.deepEqual(rows.map((r) => r.variant), ['public', 'internal']);
+ assert.ok(rows.every((r) => r.source === 'dw_unified' && r.pairSlug === 'astek'));
+ assert.ok(rows.every((r) => r.created_at === '2026-01-02T03:04:00Z'));
+});
+
+test('pair coverage only counts a vendor complete when both variants are up', () => {
+ assert.deepEqual(summarizePairs([
+ { pairSlug: 'astek', variant: 'public', up: true },
+ { pairSlug: 'astek', variant: 'internal', up: true },
+ { pairSlug: 'arte', variant: 'public', up: true },
+ { pairSlug: 'arte', variant: 'internal', up: false },
+ ]), { vendors: 2, complete: 1, public_up: 2, internal_up: 1 });
+});
diff --git a/verification/e2e-proof.json b/verification/e2e-proof.json
new file mode 100644
index 0000000..6105dc0
--- /dev/null
+++ b/verification/e2e-proof.json
@@ -0,0 +1,34 @@
+{
+ "schema_version": "1.0",
+ "feature": "Unified vendor public/internal hostname pairs in all.dw",
+ "risk_tier": 2,
+ "source_of_truth": "dw_unified via data/vendors.json generated by scripts/build-data.py",
+ "before": {
+ "live_candidates": 258,
+ "live_up": 179,
+ "live_internal_suffix_records": 5,
+ "observed_at": "2026-08-28T15:38:24.739Z"
+ },
+ "proof": [
+ {
+ "scenario": "positive-pair-expansion",
+ "command": "npm test",
+ "result": "pass",
+ "assertion": "A unified vendor expands to vendorname and vendorname-internal records."
+ },
+ {
+ "scenario": "negative-incomplete-pair",
+ "command": "npm test",
+ "result": "pass",
+ "assertion": "A vendor is not counted complete when either public or internal is down."
+ },
+ {
+ "scenario": "browser-script-syntax",
+ "command": "compile inline scripts with vm.Script",
+ "result": "pass",
+ "assertion": "The microsite directory UI parses after adding pair coverage and created timestamps."
+ }
+ ],
+ "production_verification_pending": true,
+ "pending_reason": "Targeted deploy, snapshot rebuild, and live crawl are production-gated."
+}
← ce50840 creds-safe fetch guard: resolve relative fetch vs credential
·
back to All Designerwallcoverings
·
keep internal vendor twins distinct in the crawl 3ae884e →