← back to All Designerwallcoverings

keep internal vendor twins distinct in the crawl

3ae884efabf890fd110351f708a53d259d779349 · 2026-08-28 08:59:20 -0700 · Steve Abrams

Files touched

• M scripts/crawl-microsites.js
• M test/vendor-pairs.test.js

Diff

commit 3ae884efabf890fd110351f708a53d259d779349
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Fri Aug 28 08:59:20 2026 -0700

    keep internal vendor twins distinct in the crawl
---
 scripts/crawl-microsites.js | 10 ++++++++--
 test/vendor-pairs.test.js   |  7 +++++++
 2 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/scripts/crawl-microsites.js b/scripts/crawl-microsites.js
index 97834e2..0e86543 100644
--- a/scripts/crawl-microsites.js
+++ b/scripts/crawl-microsites.js
@@ -211,7 +211,11 @@ function mergeSeeds(reg, db, cf, pm2) {
   for (const s of [...reg, ...db, ...(cf || []), ...(pm2 || [])]) {
     if (!s.slug || BANNED.test(s.slug) || BANNED.test(s.vendor || '')) continue;
     const n = norm(s.slug);
-    const key = (s.vendor && vendorToSlug.get(String(s.vendor).toLowerCase())) || normToKey.get(n) || s.slug;
+    // The internal twin must remain a distinct record. Vendor folding is correct for
+    // public aliases, but would otherwise collapse vendorname-internal onto vendorname.
+    const key = s.variant === 'internal'
+      ? s.slug
+      : ((s.vendor && vendorToSlug.get(String(s.vendor).toLowerCase())) || normToKey.get(n) || s.slug);
     if (!normToKey.has(n)) normToKey.set(n, key);
     const prev = bySlug.get(key);
     bySlug.set(key, prev
@@ -384,6 +388,8 @@ async function crawlOne(seed) {
   const rec = {
     slug: seed.slug, host, url: `https://${host}/`, type: seed.type,
     vendor: clean(seed.vendor), note: seed.note || null,
+    pairSlug: seed.pairSlug || null, variant: seed.variant || null,
+    source: seed.source || null, created_at: seed.created_at || null,
     activeProductsDB: seed.activeProductsDB ?? null,
     up: false, status: 0, finalUrl: null, title: null, hero: seed.dbSampleImage || null,
     productCount: 0, feed: false, products: [],
@@ -470,7 +476,7 @@ async function crawlMicrosites() {
   return snapshot;
 }
 
-module.exports = { crawlMicrosites, OUT };
+module.exports = { crawlMicrosites, mergeSeeds, OUT };
 
 if (require.main === module) {
   crawlMicrosites().then(() => process.exit(0)).catch((e) => { console.error('crawl failed:', e.message); process.exit(1); });
diff --git a/test/vendor-pairs.test.js b/test/vendor-pairs.test.js
index 611c401..e4ce46f 100644
--- a/test/vendor-pairs.test.js
+++ b/test/vendor-pairs.test.js
@@ -3,6 +3,7 @@
 const test = require('node:test');
 const assert = require('node:assert/strict');
 const { hostSlugify, buildVendorPairSeeds, summarizePairs } = require('../lib/vendor-pairs');
+const { mergeSeeds } = require('../scripts/crawl-microsites');
 
 test('hostname slug keeps words readable and stable', () => {
   assert.equal(hostSlugify('Farrow & Ball'), 'farrow-and-ball');
@@ -25,3 +26,9 @@ test('pair coverage only counts a vendor complete when both variants are up', ()
     { pairSlug: 'arte', variant: 'internal', up: false },
   ]), { vendors: 2, complete: 1, public_up: 2, internal_up: 1 });
 });
+
+test('seed merging never collapses the internal twin onto the public vendor', () => {
+  const paired = buildVendorPairSeeds([{ name: 'Astek' }]);
+  const merged = mergeSeeds(paired, [{ slug: 'astek', vendor: 'Astek', type: 'vendor' }], [], []);
+  assert.deepEqual(merged.filter((r) => r.pairSlug === 'astek').map((r) => r.slug).sort(), ['astek', 'astek-internal']);
+});

← 88bd22e load every unified vendor as a public and internal site pair  ·  back to All Designerwallcoverings  ·  preserve public vendor keys while pairing internal sites 59dee87 →