← back to All Designerwallcoverings
This Item deep-link: capture full per-microsite handle set + resolve by membership (kills ~15-25% dead-ends)
94d8f0aceec35b73a7e58272a05d6fe46efa853a · 2026-07-07 08:31:33 -0700 · Steve
Files touched
M scripts/crawl-microsites.jsM server.js
Diff
commit 94d8f0aceec35b73a7e58272a05d6fe46efa853a
Author: Steve <steve@designerwallcoverings.com>
Date: Tue Jul 7 08:31:33 2026 -0700
This Item deep-link: capture full per-microsite handle set + resolve by membership (kills ~15-25% dead-ends)
---
scripts/crawl-microsites.js | 81 +++++++++++++++++++++++++++++++--------------
server.js | 34 +++++++++++--------
2 files changed, 76 insertions(+), 39 deletions(-)
diff --git a/scripts/crawl-microsites.js b/scripts/crawl-microsites.js
index 3b5b507..ef16822 100644
--- a/scripts/crawl-microsites.js
+++ b/scripts/crawl-microsites.js
@@ -23,8 +23,10 @@ const SEED = path.join(__dirname, '..', 'config', 'known-subdomains.json');
const BASE = 'designerwallcoverings.com';
const CONCURRENCY = 8;
const ROOT_TIMEOUT_MS = 8000;
-const FEED_TIMEOUT_MS = 6000;
-const SAMPLE_PER_SITE = 24; // products carried per site into the merged feed
+const FEED_TIMEOUT_MS = 6000; // per Shopify page / initial feed probe
+const FULL_FEED_TIMEOUT_MS = 15000; // DW-family full-set fetch (returns everything in one shot)
+const SHOPIFY_PAGE_CAP = 40; // /products.json pages (≤10k products) — politeness ceiling
+const SAMPLE_PER_SITE = 24; // products carried per site into the merged feed (directory UI)
const MERGED_CAP = 4000; // hard ceiling on the merged products array
// Standing rule: banned upstream/private-label names never ship on a public surface.
@@ -137,27 +139,48 @@ function normalizeRow(p, host) {
};
}
+// Turn a raw feed array into a feed result: the COMPLETE handle set for the site
+// (the membership source the all- server resolves "This Item" against — internal keys,
+// never displayed) PLUS a scrubbed ≤24-product sample for the directory UI.
+function buildFeedResult(arr, host, reportedTotal, truncated) {
+ const handles = [...new Set(arr.map((p) => p.handle).filter(Boolean).map(String))];
+ const products = arr
+ .filter((p) => !BANNED.test(p.title || '') && !BANNED.test(p.vendor || '') && !BANNED.test((p.tags || []).join(' ')))
+ .slice(0, SAMPLE_PER_SITE)
+ .map((p) => normalizeRow(p, host));
+ return { has: true, count: reportedTotal != null ? reportedTotal : arr.length, truncated: !!truncated, handles, products };
+}
+
async function feedProducts(host) {
- // DW-family landings serve /api/products; Shopify sites serve /products.json.
- // Try the family endpoint first, then the Shopify one. ?limit=250 gives a count signal.
- const endpoints = [
- { url: `https://${host}/api/products?limit=250`, key: (j) => (Array.isArray(j.rows) ? j.rows : Array.isArray(j.products) ? j.products : []), total: (j) => j.total ?? j.count ?? null },
- { url: `https://${host}/products.json?limit=250`, key: (j) => (Array.isArray(j.products) ? j.products : []), total: () => null },
- ];
- for (const ep of endpoints) {
- try {
- const { status, body } = await getText(ep.url, FEED_TIMEOUT_MS);
- if (status !== 200) continue;
+ // Capture the FULL handle set per microsite (not a 24-sample) so membership is exact.
+ // • DW-family /api/products → returns the WHOLE catalog in one shot (ignores limit);
+ // shape { count|total, rows|products:[…] }.
+ // • Shopify /products.json → 250/page, paginate to completion.
+ // 1. DW-family family endpoint first.
+ try {
+ const { status, body } = await getText(`https://${host}/api/products?limit=100000`, FULL_FEED_TIMEOUT_MS);
+ if (status === 200) {
const j = JSON.parse(body);
- const arr = ep.key(j);
- if (!arr.length) continue;
- const products = arr
- .filter((p) => !BANNED.test(p.title || '') && !BANNED.test(p.vendor || '') && !BANNED.test((p.tags || []).join(' ')))
- .map((p) => normalizeRow(p, host));
- const reported = ep.total(j);
- return { has: true, count: reported != null ? reported : products.length, sawMax: arr.length >= 250, products };
- } catch { /* try next endpoint */ }
- }
+ const arr = Array.isArray(j.rows) ? j.rows : Array.isArray(j.products) ? j.products : null;
+ if (arr && arr.length) return buildFeedResult(arr, host, j.total ?? j.count ?? null, false);
+ }
+ } catch { /* fall through to Shopify */ }
+ // 2. Shopify storefront — follow /products.json to completion.
+ try {
+ const all = [];
+ let truncated = false;
+ for (let page = 1; page <= SHOPIFY_PAGE_CAP; page++) {
+ const { status, body } = await getText(`https://${host}/products.json?limit=250&page=${page}`, FEED_TIMEOUT_MS);
+ if (status !== 200) break;
+ const j = JSON.parse(body);
+ const arr = Array.isArray(j.products) ? j.products : [];
+ if (!arr.length) break;
+ all.push(...arr);
+ if (arr.length < 250) break;
+ if (page === SHOPIFY_PAGE_CAP) truncated = true; // hit the politeness ceiling
+ }
+ if (all.length) return buildFeedResult(all, host, all.length, truncated);
+ } catch { /* no feed */ }
return { has: false };
}
@@ -169,6 +192,10 @@ async function crawlOne(seed) {
activeProductsDB: seed.activeProductsDB ?? null,
up: false, status: 0, finalUrl: null, title: null, hero: seed.dbSampleImage || null,
productCount: 0, feed: false, products: [],
+ // Full handle set for this microsite — the membership source the server resolves
+ // "This Item" deep-links against. Empty for gated (401) or broken/empty (up-200-but-no-feed,
+ // e.g. daisybennett serving the 1890swallpaper vhost) sites → server suppresses cleanly.
+ handles: [], handleCount: 0,
};
try {
const root = await getText(rec.url, ROOT_TIMEOUT_MS);
@@ -182,9 +209,11 @@ async function crawlOne(seed) {
const f = await feedProducts(host);
if (f.has) {
rec.feed = true;
- rec.productCount = f.sawMax ? f.count : f.count; // 250 cap noted via sawMax
- rec.feedTruncated = !!f.sawMax;
- rec.products = f.products.slice(0, SAMPLE_PER_SITE);
+ rec.productCount = f.count;
+ rec.feedTruncated = !!f.truncated;
+ rec.handles = f.handles; // COMPLETE handle set (membership source)
+ rec.handleCount = f.handles.length;
+ rec.products = f.products; // already scrubbed + ≤SAMPLE_PER_SITE
if (!rec.hero && rec.products[0]) rec.hero = rec.products[0].image;
}
}
@@ -220,13 +249,15 @@ async function crawlMicrosites() {
total_candidates: sites.length,
up: up.length,
with_feed: sites.filter((s) => s.feed).length,
+ with_handles: sites.filter((s) => s.handleCount > 0).length,
+ total_handles: sites.reduce((n, s) => n + (s.handleCount || 0), 0),
products: merged.length,
sites: sites.sort((a, b) => (b.up - a.up) || (b.productCount - a.productCount) || a.slug.localeCompare(b.slug)),
merged,
};
fs.mkdirSync(path.dirname(OUT), { recursive: true });
fs.writeFileSync(OUT, JSON.stringify(snapshot));
- console.log(`microsite crawl: ${up.length}/${sites.length} up, ${snapshot.with_feed} with feeds, ${merged.length} products in ${snapshot.took_ms}ms [$0 local]`);
+ console.log(`microsite crawl: ${up.length}/${sites.length} up, ${snapshot.with_feed} with feeds (${snapshot.with_handles} handle-sets, ${snapshot.total_handles.toLocaleString()} handles), ${merged.length} sample products in ${snapshot.took_ms}ms [$0 local]`);
return snapshot;
}
diff --git a/server.js b/server.js
index 1c0f06a..2afa537 100644
--- a/server.js
+++ b/server.js
@@ -115,7 +115,8 @@ let LOADED_AT = null;
let pool = null;
let STOCK = new Map(); // upper(sku) → public-safe stored stock snapshot (the /livestock FREE path)
let VENDOR_VIEWER = new Map(); // slugify(vendor) / slug → internal line-viewer URL (the vendor's live microsite)
-let VENDOR_ITEM = new Map(); // same, but ONLY hosts safe for a /product/<handle> deep link (broken/empty vhosts excluded)
+let VENDOR_ITEM = new Map(); // slugify(vendor) / slug → { url, handles:Set } — the microsite's FULL handle set;
+ // "This Item" deep-links resolve ONLY when the row's handle is IN that set.
// Stored stock snapshot — the FREE /livestock path. PUBLIC-SAFE availability ONLY
// (in_stock / quantity / lead_time / as-of); cost & price columns are NEVER selected.
@@ -164,20 +165,22 @@ async function loadStock() {
// vendors without a live line viewer degrade gracefully (the control dims).
function buildVendorViewer() {
const map = new Map();
- const item = new Map(); // slug → host base for the per-PRODUCT deep link (This Item), broken hosts excluded
+ const item = new Map(); // slug → { url, handles:Set } for per-PRODUCT deep-link membership
try {
const d = JSON.parse(fs.readFileSync(MICROSITES, 'utf8'));
for (const s of (d.sites || [])) {
if (!s.up || !s.host || BANNED.test(s.host)) continue;
const url = `https://${s.host}/`;
- // "This Item" deep-links to /product/<handle> — only safe when the microsite is NOT a
- // misconfigured/empty vhost. A site that is up + status 200 but serves ZERO products is
- // broken (e.g. daisybennett serving the 1890swallpaper vhost) and 404s on /product/<handle>.
- // Gated (401) sites are fine — the page exists behind Steve's auth. So: safe when gated OR
- // it actually has products.
- const productful = (s.productCount > 0) || (Array.isArray(s.products) && s.products.length > 0);
- const itemSafe = s.status === 401 || productful;
- const setBoth = (k) => { map.set(k, url); if (itemSafe) item.set(k, url); };
+ // "This Item" deep-links to /product/<handle>. Since microsite handle == dw_unified.handle,
+ // the link is guaranteed live ONLY when THIS product's handle is actually in the microsite's
+ // FULL handle set (captured by the crawl). We never trust a 24-sample or a bare vendor match:
+ // • gated (401) sites → handles:[] (feed not fetchable) → dims, never a dead-end
+ // • broken/empty vhosts → handles:[] (up-200-but-no-feed, e.g. daisybennett) → dims
+ // • a dw product not carried by → handle ∉ set → dims (the ~15-25% dead-end this fixes)
+ // that particular microsite
+ const handles = Array.isArray(s.handles) ? s.handles : [];
+ const entry = handles.length ? { url, handles: new Set(handles) } : null;
+ const setBoth = (k) => { map.set(k, url); if (entry) item.set(k, entry); };
if (s.vendor && !BANNED.test(s.vendor)) setBoth(slugify(s.vendor));
if (s.slug) setBoth(String(s.slug).toLowerCase());
}
@@ -231,11 +234,14 @@ function deriveRow(r) {
internal_url: adminUrl(r.shopify_id),
line_viewer_url: r.vendor ? (VENDOR_VIEWER.get(slugify(r.vendor)) || null) : null,
// This Item = the SAME internal microsite, deep-linked to THIS product's /product/<handle>
- // page (one item, not the whole line). Uses VENDOR_ITEM (broken/empty vhosts excluded) so the
- // control dims rather than dead-ending on a 404 when a microsite is misconfigured or handle-less.
+ // page (one item, not the whole line). Resolves ONLY when the row's handle is actually IN
+ // the microsite's full handle set — since microsite handle == dw_unified.handle, that makes
+ // the link guaranteed-live. Handle absent (site doesn't carry this product, gated, or broken)
+ // → null, so the control dims rather than dead-ending on the SPA's client-side "not found".
internal_product_url: (function () {
- const base = r.vendor ? VENDOR_ITEM.get(slugify(r.vendor)) : null;
- return (base && r.handle) ? `${base}product/${encodeURIComponent(r.handle)}` : null;
+ if (!r.handle || !r.vendor) return null;
+ const entry = VENDOR_ITEM.get(slugify(r.vendor));
+ return (entry && entry.handles.has(r.handle)) ? `${entry.url}product/${encodeURIComponent(r.handle)}` : null;
})(),
stock: (sku && STOCK.get(String(sku).toUpperCase())) || null,
// live_capable = this SKU's display-vendor has a wired portal adapter (WallQuest-backed).
← 9769200 all-dw This Item: suppress deep-link for broken/empty vhosts
·
back to All Designerwallcoverings
·
all-dw: show Mfr # (mfr_sku) + Mfr Name (supplier_name) as c 8c24b22 →