← back to All Designerwallcoverings
all.dw: surface 4 luxury lines (Fromental/Gracie/Zuber/Crezana) via microsite full-feeds
b2fc39c689d3317c0d1cbf94d0a5e08783262627 · 2026-07-09 16:17:55 -0700 · Steve
Ingest the FULL /api/products feed of the 4 DW-family microsites over localhost
(bypassing the public 401 with the shared internal cred), normalize each feed's
heterogeneous shape to the grid row shape (colors/styles/materials via the same
tag lexicons, quote->null price), tag source='microsite', dedupe by vendor+handle,
and append to ROWS at each snapshot cycle. No Shopify import, no dw_unified write.
Additive + guarded: a failed feed keeps the prior cycle's rows for that line.
Verified locally: /api/facets vendor = Fromental 735 / Gracie 150 / Zuber 192 /
Crezana Design 443; products filter + color/style facets cross-filter; internal
/api/catalog-full unaffected (0 microsite rows).
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Files touched
Diff
commit b2fc39c689d3317c0d1cbf94d0a5e08783262627
Author: Steve <steve@designerwallcoverings.com>
Date: Thu Jul 9 16:17:55 2026 -0700
all.dw: surface 4 luxury lines (Fromental/Gracie/Zuber/Crezana) via microsite full-feeds
Ingest the FULL /api/products feed of the 4 DW-family microsites over localhost
(bypassing the public 401 with the shared internal cred), normalize each feed's
heterogeneous shape to the grid row shape (colors/styles/materials via the same
tag lexicons, quote->null price), tag source='microsite', dedupe by vendor+handle,
and append to ROWS at each snapshot cycle. No Shopify import, no dw_unified write.
Additive + guarded: a failed feed keeps the prior cycle's rows for that line.
Verified locally: /api/facets vendor = Fromental 735 / Gracie 150 / Zuber 192 /
Crezana Design 443; products filter + color/style facets cross-filter; internal
/api/catalog-full unaffected (0 microsite rows).
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
server.js | 166 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-
1 file changed, 164 insertions(+), 2 deletions(-)
diff --git a/server.js b/server.js
index ae2f6eb..e396bc1 100644
--- a/server.js
+++ b/server.js
@@ -187,6 +187,9 @@ async function snapSql() {
let ROWS = [];
let ROWS_INTERNAL = []; // internal full-catalog feed (every non-archived SKU, leak-sanitized)
+let MICROSITE_ROWS = []; // extra searchable rows sourced from DW-family microsite full-feeds
+ // (customer-facing luxury lines NOT in shopify_products — e.g. Fromental,
+ // Gracie, Zuber, Crezana Design). source='microsite'. See loadMicrositeExtras().
let LOADED_AT = null;
let pool = null;
let FM = new Map(); // fmKey(dw_sku) → FileMaker WALLPAPER mirror row (source of truth)
@@ -433,6 +436,159 @@ function deriveRow(r) {
};
}
+// ── DW-family microsite full-feed ingest (customer-facing luxury lines) ────────────
+// A handful of luxury vendors (Fromental, Gracie, Zuber, Crezana Design) live ONLY in
+// their dw_unified staging tables + their own co-located microsites — they were never
+// Shopify-imported, so shopify_products has no rows and the main grid can't see them.
+// Steve authorized surfacing them INTERNALLY (no Shopify import, no canonical dw_unified
+// write) by pulling each microsite's FULL /api/products feed over localhost (bypassing the
+// public 401), normalizing to the grid row shape, and merging into the searchable set.
+// These are CUSTOMER-FACING brands (not private-label), so their real vendor names are fine.
+// Fully additive + guarded: an unreachable feed is skipped, never breaks the grid.
+const MICROSITE_FEEDS = [
+ { vendor: 'Fromental', slug: 'fromental', url: 'http://127.0.0.1:10071/api/products?limit=100000', type: 'Wallcovering' },
+ { vendor: 'Gracie', slug: 'gracie', url: 'http://127.0.0.1:10073/api/products?limit=100000', type: 'Wallcovering' },
+ { vendor: 'Zuber', slug: 'zuber', url: 'http://127.0.0.1:10074/api/products?limit=100000', type: 'Wallcovering' },
+ { vendor: 'Crezana Design', slug: 'crezana', url: 'http://127.0.0.1:10072/api/products?limit=100000', type: 'Fabric' },
+];
+const MICROSITE_PER_SITE_CAP = 800; // full catalog per site is ≤735 — well under this
+// Shared internal cred so the localhost feeds (Basic-Auth gated, the 401 is the healthy signal)
+// are readable. Same override the crawler uses; defaults to the standard admin cred.
+const MICROSITE_FEED_AUTH = process.env.MICROSITE_BASIC_AUTH || 'admin:DW2024!';
+const MICROSITE_FEED_HDR = 'Basic ' + Buffer.from(MICROSITE_FEED_AUTH).toString('base64');
+const MICROSITE_FEED_TIMEOUT_MS = 12000;
+
+// Map a heterogeneous microsite feed row (each of the 4 feeds has a slightly different
+// shape) onto the SAME row shape the grid's deriveRow emits, so it's searchable + facetable
+// with zero frontend change. Colors/styles/materials run through the SAME tag lexicons the
+// DB rows use, seeded from the feed's own AI color/style/tag fields.
+function deriveMicrositeRow(p, feed) {
+ const sku = p.sku || p.dw_sku || p.mfr_sku || null;
+ const handle = p.handle || null;
+ const title = [p.pattern || p.pattern_name, (p.color || p.color_name)].filter(Boolean).join(' — ')
+ || p.pattern || p.pattern_name || p.title || '';
+ // gather candidate facet terms from every tag-ish field the feed carries
+ const rawColorNames = []
+ .concat(Array.isArray(p.colors) ? p.colors.map((c) => (typeof c === 'string' ? c : c && c.name)) : [])
+ .concat(Array.isArray(p.ai_colors) ? p.ai_colors.map((c) => (typeof c === 'string' ? c : c && c.name)) : [])
+ .concat(p.color || p.color_name || []);
+ const rawStyleNames = []
+ .concat(Array.isArray(p.styles) ? p.styles : [])
+ .concat(Array.isArray(p.ai_styles) ? p.ai_styles : []);
+ const rawTagNames = []
+ .concat(Array.isArray(p.tags) ? p.tags : [])
+ .concat(Array.isArray(p.aiTags) ? p.aiTags : [])
+ .concat(Array.isArray(p.ai_tags) ? p.ai_tags : [])
+ .concat(p.material ? [p.material] : []);
+ const colors = [], styles = [], materials = [];
+ for (const c of rawColorNames) { const lc = String(c || '').toLowerCase().trim(); if (COLOR_TAGS.has(lc)) colors.push(titleCase(lc)); }
+ for (const s of rawStyleNames) { const lc = String(s || '').toLowerCase().trim(); if (STYLE_TAGS.has(lc)) styles.push(titleCase(lc)); }
+ // styles + tags can both name a material or a style — match each against both lexicons
+ for (const t of rawStyleNames.concat(rawTagNames)) {
+ const lc = String(t || '').toLowerCase().trim();
+ if (STYLE_TAGS.has(lc)) styles.push(titleCase(lc));
+ if (MATERIAL_TAGS.has(lc)) materials.push(titleCase(lc));
+ }
+ const price = (p.quote === true) ? null : (p.price != null ? parseFloat(p.price) : null);
+ const gallery = Array.isArray(p.images) ? p.images : Array.isArray(p.gallery) ? p.gallery : [];
+ const primaryImg = p.image || p.image_url || gallery[0] || null;
+ const images = primaryImg ? [primaryImg, ...gallery.filter((u) => u && u !== primaryImg)] : [];
+ const created = p.createdAt || p.created_at || null;
+ const heur = [sku, p.mfr_sku, title, p.pattern, p.pattern_name, p.collection, feed.vendor,
+ ...rawColorNames, ...rawStyleNames, ...rawTagNames].filter(Boolean).join(' ').toLowerCase();
+ return {
+ id: `ms:${feed.slug}:${handle || sku || heur.slice(0, 40)}`,
+ source: 'microsite',
+ sku,
+ series: seriesOf(sku),
+ title,
+ vendor: feed.vendor,
+ type: p.product_type || p.type || feed.type || null,
+ pattern: p.pattern || p.pattern_name || null,
+ mfr_number: p.mfr_sku || null,
+ mfr_name: feed.vendor,
+ // FileMaker enrichment fields are all null (these lines aren't in the WALLPAPER file)
+ fm_present: false, fm_mfr_number: null, fm_discontinued: false, fm_disco_date: null,
+ fm_name: null, fm_width: null, fm_specs: null, fm_handle: null, mfr_mismatch: false,
+ status: 'STAGED',
+ lifecycle: 'Staged', // these lines are staged (not on the customer store)
+ image: primaryImg,
+ images,
+ image_count: images.length,
+ price: price && price > 0 ? price : null,
+ price_band: priceBand(price),
+ colors: [...new Set(colors)],
+ styles: [...new Set(styles)],
+ materials: [...new Set(materials)],
+ image_state: primaryImg ? 'Has image' : 'No image',
+ // Website = the vendor's own product page when the feed carries an absolute one.
+ url: (typeof (p.productUrl || p.product_url) === 'string' && /^https?:\/\//.test(p.productUrl || p.product_url)) ? (p.productUrl || p.product_url) : null,
+ internal_url: null,
+ // Line Viewer = the vendor's live internal microsite (resolves via the same crawl artifact).
+ line_viewer_url: VENDOR_VIEWER.get(slugify(feed.vendor)) || VENDOR_VIEWER.get(feed.slug) || null,
+ internal_product_url: (function () {
+ if (!handle) return null;
+ const entry = VENDOR_ITEM.get(slugify(feed.vendor)) || VENDOR_ITEM.get(feed.slug);
+ return (entry && entry.handles.has(handle)) ? `${entry.url}product/${encodeURIComponent(handle)}` : null;
+ })(),
+ stock: null,
+ live_capable: false, live_tier: null, live_reason: 'live check unavailable for this line',
+ created: created ? new Date(created).getTime() || 0 : 0,
+ updated: created ? new Date(created).getTime() || 0 : 0,
+ hay: heur,
+ };
+}
+
+async function fetchMicrositeFeed(feed) {
+ const r = await fetch(feed.url, {
+ redirect: 'follow', signal: AbortSignal.timeout(MICROSITE_FEED_TIMEOUT_MS),
+ headers: { 'user-agent': 'all-dw/1.0', Authorization: MICROSITE_FEED_HDR },
+ });
+ if (r.status !== 200) throw new Error(`HTTP ${r.status}`);
+ const j = JSON.parse(await r.text());
+ const arr = Array.isArray(j.rows) ? j.rows : Array.isArray(j.products) ? j.products : [];
+ return arr;
+}
+
+// Pull all 4 DW-family microsite full-feeds, normalize + dedupe (by vendor+handle, then
+// vendor+sku), and populate MICROSITE_ROWS. Never throws — a failed feed just contributes
+// nothing this cycle (the grid keeps whatever the DB provided). Runs on the snapshot cycle.
+async function loadMicrositeExtras() {
+ const perSite = {};
+ // Per-site results keyed by vendor; on a per-site failure we KEEP the prior cycle's rows for
+ // that vendor (a transient feed blip must not silently vanish an entire luxury line).
+ const prevByVendor = {};
+ for (const r of MICROSITE_ROWS) (prevByVendor[r.vendor] ||= []).push(r);
+ const nextByVendor = {};
+ await Promise.all(MICROSITE_FEEDS.map(async (feed) => {
+ try {
+ const arr = await fetchMicrositeFeed(feed);
+ const seen = new Set();
+ const rows = [];
+ for (const p of arr) {
+ if (rows.length >= MICROSITE_PER_SITE_CAP) break;
+ // leak-safety: these are customer-facing lines, but still scrub the standing banned set
+ if (BANNED.test(feed.vendor) || BANNED.test(p.title || '') || BANNED.test(p.pattern || p.pattern_name || '')) continue;
+ const row = deriveMicrositeRow(p, feed);
+ const dk = `${feed.slug}:${(row.sku || '').toLowerCase()}|${(p.handle || '').toLowerCase()}`;
+ if (seen.has(dk)) continue;
+ seen.add(dk);
+ rows.push(row);
+ }
+ nextByVendor[feed.vendor] = rows;
+ perSite[feed.vendor] = rows.length;
+ } catch (e) {
+ // keep prior rows for this vendor (may be empty on first boot)
+ nextByVendor[feed.vendor] = prevByVendor[feed.vendor] || [];
+ perSite[feed.vendor] = `kept ${nextByVendor[feed.vendor].length} (feed ${e.message})`;
+ console.error(`microsite extras: ${feed.vendor} feed unreachable (non-fatal, kept prior):`, e.message);
+ }
+ }));
+ MICROSITE_ROWS = MICROSITE_FEEDS.flatMap((f) => nextByVendor[f.vendor] || []);
+ console.log(`microsite extras: ${MICROSITE_ROWS.length} rows merged — ` +
+ MICROSITE_FEEDS.map((f) => `${f.vendor}=${perSite[f.vendor] ?? 0}`).join(' · '));
+}
+
async function loadSnapshot() {
if (!pool) pool = new Pool({ connectionString: DSN, max: 2 });
const t0 = Date.now();
@@ -444,11 +600,17 @@ async function loadSnapshot() {
try { await loadMfr(); } catch (e) { console.error('mfr load failed (non-fatal):', e.message); }
try { await loadFileMaker(); } catch (e) { console.error('filemaker load failed (non-fatal):', e.message); }
try { await loadImages(); } catch (e) { console.error('images load failed (non-fatal):', e.message); }
+ // DW-family microsite full-feeds (Fromental/Gracie/Zuber/Crezana) — additive searchable rows
+ // for the customer-facing luxury lines that are NOT in shopify_products. Non-fatal: on any
+ // failure MICROSITE_ROWS just stays whatever it was (or empty on first boot) — grid unaffected.
+ try { await loadMicrositeExtras(); } catch (e) { console.error('microsite extras load failed (non-fatal):', e.message); }
FM_MATCHED = new Set(); // repopulated by deriveRow as it joins each row this cycle
const res = await pool.query(await snapSql());
ROWS = res.rows
.filter((r) => !BANNED.test(r.vendor || '') && !BANNED.test(r.title || ''))
- .map(deriveRow);
+ .map(deriveRow)
+ // ...then append the microsite-sourced rows so they're searchable + facetable in the same grid.
+ .concat(MICROSITE_ROWS);
// INTERNAL feed — EVERY non-archived SKU (incl. private-label, NO banned-exclusion), leak-sanitized,
// plus the extra fields internal consumers (photo + substitute apps) need but the public API omits:
// product_id (numeric, for Shopify photo push), handle (for product-page URLs), tags (for material/
@@ -478,7 +640,7 @@ async function loadSnapshot() {
for (const [k, fm] of FM) { if (!fm.is_discontinued && (!fm.record_type || fm.record_type === 'Master') && !FM_MATCHED.has(k)) fmOnly++; }
FM_STATS.fm_only_live = fmOnly;
LOADED_AT = new Date().toISOString();
- console.log(`snapshot: ${ROWS.length.toLocaleString()} public + ${ROWS_INTERNAL.length.toLocaleString()} internal products in ${Date.now() - t0}ms (stock ${STOCK.size.toLocaleString()} · viewers ${VENDOR_VIEWER.size})`);
+ console.log(`snapshot: ${ROWS.length.toLocaleString()} public (incl. ${MICROSITE_ROWS.length.toLocaleString()} microsite) + ${ROWS_INTERNAL.length.toLocaleString()} internal products in ${Date.now() - t0}ms (stock ${STOCK.size.toLocaleString()} · viewers ${VENDOR_VIEWER.size})`);
console.log(` filemaker: ${FM_STATS.rows.toLocaleString()} mirror rows · ${FM_STATS.matched.toLocaleString()} joined · ${FM_STATS.mismatch.toLocaleString()} mfr# mismatches · ${discoHidden.toLocaleString()} disco-hidden · ${fmOnly.toLocaleString()} FM-only live`);
}
← 5def1fd all.dw: register Fromental/Gracie/Zuber/Crezana brands (onbo
·
back to All Designerwallcoverings
·
all.dw: make microsite feed ports env-overridable (MICROSITE c22ea62 →