all.dw: dedupe grid — suppress sample-twin + cross-source-vendor duplicates (TK-10841)

cb52b7ac67e45b3e14165874a510574c84d58e84 · 2026-08-25 10:16:58 -0700 · Steve

Add dedupeRows() post-concat pass in loadSnapshot(): drop a -sample card only
when its non-sample base twin also renders (sample-only products untouched), and
drop microsite/catalog rows whose vendor is already a live Shopify vendor (incl.
Bespoke->DW Bespoke Studio alias). No DB writes; in-RAM only; git-revertable.
Verified on running :9958: default grid 214,081 -> 188,434, 0 dupes remaining,
38,245 sample-only + Fromental(shopify) preserved.

Files touched

M server.js

Diff

commit cb52b7ac67e45b3e14165874a510574c84d58e84
Author: Steve <steve@designerwallcoverings.com>
Date:   Tue Aug 25 10:16:58 2026 -0700

    all.dw: dedupe grid — suppress sample-twin + cross-source-vendor duplicates (TK-10841)
    
    Add dedupeRows() post-concat pass in loadSnapshot(): drop a -sample card only
    when its non-sample base twin also renders (sample-only products untouched), and
    drop microsite/catalog rows whose vendor is already a live Shopify vendor (incl.
    Bespoke->DW Bespoke Studio alias). No DB writes; in-RAM only; git-revertable.
    Verified on running :9958: default grid 214,081 -> 188,434, 0 dupes remaining,
    38,245 sample-only + Fromental(shopify) preserved.
---
 server.js | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 50 insertions(+), 2 deletions(-)

diff --git a/server.js b/server.js
index 7475243..663c758 100644
--- a/server.js
+++ b/server.js
@@ -809,6 +809,53 @@ function toInternalRow(d) {
   };
 }
 
+// ── aggregator-level dedup (TK-10841) ──────────────────────────────────────────
+// The grid concatenates 3 sources (shopify_products rows, MICROSITE_ROWS, CATALOG_ROWS)
+// and /api/products renders them 1:1 with no dedup. Two TRUE-duplicate classes result,
+// removed here as a single post-concat pass (in-RAM only; git-revertable; no DB writes):
+//
+//  1. SAMPLE-TWIN — DW models a memo sample as its OWN shopify product whose dw_sku ends
+//     "-sample" (base "488-401" Active + "488-401-sample" Staged, same image/pattern).
+//     Both render as separate cards = visual dupe. We drop a "-sample" row ONLY when a
+//     non-sample row with the SAME bare SKU also renders — so the ~44k sample-ONLY products
+//     (the sample IS the sole card) are untouched. Canonical row = the non-sample one.
+//  2. CROSS-SOURCE VENDOR — a vendor now fully on Shopify (Fromental/Gracie/Zuber) is STILL
+//     injected as microsite rows (the "not in shopify_products" comment on those feeds is
+//     stale), and a catalog line whose name is an alias of a live Shopify vendor ("Bespoke"
+//     ↔ "DW Bespoke Studio") double-lists because the coarse table-level guard is exact-match.
+//     We drop any microsite/catalog row whose (alias-normalized) vendor is already a live
+//     Shopify vendor. Canonical row = the Shopify one.
+//
+// Deliberately NOT deduping by shared image alone: vendors reuse collection/lifestyle imagery
+// across genuinely distinct SKUs (verified — distinct Fromental patterns share one hero image),
+// so image-equality would over-suppress. Identity is SKU + vendor, never image.
+const CROSS_SOURCE_VENDOR_ALIAS = new Map([
+  // catalog/microsite vendor name (alnum-normalized) → the live Shopify vendor it duplicates
+  ['bespoke', 'dwbespokestudio'],
+]);
+function dedupeRows(rows, liveVendorKeys) {
+  const normV = (s) => String(s || '').toLowerCase().replace(/[^a-z0-9]/g, '');
+  const bareSku = (s) => String(s || '').toLowerCase().replace(/-sample$/, '').trim();
+  const isSample = (s) => /-sample$/i.test(String(s || ''));
+  // base (non-sample) SKUs that render, so a sample twin can be identified. Normalized.
+  const baseSkus = new Set();
+  for (const r of rows) { if (r.sku && !isSample(r.sku)) baseSkus.add(normV(bareSku(r.sku))); }
+  let sampleTwins = 0, crossSourceVendor = 0;
+  const out = rows.filter((r) => {
+    // rule 2 — a microsite/catalog row whose vendor is already a live Shopify vendor (or an alias of one)
+    if (r.source === 'microsite' || r.source === 'catalog') {
+      const vk = normV(r.vendor);
+      const canon = CROSS_SOURCE_VENDOR_ALIAS.get(vk) || vk;
+      if (liveVendorKeys.has(vk) || liveVendorKeys.has(canon)) { crossSourceVendor++; return false; }
+    }
+    // rule 1 — a "-sample" card whose non-sample base sibling also renders
+    if (r.sku && isSample(r.sku) && baseSkus.has(normV(bareSku(r.sku)))) { sampleTwins++; return false; }
+    return true;
+  });
+  console.log(`dedupe: removed ${sampleTwins.toLocaleString()} sample-twin + ${crossSourceVendor.toLocaleString()} cross-source-vendor rows (${rows.length.toLocaleString()} → ${out.length.toLocaleString()})`);
+  return out;
+}
+
 async function loadSnapshot() {
   if (!pool) pool = new Pool({ connectionString: DSN, max: 2 });
   const t0 = Date.now();
@@ -836,13 +883,14 @@ async function loadSnapshot() {
   // Every genuine catalog-only vendor line → additive searchable rows. Non-fatal: on failure
   // CATALOG_ROWS keeps its prior value (or empty on first boot) — the grid is unaffected.
   try { await loadCatalogVendors(liveVendorKeys); } catch (e) { console.error('catalog vendors load failed (non-fatal):', e.message); }
-  ROWS = res.rows
+  ROWS = dedupeRows(res.rows
     .filter((r) => !BANNED.test(r.vendor || '') && !BANNED.test(r.title || ''))
     .map(deriveRow)
     // ...then append the microsite-sourced rows so they're searchable + facetable in the same grid.
     .concat(MICROSITE_ROWS)
     // ...and every genuine catalog-only vendor line, minus the banned private-label mills.
-    .concat(CATALOG_ROWS.filter((r) => !BANNED.test(r.vendor || '') && !BANNED.test(r.title || '')));
+    .concat(CATALOG_ROWS.filter((r) => !BANNED.test(r.vendor || '') && !BANNED.test(r.title || ''))),
+    liveVendorKeys);
   // INTERNAL feed — EVERY non-archived SKU (incl. private-label, NO banned-exclusion), leak-sanitized,
   // plus the extra fields internal consumers (photo + substitute apps) need but the public API omits:
   // product_id (numeric, for Shopify photo push), handle (for product-page URLs), tags (for material/

← 5f3f6eb auto-data-snapshot: 2026-08-24T10:08:33 (1 data files) — pac · back to All Designerwallcoverings · creds-safe fetch guard: resolve relative fetch vs credential ce50840 →