Content Studio: cross-source dedup in feed (strict, Cody-hardened — no false merges) + '+N more sources' chip

dd0317e86f7d0ac321ad3dd7dd865eb6fe743cfb · 2026-08-04 17:31:26 -0700 · steve
Files touched

M public/admin/pr-intelligence/content-studio.html
M server.js
Diff

commit dd0317e86f7d0ac321ad3dd7dd865eb6fe743cfb
Author: steve <steve@designerwallcoverings.com>
Date:   Tue Aug 4 17:31:26 2026 -0700

    Content Studio: cross-source dedup in feed (strict, Cody-hardened — no false merges) + '+N more sources' chip
---
 public/admin/pr-intelligence/content-studio.html |  2 +-
 server.js                                        | 52 +++++++++++++++++++++++-
 2 files changed, 51 insertions(+), 3 deletions(-)

diff --git a/public/admin/pr-intelligence/content-studio.html b/public/admin/pr-intelligence/content-studio.html
index c1e3dcf5..def1f708 100644
--- a/public/admin/pr-intelligence/content-studio.html
+++ b/public/admin/pr-intelligence/content-studio.html
@@ -81,7 +81,7 @@
       return `<div class="cs-art${selected && String(selected.id) === String(a.id) ? ' on' : ''}" data-id="${PR.esc(a.id)}" title="${PR.esc(a.first_seen || a.date || '')}">
         ${img ? `<img src="${PR.esc(img)}" onerror="this.style.visibility='hidden'">` : '<div style="width:60px"></div>'}
         <div><div class="t">${PR.esc(a.title || '')}</div>
-          <div class="m"><span class="cat">${PR.esc(a.cat || 'RENTV')}</span><span>📰 ${PR.esc(a.source || 'RENTV')}</span><span>🕓 ${PR.esc(dayLabel(a))}</span></div></div></div>`;
+          <div class="m"><span class="cat">${PR.esc(a.cat || 'RENTV')}</span><span>📰 ${PR.esc(a.source || 'RENTV')}</span><span>🕓 ${PR.esc(dayLabel(a))}</span>${a.related_count ? `<span style="color:#0a66c2;font-weight:600" title="Also covered by: ${PR.esc((a.related||[]).map(r=>r.source).join(', '))}">+${a.related_count} more source${a.related_count>1?'s':''}</span>` : ''}</div></div></div>`;
     }).join('') : '<div class="empty">No matches</div>';
     listEl.querySelectorAll('[data-id]').forEach(el => el.addEventListener('click', () => selectArticle(el.dataset.id)));
   }
diff --git a/server.js b/server.js
index 0cf92c78..2e3eaade 100644
--- a/server.js
+++ b/server.js
@@ -419,12 +419,60 @@ ${body || '(no body available — work from the title)'}`;
 // Deep, date-ordered news archive — accumulates FAR beyond the rolling 30-item live feed.
 // SSOT = data/news-archive.json (seeded by scripts/build-news-archive.mjs, grown by pull-news).
 // Sorted newest-first by real publish date (falls back to first_seen). Admin-only (content tool).
+// dedup=1 (default) collapses the SAME deal reported by RENTV + wires into one primary row
+// (RENTV preferred) with a `related` list — same story from 4 outlets shouldn't be 4 rows.
+// Stopwords for same-deal detection: articles, generic CRE nouns, deal verbs, AND broker/firm
+// names (a broker brokers MANY deals — its name must not create false "same deal" overlap).
+const NEWS_STOP = new Set(('the a an of for to in on at and or with from into buys sells acquires acquired purchase purchased sale sells sold leases leased lands gets adds new deal deals million billion mil its inc llc lp co group properties property real estate commercial community communities totaling near arranges arranged brokers brokered negotiates negotiated secures secured closes closed provides provided spends fetches works portfolio building buildings apartment apartments multifamily retail industrial office seniors housing units unit tenant tenants center centre residential res assets asset investment investments capital partners realty advisors advisers associates group holdings '
+  + 'marcus millichap cbre jll newmark colliers cushman wakefield berkadia hanley walker dunlop kidder mathews lee institutional stepp northmarq greystone eastdil '
+  + 'avenue street road boulevard drive court place plaza tower towers sales').split(/\s+/).filter(Boolean));
+function newsSig(title) {
+  const t = String(title || '').toLowerCase();
+  const money = t.match(/\$\s?([\d.]+)\s?(billion|bil|b|million|mil|m|k)?/);
+  let m = '';
+  if (money && money[1]) { let n = parseFloat(money[1]); const u = money[2] || ''; if (/^b/.test(u)) n *= 1000; if (u === 'k') n /= 1000; if (!isNaN(n)) m = (Math.round(n * 10) / 10).toString(); } // keep 1 decimal so $47.5M ≠ $48.2M
+  const units = ((t.match(/([\d,]{2,})\s?-?\s?unit/) || [])[1] || '').replace(/,/g, '');
+  // distinctive tokens = property names, cities, unusual terms (brokers/generics already removed)
+  const toks = new Set(t.replace(/[^a-z0-9 ]/g, ' ').split(/\s+/).filter(w => w.length >= 4 && !NEWS_STOP.has(w)));
+  return { m, units, toks };
+}
+function newsSimilar(a, b) {
+  const inter = [...a.toks].filter(x => b.toks.has(x)).length;
+  const uni = new Set([...a.toks, ...b.toks]).size || 1;
+  // Same deal is a STRONG claim — err toward NOT merging (hiding a distinct story is worse than a dup row):
+  // (1) identical dollar (to 1 decimal) AND identical unit count, plus >=2 shared distinctive nouns, or
+  // (2) very high distinctive-token overlap (same property/city terms).
+  // NOTE: single-linkage clustering (dedupeNews) can split a large same-deal cluster by ingest order at
+  // scale (~2k+ articles) — acceptable at current size; revisit with union-find if the archive grows big.
+  if (a.m && b.m && a.m === b.m && a.units && b.units && a.units === b.units && inter >= 2) return true;
+  return inter >= 3 && inter / uni >= 0.6;
+}
+function dedupeNews(items) {
+  const sigs = items.map(it => ({ it, sig: newsSig(it.title) }));
+  const used = new Array(sigs.length).fill(false), out = [];
+  for (let i = 0; i < sigs.length; i++) {
+    if (used[i]) continue;
+    const cluster = [sigs[i]]; used[i] = true;
+    for (let j = i + 1; j < sigs.length; j++) {
+      if (used[j]) continue;
+      if (cluster.some(c => newsSimilar(c.sig, sigs[j].sig))) { cluster.push(sigs[j]); used[j] = true; }
+    }
+    // primary: prefer RENTV, then the longest (most descriptive) title
+    cluster.sort((x, y) => (((y.it.source === 'RENTV') - (x.it.source === 'RENTV')) || (String(y.it.title).length - String(x.it.title).length)));
+    const primary = cluster[0].it;
+    const related = cluster.slice(1).map(c => ({ source: c.it.source || 'RENTV', title: c.it.title, url: c.it.url, date: c.it.date }));
+    out.push(related.length ? { ...primary, related, related_count: related.length } : primary);
+  }
+  return out;
+}
 app.get('/api/news/archive', adminOnly, (req, res) => {
   const raw = readJSON('news-archive.json', []);
-  const items = Array.isArray(raw) ? raw : (raw.items || []);
+  let items = Array.isArray(raw) ? raw : (raw.items || []);
   items.sort((a, b) => String(b.date || b.first_seen || '').localeCompare(String(a.date || a.first_seen || '')));
+  const rawCount = items.length;
+  if (req.query.dedup !== '0') items = dedupeNews(items);
   const limit = Math.min(Number(req.query.limit) || 500, 2000);
-  res.json({ count: items.length, items: items.slice(0, limit) });
+  res.json({ count: items.length, raw_count: rawCount, items: items.slice(0, limit) });
 });
 
 // ── Topical-authority hubs (SEO basis: re-props CRE keyword library) ─────────────
← 4d3751bd Content Studio hardening: localize blog images on publish/ed · back to Rentv 2026 · auto-save: 2026-08-04T17:35:24 (3 files) — data/markets.json 9f00a2c9 →