PR intel: safe org dedup (page-title captures → canonical brand)

ea4a72c8889ef4429d2e57c550b1bb48a76fca7c · 2026-08-05 15:40:52 -0700 · Steve

Steve-approved safe subset: within each same-domain group, merge ONLY rows
whose name is an unambiguous scraped page/article title INTO the canonical
row that MATCHES the domain brand (people_count-ranked). Never merges legit
metro-office rows or picks a stub as canonical (dry-run caught + fixed that:
'Radius Commercial RE'→'Ventura Office' inversion). Excludes bizjournals
(metro-distinct ACBJ editions). Reversible (loser→duplicate). 20 merged.

Files touched

A src/pr/tools/org-dedup.js

Diff

commit ea4a72c8889ef4429d2e57c550b1bb48a76fca7c
Author: Steve <steve@designerwallcoverings.com>
Date:   Wed Aug 5 15:40:52 2026 -0700

    PR intel: safe org dedup (page-title captures → canonical brand)
    
    Steve-approved safe subset: within each same-domain group, merge ONLY rows
    whose name is an unambiguous scraped page/article title INTO the canonical
    row that MATCHES the domain brand (people_count-ranked). Never merges legit
    metro-office rows or picks a stub as canonical (dry-run caught + fixed that:
    'Radius Commercial RE'→'Ventura Office' inversion). Excludes bizjournals
    (metro-distinct ACBJ editions). Reversible (loser→duplicate). 20 merged.
---
 src/pr/tools/org-dedup.js | 66 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 66 insertions(+)

diff --git a/src/pr/tools/org-dedup.js b/src/pr/tools/org-dedup.js
new file mode 100644
index 00000000..3dfebc1c
--- /dev/null
+++ b/src/pr/tools/org-dedup.js
@@ -0,0 +1,66 @@
+'use strict';
+// Safe org dedup (Steve-approved subset, yoloforever): within each same-domain group,
+// merge ONLY the rows whose name is an unambiguous non-org PAGE/ARTICLE title into the
+// canonical brand row. Never merges a row that could be a legitimate metro office or a
+// real brand name — those are left for human review. Reversible: organizations.merge
+// reassigns people/evidence to the winner and marks the loser 'duplicate' (not deleted).
+// Dry-run by default; --apply writes. $0.
+const API = process.env.PR_API || 'https://rentv.agentabrams.com';
+const AUTH = 'Basic ' + Buffer.from(process.env.PR_AUTH || 'admin:DW2024!').toString('base64');
+const APPLY = process.argv.includes('--apply');
+
+async function api(p, opts = {}) {
+  const r = await fetch(API + '/api/pr' + p, { ...opts, headers: { Authorization: AUTH, 'Content-Type': 'application/json', ...(opts.headers || {}) }, body: opts.body ? JSON.stringify(opts.body) : undefined });
+  if (!r.ok) throw new Error(p + ' → ' + r.status + ': ' + (await r.text()).slice(0, 140));
+  return r.json();
+}
+
+// normalize display_name locally (the list API returns display_name, not normalized_name)
+const norm = (s) => String(s || '').toLowerCase().replace(/[^a-z0-9 ]/g, ' ').replace(/\s+/g, ' ').trim();
+// A name that is clearly a scraped page/article title, NOT an organization.
+const GARBAGE = /\b(top \d|best |for sale|for lease|properties|property market|^about$|^home$|listings|^news$|brief|report|unpacking|outstanding|steady worker|firms report|specializing in|services in|real estate in|commercial real estate news|s pr firms|s property|tenant representation|crisis communications|you matter|for real estate|in california|in los angeles|in san francisco|in orange county|in san diego|s best| market$)\b/i;
+const looksGarbage = (nn) => GARBAGE.test(nn) || /\b(19|20)\d{2}\b/.test(nn) || /^\d/.test(nn) || nn.split(' ').length >= 6;
+
+async function main() {
+  const seen = new Set(); const all = [];
+  for (const offset of [0, 500, 1000]) {
+    const page = await api(`/organizations?limit=500&offset=${offset}`);
+    const rows = page.rows || page.organizations || [];
+    for (const o of rows) { if (!seen.has(o.id)) { seen.add(o.id); o._nn = norm(o.display_name); all.push(o); } }
+    if (rows.length < 500) break;
+  }
+  const byDomain = {};
+  for (const o of all) {
+    if (!o.domain || o.lifecycle_status === 'duplicate') continue;
+    (byDomain[o.domain] = byDomain[o.domain] || []).push(o);
+  }
+  // metro-distinct networks Steve said to leave (editions are legit, not dupes)
+  const EXCLUDE = new Set(['bizjournals.com']);
+  const plan = [];
+  for (const [domain, group] of Object.entries(byDomain)) {
+    if (group.length < 2 || EXCLUDE.has(domain)) continue;
+    const clean = group.filter((o) => !looksGarbage(o._nn));
+    const garbage = group.filter((o) => looksGarbage(o._nn));
+    if (!clean.length || !garbage.length) continue;
+    // Canonical MUST match the domain brand (high confidence it's the real org, not a
+    // short stub like "Fresno"/"Ventura Office"). brand = first domain label; match if
+    // the brand and the space-stripped name share a >=4-char containment.
+    const brand = domain.split('.')[0].replace(/[^a-z0-9]/g, '');
+    const brandMatch = (o) => { const c = o._nn.replace(/[^a-z0-9]/g, ''); return c.length >= 4 && brand.length >= 4 && (c.includes(brand) || brand.includes(c) || o._nn.split(' ').some((t) => t.length >= 4 && brand.includes(t))); };
+    const canon = clean.filter(brandMatch);
+    if (!canon.length) continue; // no brand-matching canonical → too ambiguous, leave for human review
+    canon.sort((a, b) => (b.people_count || 0) - (a.people_count || 0) || a._nn.length - b._nn.length || a.id - b.id);
+    const winner = canon[0];
+    for (const loser of garbage) plan.push({ domain, winner, loser });
+  }
+  console.log(`[org-dedup] domains scanned ${Object.keys(byDomain).length} · safe page-title merges planned ${plan.length} · ${APPLY ? 'APPLYING' : 'DRY-RUN (--apply to write)'}`);
+  for (const m of plan.slice(0, 40)) console.log(`  ${m.domain}: "${m.loser.display_name}" (id ${m.loser.id}) → "${m.winner.display_name}" (id ${m.winner.id})`);
+  if (plan.length > 40) console.log(`  … +${plan.length - 40} more`);
+  if (APPLY) {
+    let done = 0;
+    for (const m of plan) { try { await api('/organizations/' + m.winner.id + '/merge', { method: 'POST', body: { duplicate_id: m.loser.id } }); done++; } catch (e) { console.log(`  ! ${m.loser.id}→${m.winner.id}: ${e.message.slice(0, 60)}`); } }
+    console.log(`[org-dedup] merged ${done}/${plan.length}`);
+  }
+  process.exit(0);
+}
+main().catch((e) => { console.error('fatal:', e.message); process.exit(1); });

← 02cf415b PR intel: yoloforever cycle 2 — enrich lockfile + linkedin r · back to Rentv · auto-save: 2026-08-05T15:43:31 (7 files) — data/deals-regist 96574cc1 →