← back to Nationalrealestate

usre: harden firm-phone enrichment for accuracy (name-match gate, Bing, dedup) (TK-10687)

e72841d03f0b6c79cd68594ee95786d3a685e344 · 2026-08-18 13:43:23 -0700 · Steve Abrams

A call directory must not guess — a wrong number wastes Frank's time. Hardening:
- NAME-MATCH GATE: trust a discovered phone ONLY from the firm's own known site or a
  DOMAIN carrying the firm's distinctive name token (serhant.com, colliers.com,
  axencareers.com). Kills the Bing false matches (baseball-reference / cabq.gov /
  southpointcasino / visitpasadena) that common-word firm names pulled in.
- Expanded stop-list (geo/directional/generic words) + NONBIZ blocklist (tourism/gov/
  wiki/social) so place-named firms don't match a city tourism site.
- Prefer OWN domain over aggregators; drop aggregator TOLL-FREE lead-lines; confidence
  tag in phone_source (web-own / web-agg).
- DEDUP: never stamp a phone already assigned to a different firm (shared switchboard).
- Bing primary discovery (DuckDuckGo rate-limited the IP) + DDG fallback; plausible-
  area-code filter. Precision-first: generic-named firms stay 'unverified' (honest)
  rather than get a wrong number. $0 (local model + local fetch + free search).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Files touched

• M scripts/enrich-firm-contacts.mjs

Diff

commit e72841d03f0b6c79cd68594ee95786d3a685e344
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Tue Aug 18 13:43:23 2026 -0700

    usre: harden firm-phone enrichment for accuracy (name-match gate, Bing, dedup) (TK-10687)
    
    A call directory must not guess — a wrong number wastes Frank's time. Hardening:
    - NAME-MATCH GATE: trust a discovered phone ONLY from the firm's own known site or a
      DOMAIN carrying the firm's distinctive name token (serhant.com, colliers.com,
      axencareers.com). Kills the Bing false matches (baseball-reference / cabq.gov /
      southpointcasino / visitpasadena) that common-word firm names pulled in.
    - Expanded stop-list (geo/directional/generic words) + NONBIZ blocklist (tourism/gov/
      wiki/social) so place-named firms don't match a city tourism site.
    - Prefer OWN domain over aggregators; drop aggregator TOLL-FREE lead-lines; confidence
      tag in phone_source (web-own / web-agg).
    - DEDUP: never stamp a phone already assigned to a different firm (shared switchboard).
    - Bing primary discovery (DuckDuckGo rate-limited the IP) + DDG fallback; plausible-
      area-code filter. Precision-first: generic-named firms stay 'unverified' (honest)
      rather than get a wrong number. $0 (local model + local fetch + free search).
    
    Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 scripts/enrich-firm-contacts.mjs | 103 ++++++++++++++++++++++++++++++++-------
 1 file changed, 85 insertions(+), 18 deletions(-)

diff --git a/scripts/enrich-firm-contacts.mjs b/scripts/enrich-firm-contacts.mjs
index cf3b501..40cae78 100644
--- a/scripts/enrich-firm-contacts.mjs
+++ b/scripts/enrich-firm-contacts.mjs
@@ -101,21 +101,65 @@ function stripHtml(html) {
     .replace(/<[^>]+>/g, ' ').replace(/&nbsp;/g, ' ').replace(/&amp;/g, '&').replace(/\s+/g, ' ').trim();
 }
 function hostOf(u) { try { return new URL(u).hostname.replace(/^www\./, ''); } catch { return ''; } }
+// Distinctive firm-name tokens (drop generic real-estate words) used to VERIFY a page is actually
+// about this firm before trusting its phone — a wrong number is worse than none for a call directory.
+const NAME_STOP = new Set([
+  'real','estate','realty','realtors','realtor','inc','corp','llc','group','properties','property',
+  'company','co','the','and','of','ii','iii','homes','home','brokers','broker','brokerage','associates',
+  'partners','enterprises','services','international','california','holdings','management','investments',
+  'investment','investors','investor','financial','finance','capital','ventures','venture','solutions',
+  // common geographic / directional / generic words that are too weak to verify a domain on their own
+  'south','north','east','west','central','pacific','valley','coast','coastal','bay','city','county',
+  'star','sun','gold','golden','blue','first','prime','elite','premier','best','hall','park','sales',
+  'network','team','national','american','united','global','metro','urban','land','house','living','usa']);
+function firmTokens(name) { return String(name || '').toLowerCase().replace(/[^a-z0-9 ]/g, ' ').split(/\s+/).filter(w => w.length >= 4 && !NAME_STOP.has(w)); }
+function pageMatchesFirm(page, tokens) {
+  if (!tokens.length) return false;                     // no distinctive token → unverifiable → don't trust
+  const host = (page.host || '').toLowerCase();
+  // Trust ONLY a domain that carries the firm's distinctive name token (e.g. colliers.com,
+  // axencareers.com, serhant.com). Text-only matching is too noisy for common-word firm names
+  // ("Park and Refer" hitting a city-parks page) — a call directory must not guess. The firm's own
+  // known website is trusted separately at the call site.
+  return tokens.some(t => t.length >= 4 && host.includes(t));
+}
+// NANP sanity: area/exchange codes start 2-9 (already in PHONE_RE); reject a few impossible ones.
+function plausiblePhone(p) { const m = p.match(/\((\d{3})\)/); return m && /^[2-9]\d{2}$/.test(m[1]) && !/(^0|555$)/.test(m[1]); }
 
 // ── free web discovery (DuckDuckGo HTML) ─────────────────────────────────────────────────────────
-const AGG = /(zillow|redfin|realtor\.com|loopnet|crexi|yelp|mapquest|bbb\.org|facebook|linkedin|indeed|glassdoor|manta|dnb\.com|bizapedia|buzzfile)/i;
-async function discoverSite(name, city) {
-  const q = encodeURIComponent(`${name} ${city || ''} California real estate brokerage`);
-  const html = await fetchText(`https://html.duckduckgo.com/html/?q=${q}`);
+const AGG = /(zillow|redfin|realtor\.com|realty\.com|loopnet|crexi|yelp|mapquest|bbb\.org|facebook|linkedin|indeed|glassdoor|manta|dnb\.com|bizapedia|buzzfile|nestfully|mlslistings|homes\.com|point2homes|trulia)/i;
+const TOLLFREE = /^\((?:800|833|844|855|866|877|888)\)/;   // aggregator toll-free = almost always a lead-routing line, not the firm's desk
+// Never a brokerage — tourism / gov / wiki / social / directory noise that a place-or-generic firm
+// name (e.g. "Pasadena Market Center" → visitpasadena.com) false-matches on.
+const NONBIZ = /(^|\.)(visit[a-z]+\.(com|org)|cityof[a-z]+|[a-z]+chamber|wikipedia\.org|tripadvisor|casino|hotels?\.com|whatsapp|youtube|reddit|pinterest|instagram|twitter|x\.com|tiktok|amazon|ebay|craigslist)|\.gov(\/|$|\.)|\.edu(\/|$)/i;
+function decodeBing(u) {
+  const m = u.match(/[?&]u=a1([^&]+)/); if (!m) return u;
+  try { return Buffer.from(m[1].replace(/-/g, '+').replace(/_/g, '/'), 'base64').toString('utf8'); } catch { return u; }
+}
+async function searchBing(q) {
+  const html = await fetchText('https://www.bing.com/search?q=' + encodeURIComponent(q));
+  if (!html) return [];
+  const urls = [];
+  for (const m of html.matchAll(/href="(https?:\/\/www\.bing\.com\/ck\/a\?[^"]*&(?:amp;)?u=a1[^"]+)"/g)) {
+    const u = decodeBing(m[1].replace(/&amp;/g, '&')); if (/^https?:\/\//.test(u) && !/bing\.com/.test(u)) urls.push(u);
+  }
+  for (const m of html.matchAll(/<h2><a[^>]+href="(https?:\/\/(?!www\.bing)[^"]+)"/g)) urls.push(m[1]);
+  return [...new Set(urls)];
+}
+async function searchDDG(q) {
+  const html = await fetchText('https://html.duckduckgo.com/html/?q=' + encodeURIComponent(q));
   if (!html) return [];
   const urls = [];
-  const re = /<a[^>]+class="result__a"[^>]+href="([^"]+)"/gi; let m;
-  while ((m = re.exec(html)) && urls.length < 6) {
-    let u = m[1];
-    const dd = u.match(/uddg=([^&]+)/); if (dd) { try { u = decodeURIComponent(dd[1]); } catch {} }
+  for (const m of html.matchAll(/<a[^>]+class="result__a"[^>]+href="([^"]+)"/gi)) {
+    let u = m[1]; const dd = u.match(/uddg=([^&]+)/); if (dd) { try { u = decodeURIComponent(dd[1]); } catch {} }
     if (/^https?:\/\//.test(u)) urls.push(u);
   }
-  // own-site first (non-aggregator), then aggregator listings (still carry the phone)
+  return [...new Set(urls)];
+}
+async function discoverSite(name, city) {
+  const q = `${name} ${city || ''} California real estate brokerage`;
+  let urls = await searchBing(q);                       // Bing = most resilient to rate-limits
+  if (!urls.length) urls = await searchDDG(q);          // DDG fallback
+  // own-site first (non-aggregator), then aggregator listings (a last resort — still carry a phone)
   return [...urls.filter(u => !AGG.test(u)), ...urls.filter(u => AGG.test(u))].slice(0, 4);
 }
 
@@ -170,25 +214,42 @@ async function enrichFirm(firm) {
   }
   if (!pages.length) return { status: 'unreachable' };
 
-  // rank the pages: the one with the most phone candidates likely IS the contact page
-  const scored = pages.map(p => ({ ...p, phones: phoneCandidates(p.text) })).sort((a, b) => b.phones.length - a.phones.length);
-  const best = scored[0];
-  const domain = hostOf(best.url);
-  const phones = [...new Set(scored.flatMap(p => p.phones))].slice(0, 12);
-  const emails = emailCandidates(scored.map(p => p.text).join(' '), AGG.test(best.url) ? '' : domain);
+  // NAME-MATCH GATE (correctness > coverage): only trust a page that is actually ABOUT this firm.
+  // A page from the firm's own known website is trusted; a DISCOVERED page must carry the firm's
+  // distinctive name token(s) in its domain or text. This kills the baseball-reference / cabq.gov /
+  // random-quiz-site false matches Bing returns for generic firm names.
+  const tokens = firmTokens(firm.name);
+  const knownHost = firm.website ? hostOf(/^https?:/.test(firm.website) ? firm.website : 'https://' + firm.website) : '';
+  let scored = pages.map(p => ({ ...p, host: hostOf(p.url), agg: AGG.test(p.url), phones: phoneCandidates(p.text).filter(plausiblePhone) }));
+  scored = scored.filter(p => !NONBIZ.test(p.host));    // drop tourism/gov/wiki/social noise first
+  scored = scored.filter(p => (knownHost && p.host === knownHost) || pageMatchesFirm(p, tokens));
+  if (!scored.length) return { status: 'unverified' };  // found pages, but none provably this firm → don't guess
+
+  // Prefer the firm's OWN domain over aggregators — an aggregator page often shows a lead-routing
+  // number or a specific agent's cell, not the firm's desk. Own-domain phones win; only fall back to
+  // NON-toll-free aggregator numbers (toll-free from an aggregator ≈ a lead line, dropped).
+  const own = scored.filter(p => !p.agg).sort((a, b) => b.phones.length - a.phones.length);
+  const agg = scored.filter(p => p.agg).sort((a, b) => b.phones.length - a.phones.length);
+  const ownPhones = [...new Set(own.flatMap(p => p.phones))].slice(0, 12);
+  const aggPhones = [...new Set(agg.flatMap(p => p.phones))].filter(p => !TOLLFREE.test(p)).slice(0, 12);
+  const best = own.find(p => p.phones.length) || own[0] || agg.find(p => p.phones.length) || agg[0];
+  if (!best) return { status: 'unreachable' };
+  const confidence = best.agg ? 'agg' : 'own';
+  const phones = ownPhones.length ? ownPhones : aggPhones;                 // own-domain phones preferred
+  const emails = emailCandidates([...own, ...agg].map(p => p.text).join(' '), best.agg ? '' : best.host);
   if (!phones.length && !emails.length) return { status: 'no_contact', source_url: best.url };
 
   const pick = await modelPick(firm.name, firm.hq_city, best.text, phones, emails);
   return {
-    status: pick.phone ? 'found' : (emails.length ? 'email_only' : 'no_contact'),
+    status: pick.phone ? 'found' : (pick.email ? 'email_only' : 'no_contact'),
     phone: pick.phone || null, email: pick.email || null, address: pick.address || null,
-    source_url: best.url, source_host: domain,
+    source_url: best.url, source_host: best.host, confidence,
   };
 }
 
 // ── DB writes ─────────────────────────────────────────────────────────────────────────────────────
 async function writeFirm(client, firm, r) {
-  const src = r.source_host ? `web:${r.source_host}` : 'web';
+  const src = r.source_host ? `web-${r.confidence || 'own'}:${r.source_host}` : 'web';
   await client.query(
     `UPDATE firm SET
         phone = COALESCE(NULLIF(phone,''), $2),
@@ -229,6 +290,12 @@ async function main() {
   for (const f of firms) {
     attempted++;
     let r; try { r = await enrichFirm(f); } catch (e) { r = { status: 'error' }; }
+    // Dedup: a phone already assigned to a DIFFERENT firm is a shared switchboard / lead line — drop
+    // it (don't stamp the same number onto many firms). Keeps the directory honest.
+    if (r.phone) {
+      const dup = await pool.query(`SELECT 1 FROM firm WHERE phone=$1 AND id<>$2 LIMIT 1`, [r.phone, f.id]);
+      if (dup.rowCount) { r.phone = null; r.status = r.email ? 'email_only' : 'dup_phone'; }
+    }
     if (r.phone) found++; if (r.email) email++;
     const tag = r.phone ? '📞 ' + r.phone : (r.email ? '✉ ' + r.email : '· ' + r.status);
     console.log(`  [${attempted}/${firms.length}] ${f.name} (${f.active_brokers} active) → ${tag}${r.address ? '  📍 ' + r.address : ''}${r.source_host ? '  <' + r.source_host + '>' : ''}`);

← b3eef68 auto-data-snapshot: 2026-08-18T13:28:17 (1 data files) — dat  ·  back to Nationalrealestate  ·  usre: Tier-1 accurate firm-phone backfill from cre office li 88312fc →