`scripts/enrich-condo-emails.js`

98 lines
#!/usr/bin/env node
/*
 * enrich-condo-emails.js — best-effort listing-agent EMAIL for the condo set (TK-10243).
 *
 * WHY this is best-effort (honest caveats):
 *   - Redfin does NOT publish agent email (0/115 in the scrape), so there's no source field to copy.
 *   - The shared email-hunt engine ($0 plain-fetch) needs a WEBSITE and, by design, REFUSES to guess
 *     a personal email on a shared corporate brokerage megasite (Compass/CBRE/KW/eXp/Sotheby's…) —
 *     a stranger's email is worse than none. Most condo agents ARE on those big sites, so expected
 *     yield is LOW. This script never fabricates an address: it only records what the guarded engine
 *     returns (a name-matched personal email, or a role inbox on the firm's own domain).
 *
 * Pipeline (per condo broker in data/condo-brokers.json, missing email):
 *   1) resolve a candidate website — CURATED brand→domain map only (no domain guessing).
 *   2) run emailHunt.huntEmails({name, website}) — the SAME name-guarded engine as the ✉ button.
 *   3) on a guarded hit, write agent_email (+ agents[0].email + _email_basis) back into condo-brokers.json.
 *
 * Cost: $0 (plain fetch of public firm sites, no API, no browser). Read-only outbound GETs.
 * Usage: node scripts/enrich-condo-emails.js [--limit N] [--ids a,b] [--only-missing] [--dry-run]
 */
'use strict';
const fs = require('fs');
const path = require('path');
const emailHunt = require('./lib/email-hunt');

const ROOT = path.join(__dirname, '..');
const BROKERS = path.join(ROOT, 'data', 'condo-brokers.json');
const arg = (k, d) => { const i = process.argv.indexOf(k); return i > -1 ? (process.argv[i + 1] ?? true) : d; };
const LIMIT = +arg('--limit', 0) || Infinity;
const ONLY_IDS = (arg('--ids', '') || '').split(',').filter(Boolean);
const ONLY_MISSING = !process.argv.includes('--all'); // default: skip rows already emailed
const DRY = process.argv.includes('--dry-run');
const jload = (p, d) => { try { return JSON.parse(fs.readFileSync(p, 'utf8')); } catch { return d; } };
const sleep = ms => new Promise(r => setTimeout(r, ms));

// Curated firm→domain for the recognizable brands seen in the condo set. For a corporate megasite the
// engine will typically return only a role inbox (or nothing) — that's expected and correct.
const BRAND = {
  'compass': 'compass.com', 'coldwell banker': 'coldwellbankerhomes.com', 'keller williams': 'kw.com',
  'century 21': 'century21.com', 'exp realty': 'exprealty.com', 'sotheby': 'sothebysrealty.com',
  'redfin': 'redfin.com', 'equity union': 'equityunion.com', 'first team': 'firstteam.com',
  'pinnacle estate': 'pinnacleestate.com', 'johnhart': 'johnhartrealestate.com',
  'douglas elliman': 'elliman.com', 'berkshire hathaway': 'bhhscalifornia.com', 're/max': 'remax.com',
  'the agency': 'theagencyre.com', 'rodeo realty': 'rodeorealty.com',
};
// Resolve a candidate website from the firm name — CURATED BRAND MAP ONLY. The firm-slug→.com
// heuristic was REMOVED (TK-10243): it resolved firm names to unrelated/parked domains (e.g.
// "Summit" → summit.com, "…" → domainmarket.com) and the engine then grabbed a stranger's role
// inbox — false emails, worse than none. We now only crawl a KNOWN-REAL brokerage domain.
function firmToSite(firm) {
  if (!firm) return '';
  const low = firm.toLowerCase();
  for (const [k, dom] of Object.entries(BRAND)) if (low.includes(k)) return 'https://' + dom;
  return '';   // unknown firm → don't guess a domain
}
// Only a name-matched PERSONAL email is trustworthy enough to record as "the agent's email".
// Reject role-inbox (generic info@) and sole-email (one-address-on-page faith-grab) — those are how
// garbage got written. A real name match on a real brokerage domain is the only accept.
function accept(r) { return r && r.found && /^name-match/.test(r.basis || ''); }

async function main() {
  const brokers = jload(BROKERS, {});
  let ids = Object.keys(brokers);
  if (ONLY_IDS.length) ids = ids.filter(id => ONLY_IDS.includes(id));
  if (ONLY_MISSING) ids = ids.filter(id => !brokers[id].agent_email);
  ids = ids.filter(id => brokers[id].broker_name || brokers[id].firm_name);

  console.log(`condo-email enrich: ${Math.min(ids.length, LIMIT)} target(s) · $0 local plain-fetch${DRY ? ' · DRY-RUN' : ''}`);
  let done = 0, found = 0, noSite = 0;
  for (const id of ids) {
    if (done >= LIMIT) break;
    done++;
    const b = brokers[id];
    const website = firmToSite(b.firm_name);
    if (!website) { noSite++; process.stdout.write(`\r  ${done}  found:${found}  noSite:${noSite}   `); continue; }
    let r;
    try {
      r = await emailHunt.withTimeout(emailHunt.huntEmails({ id, name: b.broker_name || '', website }), 30000,
        () => ({ found: null, basis: 'timeout' }));
    } catch (e) { r = { found: null, basis: 'err:' + String(e.message).split('\n')[0] }; }
    if (accept(r)) {
      found++;
      if (!DRY) {
        b.agent_email = r.found; b._email_basis = r.basis; b._email_site = website;
        if (Array.isArray(b.agents) && b.agents[0] && !b.agents[0].email) b.agents[0].email = r.found;
        fs.writeFileSync(BROKERS, JSON.stringify(brokers, null, 1));
      }
      process.stdout.write(`\r  ${done}  ✉ ${b.broker_name || b.firm_name} → ${r.found} (${r.basis})\n`);
    }
    process.stdout.write(`\r  ${done}  found:${found}  noSite:${noSite}   `);
    await sleep(400);
  }
  process.stdout.write('\n');
  console.log(`done: ${done} tried · ${found} email(s) found · ${noSite} had no resolvable site · $0 (local plain-fetch)`);
  if (found && !DRY) console.log('  → wrote agent_email into data/condo-brokers.json (surfaces via /api/condos overlay)');
}
main().catch(e => { console.error(e); process.exit(1); });