← back to Commercialrealestate
scripts/lib/email-hunt.js
183 lines
/*
* email-hunt.js — the ONE broker-email hunting engine ($0 plain fetch, no DB, no browser).
* Extracted from broker-email-finder.js so the CLI batch runner AND serve.js's on-demand
* "✉ find email" button share the same guards:
* - mailto: trusted; text emails only when on the site's own domain
* - this broker's NAME in the local-part always wins
* - refuses to guess on shared corporate pages (a stranger's email is worse than none)
* - junk/tracking + fake-TLD noise filtered
* Optional opts.browserHtml(url) hook lets the CLI escalate to headless Chrome.
*/
'use strict';
const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36';
const HEADERS = { 'User-Agent': UA, 'Accept': 'text/html,application/xhtml+xml', 'Accept-Language': 'en-US,en;q=0.9', 'Referer': 'https://www.google.com/' };
const EMAIL_RE = /[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/g;
// asset/tracking junk that happens to match the email shape
const JUNK = /(\.(png|jpg|jpeg|gif|webp|svg|css|js|ico)|@2x|@3x|sentry|wixpress|example\.|core-js|@types|@sha256|u00|domain\.com|email\.com|yourdomain|@version|placeholder)/i;
// real-ish TLDs (incl. common real-estate vanity TLDs). Rejects JS-property "TLDs" like
// .push/.protocol/.location/.length/.score/.ion that minified scripts produce.
const TLD_OK = new Set(('com net org edu gov mil io co us biz info me tv realtor realty homes house properties ' +
'estate realestate group team agency partners capital ventures llc inc law cpa dental health ' +
'ca uk au nz de fr es it nl eu global online site pro top xyz live life world today').split(' '));
const sleep = ms => new Promise(r => setTimeout(r, ms));
// Hard ceiling — a single site's fetch/body-read can occasionally evade the AbortController
// (slow-trickle body, keep-alive) and wedge the caller. This guarantees the caller always advances.
function withTimeout(promise, ms, onTimeout) {
return Promise.race([promise, new Promise(res => setTimeout(() => res(onTimeout()), ms))]);
}
function validEmail(e) {
if (!e || e.length > 60 || (e.match(/@/g) || []).length !== 1 || JUNK.test(e)) return false;
const dom = e.split('@')[1] || ''; const tld = dom.split('.').pop();
if (!tld || !TLD_OK.has(tld.toLowerCase())) return false; // kills document.loc@ion.protocol etc.
if (!/^[a-z0-9._%+-]+@([a-z0-9-]+\.)+[a-z]{2,}$/i.test(e)) return false;
return true;
}
async function get(url, ms = 12000) {
try {
const ctl = new AbortController(); const t = setTimeout(() => ctl.abort(), ms);
const r = await fetch(url, { headers: HEADERS, redirect: 'follow', signal: ctl.signal });
clearTimeout(t);
if (!r.ok) return { ok: false, status: r.status, html: '', finalUrl: url };
const ct = r.headers.get('content-type') || '';
if (!/html|text/i.test(ct)) return { ok: false, status: r.status, html: '', finalUrl: r.url };
return { ok: true, status: r.status, html: (await r.text()).slice(0, 800000), finalUrl: r.url };
} catch (e) { return { ok: false, status: 0, err: e.name === 'AbortError' ? 'timeout' : e.message.split('\n')[0], html: '', finalUrl: url }; }
}
// Lenient-TLS fallback (node:https, rejectUnauthorized:false) for when normal fetch dies on an
// EXPIRED/self-signed cert — common on small broker marketing sites (mdrealtycorp.com's cert is
// expired, e.g.). Read-only public-page scrape, so a bad cert is acceptable HERE and only here.
// Follows up to 4 redirects manually; same return shape as get().
function getInsecure(url, ms = 12000, hops = 4) {
return new Promise(resolve => {
let u; try { u = new URL(url); } catch { return resolve({ ok: false, status: 0, err: 'bad url', html: '', finalUrl: url }); }
const mod = u.protocol === 'http:' ? require('http') : require('https');
const req = mod.get(u, { headers: HEADERS, rejectUnauthorized: false, timeout: ms }, res => {
const st = res.statusCode || 0;
if (st >= 301 && st <= 308 && res.headers.location && hops > 0) {
res.resume();
let next; try { next = new URL(res.headers.location, url).href; } catch { next = null; }
if (next) return resolve(getInsecure(next, ms, hops - 1));
}
if (st < 200 || st >= 400) { res.resume(); return resolve({ ok: false, status: st, html: '', finalUrl: url }); }
const ct = res.headers['content-type'] || '';
if (!/html|text/i.test(ct)) { res.resume(); return resolve({ ok: false, status: st, html: '', finalUrl: url }); }
let body = '';
res.setEncoding('utf8');
res.on('data', d => { body += d; if (body.length > 800000) { body = body.slice(0, 800000); req.destroy(); } });
res.on('end', () => resolve({ ok: true, status: st, html: body, finalUrl: url }));
res.on('error', () => resolve({ ok: !!body, status: st, html: body, finalUrl: url }));
});
req.on('timeout', () => { req.destroy(); resolve({ ok: false, status: 0, err: 'timeout', html: '', finalUrl: url }); });
req.on('error', e => resolve({ ok: false, status: 0, err: String(e.message || e).split('\n')[0], html: '', finalUrl: url }));
});
}
// get() + lenient-TLS rescue: normal fetch first; a hard connection failure (status 0 — DNS dead
// OR bad cert) retries once insecure. DNS-dead just fails again; bad-cert sites come back alive.
async function fetchPage(url, ms = 12000) {
const r = await get(url, ms);
if (!r.ok && r.status === 0) { const r2 = await getInsecure(url, ms); if (r2.ok) return r2; }
return r;
}
// try the URL; on connection failure retry the http<->https twin (many broker sites moved to https)
async function getResilient(url) {
let r = await fetchPage(url);
if (!r.ok && (r.status === 0 || r.status >= 500)) {
const twin = url.startsWith('https://') ? url.replace(/^https:/, 'http:') : url.replace(/^http:/, 'https:');
await sleep(200); const r2 = await fetchPage(twin); if (r2.ok) return r2;
}
return r;
}
function siteDomain(u) { try { return new URL(u).hostname.replace(/^www\./, ''); } catch { return ''; } }
// Returns { mailtos:[], texts:[] } — mailto: emails are trustworthy; text emails are used only
// when they match the site's own domain (so cross-domain JS junk / other companies are dropped).
function extractEmails(html) {
const mailtos = new Set(), texts = new Set();
for (const m of html.matchAll(/mailto:([^"'?>\s]+)/gi)) { const e = decodeURIComponent(m[1]).toLowerCase(); if (validEmail(e)) mailtos.add(e); }
for (const m of (html.match(EMAIL_RE) || [])) { const e = m.toLowerCase(); if (validEmail(e)) texts.add(e); }
return { mailtos: [...mailtos], texts: [...texts] };
}
function findContactLinks(html, base) {
const links = new Set();
for (const m of html.matchAll(/href\s*=\s*["']([^"']+)["']/gi)) {
const href = m[1];
if (/contact|about|team|reach|connect|get-in-touch/i.test(href)) {
try { links.add(new URL(href, base).href); } catch {}
}
}
return [...links].slice(0, 3);
}
const sameDomain = (e, dom) => { const d = e.split('@')[1] || ''; return dom && (d === dom || d.endsWith('.' + dom) || dom.endsWith('.' + d)); };
// Choose the broker's email from mailto: + same-domain text emails. On a SHARED corporate domain
// (many agents on one page) the broker's NAME in the local-part is the deciding signal — never guess
// a random other agent. Falls back to a role inbox (info@/contact@) on the site's own domain.
function pickBest(mailtos, texts, dom, name) {
const pool = [...new Set([...mailtos, ...texts.filter(e => sameDomain(e, dom))])];
if (!pool.length) return { email: null, basis: null };
const parts = String(name || '').toLowerCase().split(/\s+/).filter(w => w.length > 1);
const first = parts[0] || '', last = parts[parts.length - 1] || '';
const nameKeys = [last, first + last, first + '.' + last, first[0] + last, first + '_' + last].filter(Boolean);
const role = e => /^(info|contact|hello|sales|admin|office|team|reception|frontdesk)@/.test(e);
const isName = e => { const l = e.split('@')[0]; return (last && nameKeys.some(k => l.includes(k))) || (first && first.length > 2 && l.includes(first)); };
// 1) this broker by name — always the answer when present.
const named = pool.find(isName);
if (named) return { email: named, basis: 'name-match', pool };
// 2) a role inbox on the site's own domain (info@/contact@) — the firm's real contact.
const roleInbox = pool.find(e => role(e) && sameDomain(e, dom));
if (roleInbox) return { email: roleInbox, basis: 'role-inbox', pool };
// 3) exactly ONE email on the whole site → this broker's, BUT ONLY on their own small domain.
// On a corporate megasite (marcusmillichap/cbre/kw/…) a broker's "website" is often a shared
// office page rendering ONE OTHER agent's email — sole-email there mis-assigns a stranger. Skip.
const CORP_DOM = /marcusmillichap|cbre|kw\.com|kwcommercial|remax|coldwell|century21|compass|colliers|cushwake|jll|berkadia|newmark|kidder/i;
if (pool.length === 1 && !CORP_DOM.test(dom) && !CORP_DOM.test(pool[0])) return { email: pool[0], basis: 'sole-email', pool };
// 4) Multiple PERSONAL emails, none matching this broker → it's a shared/corporate page listing
// OTHER agents. Refuse to guess — a stranger's email is worse than none.
return { email: null, basis: 'ambiguous-shared-page', pool };
}
// Hunt one broker: homepage → contact-ish pages → pickBest. b = { name, website }.
// opts.browserHtml(url) => {ok,status,html,err} — optional headless-Chrome escalation (CLI only).
async function huntEmails(b, opts = {}) {
const dom = siteDomain(b.website);
const tried = []; const mailtos = new Set(), texts = new Set();
const home = await getResilient(b.website);
tried.push({ url: b.website, status: home.status, err: home.err });
if (home.ok) {
const e0 = extractEmails(home.html); e0.mailtos.forEach(x => mailtos.add(x)); e0.texts.forEach(x => texts.add(x));
for (const link of findContactLinks(home.html, home.finalUrl)) {
await sleep(300);
const cp = await fetchPage(link);
tried.push({ url: link, status: cp.status, err: cp.err });
if (cp.ok) { const e = extractEmails(cp.html); e.mailtos.forEach(x => mailtos.add(x)); e.texts.forEach(x => texts.add(x)); }
}
}
let { email, basis, pool } = pickBest([...mailtos], [...texts], dom, b.name);
// Escalate to the real browser when no email found: either the site blocked us (403/0) or it
// rendered fine but published no email (JS-rendered contact widgets). Renders JS + carries a
// genuine browser fingerprint.
const shouldEscalate = !email && (tried.some(t => t.status === 403 || t.status === 0) || tried.every(t => t.status >= 200));
if (opts.browserHtml && shouldEscalate) {
const bh = await opts.browserHtml(b.website);
tried.push({ url: b.website + ' [browser]', status: bh.status, err: bh.err });
if (bh.ok) {
const e = extractEmails(bh.html); e.mailtos.forEach(x => mailtos.add(x)); e.texts.forEach(x => texts.add(x));
for (const link of findContactLinks(bh.html, b.website)) {
const cb = await opts.browserHtml(link);
tried.push({ url: link + ' [browser]', status: cb.status, err: cb.err });
if (cb.ok) { const e2 = extractEmails(cb.html); e2.mailtos.forEach(x => mailtos.add(x)); e2.texts.forEach(x => texts.add(x)); }
}
({ email, basis, pool } = pickBest([...mailtos], [...texts], dom, b.name));
if (email) basis = basis + '+browser';
}
}
return { id: b.id, name: b.name, website: b.website, domain: dom, found: email, basis, all: pool || [], tried };
}
module.exports = { UA, HEADERS, validEmail, get, getInsecure, fetchPage, getResilient, siteDomain, extractEmails, findContactLinks, sameDomain, pickBest, huntEmails, withTimeout, sleep };