← back to Commercialrealestate
find-email button: on-demand broker email hunt from their website (CRCP + broker grid)
47b46629a38e731453833854f98768d1c38b6296 · 2026-07-13 01:34:00 -0700 · Steve Abrams
- lib/email-hunt.js: engine extracted from broker-email-finder.js (shared guards:
name-match wins, refuse shared corporate pages, junk/TLD filter) + lenient-TLS
fallback for expired-cert broker sites (mdrealtycorp.com)
- lib/email-hunt-browser.js: headless-Chrome tier (ignoreHTTPSErrors) shared by CLI+server
- serve.js: POST /api/broker/find-email — hunts the broker's own site, saves to
broker.email (PG) or brokers-snapshot.json (DB-less prod)
- crcp.html + broker-grid.html: website links now show the SITE NAME (domain) instead
of bare 'site↗'; '✉ find email' button where email is missing; modal variant too
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Files touched
M public/broker-grid.htmlM public/crcp.htmlM scripts/broker-email-finder.jsA scripts/lib/email-hunt-browser.jsA scripts/lib/email-hunt.jsM scripts/serve.js
Diff
commit 47b46629a38e731453833854f98768d1c38b6296
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Mon Jul 13 01:34:00 2026 -0700
find-email button: on-demand broker email hunt from their website (CRCP + broker grid)
- lib/email-hunt.js: engine extracted from broker-email-finder.js (shared guards:
name-match wins, refuse shared corporate pages, junk/TLD filter) + lenient-TLS
fallback for expired-cert broker sites (mdrealtycorp.com)
- lib/email-hunt-browser.js: headless-Chrome tier (ignoreHTTPSErrors) shared by CLI+server
- serve.js: POST /api/broker/find-email — hunts the broker's own site, saves to
broker.email (PG) or brokers-snapshot.json (DB-less prod)
- crcp.html + broker-grid.html: website links now show the SITE NAME (domain) instead
of bare 'site↗'; '✉ find email' button where email is missing; modal variant too
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
public/broker-grid.html | 39 +++++++-
public/crcp.html | 47 +++++++++-
scripts/broker-email-finder.js | 184 ++------------------------------------
scripts/lib/email-hunt-browser.js | 55 ++++++++++++
scripts/lib/email-hunt.js | 181 +++++++++++++++++++++++++++++++++++++
scripts/serve.js | 80 +++++++++++++++++
6 files changed, 399 insertions(+), 187 deletions(-)
diff --git a/public/broker-grid.html b/public/broker-grid.html
index 50b424d..95c8894 100644
--- a/public/broker-grid.html
+++ b/public/broker-grid.html
@@ -32,6 +32,9 @@
.atype.res{background:rgba(88,166,255,.16);color:var(--blue);} .atype.com{background:rgba(200,162,75,.16);color:var(--gold);}
.atype.arc{background:rgba(127,174,106,.2);color:#8fd06f;border:1px solid rgba(127,174,106,.55);margin-left:5px;}
.mut{color:var(--mut);} .miss{color:#5A6472;}
+ .findmail{background:transparent;border:1px solid #5B9BD5;color:#5B9BD5;border-radius:9px;padding:2px 8px;font-size:11px;cursor:pointer;white-space:nowrap;}
+ .findmail:hover{background:rgba(91,155,213,.12);}
+ .hunting{color:#E0A33E;font-size:12px;}
.geo{display:inline-block;font-size:10px;font-weight:700;padding:1px 7px;border-radius:8px;white-space:nowrap;}
.geo.ca{background:rgba(127,174,106,.18);color:#8fd06f;border:1px solid rgba(127,174,106,.5);}
.geo.non{background:rgba(248,81,73,.16);color:#f77;border:1px solid rgba(248,81,73,.5);}
@@ -119,6 +122,34 @@ const num=n=>(n==null||n==='')?'—':Number(n).toLocaleString();
const tel=v=>v?`<a href="tel:${esc(v)}">${esc(v)}</a>`:'<span class="miss">—</span>';
const mail=v=>v?`<a href="mailto:${esc(v)}">${esc(v)}</a>`:'<span class="miss">—</span>';
const url=(v,lbl)=>{ if(!v) return '<span class="miss">—</span>'; const h=/^https?:/.test(v)?v:'https://'+v; return `<a href="${esc(h)}" target="_blank" rel="noopener noreferrer">${lbl}</a>`; };
+// Website links show the SITE NAME (domain), not a bare "site↗" — Steve 2026-07-13.
+const siteName=u=>{ try{ return new URL(/^https?:/.test(u)?u:'https://'+u).hostname.replace(/^www\./,''); }catch(e){ return u; } };
+const web=v=>{ if(!v) return '<span class="miss">—</span>'; const h=/^https?:/.test(v)?v:'https://'+v; return `<a href="${esc(h)}" target="_blank" rel="noopener noreferrer" title="${esc(h)}">${esc(siteName(v))}↗</a>`; };
+// On-demand email hunt (the "✉ find email" button) — POSTs /api/broker/find-email, which visits
+// the broker's site and extracts an email with the batch finder's guards. hunt[] keeps per-row
+// state across re-renders; a success also writes the row in DATA so every view picks it up.
+const hunt={};
+function mailCell(r){
+ if(r.email) return mail(r.email);
+ const h=hunt[r.id];
+ if(h){
+ if(h.state==='hunting') return '<span class="hunting">⏳ hunting…</span>';
+ if(h.email) return mail(h.email);
+ return `<span class="miss" title="${esc(h.miss||'')}">not found</span>`;
+ }
+ if(r.website) return `<button class="findmail" data-id="${r.id}" title="Visit ${esc(siteName(r.website))} and hunt down an email address">✉ find email</button>`;
+ return '<span class="miss">—</span>';
+}
+async function findEmail(id){
+ const r=DATA.find(x=>+x.id===+id); if(!r) return;
+ hunt[id]={state:'hunting'}; render();
+ try{
+ const d=await (await fetch('/api/broker/find-email',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({id:+id,name:r.name,website:r.website})})).json();
+ if(d.email){ hunt[id]={email:d.email}; r.email=d.email; }
+ else hunt[id]={miss:d.reason||d.error||'not found'};
+ }catch(e){ hunt[id]={miss:e.message}; }
+ render();
+}
const dt=s=>{ if(!s) return '—'; try{ return new Date(s).toLocaleDateString(undefined,{year:'numeric',month:'short',day:'numeric'}); }catch(e){ return '—'; } };
// Every field carries a group `g` (for the sort menu optgroups + organized rail toggles).
// Order below IS the column order. Every field is sortable (raw-value sort, any type).
@@ -190,8 +221,8 @@ function cell(r,c){
if(c.k==='name') return esc(v==null?'':v)+(r.arcstone_flag?' <span class="atype arc" title="Prior Arcstone relationship">★ Arcstone</span>':'');
if(c.t==='n') return num(v);
if(c.t==='tel') return tel(v);
- if(c.t==='mail') return mail(v);
- if(c.t==='web') return url(v,'site↗');
+ if(c.t==='mail') return mailCell(r);
+ if(c.t==='web') return web(v);
if(c.t==='li') return url(v,'in↗');
if(c.t==='date') return dt(v);
if(c.t==='atype') return v==='residential'?'<span class="atype res">Residential</span>':'<span class="atype com">Commercial</span>';
@@ -235,7 +266,7 @@ function render(){
<div class="nm">${esc(r.name)} ${r.agent_type==='residential'?'<span class="atype res">Res</span>':'<span class="atype com">Com</span>'}${r.arcstone_flag?'<span class="atype arc" title="Prior Arcstone relationship">★ Arcstone</span>':''}</div>
<div class="fm">${esc(r.firm||'—')}${r.title?' · '+esc(r.title):''}</div>`
+bodyCols.map(c=>`<div class="row"><span class="k">${esc(c.l)}</span><span>${cell(r,c)}</span></div>`).join('')
- +`<div class="ci">${tel(r.phone)} ${mail(r.email)} ${url(r.website,'site↗')} ${r.linkedin?url(r.linkedin,'in↗'):''}</div>
+ +`<div class="ci">${tel(r.phone)} ${mailCell(r)} ${web(r.website)} ${r.linkedin?url(r.linkedin,'in↗'):''}</div>
</div>`).join('');
}
}
@@ -268,7 +299,7 @@ async function genPitch(name,id){
out.innerHTML=r.error?('<div class="mut">'+esc(r.error)+'</div>'):renderPitch(r.pitch);
}catch(e){ out.innerHTML='<div class="mut">'+esc(e.message)+'</div>'; }
}
-document.addEventListener('click',e=>{ const row=e.target.closest('.brow'); if(row && e.target.tagName!=='A'){ openContact(row.dataset.name,row.dataset.id); } if(e.target.id==='ov') $('#ov').classList.remove('on'); });
+document.addEventListener('click',e=>{ const fm=e.target.closest('.findmail'); if(fm){ findEmail(fm.dataset.id); return; } const row=e.target.closest('.brow'); if(row && e.target.tagName!=='A' && e.target.tagName!=='BUTTON'){ openContact(row.dataset.name,row.dataset.id); } if(e.target.id==='ov') $('#ov').classList.remove('on'); });
$('#thead').addEventListener('click',e=>{const th=e.target.closest('th');if(!th)return;const k=th.dataset.k;
if(sortKey===k)sortDir*=-1;else{sortKey=k;sortDir=(k==='listings'||k==='total_assets'||k==='created_at')?-1:1;}
localStorage.setItem('brokSortKey',sortKey);localStorage.setItem('brokSortDir',sortDir);if(window.__syncSortSel)window.__syncSortSel();render();});
diff --git a/public/crcp.html b/public/crcp.html
index d5bf9ad..50560b8 100644
--- a/public/crcp.html
+++ b/public/crcp.html
@@ -76,6 +76,10 @@
.pitch{background:#0a0d13;border:1px solid var(--line);border-radius:10px;padding:12px;font-size:13px;white-space:pre-wrap;margin-top:10px;}
.pitch .subj{color:var(--gold);font-weight:700;margin-bottom:6px;}
.gate{font-size:11px;color:var(--staged);margin-top:6px;}
+ .findmail{background:transparent;border:1px solid var(--draft);color:var(--draft);border-radius:9px;padding:2px 8px;font-size:11px;cursor:pointer;white-space:nowrap;}
+ .findmail:hover{background:rgba(91,155,213,.12);}
+ .findmail:disabled{opacity:.5;cursor:default;}
+ .hunting{color:var(--staged);font-size:12px;}
</style>
</head>
<body>
@@ -129,7 +133,7 @@ async function openContact(name, id, seg){
mb.innerHTML=`<h2>${b.name}${atype(b.agent_type)}${b.arcstone_flag?'<span class="atype arc" title="Arcstone has a prior relationship with this agent/firm (pitched, co-listed, or Arcstone firm)">★ Prior Arcstone</span>':''}</h2><div class="mfirm">${b.firm||'—'}${b.title?' · '+b.title:''}${b.license?' · DRE/MLS '+b.license:''}</div>
<div class="cinfo">
${b.phone?`<a href="tel:${b.phone}">☎ ${b.phone}</a>`:'<span class="m">☎ —</span>'}
- ${b.email?`<a href="mailto:${b.email}">✉ ${b.email}</a>`:'<span class="m">✉ —</span>'}
+ ${b.email?`<a href="mailto:${b.email}">✉ ${b.email}</a>`:(w?`<span class="m" id="memail">✉ — <button class="findmail" onclick="findEmailModal(${b.id})" title="Visit the site and hunt down an email address">find email on site</button></span>`:'<span class="m">✉ —</span>')}
${w?`<a href="${w}" target="_blank" rel="noopener noreferrer">🌐 ${w.replace(/^https?:\/\/(www\.)?/,'').replace(/\/$/,'')}</a>`:'<span class="m">🌐 —</span>'}
<span>📦 ${b.agent_type==='residential'?(d.current.length+' condo listing'+(d.current.length===1?'':'s')+' (Redfin)'):((b.total_assets||'?')+' total listings (Crexi)')}</span>
</div>
@@ -144,6 +148,16 @@ async function openContact(name, id, seg){
</div>`;
}
function renderPitch(p){ return `<div class="pitch">${p.subject?`<div class="subj">${p.subject}</div>`:''}${(p.body||'').replace(/&/g,'&').replace(/</g,'<')}</div><div class="gate">⚑ DRAFT — sending is George-gated + Steve-approved; not sent from here.</div>`; }
+// Modal variant of the email hunt — same endpoint, updates the ✉ chip in place.
+async function findEmailModal(id){
+ const el=document.getElementById('memail'); if(el) el.innerHTML='✉ <span class="hunting">⏳ hunting on the site…</span>';
+ try{
+ const r=await (await fetch('/api/broker/find-email',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({id:+id})})).json();
+ if(r.email){ hunt[id]={email:r.email,basis:r.basis}; if(el) el.innerHTML=`✉ <a href="mailto:${r.email}">${r.email}</a> <span style="color:var(--muted);font-size:11px">(${r.basis||'found'}${r.site?' · '+r.site:''})</span>`; }
+ else { hunt[id]={miss:r.reason||r.error||'not found'}; if(el) el.innerHTML=`✉ not found <span style="color:var(--muted);font-size:11px" title="${((r.reason||r.error||'')+'').replace(/"/g,'"')}">(${r.reason||r.error||'no email on the site'})</span>`; }
+ renderBrokers(lastStats);
+ }catch(e){ if(el) el.innerHTML='✉ error: '+e.message; }
+}
async function openFirm(name){
const ov=$('#ov'), mb=$('#mbody'); ov.classList.add('on'); mb.innerHTML='<div class="meta">loading '+name+'…</div>';
let d; try{ d=await (await fetch('/api/firm?name='+encodeURIComponent(name))).json(); }catch(e){ mb.innerHTML='error'; return; }
@@ -213,16 +227,41 @@ function srt(arr,col,dir){ return arr.slice().sort((a,b)=>{ let x=a[col],y=b[col
if(typeof x==='string'||typeof y==='string') return dir*String(x||'').localeCompare(String(y||''));
return dir*((+x||0)-(+y||0)); }); }
function markHdr(tbl,sort){ document.querySelectorAll('#'+tbl+' th.srt').forEach(th=>{ th.classList.remove('asc','desc'); if(th.dataset.col===sort.col) th.classList.add(sort.dir>0?'asc':'desc'); }); }
-const webLink=u=>{ if(!u) return '<span class="miss">—</span>'; const h=/^https?:/.test(u)?u:'https://'+u; return `<a href="${h}" target="_blank" rel="noopener noreferrer" style="color:var(--gold)">site↗</a>`; };
+// Web column shows the SITE NAME (domain), not a bare "site↗" — Steve 2026-07-13.
+const siteName=u=>{ try{ return new URL(/^https?:/.test(u)?u:'https://'+u).hostname.replace(/^www\./,''); }catch(e){ return u; } };
+const webLink=u=>{ if(!u) return '<span class="miss">—</span>'; const h=/^https?:/.test(u)?u:'https://'+u; return `<a href="${h}" target="_blank" rel="noopener noreferrer" style="color:var(--gold)" title="${h}">${siteName(u)}↗</a>`; };
+// On-demand email hunt (the "✉ find email" button). hunt[] survives the 3s poll re-renders;
+// a found email also lands in broker.email server-side, so the next poll carries it natively.
+const hunt={};
+async function findEmail(id,name,website){
+ hunt[id]={state:'hunting'}; renderBrokers(lastStats);
+ try{
+ const r=await (await fetch('/api/broker/find-email',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({id:+id,name,website})})).json();
+ hunt[id]=r.email?{email:r.email,basis:r.basis}:{miss:r.reason||r.error||'not found'};
+ }catch(e){ hunt[id]={miss:e.message}; }
+ renderBrokers(lastStats);
+}
+function emailCell(b){
+ if(b.email) return `<a href="mailto:${b.email}" class="has" style="color:var(--draft)">${b.email}</a>`;
+ const h=hunt[b.id];
+ if(h){
+ if(h.state==='hunting') return '<span class="hunting">⏳ hunting…</span>';
+ if(h.email) return `<a href="mailto:${h.email}" class="has" style="color:var(--draft)" title="found on site (${h.basis||''})">${h.email}</a>`;
+ return `<span class="miss" title="${(h.miss||'').replace(/"/g,'"')}">not found</span>`;
+ }
+ if(b.website) return `<button class="findmail" data-id="${b.id}" data-name="${(b.name||'').replace(/"/g,'"')}" data-web="${(b.website||'').replace(/"/g,'"')}" title="Visit ${siteName(b.website)} and hunt down an email address">✉ find email</button>`;
+ return '<span class="miss">—</span>';
+}
function renderBrokers(s){ if(!s) return; const rows=srt(s.topBrokers||[], brSort.col, brSort.dir);
$('#brC').textContent=rows.length+' shown · click a broker · click a header to sort';
$('#brTbl tbody').innerHTML=rows.map(b=>`<tr class="brow" data-id="${b.id||''}" data-name="${(b.name||'').replace(/"/g,'"')}" style="cursor:pointer">
<td>${b.name}${atype(b.agent_type)}<div class="firm">${b.firm||'—'}</div></td>
<td class="n">${b.listings}</td>
<td>${b.phone?`<a href="tel:${b.phone}" class="has" style="color:var(--active)">${b.phone}</a>`:'<span class="miss">—</span>'}</td>
- <td>${b.email?`<a href="mailto:${b.email}" class="has" style="color:var(--draft)">${b.email}</a>`:'<span class="miss">—</span>'}</td>
+ <td>${emailCell(b)}</td>
<td>${webLink(b.website)}</td></tr>`).join('');
- document.querySelectorAll('.brow').forEach(r=>r.onclick=e=>{ if(e.target.tagName==='A') return; openContact(r.dataset.name, r.dataset.id); });
+ document.querySelectorAll('.brow').forEach(r=>r.onclick=e=>{ if(e.target.tagName==='A'||e.target.tagName==='BUTTON') return; openContact(r.dataset.name, r.dataset.id); });
+ document.querySelectorAll('#brTbl .findmail').forEach(x=>x.onclick=e=>{ e.stopPropagation(); findEmail(x.dataset.id, x.dataset.name, x.dataset.web); });
markHdr('brTbl',brSort); }
function renderFirms(s){ if(!s) return; const rows=srt(s.topFirms||[], fmSort.col, fmSort.dir);
$('#fmC').textContent='click a firm · click a header to sort';
diff --git a/scripts/broker-email-finder.js b/scripts/broker-email-finder.js
index 4e80bc1..4c28c5d 100644
--- a/scripts/broker-email-finder.js
+++ b/scripts/broker-email-finder.js
@@ -7,9 +7,12 @@
* node broker-email-finder.js --limit 50 --commit # write found emails to broker.email
* Strategy: fetch homepage → collect mailto: + text emails → also fetch up to 3 "contact"-ish
* linked pages → score candidates (prefer address on the site's own domain; drop junk/role-y noise).
+ * The engine (fetch/extract/pick guards) lives in lib/email-hunt.js — shared with serve.js's
+ * on-demand /api/broker/find-email button so the batch and the button can never diverge.
*/
'use strict';
const { pool } = require('./db/brokers-db');
+const { huntEmails, siteDomain, withTimeout, sleep } = require('./lib/email-hunt');
const args = process.argv.slice(2);
const COMMIT = args.includes('--commit');
const ti = args.indexOf('--test'); const TEST_N = ti > -1 ? +args[ti + 1] : 0;
@@ -19,184 +22,7 @@ const li = args.indexOf('--limit'); const LIMIT = li > -1 ? +args[li + 1] : 25;
// retry with a local headless Chrome (Playwright, $0). Real browser fingerprint + JS execution
// recovers a big chunk of the plain-fetch misses. Shared browser instance, reused across brokers.
const USE_BROWSER = args.includes('--browser');
-let _browser = null, _pw = null;
-function loadPlaywright() {
- const path = require('path'), os = require('os');
- const cands = [
- path.join(os.homedir(), 'Projects/Designer-Wallcoverings/DW-Programming/ImportNewSkufromURL/node_modules/playwright'),
- 'playwright',
- ];
- for (const c of cands) { try { return require(c); } catch {} }
- throw new Error('playwright not found — install it or drop --browser');
-}
-const CHROME_APP = '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome';
-const CHROME_SHELL = require('os').homedir() + '/Library/Caches/ms-playwright/chromium_headless_shell-1228/chrome-headless-shell-mac-arm64/chrome-headless-shell';
-async function launchBrowser() {
- const fs = require('fs');
- const opts = { headless: true, args: ['--no-sandbox', '--disable-dev-shm-usage'] };
- // Prefer REAL system Chrome (best anti-bot fingerprint); fall back to the installed headless-shell.
- if (fs.existsSync(CHROME_APP)) { try { return await _pw.chromium.launch({ ...opts, executablePath: CHROME_APP }); } catch {} }
- if (fs.existsSync(CHROME_SHELL)) { try { return await _pw.chromium.launch({ ...opts, executablePath: CHROME_SHELL }); } catch {} }
- return await _pw.chromium.launch(opts);
-}
-async function browserHtml(url, ms = 20000) {
- try {
- if (!_pw) _pw = loadPlaywright();
- if (!_browser) _browser = await launchBrowser();
- const ctx = await _browser.newContext({ userAgent: UA, viewport: { width: 1280, height: 900 } });
- const page = await ctx.newPage();
- let html = '';
- try {
- await page.goto(url, { waitUntil: 'domcontentloaded', timeout: ms });
- await page.waitForTimeout(1800); // let JS hydrate contact widgets
- html = await page.content();
- } catch (e) { /* return whatever rendered */ try { html = await page.content(); } catch {} }
- await ctx.close();
- return { ok: !!html, status: html ? 200 : 0, html: (html || '').slice(0, 800000), finalUrl: page.url ? url : url };
- } catch (e) { return { ok: false, status: 0, err: e.message.split('\n')[0], html: '', finalUrl: url }; }
-}
-async function closeBrowser() { try { if (_browser) await _browser.close(); } catch {} }
-
-const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36';
-const HEADERS = { 'User-Agent': UA, 'Accept': 'text/html,application/xhtml+xml', 'Accept-Language': 'en-US,en;q=0.9', 'Referer': 'https://www.google.com/' };
-const EMAIL_RE = /[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/g;
-// asset/tracking junk that happens to match the email shape
-const JUNK = /(\.(png|jpg|jpeg|gif|webp|svg|css|js|ico)|@2x|@3x|sentry|wixpress|example\.|core-js|@types|@sha256|u00|domain\.com|email\.com|yourdomain|@version|placeholder)/i;
-// real-ish TLDs (incl. common real-estate vanity TLDs). Rejects JS-property "TLDs" like
-// .push/.protocol/.location/.length/.score/.ion that minified scripts produce.
-const TLD_OK = new Set(('com net org edu gov mil io co us biz info me tv realtor realty homes house properties ' +
- 'estate realestate group team agency partners capital ventures llc inc law cpa dental health ' +
- 'ca uk au nz de fr es it nl eu global online site pro top xyz live life world today').split(' '));
-const sleep = ms => new Promise(r => setTimeout(r, ms));
-// Hard per-broker ceiling — a single site's fetch/body-read can occasionally evade the AbortController
-// (slow-trickle body, keep-alive) and wedge the whole batch. This guarantees the run always advances.
-function withTimeout(promise, ms, onTimeout) {
- return Promise.race([promise, new Promise(res => setTimeout(() => res(onTimeout()), ms))]);
-}
-
-function validEmail(e) {
- if (!e || e.length > 60 || (e.match(/@/g) || []).length !== 1 || JUNK.test(e)) return false;
- const dom = e.split('@')[1] || ''; const tld = dom.split('.').pop();
- if (!tld || !TLD_OK.has(tld.toLowerCase())) return false; // kills document.loc@ion.protocol etc.
- if (!/^[a-z0-9._%+-]+@([a-z0-9-]+\.)+[a-z]{2,}$/i.test(e)) return false;
- return true;
-}
-
-async function get(url, ms = 12000) {
- try {
- const ctl = new AbortController(); const t = setTimeout(() => ctl.abort(), ms);
- const r = await fetch(url, { headers: HEADERS, redirect: 'follow', signal: ctl.signal });
- clearTimeout(t);
- if (!r.ok) return { ok: false, status: r.status, html: '', finalUrl: url };
- const ct = r.headers.get('content-type') || '';
- if (!/html|text/i.test(ct)) return { ok: false, status: r.status, html: '', finalUrl: r.url };
- return { ok: true, status: r.status, html: (await r.text()).slice(0, 800000), finalUrl: r.url };
- } catch (e) { return { ok: false, status: 0, err: e.name === 'AbortError' ? 'timeout' : e.message.split('\n')[0], html: '', finalUrl: url }; }
-}
-// try the URL; on connection failure retry the http<->https twin (many broker sites moved to https)
-async function getResilient(url) {
- let r = await get(url);
- if (!r.ok && (r.status === 0 || r.status >= 500)) {
- const twin = url.startsWith('https://') ? url.replace(/^https:/, 'http:') : url.replace(/^http:/, 'https:');
- await sleep(200); const r2 = await get(twin); if (r2.ok) return r2;
- }
- return r;
-}
-
-function siteDomain(u) { try { return new URL(u).hostname.replace(/^www\./, ''); } catch { return ''; } }
-
-// Returns { mailtos:[], texts:[] } — mailto: emails are trustworthy; text emails are used only
-// when they match the site's own domain (so cross-domain JS junk / other companies are dropped).
-function extractEmails(html) {
- const mailtos = new Set(), texts = new Set();
- for (const m of html.matchAll(/mailto:([^"'?>\s]+)/gi)) { const e = decodeURIComponent(m[1]).toLowerCase(); if (validEmail(e)) mailtos.add(e); }
- for (const m of (html.match(EMAIL_RE) || [])) { const e = m.toLowerCase(); if (validEmail(e)) texts.add(e); }
- return { mailtos: [...mailtos], texts: [...texts] };
-}
-
-function findContactLinks(html, base) {
- const links = new Set();
- for (const m of html.matchAll(/href\s*=\s*["']([^"']+)["']/gi)) {
- const href = m[1];
- if (/contact|about|team|reach|connect|get-in-touch/i.test(href)) {
- try { links.add(new URL(href, base).href); } catch {}
- }
- }
- return [...links].slice(0, 3);
-}
-
-const sameDomain = (e, dom) => { const d = e.split('@')[1] || ''; return dom && (d === dom || d.endsWith('.' + dom) || dom.endsWith('.' + d)); };
-// Choose the broker's email from mailto: + same-domain text emails. On a SHARED corporate domain
-// (many agents on one page) the broker's NAME in the local-part is the deciding signal — never guess
-// a random other agent. Falls back to a role inbox (info@/contact@) on the site's own domain.
-function pickBest(mailtos, texts, dom, name) {
- const pool = [...new Set([...mailtos, ...texts.filter(e => sameDomain(e, dom))])];
- if (!pool.length) return { email: null, basis: null };
- const parts = String(name || '').toLowerCase().split(/\s+/).filter(w => w.length > 1);
- const first = parts[0] || '', last = parts[parts.length - 1] || '';
- const nameKeys = [last, first + last, first + '.' + last, first[0] + last, first + '_' + last].filter(Boolean);
- const role = e => /^(info|contact|hello|sales|admin|office|team|reception|frontdesk)@/.test(e);
- const score = e => {
- const local = e.split('@')[0];
- let s = 0;
- if (last && nameKeys.some(k => local.includes(k))) s += 100; // this broker by name — the strong signal
- else if (first && local.includes(first)) s += 30;
- if (sameDomain(e, dom)) s += 40; // on the site's own domain
- if (role(e)) s += 15; // real contact inbox
- if (/gmail|yahoo|hotmail|outlook|aol/.test(e)) s -= 10;
- return s;
- };
- const isName = e => { const l = e.split('@')[0]; return (last && nameKeys.some(k => l.includes(k))) || (first && first.length > 2 && l.includes(first)); };
- // 1) this broker by name — always the answer when present.
- const named = pool.find(isName);
- if (named) return { email: named, basis: 'name-match', pool };
- // 2) a role inbox on the site's own domain (info@/contact@) — the firm's real contact.
- const roleInbox = pool.find(e => role(e) && sameDomain(e, dom));
- if (roleInbox) return { email: roleInbox, basis: 'role-inbox', pool };
- // 3) exactly ONE email on the whole site → this broker's, BUT ONLY on their own small domain.
- // On a corporate megasite (marcusmillichap/cbre/kw/…) a broker's "website" is often a shared
- // office page rendering ONE OTHER agent's email — sole-email there mis-assigns a stranger. Skip.
- const CORP_DOM = /marcusmillichap|cbre|kw\.com|kwcommercial|remax|coldwell|century21|compass|colliers|cushwake|jll|berkadia|newmark|kidder/i;
- if (pool.length === 1 && !CORP_DOM.test(dom) && !CORP_DOM.test(pool[0])) return { email: pool[0], basis: 'sole-email', pool };
- // 4) Multiple PERSONAL emails, none matching this broker → it's a shared/corporate page listing
- // OTHER agents. Refuse to guess — a stranger's email is worse than none.
- return { email: null, basis: 'ambiguous-shared-page', pool };
-}
-
-async function findForBroker(b) {
- const dom = siteDomain(b.website);
- const tried = []; const mailtos = new Set(), texts = new Set();
- const home = await getResilient(b.website);
- tried.push({ url: b.website, status: home.status, err: home.err });
- if (home.ok) {
- const e0 = extractEmails(home.html); e0.mailtos.forEach(x => mailtos.add(x)); e0.texts.forEach(x => texts.add(x));
- for (const link of findContactLinks(home.html, home.finalUrl)) {
- await sleep(300);
- const cp = await get(link);
- tried.push({ url: link, status: cp.status, err: cp.err });
- if (cp.ok) { const e = extractEmails(cp.html); e.mailtos.forEach(x => mailtos.add(x)); e.texts.forEach(x => texts.add(x)); }
- }
- }
- let { email, basis, pool } = pickBest([...mailtos], [...texts], dom, b.name);
- // Escalate to the real browser only when plain fetch struck out (blocked or JS-rendered) AND we
- // don't already have a confident hit. Renders JS + carries a genuine browser fingerprint.
- const blockedOrEmpty = !email && (tried.some(t => t.status === 403 || t.status === 0) || tried.every(t => t.status >= 200));
- if (USE_BROWSER && blockedOrEmpty) {
- const bh = await browserHtml(b.website);
- tried.push({ url: b.website + ' [browser]', status: bh.status, err: bh.err });
- if (bh.ok) {
- const e = extractEmails(bh.html); e.mailtos.forEach(x => mailtos.add(x)); e.texts.forEach(x => texts.add(x));
- for (const link of findContactLinks(bh.html, b.website)) {
- const cb = await browserHtml(link);
- tried.push({ url: link + ' [browser]', status: cb.status, err: cb.err });
- if (cb.ok) { const e2 = extractEmails(cb.html); e2.mailtos.forEach(x => mailtos.add(x)); e2.texts.forEach(x => texts.add(x)); }
- }
- ({ email, basis, pool } = pickBest([...mailtos], [...texts], dom, b.name));
- if (email) basis = basis + '+browser';
- }
- }
- return { id: b.id, name: b.name, website: b.website, domain: dom, found: email, basis, all: pool || [], tried };
-}
+const { browserHtml, closeBrowser } = require('./lib/email-hunt-browser');
(async () => {
let sql, params = [];
@@ -213,7 +39,7 @@ async function findForBroker(b) {
let found = 0, wrote = 0;
for (const b of rows) {
const cap = USE_BROWSER ? 75000 : 45000; // browser tier legitimately needs longer
- const r = await withTimeout(findForBroker(b), cap, () => ({ id: b.id, name: b.name, website: b.website, domain: siteDomain(b.website), found: null, basis: 'timeout', all: [], tried: [{ status: 'HANG>cap' }] }));
+ const r = await withTimeout(huntEmails(b, USE_BROWSER ? { browserHtml } : {}), cap, () => ({ id: b.id, name: b.name, website: b.website, domain: siteDomain(b.website), found: null, basis: 'timeout', all: [], tried: [{ status: 'HANG>cap' }] }));
if (r.found) {
found++;
console.log(`✓ ${r.name} (#${r.id}) [${r.basis}]`);
diff --git a/scripts/lib/email-hunt-browser.js b/scripts/lib/email-hunt-browser.js
new file mode 100644
index 0000000..8f3443a
--- /dev/null
+++ b/scripts/lib/email-hunt-browser.js
@@ -0,0 +1,55 @@
+/*
+ * email-hunt-browser.js — the headless-Chrome escalation tier for the email hunt ($0, local).
+ * When plain fetch strikes out (403 anti-bot or JS-rendered contact info), a real browser
+ * fingerprint + JS execution recovers a big chunk of the misses (e.g. steppcommercial.com's
+ * agent emails only exist after hydration). Shared browser instance, reused across calls.
+ * Requiring this module is safe everywhere; loadable() says whether Playwright actually exists
+ * (Kamatera prod has no Playwright → callers just skip the browser tier).
+ */
+'use strict';
+const path = require('path');
+const os = require('os');
+const fs = require('fs');
+const { UA } = require('./email-hunt');
+
+let _browser = null, _pw = null;
+function loadPlaywright() {
+ const cands = [
+ path.join(os.homedir(), 'Projects/Designer-Wallcoverings/DW-Programming/ImportNewSkufromURL/node_modules/playwright'),
+ 'playwright',
+ ];
+ for (const c of cands) { try { return require(c); } catch {} }
+ throw new Error('playwright not found — install it or drop the browser tier');
+}
+function loadable() { try { loadPlaywright(); return true; } catch { return false; } }
+
+const CHROME_APP = '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome';
+const CHROME_SHELL = os.homedir() + '/Library/Caches/ms-playwright/chromium_headless_shell-1228/chrome-headless-shell-mac-arm64/chrome-headless-shell';
+async function launchBrowser() {
+ const opts = { headless: true, args: ['--no-sandbox', '--disable-dev-shm-usage'] };
+ // Prefer REAL system Chrome (best anti-bot fingerprint); fall back to the installed headless-shell.
+ if (fs.existsSync(CHROME_APP)) { try { return await _pw.chromium.launch({ ...opts, executablePath: CHROME_APP }); } catch {} }
+ if (fs.existsSync(CHROME_SHELL)) { try { return await _pw.chromium.launch({ ...opts, executablePath: CHROME_SHELL }); } catch {} }
+ return await _pw.chromium.launch(opts);
+}
+async function browserHtml(url, ms = 20000) {
+ try {
+ if (!_pw) _pw = loadPlaywright();
+ if (!_browser) _browser = await launchBrowser();
+ // ignoreHTTPSErrors: small broker sites run expired certs (mdrealtycorp.com) — read-only
+ // public-page scrape, same lenient-TLS stance as email-hunt's getInsecure fallback.
+ const ctx = await _browser.newContext({ userAgent: UA, viewport: { width: 1280, height: 900 }, ignoreHTTPSErrors: true });
+ const page = await ctx.newPage();
+ let html = '';
+ try {
+ await page.goto(url, { waitUntil: 'domcontentloaded', timeout: ms });
+ await page.waitForTimeout(1800); // let JS hydrate contact widgets
+ html = await page.content();
+ } catch (e) { /* return whatever rendered */ try { html = await page.content(); } catch {} }
+ await ctx.close();
+ return { ok: !!html, status: html ? 200 : 0, html: (html || '').slice(0, 800000), finalUrl: url };
+ } catch (e) { return { ok: false, status: 0, err: e.message.split('\n')[0], html: '', finalUrl: url }; }
+}
+async function closeBrowser() { try { if (_browser) await _browser.close(); } catch {} _browser = null; }
+
+module.exports = { loadable, browserHtml, closeBrowser };
diff --git a/scripts/lib/email-hunt.js b/scripts/lib/email-hunt.js
new file mode 100644
index 0000000..4003b76
--- /dev/null
+++ b/scripts/lib/email-hunt.js
@@ -0,0 +1,181 @@
+/*
+ * email-hunt.js — the ONE broker-email hunting engine ($0 plain fetch, no DB, no browser).
+ * Extracted from broker-email-finder.js so the CLI batch runner AND serve.js's on-demand
+ * "✉ find email" button share the same guards:
+ * - mailto: trusted; text emails only when on the site's own domain
+ * - this broker's NAME in the local-part always wins
+ * - refuses to guess on shared corporate pages (a stranger's email is worse than none)
+ * - junk/tracking + fake-TLD noise filtered
+ * Optional opts.browserHtml(url) hook lets the CLI escalate to headless Chrome.
+ */
+'use strict';
+
+const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36';
+const HEADERS = { 'User-Agent': UA, 'Accept': 'text/html,application/xhtml+xml', 'Accept-Language': 'en-US,en;q=0.9', 'Referer': 'https://www.google.com/' };
+const EMAIL_RE = /[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/g;
+// asset/tracking junk that happens to match the email shape
+const JUNK = /(\.(png|jpg|jpeg|gif|webp|svg|css|js|ico)|@2x|@3x|sentry|wixpress|example\.|core-js|@types|@sha256|u00|domain\.com|email\.com|yourdomain|@version|placeholder)/i;
+// real-ish TLDs (incl. common real-estate vanity TLDs). Rejects JS-property "TLDs" like
+// .push/.protocol/.location/.length/.score/.ion that minified scripts produce.
+const TLD_OK = new Set(('com net org edu gov mil io co us biz info me tv realtor realty homes house properties ' +
+ 'estate realestate group team agency partners capital ventures llc inc law cpa dental health ' +
+ 'ca uk au nz de fr es it nl eu global online site pro top xyz live life world today').split(' '));
+const sleep = ms => new Promise(r => setTimeout(r, ms));
+// Hard ceiling — a single site's fetch/body-read can occasionally evade the AbortController
+// (slow-trickle body, keep-alive) and wedge the caller. This guarantees the caller always advances.
+function withTimeout(promise, ms, onTimeout) {
+ return Promise.race([promise, new Promise(res => setTimeout(() => res(onTimeout()), ms))]);
+}
+
+function validEmail(e) {
+ if (!e || e.length > 60 || (e.match(/@/g) || []).length !== 1 || JUNK.test(e)) return false;
+ const dom = e.split('@')[1] || ''; const tld = dom.split('.').pop();
+ if (!tld || !TLD_OK.has(tld.toLowerCase())) return false; // kills document.loc@ion.protocol etc.
+ if (!/^[a-z0-9._%+-]+@([a-z0-9-]+\.)+[a-z]{2,}$/i.test(e)) return false;
+ return true;
+}
+
+async function get(url, ms = 12000) {
+ try {
+ const ctl = new AbortController(); const t = setTimeout(() => ctl.abort(), ms);
+ const r = await fetch(url, { headers: HEADERS, redirect: 'follow', signal: ctl.signal });
+ clearTimeout(t);
+ if (!r.ok) return { ok: false, status: r.status, html: '', finalUrl: url };
+ const ct = r.headers.get('content-type') || '';
+ if (!/html|text/i.test(ct)) return { ok: false, status: r.status, html: '', finalUrl: r.url };
+ return { ok: true, status: r.status, html: (await r.text()).slice(0, 800000), finalUrl: r.url };
+ } catch (e) { return { ok: false, status: 0, err: e.name === 'AbortError' ? 'timeout' : e.message.split('\n')[0], html: '', finalUrl: url }; }
+}
+// Lenient-TLS fallback (node:https, rejectUnauthorized:false) for when normal fetch dies on an
+// EXPIRED/self-signed cert — common on small broker marketing sites (mdrealtycorp.com's cert is
+// expired, e.g.). Read-only public-page scrape, so a bad cert is acceptable HERE and only here.
+// Follows up to 4 redirects manually; same return shape as get().
+function getInsecure(url, ms = 12000, hops = 4) {
+ return new Promise(resolve => {
+ let u; try { u = new URL(url); } catch { return resolve({ ok: false, status: 0, err: 'bad url', html: '', finalUrl: url }); }
+ const mod = u.protocol === 'http:' ? require('http') : require('https');
+ const req = mod.get(u, { headers: HEADERS, rejectUnauthorized: false, timeout: ms }, res => {
+ const st = res.statusCode || 0;
+ if (st >= 301 && st <= 308 && res.headers.location && hops > 0) {
+ res.resume();
+ let next; try { next = new URL(res.headers.location, url).href; } catch { next = null; }
+ if (next) return resolve(getInsecure(next, ms, hops - 1));
+ }
+ if (st < 200 || st >= 400) { res.resume(); return resolve({ ok: false, status: st, html: '', finalUrl: url }); }
+ const ct = res.headers['content-type'] || '';
+ if (!/html|text/i.test(ct)) { res.resume(); return resolve({ ok: false, status: st, html: '', finalUrl: url }); }
+ let body = '';
+ res.setEncoding('utf8');
+ res.on('data', d => { body += d; if (body.length > 800000) { body = body.slice(0, 800000); req.destroy(); } });
+ res.on('end', () => resolve({ ok: true, status: st, html: body, finalUrl: url }));
+ res.on('error', () => resolve({ ok: !!body, status: st, html: body, finalUrl: url }));
+ });
+ req.on('timeout', () => { req.destroy(); resolve({ ok: false, status: 0, err: 'timeout', html: '', finalUrl: url }); });
+ req.on('error', e => resolve({ ok: false, status: 0, err: String(e.message || e).split('\n')[0], html: '', finalUrl: url }));
+ });
+}
+// get() + lenient-TLS rescue: normal fetch first; a hard connection failure (status 0 — DNS dead
+// OR bad cert) retries once insecure. DNS-dead just fails again; bad-cert sites come back alive.
+async function fetchPage(url, ms = 12000) {
+ const r = await get(url, ms);
+ if (!r.ok && r.status === 0) { const r2 = await getInsecure(url, ms); if (r2.ok) return r2; }
+ return r;
+}
+// try the URL; on connection failure retry the http<->https twin (many broker sites moved to https)
+async function getResilient(url) {
+ let r = await fetchPage(url);
+ if (!r.ok && (r.status === 0 || r.status >= 500)) {
+ const twin = url.startsWith('https://') ? url.replace(/^https:/, 'http:') : url.replace(/^http:/, 'https:');
+ await sleep(200); const r2 = await fetchPage(twin); if (r2.ok) return r2;
+ }
+ return r;
+}
+
+function siteDomain(u) { try { return new URL(u).hostname.replace(/^www\./, ''); } catch { return ''; } }
+
+// Returns { mailtos:[], texts:[] } — mailto: emails are trustworthy; text emails are used only
+// when they match the site's own domain (so cross-domain JS junk / other companies are dropped).
+function extractEmails(html) {
+ const mailtos = new Set(), texts = new Set();
+ for (const m of html.matchAll(/mailto:([^"'?>\s]+)/gi)) { const e = decodeURIComponent(m[1]).toLowerCase(); if (validEmail(e)) mailtos.add(e); }
+ for (const m of (html.match(EMAIL_RE) || [])) { const e = m.toLowerCase(); if (validEmail(e)) texts.add(e); }
+ return { mailtos: [...mailtos], texts: [...texts] };
+}
+
+function findContactLinks(html, base) {
+ const links = new Set();
+ for (const m of html.matchAll(/href\s*=\s*["']([^"']+)["']/gi)) {
+ const href = m[1];
+ if (/contact|about|team|reach|connect|get-in-touch/i.test(href)) {
+ try { links.add(new URL(href, base).href); } catch {}
+ }
+ }
+ return [...links].slice(0, 3);
+}
+
+const sameDomain = (e, dom) => { const d = e.split('@')[1] || ''; return dom && (d === dom || d.endsWith('.' + dom) || dom.endsWith('.' + d)); };
+// Choose the broker's email from mailto: + same-domain text emails. On a SHARED corporate domain
+// (many agents on one page) the broker's NAME in the local-part is the deciding signal — never guess
+// a random other agent. Falls back to a role inbox (info@/contact@) on the site's own domain.
+function pickBest(mailtos, texts, dom, name) {
+ const pool = [...new Set([...mailtos, ...texts.filter(e => sameDomain(e, dom))])];
+ if (!pool.length) return { email: null, basis: null };
+ const parts = String(name || '').toLowerCase().split(/\s+/).filter(w => w.length > 1);
+ const first = parts[0] || '', last = parts[parts.length - 1] || '';
+ const nameKeys = [last, first + last, first + '.' + last, first[0] + last, first + '_' + last].filter(Boolean);
+ const role = e => /^(info|contact|hello|sales|admin|office|team|reception|frontdesk)@/.test(e);
+ const isName = e => { const l = e.split('@')[0]; return (last && nameKeys.some(k => l.includes(k))) || (first && first.length > 2 && l.includes(first)); };
+ // 1) this broker by name — always the answer when present.
+ const named = pool.find(isName);
+ if (named) return { email: named, basis: 'name-match', pool };
+ // 2) a role inbox on the site's own domain (info@/contact@) — the firm's real contact.
+ const roleInbox = pool.find(e => role(e) && sameDomain(e, dom));
+ if (roleInbox) return { email: roleInbox, basis: 'role-inbox', pool };
+ // 3) exactly ONE email on the whole site → this broker's, BUT ONLY on their own small domain.
+ // On a corporate megasite (marcusmillichap/cbre/kw/…) a broker's "website" is often a shared
+ // office page rendering ONE OTHER agent's email — sole-email there mis-assigns a stranger. Skip.
+ const CORP_DOM = /marcusmillichap|cbre|kw\.com|kwcommercial|remax|coldwell|century21|compass|colliers|cushwake|jll|berkadia|newmark|kidder/i;
+ if (pool.length === 1 && !CORP_DOM.test(dom) && !CORP_DOM.test(pool[0])) return { email: pool[0], basis: 'sole-email', pool };
+ // 4) Multiple PERSONAL emails, none matching this broker → it's a shared/corporate page listing
+ // OTHER agents. Refuse to guess — a stranger's email is worse than none.
+ return { email: null, basis: 'ambiguous-shared-page', pool };
+}
+
+// Hunt one broker: homepage → contact-ish pages → pickBest. b = { name, website }.
+// opts.browserHtml(url) => {ok,status,html,err} — optional headless-Chrome escalation (CLI only).
+async function huntEmails(b, opts = {}) {
+ const dom = siteDomain(b.website);
+ const tried = []; const mailtos = new Set(), texts = new Set();
+ const home = await getResilient(b.website);
+ tried.push({ url: b.website, status: home.status, err: home.err });
+ if (home.ok) {
+ const e0 = extractEmails(home.html); e0.mailtos.forEach(x => mailtos.add(x)); e0.texts.forEach(x => texts.add(x));
+ for (const link of findContactLinks(home.html, home.finalUrl)) {
+ await sleep(300);
+ const cp = await fetchPage(link);
+ tried.push({ url: link, status: cp.status, err: cp.err });
+ if (cp.ok) { const e = extractEmails(cp.html); e.mailtos.forEach(x => mailtos.add(x)); e.texts.forEach(x => texts.add(x)); }
+ }
+ }
+ let { email, basis, pool } = pickBest([...mailtos], [...texts], dom, b.name);
+ // Escalate to the real browser only when plain fetch struck out (blocked or JS-rendered) AND we
+ // don't already have a confident hit. Renders JS + carries a genuine browser fingerprint.
+ const blockedOrEmpty = !email && (tried.some(t => t.status === 403 || t.status === 0) || tried.every(t => t.status >= 200));
+ if (opts.browserHtml && blockedOrEmpty) {
+ const bh = await opts.browserHtml(b.website);
+ tried.push({ url: b.website + ' [browser]', status: bh.status, err: bh.err });
+ if (bh.ok) {
+ const e = extractEmails(bh.html); e.mailtos.forEach(x => mailtos.add(x)); e.texts.forEach(x => texts.add(x));
+ for (const link of findContactLinks(bh.html, b.website)) {
+ const cb = await opts.browserHtml(link);
+ tried.push({ url: link + ' [browser]', status: cb.status, err: cb.err });
+ if (cb.ok) { const e2 = extractEmails(cb.html); e2.mailtos.forEach(x => mailtos.add(x)); e2.texts.forEach(x => texts.add(x)); }
+ }
+ ({ email, basis, pool } = pickBest([...mailtos], [...texts], dom, b.name));
+ if (email) basis = basis + '+browser';
+ }
+ }
+ return { id: b.id, name: b.name, website: b.website, domain: dom, found: email, basis, all: pool || [], tried };
+}
+
+module.exports = { UA, HEADERS, validEmail, get, getInsecure, fetchPage, getResilient, siteDomain, extractEmails, findContactLinks, sameDomain, pickBest, huntEmails, withTimeout, sleep };
diff --git a/scripts/serve.js b/scripts/serve.js
index c0d9346..de6d211 100644
--- a/scripts/serve.js
+++ b/scripts/serve.js
@@ -772,6 +772,86 @@ app.post('/api/broker/pitch', async (req, res) => {
} catch (e) { res.status(502).json({ error: String(e.message).split('\n')[0] }); }
});
+// On-demand email hunt for ONE broker — backs the "✉ find email" button on the CRCP Top-brokers
+// table + broker grid + contact modal. Visits the broker's own website (homepage + up to 3
+// contact-ish pages), extracts mailto:/text emails, and picks with the SAME guards as the
+// broker-email-finder.js batch (shared lib/email-hunt.js): the broker's name in the local-part
+// wins; refuses to guess on shared corporate pages — a stranger's email is worse than none.
+// $0 (plain local fetch, no paid API). Found email persists: broker.email locally (Postgres),
+// or written back into data/brokers-snapshot.json on DB-less prod (sticks until next refresh).
+const emailHunt = require('./lib/email-hunt');
+// Browser escalation (headless Chrome) when Playwright is available (the Mac); Kamatera prod has
+// no Playwright → loadable() false → the button quietly runs fetch-only. Browser is closed after
+// each hunt — a button click is rare enough that a ~1s relaunch beats a resident Chrome.
+let emailHuntBrowser = null; try { const b = require('./lib/email-hunt-browser'); if (b.loadable()) emailHuntBrowser = b; } catch (_) {}
+const _huntInFlight = new Set();
+app.post('/api/broker/find-email', async (req, res) => {
+ const bid = (req.body || {}).id ? +(req.body || {}).id : null;
+ let name = String((req.body || {}).name || '').trim();
+ let website = String((req.body || {}).website || '').trim();
+ try {
+ // Resolve name/website server-side (DB locally, snapshot on prod) so the hunt + any write
+ // target the broker ON FILE — the client can't point a broker id at an arbitrary site.
+ if (bid && brokerdb) {
+ const b = (await brokerdb.pool.query(`SELECT id, name, website, email FROM broker WHERE id=$1`, [bid])).rows[0];
+ if (!b) return res.status(404).json({ error: 'broker not found' });
+ if (b.email) return res.json({ email: b.email, basis: 'already-on-file', saved: false });
+ name = b.name; website = b.website || '';
+ } else if (bid) {
+ const s = readBrokerSnap(); const b = s && (s.brokers || []).find(x => +x.id === bid);
+ if (b) {
+ if (b.email) return res.json({ email: b.email, basis: 'already-on-file', saved: false });
+ name = b.name || name; website = b.website || '';
+ }
+ }
+ if (!website) return res.status(400).json({ error: 'no website on file for this broker' });
+ if (!/^https?:\/\//i.test(website)) website = 'https://' + website;
+ const key = bid || website;
+ if (_huntInFlight.has(key)) return res.status(429).json({ error: 'a hunt is already running for this broker' });
+ _huntInFlight.add(key);
+ let r;
+ try {
+ const opts = emailHuntBrowser ? { browserHtml: emailHuntBrowser.browserHtml } : {};
+ const cap = emailHuntBrowser ? 75000 : 45000; // browser tier legitimately needs longer
+ r = await emailHunt.withTimeout(emailHunt.huntEmails({ id: bid, name, website }, opts), cap,
+ () => ({ found: null, basis: 'timeout', all: [], tried: [] }));
+ } finally {
+ _huntInFlight.delete(key);
+ if (emailHuntBrowser) emailHuntBrowser.closeBrowser();
+ }
+ let saved = false;
+ if (r.found && bid) {
+ if (brokerdb) {
+ try {
+ const u = await brokerdb.pool.query(`UPDATE broker SET email=$1 WHERE id=$2 AND (email IS NULL OR email='')`, [r.found, bid]);
+ saved = u.rowCount > 0;
+ } catch (_) {}
+ } else {
+ try { // DB-less prod: write into the snapshot so the find survives reloads until the nightly refresh
+ const s = readBrokerSnap();
+ if (s) {
+ for (const arr of [s.brokers, s.top]) (arr || []).forEach(x => { if (+x.id === bid && !x.email) { x.email = r.found; saved = true; } });
+ if (saved) fs.writeFileSync(path.join(ROOT, 'data', 'brokers-snapshot.json'), JSON.stringify(s));
+ }
+ } catch (_) { saved = false; }
+ }
+ }
+ // "reached" = ANY page fetched OK (a failed sub-page must not read as "site unreachable")
+ const reachedOk = (r.tried || []).some(t => t.status >= 200 && t.status < 400);
+ res.json({
+ email: r.found || null, basis: r.basis || null, saved,
+ others: (r.all || []).filter(e => e !== r.found).slice(0, 5),
+ site: emailHunt.siteDomain(website),
+ reason: r.found ? null
+ : r.basis === 'ambiguous-shared-page' ? `shared page — ${(r.all || []).length} other agents' emails, none match ${name}; refused to guess`
+ : r.basis === 'timeout' ? 'site too slow (45s cap)'
+ : reachedOk ? 'reached the site but no email is published on it'
+ : 'site unreachable/blocked (' + (r.tried || []).map(t => t.status || t.err || '?').join('/') + ')',
+ cost: '$0 (local fetch)'
+ });
+ } catch (e) { res.status(502).json({ error: String(e.message).split('\n')[0] }); }
+});
+
// Closed-sales comps (Redfin sold, ~5yr) — browsable market dataset + aggregates. Read-only.
app.get('/api/closed-sales', async (req, res) => {
const city = String(req.query.city || '').trim();
← eec551f auto-save: 2026-07-13T00:51:43 (4 files) — data/condos-redfi
·
back to Commercialrealestate
·
prod CRCP: snapshot fallback for stats top-brokers + id/webs 51399ad →