← back to Commercialrealestate
scripts/discover-agent-sites.js
183 lines
#!/usr/bin/env node
/**
* CRCP agent-site + phone discovery — bounded ($0, free SERP + local Ollama). CommonJS.
*
* Unlike the national usre sweep (2M agents → throttle-walls), CRCP has a BOUNDED
* agent set: the distinct broker_agent × broker_firm pairs on ranked.json's 3,105
* deals (~2,022 agents). Small enough that a $0 pass — even throttle-limited, run in
* batches — actually completes. For each agent: free SERP query → local qwen3:14b
* identity gate (is this hit THIS agent's own site, not a namesake / directory / the
* firm's generic site) → if found, fetch the page and extract a phone. Results land
* in data/agent-sites.json keyed by norm(agent)|norm(firm), merged onto deals at
* render time. broker_url (the FIRM site) is left untouched — this fills the missing
* agent-OWN-site + phone.
*
* Run: node scripts/discover-agent-sites.js --limit=40
* node scripts/discover-agent-sites.js --agent="Michael Yue" --firm="Compass" # one, on-demand
*
* Polite 2.5-4s/query, aborts after 3 consecutive throttles (resume picks up where
* it left off — done agents are skipped). Linking to the agent's own page is allowed
* per the 2026-08-06 policy; we never re-host their content.
*/
const fs = require('fs');
const path = require('path');
const ROOT = path.join(__dirname, '..');
const RANKED = path.join(ROOT, 'data', 'ranked.json');
const OUT = path.join(ROOT, 'data', 'agent-sites.json');
const OLLAMA = (process.env.OLLAMA_URL || 'http://127.0.0.1:11434') + '/api/generate';
const MODEL = process.env.OLLAMA_MODEL || 'qwen3:14b';
const MIN_CONFIDENCE = 0.6;
const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36';
const BLOCK = new Set([
'zillow.com','realtor.com','redfin.com','homes.com','trulia.com','loopnet.com','crexi.com',
'costar.com','commercialsearch.com','cityfeet.com','showcase.com','brevitas.com','myelisting.com',
'apartments.com','point2homes.com','streeteasy.com','compass.com','cbre.com','jll.com','colliers.com',
'facebook.com','instagram.com','linkedin.com','twitter.com','x.com','youtube.com','tiktok.com',
'yelp.com','bbb.org','yellowpages.com','whitepages.com','zoominfo.com','crunchbase.com','manta.com',
'wikipedia.org','google.com','duckduckgo.com','brave.com','mapquest.com','ratemyagent.com',
]);
function host(u){ try { return new URL(u).hostname.replace(/^www\./,'').toLowerCase(); } catch { return null; } }
function blocked(h){
for (const r of BLOCK) if (h === r || h.endsWith('.' + r)) return true;
if (h.endsWith('.gov') || h.includes('mls')) return true;
if (/realtor/.test(h) && /(board|assoc)/.test(h)) return true;
return false;
}
const norm = s => String(s || '').toLowerCase().replace(/[^a-z0-9]+/g, ' ').trim();
const key = (agent, firm) => norm(agent) + '|' + norm(firm);
class ThrottleError extends Error {}
async function fetchText(url, ms = 10000){
const res = await fetch(url, { headers: { 'User-Agent': UA, 'Accept': 'text/html,application/xhtml+xml', 'Accept-Language': 'en-US,en;q=0.9' }, redirect: 'follow', signal: AbortSignal.timeout(ms) });
if (res.status === 429 || res.status === 403 || res.status === 202) throw new ThrottleError('http ' + res.status);
if (!res.ok) throw new Error('http ' + res.status);
return res.text();
}
async function braveSearch(q){
const html = await fetchText('https://search.brave.com/search?q=' + encodeURIComponent(q));
if (/verifying you are human|challenge-platform/i.test(html)) throw new ThrottleError('brave challenge');
const out = [];
for (const blk of html.split('data-type="web"').slice(1)) {
const hm = blk.match(/href="(https?:\/\/[^"]+)"/);
if (!hm) continue;
const tm = blk.match(/>([^<]{8,120})<\/a>/);
out.push({ url: hm[1].replace(/&/g, '&'), title: (tm ? tm[1] : '').trim() });
}
return out.slice(0, 10);
}
async function ddgSearch(q){
const html = await fetchText('https://html.duckduckgo.com/html/?q=' + encodeURIComponent(q));
if (/anomaly-modal|anomaly\.js|challenge-form/i.test(html)) throw new ThrottleError('ddg anomaly');
const out = [];
const re = /<a[^>]+class="[^"]*result__a[^"]*"[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/g;
let m;
while ((m = re.exec(html)) !== null) {
let abs = m[1].replace(/&/g, '&');
try { const u = new URL(abs, 'https://duckduckgo.com'); const g = u.searchParams.get('uddg'); if (g) abs = decodeURIComponent(g); } catch {}
if (/^https?:\/\//i.test(abs)) out.push({ url: abs, title: m[2].replace(/<[^>]+>/g, '').trim() });
}
return out.slice(0, 10);
}
async function searchWeb(q){
try { return { hits: await braveSearch(q), engine: 'brave' }; }
catch (e) { if (!(e instanceof ThrottleError)) throw e; }
return { hits: await ddgSearch(q), engine: 'ddg' };
}
async function verify(agent, firm, city, cands){
if (!cands.length) return { index: -1, confidence: 0 };
const list = cands.map((c, i) => `${i}. ${host(c.url)} — "${(c.title || '').slice(0, 90)}"`).join('\n');
const prompt = `Verifying a COMMERCIAL real-estate agent's own website.
agent: ${agent}
firm: ${firm || '(unknown)'}${city ? '\n area: ' + city : ''}
Candidates (index. domain — title):
${list}
Pick the single index that is THIS SPECIFIC agent's OWN professional page — their personal agent site or their bio/profile page on their firm's own site. Must be THIS person (name matches), NOT a namesake, a listing portal/aggregator, a REALTOR board/association, or a generic directory. Most agents have no personal site — -1 is the common, correct answer.
Respond ONLY compact JSON: {"index": <n>, "confidence": <0..1>}`;
const res = await fetch(OLLAMA, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ model: MODEL, prompt, stream: false, options: { temperature: 0 } }), signal: AbortSignal.timeout(60000) });
if (!res.ok) throw new Error('ollama ' + res.status);
const data = await res.json();
const m = (data.response || '').match(/\{[\s\S]*?\}/);
if (!m) return { index: -1, confidence: 0 };
try { const j = JSON.parse(m[0]); const i = Number.isInteger(j.index) ? j.index : -1; const c = typeof j.confidence === 'number' ? j.confidence : 0; return (i < 0 || i >= cands.length) ? { index: -1, confidence: 0 } : { index: i, confidence: c }; }
catch { return { index: -1, confidence: 0 }; }
}
async function extractPhone(url){
try {
const html = await fetchText(url, 8000);
const tel = html.match(/href=["']tel:\+?([0-9().\-\s]{10,20})["']/i);
if (tel) return tel[1].replace(/[^0-9]/g, '').replace(/^1?(\d{3})(\d{3})(\d{4})$/, '($1) $2-$3');
const txt = html.match(/(\(?\d{3}\)?[.\-\s]\d{3}[.\-\s]\d{4})/);
return txt ? txt[1] : null;
} catch { return null; }
}
// Discover one agent — the shared core (used by batch + the serve.js on-demand endpoint).
async function discoverAgent(agent, firm, city){
const q = `"${agent}" ${firm || ''} ${city || ''} commercial real estate broker`.replace(/\s+/g, ' ').trim();
let hits;
try { ({ hits } = await searchWeb(q)); }
catch (e) { if (e instanceof ThrottleError) return { status: 'throttled', website: null, phone: null, confidence: 0 }; throw e; }
const cands = hits.filter(h => { const hh = host(h.url); return hh && !blocked(hh); }).slice(0, 8);
const v = cands.length ? await verify(agent, firm, city, cands) : { index: -1, confidence: 0 };
if (v.index >= 0 && v.confidence >= MIN_CONFIDENCE) {
const website = 'https://' + host(cands[v.index].url);
const phone = await extractPhone(cands[v.index].url);
return { status: 'found', website, phone, confidence: v.confidence };
}
return { status: 'no_url', website: null, phone: null, confidence: 0 };
}
async function main(){
const arg = k => { const a = process.argv.find(x => x.startsWith('--' + k + '=')); return a ? a.split('=')[1] : null; };
const store = (() => { try { return JSON.parse(fs.readFileSync(OUT, 'utf8')); } catch { return {}; } })();
const save = () => { const tmp = OUT + '.tmp'; fs.writeFileSync(tmp, JSON.stringify(store, null, 2)); fs.renameSync(tmp, OUT); };
// On-demand single-agent mode
if (arg('agent')) {
const r = await discoverAgent(arg('agent'), arg('firm') || '', arg('city') || '');
if (r.status !== 'throttled') { store[key(arg('agent'), arg('firm'))] = { ...r, agent: arg('agent'), firm: arg('firm') || null, discovered_at: new Date().toISOString() }; save(); }
console.log('[agent-discover]', arg('agent'), '→', JSON.stringify(r));
return;
}
const limit = parseInt(arg('limit') || '40', 10);
const { ranked } = JSON.parse(fs.readFileSync(RANKED, 'utf8'));
// distinct agent×firm, prefer those with a firm; skip already-attempted
const seen = new Set(), queue = [];
for (const d of ranked) {
if (!d.broker_agent) continue;
const k = key(d.broker_agent, d.broker_firm);
if (seen.has(k) || store[k]) continue;
seen.add(k);
queue.push({ agent: d.broker_agent, firm: d.broker_firm || '', city: d.city || '' });
}
queue.sort((a, b) => (b.firm ? 1 : 0) - (a.firm ? 1 : 0));
const batch = queue.slice(0, limit);
console.log(`[agent-discover] ${batch.length} agents this batch (${queue.length} remaining of ${seen.size} undone) · SERP + ${MODEL}`);
let ok = 0, miss = 0, thr = 0, consec = 0;
for (let i = 0; i < batch.length; i++) {
const a = batch[i];
const r = await discoverAgent(a.agent, a.firm, a.city);
if (r.status === 'throttled') {
consec++; thr++;
console.log(` [${i + 1}/${batch.length}] ⚠ ${a.agent} (throttled)`);
if (consec >= 3) { console.log('[agent-discover] 3 consecutive throttles — stopping (resume later)'); break; }
await new Promise(r => setTimeout(r, 30000));
continue;
}
consec = 0;
store[key(a.agent, a.firm)] = { ...r, agent: a.agent, firm: a.firm || null, discovered_at: new Date().toISOString() };
save();
if (r.status === 'found') { ok++; console.log(` [${i + 1}/${batch.length}] ✓ ${a.agent} @ ${a.firm} → ${r.website}${r.phone ? ' · ' + r.phone : ''} (${r.confidence.toFixed(2)})`); }
else { miss++; console.log(` [${i + 1}/${batch.length}] · ${a.agent} (no verified site)`); }
await new Promise(r => setTimeout(r, 2500 + Math.random() * 1500));
}
console.log(`[agent-discover] done · ${ok} found · ${miss} no-site · ${thr} throttled`);
}
module.exports = { discoverAgent, key };
if (require.main === module) main().catch(e => { console.error('[agent-discover]', e); process.exit(1); });