← back to Commercialrealestate
CRCP: bounded $0 agent-site+phone discovery (2022 agents, free SERP + local qwen3:14b) → data/agent-sites.json; /api/agent-sites + on-demand /api/agent/discover; mls.html renders 📞 phone · 👤 agent site (or find btn) · 🔗 listing per deal
193f7b3ed76b830c5138a5778a252291596b7e2a · 2026-08-06 11:10:48 -0700 · Steve Abrams
Files touched
A data/agent-sites.jsonM public/mls.htmlA scripts/discover-agent-sites.jsM scripts/serve.js
Diff
commit 193f7b3ed76b830c5138a5778a252291596b7e2a
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Thu Aug 6 11:10:48 2026 -0700
CRCP: bounded $0 agent-site+phone discovery (2022 agents, free SERP + local qwen3:14b) → data/agent-sites.json; /api/agent-sites + on-demand /api/agent/discover; mls.html renders 📞 phone · 👤 agent site (or find btn) · 🔗 listing per deal
---
data/agent-sites.json | 11 +++
public/mls.html | 44 +++++++++-
scripts/discover-agent-sites.js | 182 ++++++++++++++++++++++++++++++++++++++++
scripts/serve.js | 20 +++++
4 files changed, 253 insertions(+), 4 deletions(-)
diff --git a/data/agent-sites.json b/data/agent-sites.json
new file mode 100644
index 0000000..313dab0
--- /dev/null
+++ b/data/agent-sites.json
@@ -0,0 +1,11 @@
+{
+ "michael yue|compass": {
+ "status": "no_url",
+ "website": null,
+ "phone": null,
+ "confidence": 0,
+ "agent": "Michael Yue",
+ "firm": "Compass",
+ "discovered_at": "2026-08-06T18:10:26.525Z"
+ }
+}
\ No newline at end of file
diff --git a/public/mls.html b/public/mls.html
index b78493e..37fba9f 100644
--- a/public/mls.html
+++ b/public/mls.html
@@ -345,6 +345,17 @@ const cval=(r,c)=>c.calc?c.calc(r):r[c.k];
const _AGG=['crexi','redfin','zillow','costar','loopnet','realtor','myelisting'];
const _host=u=>{try{return new URL(u).hostname.replace(/^www\./,'');}catch{return null;}};
const _isAgg=u=>{const h=_host(u);return !!(h&&_AGG.some(a=>h.includes(a)));};
+// Agent OWN-site + phone bits (from the $0 agent-sites discovery), appended to the
+// listing cell so every displayed agent ties to firm AND shows 📞 phone · 👤 own site
+// (or a free on-demand "find" button). broker_url = firm site, handled separately.
+function agentBits(r){
+ if(!r.broker_agent) return '';
+ const out=[];
+ if(r.agent_phone) out.push(`<a href="tel:${esc(r.agent_phone)}" title="the agent's phone">📞 ${esc(r.agent_phone)}</a>`);
+ if(r.agent_site) out.push(`<a href="${esc(r.agent_site)}" target="_blank" rel="noopener noreferrer" title="the agent's own website">👤 agent ↗</a>`);
+ else if(r.agent_site_status!=='no_url') out.push(`<button class="agent-find" data-a="${esc(r.broker_agent)}" data-f="${esc(r.broker_firm||'')}" data-c="${esc(r.city||'')}" title="find this agent's own site (free, local)" style="background:none;border:1px solid currentColor;border-radius:4px;padding:0 5px;cursor:pointer;font:inherit;color:inherit;opacity:.7">🔎 agent</button>`);
+ return out.length?`<div style="margin-top:3px;display:flex;gap:8px;flex-wrap:wrap;font-size:11px">${out.join('')}</div>`:'';
+}
function brokerListing(r){
// 0. Harvested firm-direct deep link (broker's own property page) — highest-confidence source of truth
if(r.firm_direct_url && !_isAgg(r.firm_direct_url)) return {mode:'listing',href:r.firm_direct_url,label:'View firm listing ↗',title:"the broker's own property page ("+(r.firm_direct_source||'firm-direct')+")"};
@@ -363,11 +374,11 @@ function cellHTML(r,c){
if(c.t==='pct') return v==null||v===''?'—':v+'%';
if(c.t==='n') return num(v);
if(c.t==='status'){ const a=String(v||'').startsWith('Active'); return `<span class="pill ${a?'active':'other'}">${esc(v||'—')}</span>`; }
- if(c.t==='link'){ const u=brokerListing(r);
- if(u.mode==='listing') return `<a href="${esc(u.href)}" target="_blank" rel="noopener noreferrer" title="${esc(u.title)}">${esc(u.label)}</a>`;
- // mode==='text': firm site (if non-agg) + always a find link
+ if(c.t==='link'){ const u=brokerListing(r); const ab=agentBits(r);
+ if(u.mode==='listing') return `<a href="${esc(u.href)}" target="_blank" rel="noopener noreferrer" title="${esc(u.title)}">🔗 ${esc(u.label)}</a>`+ab;
+ // mode==='text': firm site (if non-agg) + always a find link, then agent phone/own-site
const fs=u.firmSite?`<a href="${esc(u.firmSite)}" target="_blank" rel="noopener noreferrer" title="the listing broker's own site">🏢 site ↗</a> `:'';
- return fs+`<a href="${esc(u.find)}" target="_blank" rel="noopener noreferrer" title="find this listing on the web">🔎 find</a>`; }
+ return fs+`<a href="${esc(u.find)}" target="_blank" rel="noopener noreferrer" title="find this listing on the web">🔎 find</a>`+ab; }
// Firm name links to the broker's own site (only when non-aggregator).
if(c.k==='firm' && r.broker_url && !_isAgg(r.broker_url)) return `<a href="${esc(r.broker_url)}" target="_blank" rel="noopener noreferrer" title="the listing broker's own site">${esc(v==null||v===''?'—':v)} ↗</a>`;
return esc(v==null||v===''?'—':v);
@@ -569,8 +580,33 @@ fetch('/data/ranked.json').then(r=>r.json()).then(async d=>{ DATA=(d.ranked||d)|
r.last_bor_observed_at=e.observed_at||null;
});
}catch(e){}
+ // Agent OWN-site + phone (bounded $0 discovery, /api/agent-sites keyed norm(agent)|norm(firm)).
+ try{ const as=await (await fetch('/api/agent-sites')).json(); const SITES=(as&&as.sites)||{};
+ const nrm=s=>String(s||'').toLowerCase().replace(/[^a-z0-9]+/g,' ').trim();
+ DATA.forEach(r=>{ const e=SITES[nrm(r.broker_agent)+'|'+nrm(r.broker_firm)]; if(!e) return;
+ if(e.website) r.agent_site=e.website;
+ if(e.phone) r.agent_phone=e.phone;
+ r.agent_site_status=e.status||null; // found | no_url
+ });
+ }catch(e){}
autoCols(); buildRail(); render(); })
.catch(()=>{ $('#count').textContent='failed to load /data/ranked.json'; });
+
+// $0 on-demand: click "🔎 agent" → discover that one agent's own site+phone (local, free).
+document.addEventListener('click', async e=>{
+ const b=e.target.closest('.agent-find'); if(!b) return;
+ e.preventDefault(); b.disabled=true; const t=b.textContent; b.textContent='🔎 …';
+ try{
+ const r=await (await fetch('/api/agent/discover',{method:'POST',headers:{'Content-Type':'application/json'},
+ body:JSON.stringify({agent:b.dataset.a,firm:b.dataset.f,city:b.dataset.c})})).json();
+ if(r.status==='found'&&r.website){
+ const bits=[]; if(r.phone) bits.push(`<a href="tel:${esc(r.phone)}">📞 ${esc(r.phone)}</a>`);
+ bits.push(`<a href="${esc(r.website)}" target="_blank" rel="noopener noreferrer">👤 agent ↗</a>`);
+ b.outerHTML=bits.join(' ');
+ } else if(r.status==='throttled'){ b.disabled=false; b.textContent='🔎 busy — retry'; }
+ else b.outerHTML='<span style="opacity:.6;font-size:11px">👤 no site</span>';
+ }catch(_){ b.disabled=false; b.textContent=t; }
+});
</script>
<script src="/column-manager.js" defer></script>
<script src="/col-resize.js" defer></script>
diff --git a/scripts/discover-agent-sites.js b/scripts/discover-agent-sites.js
new file mode 100644
index 0000000..70727fa
--- /dev/null
+++ b/scripts/discover-agent-sites.js
@@ -0,0 +1,182 @@
+#!/usr/bin/env node
+/**
+ * CRCP agent-site + phone discovery — bounded ($0, free SERP + local Ollama). CommonJS.
+ *
+ * Unlike the national usre sweep (2M agents → throttle-walls), CRCP has a BOUNDED
+ * agent set: the distinct broker_agent × broker_firm pairs on ranked.json's 3,105
+ * deals (~2,022 agents). Small enough that a $0 pass — even throttle-limited, run in
+ * batches — actually completes. For each agent: free SERP query → local qwen3:14b
+ * identity gate (is this hit THIS agent's own site, not a namesake / directory / the
+ * firm's generic site) → if found, fetch the page and extract a phone. Results land
+ * in data/agent-sites.json keyed by norm(agent)|norm(firm), merged onto deals at
+ * render time. broker_url (the FIRM site) is left untouched — this fills the missing
+ * agent-OWN-site + phone.
+ *
+ * Run: node scripts/discover-agent-sites.js --limit=40
+ * node scripts/discover-agent-sites.js --agent="Michael Yue" --firm="Compass" # one, on-demand
+ *
+ * Polite 2.5-4s/query, aborts after 3 consecutive throttles (resume picks up where
+ * it left off — done agents are skipped). Linking to the agent's own page is allowed
+ * per the 2026-08-06 policy; we never re-host their content.
+ */
+const fs = require('fs');
+const path = require('path');
+
+const ROOT = path.join(__dirname, '..');
+const RANKED = path.join(ROOT, 'data', 'ranked.json');
+const OUT = path.join(ROOT, 'data', 'agent-sites.json');
+const OLLAMA = (process.env.OLLAMA_URL || 'http://127.0.0.1:11434') + '/api/generate';
+const MODEL = process.env.OLLAMA_MODEL || 'qwen3:14b';
+const MIN_CONFIDENCE = 0.6;
+const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36';
+
+const BLOCK = new Set([
+ 'zillow.com','realtor.com','redfin.com','homes.com','trulia.com','loopnet.com','crexi.com',
+ 'costar.com','commercialsearch.com','cityfeet.com','showcase.com','brevitas.com','myelisting.com',
+ 'apartments.com','point2homes.com','streeteasy.com','compass.com','cbre.com','jll.com','colliers.com',
+ 'facebook.com','instagram.com','linkedin.com','twitter.com','x.com','youtube.com','tiktok.com',
+ 'yelp.com','bbb.org','yellowpages.com','whitepages.com','zoominfo.com','crunchbase.com','manta.com',
+ 'wikipedia.org','google.com','duckduckgo.com','brave.com','mapquest.com','ratemyagent.com',
+]);
+function host(u){ try { return new URL(u).hostname.replace(/^www\./,'').toLowerCase(); } catch { return null; } }
+function blocked(h){
+ for (const r of BLOCK) if (h === r || h.endsWith('.' + r)) return true;
+ if (h.endsWith('.gov') || h.includes('mls')) return true;
+ if (/realtor/.test(h) && /(board|assoc)/.test(h)) return true;
+ return false;
+}
+const norm = s => String(s || '').toLowerCase().replace(/[^a-z0-9]+/g, ' ').trim();
+const key = (agent, firm) => norm(agent) + '|' + norm(firm);
+
+class ThrottleError extends Error {}
+async function fetchText(url, ms = 10000){
+ const res = await fetch(url, { headers: { 'User-Agent': UA, 'Accept': 'text/html,application/xhtml+xml', 'Accept-Language': 'en-US,en;q=0.9' }, redirect: 'follow', signal: AbortSignal.timeout(ms) });
+ if (res.status === 429 || res.status === 403 || res.status === 202) throw new ThrottleError('http ' + res.status);
+ if (!res.ok) throw new Error('http ' + res.status);
+ return res.text();
+}
+async function braveSearch(q){
+ const html = await fetchText('https://search.brave.com/search?q=' + encodeURIComponent(q));
+ if (/verifying you are human|challenge-platform/i.test(html)) throw new ThrottleError('brave challenge');
+ const out = [];
+ for (const blk of html.split('data-type="web"').slice(1)) {
+ const hm = blk.match(/href="(https?:\/\/[^"]+)"/);
+ if (!hm) continue;
+ const tm = blk.match(/>([^<]{8,120})<\/a>/);
+ out.push({ url: hm[1].replace(/&/g, '&'), title: (tm ? tm[1] : '').trim() });
+ }
+ return out.slice(0, 10);
+}
+async function ddgSearch(q){
+ const html = await fetchText('https://html.duckduckgo.com/html/?q=' + encodeURIComponent(q));
+ if (/anomaly-modal|anomaly\.js|challenge-form/i.test(html)) throw new ThrottleError('ddg anomaly');
+ const out = [];
+ const re = /<a[^>]+class="[^"]*result__a[^"]*"[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/g;
+ let m;
+ while ((m = re.exec(html)) !== null) {
+ let abs = m[1].replace(/&/g, '&');
+ try { const u = new URL(abs, 'https://duckduckgo.com'); const g = u.searchParams.get('uddg'); if (g) abs = decodeURIComponent(g); } catch {}
+ if (/^https?:\/\//i.test(abs)) out.push({ url: abs, title: m[2].replace(/<[^>]+>/g, '').trim() });
+ }
+ return out.slice(0, 10);
+}
+async function searchWeb(q){
+ try { return { hits: await braveSearch(q), engine: 'brave' }; }
+ catch (e) { if (!(e instanceof ThrottleError)) throw e; }
+ return { hits: await ddgSearch(q), engine: 'ddg' };
+}
+async function verify(agent, firm, city, cands){
+ if (!cands.length) return { index: -1, confidence: 0 };
+ const list = cands.map((c, i) => `${i}. ${host(c.url)} — "${(c.title || '').slice(0, 90)}"`).join('\n');
+ const prompt = `Verifying a COMMERCIAL real-estate agent's own website.
+ agent: ${agent}
+ firm: ${firm || '(unknown)'}${city ? '\n area: ' + city : ''}
+Candidates (index. domain — title):
+${list}
+Pick the single index that is THIS SPECIFIC agent's OWN professional page — their personal agent site or their bio/profile page on their firm's own site. Must be THIS person (name matches), NOT a namesake, a listing portal/aggregator, a REALTOR board/association, or a generic directory. Most agents have no personal site — -1 is the common, correct answer.
+Respond ONLY compact JSON: {"index": <n>, "confidence": <0..1>}`;
+ const res = await fetch(OLLAMA, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ model: MODEL, prompt, stream: false, options: { temperature: 0 } }), signal: AbortSignal.timeout(60000) });
+ if (!res.ok) throw new Error('ollama ' + res.status);
+ const data = await res.json();
+ const m = (data.response || '').match(/\{[\s\S]*?\}/);
+ if (!m) return { index: -1, confidence: 0 };
+ try { const j = JSON.parse(m[0]); const i = Number.isInteger(j.index) ? j.index : -1; const c = typeof j.confidence === 'number' ? j.confidence : 0; return (i < 0 || i >= cands.length) ? { index: -1, confidence: 0 } : { index: i, confidence: c }; }
+ catch { return { index: -1, confidence: 0 }; }
+}
+async function extractPhone(url){
+ try {
+ const html = await fetchText(url, 8000);
+ const tel = html.match(/href=["']tel:\+?([0-9().\-\s]{10,20})["']/i);
+ if (tel) return tel[1].replace(/[^0-9]/g, '').replace(/^1?(\d{3})(\d{3})(\d{4})$/, '($1) $2-$3');
+ const txt = html.match(/(\(?\d{3}\)?[.\-\s]\d{3}[.\-\s]\d{4})/);
+ return txt ? txt[1] : null;
+ } catch { return null; }
+}
+
+// Discover one agent — the shared core (used by batch + the serve.js on-demand endpoint).
+async function discoverAgent(agent, firm, city){
+ const q = `"${agent}" ${firm || ''} ${city || ''} commercial real estate broker`.replace(/\s+/g, ' ').trim();
+ let hits;
+ try { ({ hits } = await searchWeb(q)); }
+ catch (e) { if (e instanceof ThrottleError) return { status: 'throttled', website: null, phone: null, confidence: 0 }; throw e; }
+ const cands = hits.filter(h => { const hh = host(h.url); return hh && !blocked(hh); }).slice(0, 8);
+ const v = cands.length ? await verify(agent, firm, city, cands) : { index: -1, confidence: 0 };
+ if (v.index >= 0 && v.confidence >= MIN_CONFIDENCE) {
+ const website = 'https://' + host(cands[v.index].url);
+ const phone = await extractPhone(cands[v.index].url);
+ return { status: 'found', website, phone, confidence: v.confidence };
+ }
+ return { status: 'no_url', website: null, phone: null, confidence: 0 };
+}
+
+async function main(){
+ const arg = k => { const a = process.argv.find(x => x.startsWith('--' + k + '=')); return a ? a.split('=')[1] : null; };
+ const store = (() => { try { return JSON.parse(fs.readFileSync(OUT, 'utf8')); } catch { return {}; } })();
+ const save = () => { const tmp = OUT + '.tmp'; fs.writeFileSync(tmp, JSON.stringify(store, null, 2)); fs.renameSync(tmp, OUT); };
+
+ // On-demand single-agent mode
+ if (arg('agent')) {
+ const r = await discoverAgent(arg('agent'), arg('firm') || '', arg('city') || '');
+ if (r.status !== 'throttled') { store[key(arg('agent'), arg('firm'))] = { ...r, agent: arg('agent'), firm: arg('firm') || null, discovered_at: new Date().toISOString() }; save(); }
+ console.log('[agent-discover]', arg('agent'), '→', JSON.stringify(r));
+ return;
+ }
+
+ const limit = parseInt(arg('limit') || '40', 10);
+ const { ranked } = JSON.parse(fs.readFileSync(RANKED, 'utf8'));
+ // distinct agent×firm, prefer those with a firm; skip already-attempted
+ const seen = new Set(), queue = [];
+ for (const d of ranked) {
+ if (!d.broker_agent) continue;
+ const k = key(d.broker_agent, d.broker_firm);
+ if (seen.has(k) || store[k]) continue;
+ seen.add(k);
+ queue.push({ agent: d.broker_agent, firm: d.broker_firm || '', city: d.city || '' });
+ }
+ queue.sort((a, b) => (b.firm ? 1 : 0) - (a.firm ? 1 : 0));
+ const batch = queue.slice(0, limit);
+ console.log(`[agent-discover] ${batch.length} agents this batch (${queue.length} remaining of ${seen.size} undone) · SERP + ${MODEL}`);
+
+ let ok = 0, miss = 0, thr = 0, consec = 0;
+ for (let i = 0; i < batch.length; i++) {
+ const a = batch[i];
+ const r = await discoverAgent(a.agent, a.firm, a.city);
+ if (r.status === 'throttled') {
+ consec++; thr++;
+ console.log(` [${i + 1}/${batch.length}] ⚠ ${a.agent} (throttled)`);
+ if (consec >= 3) { console.log('[agent-discover] 3 consecutive throttles — stopping (resume later)'); break; }
+ await new Promise(r => setTimeout(r, 30000));
+ continue;
+ }
+ consec = 0;
+ store[key(a.agent, a.firm)] = { ...r, agent: a.agent, firm: a.firm || null, discovered_at: new Date().toISOString() };
+ save();
+ if (r.status === 'found') { ok++; console.log(` [${i + 1}/${batch.length}] ✓ ${a.agent} @ ${a.firm} → ${r.website}${r.phone ? ' · ' + r.phone : ''} (${r.confidence.toFixed(2)})`); }
+ else { miss++; console.log(` [${i + 1}/${batch.length}] · ${a.agent} (no verified site)`); }
+ await new Promise(r => setTimeout(r, 2500 + Math.random() * 1500));
+ }
+ console.log(`[agent-discover] done · ${ok} found · ${miss} no-site · ${thr} throttled`);
+}
+
+module.exports = { discoverAgent, key };
+if (require.main === module) main().catch(e => { console.error('[agent-discover]', e); process.exit(1); });
diff --git a/scripts/serve.js b/scripts/serve.js
index 2bf8667..d6b912c 100644
--- a/scripts/serve.js
+++ b/scripts/serve.js
@@ -139,6 +139,26 @@ app.post('/api/contacts/:key/letter/:idx/delete', (req, res) => {
return rec;
}).then(rec => res.json({ ok: true, contact: rec })).catch(jsonErr(res));
});
+
+// Agent OWN-site + phone — the bounded $0 discovery (data/agent-sites.json, keyed
+// norm(agent)|norm(firm)). broker_url stays the FIRM site; this fills the missing
+// agent-own-site + phone. Linking to the agent's own page is allowed (policy 2026-08-06).
+const { discoverAgent, key: agentKey } = require('./discover-agent-sites.js');
+const AGENT_SITES = path.join(ROOT, 'data', 'agent-sites.json');
+const readAgentSites = () => { try { return JSON.parse(fs.readFileSync(AGENT_SITES, 'utf8')); } catch (_) { return {}; } };
+const writeAgentSites = (m) => { const t = AGENT_SITES + '.tmp'; fs.writeFileSync(t, JSON.stringify(m, null, 2)); fs.renameSync(t, AGENT_SITES); };
+app.get('/api/agent-sites', (req, res) => res.json({ sites: readAgentSites() }));
+// On-demand: the $0 lazy path a deal's broker cell fires when it has no agent site yet.
+app.post('/api/agent/discover', async (req, res) => {
+ try {
+ const agent = clip(req.body && req.body.agent, 120), firm = clip(req.body && req.body.firm, 120), city = clip(req.body && req.body.city, 80);
+ if (!agent.trim()) return res.status(400).json({ error: 'no agent' });
+ const r = await discoverAgent(agent, firm, city);
+ if (r.status !== 'throttled') { const m = readAgentSites(); m[agentKey(agent, firm)] = { ...r, agent, firm: firm || null, discovered_at: new Date().toISOString() }; writeAgentSites(m); }
+ res.json(r);
+ } catch (e) { jsonErr(res)(e); }
+});
+
// "Send via George" — creates a Gmail DRAFT (never a live send) via the George gmail-agent on
// this same box (127.0.0.1:9850 /api/drafts). GATED THREE WAYS: (1) it only ever creates a DRAFT,
// so nothing leaves until Steve reviews + sends in Gmail; (2) it is FAIL-CLOSED — with no
← b0d2cf6 auto-save: 2026-08-06T07:19:35 (1 files) — data/alerts-state
·
back to Commercialrealestate
·
auto-data-snapshot: 2026-08-06T11:23:39 (1 data files) — dat 9ddd258 →