← back to Commercialrealestate
CRE Job B Tier 3: Brave-search enrichment (~$0.004/q) for brokers-with-listings missing email — top-result fetch+harvest, last-name email attribution, firm/linkedin/crexi result ranking, $10 global cap + running spend surfaced
63ed0e263b7f9bc2dd67c491d9384df226edf4ed · 2026-06-28 07:22:04 -0700 · steve@designerwallcoverings.com
Files touched
M scripts/enrich-tier2-firmsites.jsA scripts/enrich-tier3-search.js
Diff
commit 63ed0e263b7f9bc2dd67c491d9384df226edf4ed
Author: steve@designerwallcoverings.com <steve@designerwallcoverings.com>
Date: Sun Jun 28 07:22:04 2026 -0700
CRE Job B Tier 3: Brave-search enrichment (~$0.004/q) for brokers-with-listings missing email — top-result fetch+harvest, last-name email attribution, firm/linkedin/crexi result ranking, $10 global cap + running spend surfaced
---
scripts/enrich-tier2-firmsites.js | 2 +-
scripts/enrich-tier3-search.js | 144 ++++++++++++++++++++++++++++++++++++++
2 files changed, 145 insertions(+), 1 deletion(-)
diff --git a/scripts/enrich-tier2-firmsites.js b/scripts/enrich-tier2-firmsites.js
index 830f141..b1e13e2 100644
--- a/scripts/enrich-tier2-firmsites.js
+++ b/scripts/enrich-tier2-firmsites.js
@@ -42,7 +42,7 @@ async function fetchText(url, ms = 9000) {
function harvest(html) {
const emails = new Set(), phones = new Set();
// mailto:/tel: are the highest-confidence.
- for (const m of html.matchAll(/mailto:([^"'?>\s]+)/gi)) { const e = decodeURIComponent(m[1]).toLowerCase(); if (!BAD_EMAIL.test(e)) emails.add(e); }
+ for (const m of html.matchAll(/mailto:([^"'?>\s]+)/gi)) { let e; try { e = decodeURIComponent(m[1]); } catch { e = m[1]; } e = e.toLowerCase(); if (/@/.test(e) && !BAD_EMAIL.test(e)) emails.add(e); }
for (const m of html.matchAll(/tel:([^"'>\s]+)/gi)) { const p = m[1].replace(/[^\d+]/g, ''); if (p.replace(/\D/g, '').length >= 10) phones.add(p); }
// Fallback regex over visible text (strip tags first to avoid matching asset hashes).
const text = html.replace(/<script[\s\S]*?<\/script>/gi, ' ').replace(/<style[\s\S]*?<\/style>/gi, ' ').replace(/<[^>]+>/g, ' ');
diff --git a/scripts/enrich-tier3-search.js b/scripts/enrich-tier3-search.js
new file mode 100644
index 0000000..e4f708f
--- /dev/null
+++ b/scripts/enrich-tier3-search.js
@@ -0,0 +1,144 @@
+// enrich-tier3-search.js — Job B TIER 3 (metered, Brave Search ~$0.004/query): for brokers still
+// missing contact info, run a web search "<name> <firm> email/phone/LinkedIn", fetch the top result
+// page(s), and harvest a business email/phone/LinkedIn attributable to that broker. Cheapest-first:
+// only brokers who (a) have listings in our set and (b) still lack email are searched. Each Brave
+// query cost is surfaced + a running batch total; HARD STOP at the $10 Job-B cap.
+//
+// CCPA: business contacts only; per-field source URL recorded. NO send.
+//
+// Usage: BRAVE budget: node scripts/enrich-tier3-search.js
+// CC_TIER3_MAX=300 max brokers to search this run (default: all eligible)
+// CC_BUDGET=10 Job-B hard $ cap (this tier's running total counts the prior tiers' spend)
+// CC_PRIOR_SPEND=0.04 $ already spent in Job B before tier 3 (so the cap is global)
+'use strict';
+const fs = require('fs');
+const db = require('./db/brokers-db');
+
+const BRAVE_KEY = (() => {
+ for (const p of [process.env.HOME + '/Projects/secrets-manager/.env', process.env.HOME + '/Projects/commercialrealestate/.env']) {
+ try { const m = fs.readFileSync(p, 'utf8').match(/^BRAVE_SEARCH_API_KEY=(.*)$/m); if (m) return m[1].replace(/['"]/g, '').trim(); } catch (_) {}
+ }
+ return process.env.BRAVE_SEARCH_API_KEY;
+})();
+const BRAVE_COST = 0.004;
+const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36';
+const BUDGET = +(process.env.CC_BUDGET || 10);
+const PRIOR_SPEND = +(process.env.CC_PRIOR_SPEND || 0.04);
+const TIER3_MAX = process.env.CC_TIER3_MAX ? +process.env.CC_TIER3_MAX : Infinity;
+
+const EMAIL_RE = /[a-z0-9](?:[a-z0-9._%+-]{0,40})@(?:[a-z0-9-]+\.)+[a-z]{2,8}/gi;
+const PHONE_RE = /(?:\+?1[\s.-]?)?\(?\d{3}\)?[\s.-]?\d{3}[\s.-]?\d{4}/g;
+const BAD_EMAIL = /\.(png|jpg|jpeg|gif|webp|svg|css|js)$|@(?:sentry|wix|example|email|domain|2x|godaddy|sentry)\b/i;
+
+async function braveSearch(q) {
+ const ctrl = new AbortController(); const t = setTimeout(() => ctrl.abort(), 12000);
+ try {
+ const r = await fetch('https://api.search.brave.com/res/v1/web/search?count=4&q=' + encodeURIComponent(q),
+ { headers: { accept: 'application/json', 'X-Subscription-Token': BRAVE_KEY }, signal: ctrl.signal });
+ if (!r.ok) return { results: [], status: r.status };
+ const j = await r.json();
+ return { results: (j.web && j.web.results || []).map(x => ({ url: x.url, title: x.title, desc: x.description })) };
+ } catch (e) { return { results: [], err: String(e).slice(0, 60) }; } finally { clearTimeout(t); }
+}
+
+async function fetchText(url, ms = 9000) {
+ const ctrl = new AbortController(); const t = setTimeout(() => ctrl.abort(), ms);
+ try {
+ const r = await fetch(url, { headers: { 'user-agent': UA, accept: 'text/html' }, redirect: 'follow', signal: ctrl.signal });
+ if (!r.ok) return null;
+ if (!/html|text/.test(r.headers.get('content-type') || '')) return null;
+ return (await r.text()).slice(0, 400000);
+ } catch { return null; } finally { clearTimeout(t); }
+}
+
+function harvest(html, lastName) {
+ const emails = new Set(), phones = new Set(), linkedin = new Set();
+ for (const m of html.matchAll(/mailto:([^"'?>\s]+)/gi)) { let e; try { e = decodeURIComponent(m[1]); } catch { e = m[1]; } e = e.toLowerCase(); if (/@/.test(e) && !BAD_EMAIL.test(e)) emails.add(e); }
+ for (const m of html.matchAll(/tel:([^"'>\s]+)/gi)) { const p = m[1].replace(/[^\d]/g, '').slice(-10); if (p.length === 10) phones.add(p); }
+ const text = html.replace(/<script[\s\S]*?<\/script>/gi, ' ').replace(/<style[\s\S]*?<\/style>/gi, ' ').replace(/<[^>]+>/g, ' ');
+ for (const m of text.matchAll(EMAIL_RE)) { const e = m[0].toLowerCase(); if (!BAD_EMAIL.test(e)) emails.add(e); }
+ for (const m of text.matchAll(PHONE_RE)) { const p = m[0].replace(/\D/g, ''); const d = p.length === 11 && p[0] === '1' ? p.slice(1) : p; if (d.length === 10) phones.add(d); }
+ for (const m of html.matchAll(/https?:\/\/(?:www\.)?linkedin\.com\/in\/[A-Za-z0-9_-]+/gi)) linkedin.add(m[0]);
+ // Prefer an email whose local-part contains the broker's last name (highest attribution confidence).
+ let best = null;
+ if (lastName) best = [...emails].find(e => e.split('@')[0].toLowerCase().includes(lastName.toLowerCase()));
+ return { email: best || [...emails].find(e => !/^(info|contact|sales|admin|hello|office|marketing)@/i.test(e)) || [...emails][0],
+ phone: [...phones][0], linkedin: [...linkedin][0] };
+}
+
+function fmtPhone(p) { const d = String(p).replace(/\D/g, '').slice(-10); return d.length === 10 ? `(${d.slice(0,3)}) ${d.slice(3,6)}-${d.slice(6)}` : p; }
+
+async function setField(brokerId, field, value, sourceUrl) {
+ if (value == null || value === '') return false;
+ await db.pool.query(
+ `INSERT INTO broker_field_source(broker_id, field, value, source_url, tier)
+ VALUES($1,$2,$3,$4,'tier3-search') ON CONFLICT(broker_id, field) DO NOTHING`,
+ [brokerId, field, String(value).slice(0, 500), sourceUrl]);
+ const col = { phone: 'phone', email: 'email', linkedin: 'linkedin' }[field]; if (!col) return false;
+ const r = await db.pool.query(`UPDATE broker SET ${col}=$2, enriched_at=now() WHERE id=$1 AND ${col} IS NULL`, [brokerId, String(value).slice(0,500)]);
+ return r.rowCount > 0;
+}
+
+(async () => {
+ if (!BRAVE_KEY) { console.error('No BRAVE_SEARCH_API_KEY'); process.exit(1); }
+ // Eligible: brokers WITH listings in our set that still lack email; richest book first.
+ const eligible = (await db.pool.query(
+ `SELECT b.id, b.name, b.email, b.phone, b.linkedin, b.website, f.name AS firm,
+ count(bl.listing_id) AS our_listings, b.total_assets
+ FROM broker b LEFT JOIN firm f ON f.id=b.firm_id
+ JOIN broker_listing bl ON bl.broker_id=b.id
+ WHERE b.email IS NULL
+ GROUP BY b.id, f.name
+ ORDER BY count(bl.listing_id) DESC, b.total_assets DESC NULLS LAST`)).rows;
+ process.stderr.write(`Tier 3 eligible (have listings, no email): ${eligible.length}\n`);
+
+ let searches = 0, emailFound = 0, phoneFound = 0, liFound = 0;
+ for (const b of eligible) {
+ if (searches >= TIER3_MAX) { process.stderr.write(`[CAP] tier3 max ${TIER3_MAX} reached\n`); break; }
+ const spend = PRIOR_SPEND + (searches + 1) * BRAVE_COST;
+ if (spend > BUDGET) { process.stderr.write(`[CAP] $${BUDGET} Job-B budget would be exceeded; stopping (spend ~$${spend.toFixed(3)}).\n`); break; }
+
+ const lastName = (b.name.split(/\s+/).pop() || '');
+ const q = `"${b.name}" ${b.firm || ''} commercial real estate broker email phone`.trim();
+ const { results } = await braveSearch(q);
+ searches++;
+ // Visit up to 2 most-relevant results (prefer the firm domain / linkedin / crexi profile).
+ const ranked = results.sort((a, c) => score(c, b) - score(a, b)).slice(0, 2);
+ let got = { email: null, phone: null, linkedin: null }, srcUrl = null;
+ for (const r of ranked) {
+ if (/linkedin\.com\/in\//i.test(r.url) && !got.linkedin) { got.linkedin = r.url.split('?')[0]; srcUrl = srcUrl || r.url; }
+ const html = await fetchText(r.url);
+ if (!html) continue;
+ const h = harvest(html, lastName);
+ if (h.email && !got.email) { got.email = h.email; srcUrl = r.url; }
+ if (h.phone && !got.phone) got.phone = h.phone;
+ if (h.linkedin && !got.linkedin) got.linkedin = h.linkedin;
+ if (got.email && got.phone) break;
+ }
+ if (got.email && await setField(b.id, 'email', got.email, srcUrl || q)) emailFound++;
+ if (got.phone && await setField(b.id, 'phone', fmtPhone(got.phone), srcUrl || q)) phoneFound++;
+ if (got.linkedin && await setField(b.id, 'linkedin', got.linkedin, srcUrl || q)) liFound++;
+
+ if (searches % 20 === 0) process.stderr.write(` ${searches} searched · email+${emailFound} phone+${phoneFound} · running $${(PRIOR_SPEND + searches*BRAVE_COST).toFixed(3)}\r`);
+ }
+
+ const runningCost = (searches * BRAVE_COST);
+ const st = (await db.pool.query(
+ `SELECT count(*) FILTER (WHERE phone IS NOT NULL) phone, count(*) FILTER (WHERE email IS NOT NULL) email,
+ count(*) FILTER (WHERE linkedin IS NOT NULL) linkedin, count(*) total FROM broker`)).rows[0];
+ process.stderr.write(`\nTier 3: ${searches} Brave searches ($${runningCost.toFixed(3)}) · email +${emailFound} phone +${phoneFound} linkedin +${liFound}\n`);
+ console.log(JSON.stringify({ tier: 3, searches, tierCost: '$' + runningCost.toFixed(3),
+ jobBSpend: '$' + (PRIOR_SPEND + runningCost).toFixed(3), dbState: st }));
+ await db.pool.end();
+})();
+
+function score(r, b) {
+ if (!r || !r.url) return 0;
+ let s = 0; const u = r.url.toLowerCase();
+ if (b.website && u.includes(b.website.toLowerCase().replace(/^https?:\/\//, '').replace(/^www\./, '').split('/')[0])) s += 5;
+ if (/linkedin\.com\/in\//.test(u)) s += 3;
+ if (/crexi\.com\/(broker|profile)/.test(u)) s += 3;
+ if (/loopnet|costar|biproxi|commercialcafe/.test(u)) s += 2;
+ if ((r.title || '').toLowerCase().includes((b.name || '').toLowerCase())) s += 2;
+ return s;
+}
← 22629f0 CRCP: live Commercial Real Estate Control Panel (CNCP-styled
·
back to Commercialrealestate
·
CRE Job B Tier 3: filter data-aggregator emails (signalhire/ c5507a7 →