broker-email-finder: --browser tier (real system Chrome) recovers 403/anti-bot + JS-rendered sites; name-match disambiguates multi-agent firm pages

52b71e11989d9f56257b9aa0430ed1ae3833d5e4 · 2026-07-12 10:38:36 -0700 · Steve Abrams
Files touched

M scripts/broker-email-finder.js
Diff

commit 52b71e11989d9f56257b9aa0430ed1ae3833d5e4
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Sun Jul 12 10:38:36 2026 -0700

    broker-email-finder: --browser tier (real system Chrome) recovers 403/anti-bot + JS-rendered sites; name-match disambiguates multi-agent firm pages
---
 scripts/broker-email-finder.js | 65 ++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 62 insertions(+), 3 deletions(-)

diff --git a/scripts/broker-email-finder.js b/scripts/broker-email-finder.js
index 507bf68..bcb8e75 100644
--- a/scripts/broker-email-finder.js
+++ b/scripts/broker-email-finder.js
@@ -15,6 +15,47 @@ const COMMIT = args.includes('--commit');
 const ti = args.indexOf('--test'); const TEST_N = ti > -1 ? +args[ti + 1] : 0;
 const ii = args.indexOf('--ids'); const IDS = ii > -1 ? args[ii + 1].split(',').map(Number) : null;
 const li = args.indexOf('--limit'); const LIMIT = li > -1 ? +args[li + 1] : 25;
+// --browser: when plain fetch fails (403 anti-bot) or returns no email (JS-rendered contact),
+// retry with a local headless Chrome (Playwright, $0). Real browser fingerprint + JS execution
+// recovers a big chunk of the plain-fetch misses. Shared browser instance, reused across brokers.
+const USE_BROWSER = args.includes('--browser');
+let _browser = null, _pw = null;
+function loadPlaywright() {
+  const path = require('path'), os = require('os');
+  const cands = [
+    path.join(os.homedir(), 'Projects/Designer-Wallcoverings/DW-Programming/ImportNewSkufromURL/node_modules/playwright'),
+    'playwright',
+  ];
+  for (const c of cands) { try { return require(c); } catch {} }
+  throw new Error('playwright not found — install it or drop --browser');
+}
+const CHROME_APP = '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome';
+const CHROME_SHELL = require('os').homedir() + '/Library/Caches/ms-playwright/chromium_headless_shell-1228/chrome-headless-shell-mac-arm64/chrome-headless-shell';
+async function launchBrowser() {
+  const fs = require('fs');
+  const opts = { headless: true, args: ['--no-sandbox', '--disable-dev-shm-usage'] };
+  // Prefer REAL system Chrome (best anti-bot fingerprint); fall back to the installed headless-shell.
+  if (fs.existsSync(CHROME_APP)) { try { return await _pw.chromium.launch({ ...opts, executablePath: CHROME_APP }); } catch {} }
+  if (fs.existsSync(CHROME_SHELL)) { try { return await _pw.chromium.launch({ ...opts, executablePath: CHROME_SHELL }); } catch {} }
+  return await _pw.chromium.launch(opts);
+}
+async function browserHtml(url, ms = 20000) {
+  try {
+    if (!_pw) _pw = loadPlaywright();
+    if (!_browser) _browser = await launchBrowser();
+    const ctx = await _browser.newContext({ userAgent: UA, viewport: { width: 1280, height: 900 } });
+    const page = await ctx.newPage();
+    let html = '';
+    try {
+      await page.goto(url, { waitUntil: 'domcontentloaded', timeout: ms });
+      await page.waitForTimeout(1800);                       // let JS hydrate contact widgets
+      html = await page.content();
+    } catch (e) { /* return whatever rendered */ try { html = await page.content(); } catch {} }
+    await ctx.close();
+    return { ok: !!html, status: html ? 200 : 0, html: (html || '').slice(0, 800000), finalUrl: page.url ? url : url };
+  } catch (e) { return { ok: false, status: 0, err: e.message.split('\n')[0], html: '', finalUrl: url }; }
+}
+async function closeBrowser() { try { if (_browser) await _browser.close(); } catch {} }
 
 const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36';
 const HEADERS = { 'User-Agent': UA, 'Accept': 'text/html,application/xhtml+xml', 'Accept-Language': 'en-US,en;q=0.9', 'Referer': 'https://www.google.com/' };
@@ -128,7 +169,24 @@ async function findForBroker(b) {
       if (cp.ok) { const e = extractEmails(cp.html); e.mailtos.forEach(x => mailtos.add(x)); e.texts.forEach(x => texts.add(x)); }
     }
   }
-  const { email, basis, pool } = pickBest([...mailtos], [...texts], dom, b.name);
+  let { email, basis, pool } = pickBest([...mailtos], [...texts], dom, b.name);
+  // Escalate to the real browser only when plain fetch struck out (blocked or JS-rendered) AND we
+  // don't already have a confident hit. Renders JS + carries a genuine browser fingerprint.
+  const blockedOrEmpty = !email && (tried.some(t => t.status === 403 || t.status === 0) || tried.every(t => t.status >= 200));
+  if (USE_BROWSER && blockedOrEmpty) {
+    const bh = await browserHtml(b.website);
+    tried.push({ url: b.website + ' [browser]', status: bh.status, err: bh.err });
+    if (bh.ok) {
+      const e = extractEmails(bh.html); e.mailtos.forEach(x => mailtos.add(x)); e.texts.forEach(x => texts.add(x));
+      for (const link of findContactLinks(bh.html, b.website)) {
+        const cb = await browserHtml(link);
+        tried.push({ url: link + ' [browser]', status: cb.status, err: cb.err });
+        if (cb.ok) { const e2 = extractEmails(cb.html); e2.mailtos.forEach(x => mailtos.add(x)); e2.texts.forEach(x => texts.add(x)); }
+      }
+      ({ email, basis, pool } = pickBest([...mailtos], [...texts], dom, b.name));
+      if (email) basis = basis + '+browser';
+    }
+  }
   return { id: b.id, name: b.name, website: b.website, domain: dom, found: email, basis, all: pool || [], tried };
 }
 
@@ -161,6 +219,7 @@ async function findForBroker(b) {
     }
     await sleep(500);
   }
-  console.log(`\nDONE: ${found}/${rows.length} emails found${COMMIT ? `, ${wrote} written to broker.email` : ' (dry — nothing written)'}. $0 (local fetch).`);
+  console.log(`\nDONE: ${found}/${rows.length} emails found${COMMIT ? `, ${wrote} written to broker.email` : ' (dry — nothing written)'}. $0 (local${USE_BROWSER ? ' fetch+browser' : ' fetch'}).`);
+  await closeBrowser();
   await pool.end();
-})().catch(e => { console.error(e); process.exit(1); });
+})().catch(async e => { await closeBrowser(); console.error(e); process.exit(1); });
← 8861282 CRCP gov data: regenerate snapshot with all 21 metros (top-e · back to Commercialrealestate · CRCP email-alerts: snapshot-diff engine + George send wiring 91621f0 →