`scripts/broker-email-finder.js`

61 lines
#!/usr/bin/env node
/*
 * broker-email-finder.js — for brokers that have a website but no email, visit the site,
 * find the "Contact" area, and extract an email. $0 (plain fetch, no paid API, no browser).
 *   node broker-email-finder.js --test 5          # dry: 5 candidates, print, NO db write
 *   node broker-email-finder.js --ids 293,1015     # dry: specific brokers
 *   node broker-email-finder.js --limit 50 --commit  # write found emails to broker.email
 * Strategy: fetch homepage → collect mailto: + text emails → also fetch up to 3 "contact"-ish
 * linked pages → score candidates (prefer address on the site's own domain; drop junk/role-y noise).
 * The engine (fetch/extract/pick guards) lives in lib/email-hunt.js — shared with serve.js's
 * on-demand /api/broker/find-email button so the batch and the button can never diverge.
 */
'use strict';
const { pool } = require('./db/brokers-db');
const { huntEmails, siteDomain, withTimeout, sleep } = require('./lib/email-hunt');
const args = process.argv.slice(2);
const COMMIT = args.includes('--commit');
const ti = args.indexOf('--test'); const TEST_N = ti > -1 ? +args[ti + 1] : 0;
const ii = args.indexOf('--ids'); const IDS = ii > -1 ? args[ii + 1].split(',').map(Number) : null;
const li = args.indexOf('--limit'); const LIMIT = li > -1 ? +args[li + 1] : 25;
// --browser: when plain fetch fails (403 anti-bot) or returns no email (JS-rendered contact),
// retry with a local headless Chrome (Playwright, $0). Real browser fingerprint + JS execution
// recovers a big chunk of the plain-fetch misses. Shared browser instance, reused across brokers.
const USE_BROWSER = args.includes('--browser');
const { browserHtml, closeBrowser } = require('./lib/email-hunt-browser');

(async () => {
  let sql, params = [];
  if (IDS) { sql = `SELECT id, name, website FROM broker WHERE id = ANY($1)`; params = [IDS]; }
  else {
    const n = TEST_N || LIMIT;
    sql = `SELECT id, name, website FROM broker
             WHERE website IS NOT NULL AND website<>'' AND (email IS NULL OR email='') AND website ~* '^https?://'
             ORDER BY (SELECT count(*) FROM broker_listing bl WHERE bl.broker_id=broker.id) DESC NULLS LAST
             LIMIT ${n}`;
  }
  const rows = (await pool.query(sql, params)).rows;
  console.log(`brokers to scan: ${rows.length} | mode ${COMMIT ? 'COMMIT' : 'DRY (no db write)'}\n`);
  let found = 0, wrote = 0;
  for (const b of rows) {
    const cap = USE_BROWSER ? 75000 : 45000;   // browser tier legitimately needs longer
    const r = await withTimeout(huntEmails(b, USE_BROWSER ? { browserHtml } : {}), cap, () => ({ id: b.id, name: b.name, website: b.website, domain: siteDomain(b.website), found: null, basis: 'timeout', all: [], tried: [{ status: 'HANG>cap' }] }));
    if (r.found) {
      found++;
      console.log(`✓ ${r.name}  (#${r.id})  [${r.basis}]`);
      console.log(`    site: ${r.website}`);
      console.log(`    EMAIL: ${r.found}${r.all.length > 1 ? '   [others: ' + r.all.filter(e => e !== r.found).slice(0, 4).join(', ') + ']' : ''}`);
      if (COMMIT) { await pool.query(`UPDATE broker SET email=$1 WHERE id=$2 AND (email IS NULL OR email='')`, [r.found, r.id]); wrote++; }
    } else {
      const why = r.tried.map(t => `${t.status || t.err || '?'}`).join('/');
      const reason = r.basis === 'ambiguous-shared-page' ? `refused: shared page, ${r.all.length} other-agent emails, none match ${r.name}`
        : (r.tried.every(t => t.status && t.status < 400)) ? 'reached site but no email present' : `fetch: ${why}`;
      console.log(`✗ ${r.name}  (#${r.id})  — no email  [${r.website}]  (${reason})`);
    }
    await sleep(500);
  }
  console.log(`\nDONE: ${found}/${rows.length} emails found${COMMIT ? `, ${wrote} written to broker.email` : ' (dry — nothing written)'}. $0 (local${USE_BROWSER ? ' fetch+browser' : ' fetch'}).`);
  await closeBrowser();
  await pool.end();
})().catch(async e => { await closeBrowser(); console.error(e); process.exit(1); });