`scripts/scrape-org-emails.js`

177 lines
#!/usr/bin/env node
/**
 * Org-website email scraper.
 *
 * For every organization with a populated website but no email row yet,
 * fetch the homepage + /contact + /about and extract any on-domain or
 * common-vendor email addresses. Politely rate-limited per host via
 * shared/compliance.fetchCompliant.
 *
 * Run:
 *   node scripts/scrape-org-emails.js                # all eligible orgs
 *   LIMIT=20 node scripts/scrape-org-emails.js       # smoke
 *   ONLY_DOCTORS=1 node scripts/scrape-org-emails.js # restrict to doctor-type orgs
 */
const cheerio = require('cheerio');
const { pool, query } = require('../agents/shared/db');
const { fetchCompliant, setHostRateLimit } = require('../agents/shared/compliance');

const LIMIT = process.env.LIMIT ? Number(process.env.LIMIT) : null;
const ONLY_DOCTORS = process.env.ONLY_DOCTORS === '1';

const DOCTOR_TYPES = ['medical_group','clinic','dental_office','optometrist_office','surgery_center','hospital','fqhc','tcm_clinic','acupuncture_clinic','chiropractor_office','tcm_herbalist'];

const SUBPATHS = ['', '/contact', '/contact-us', '/about', '/about-us'];

const EMAIL_RE = /\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/gi;

const DECOY_RE = /^(noreply|no-reply|donotreply|do-not-reply|example|test|wix\.com|sentry|godaddy|squarespace|cloudflare|webmaster|admin|support|hello|mail)@/i;
const VENDOR_DOMAINS = new Set([
  'sentry.io','sentry-next.wixpress.com','wix.com','squarespace.com','godaddy.com',
  'cloudflare.com','google.com','gmail.com','example.com','example.org','tld.com',
  'site.com','domain.com','website.com','someone@example.com','sample.com',
]);

function normalize(email) {
  return String(email || '').trim().toLowerCase().replace(/\.$/, '');
}

function isPlausible(email) {
  if (!email) return false;
  if (email.length > 100) return false;
  if (!email.includes('@')) return false;
  if (DECOY_RE.test(email)) return false;
  // Reject literal example domains and obvious vendors.
  const domain = email.split('@')[1];
  if (!domain) return false;
  if (VENDOR_DOMAINS.has(domain)) return false;
  if (/\.(png|jpe?g|gif|svg|webp|css|js)$/i.test(email)) return false;
  // Reject anything that looks like a sentry/analytics token.
  if (/[a-f0-9]{32,}/i.test(email.split('@')[0])) return false;
  return true;
}

function originOf(websiteRaw) {
  let s = String(websiteRaw || '').trim();
  if (!s) return null;
  if (!/^https?:\/\//i.test(s)) s = 'https://' + s;
  try { return new URL(s); } catch { return null; }
}

async function fetchHtml(url) {
  try {
    const res = await fetchCompliant(url, { accept: 'text/html' });
    if (!res.ok) return null;
    const ct = res.headers.get('content-type') || '';
    if (!/html|text/i.test(ct)) return null;
    const text = await res.text();
    if (text.length > 800_000) return text.slice(0, 800_000);
    return text;
  } catch (e) {
    if (e?.code === 'CAPTCHA_DETECTED') return null;
    return null;
  }
}

function extractEmails(html, sourceUrl) {
  if (!html) return [];
  const $ = cheerio.load(html);
  const found = new Map(); // email → page

  // Pass 1: explicit mailto: hrefs.
  $('a[href^="mailto:"]').each((_, el) => {
    const href = ($(el).attr('href') || '').replace(/^mailto:/i, '').split('?')[0];
    const e = normalize(href);
    if (isPlausible(e) && !found.has(e)) found.set(e, sourceUrl);
  });

  // Pass 2: text-mode addresses anywhere in body. Strip script/style tags first.
  $('script, style, noscript').remove();
  const text = $('body').text() || $.text();
  for (const m of text.matchAll(EMAIL_RE)) {
    const e = normalize(m[0]);
    if (isPlausible(e) && !found.has(e)) found.set(e, sourceUrl);
  }

  return [...found.entries()].map(([email, page]) => ({ email, page }));
}

async function processOrg(o) {
  const origin = originOf(o.website);
  if (!origin) return { found: 0, hit_pages: 0 };

  setHostRateLimit(origin.hostname, 1.0);

  const seen = new Map();
  let pagesHit = 0;
  for (const p of SUBPATHS) {
    const url = origin.origin + p;
    const html = await fetchHtml(url);
    if (!html) continue;
    pagesHit++;
    for (const { email, page } of extractEmails(html, url)) {
      if (!seen.has(email)) seen.set(email, page);
    }
    if (seen.size >= 5) break; // diminishing returns
  }

  let inserted = 0;
  for (const [email, page] of seen) {
    try {
      await query(
        `INSERT INTO emails (organization_id, email, email_type, source_url, last_verified_at, verification_status)
         VALUES ($1, $2, 'office', $3, NULL, 'unverified')
         ON CONFLICT DO NOTHING`,
        [o.id, email, page]
      );
      inserted++;
    } catch (e) {
      // duplicate or fk error — skip
    }
  }
  return { found: seen.size, inserted, pages: pagesHit };
}

async function main() {
  const where = [
    `o.website IS NOT NULL`,
    `o.website <> ''`,
    `NOT EXISTS (SELECT 1 FROM emails e WHERE e.organization_id = o.id)`,
  ];
  if (ONLY_DOCTORS) where.push(`o.type = ANY($1)`);
  const params = ONLY_DOCTORS ? [DOCTOR_TYPES] : [];
  const sql = `
    SELECT o.id, o.name, o.website, o.type
    FROM organizations o
    WHERE ${where.join(' AND ')}
    ORDER BY o.id
    ${LIMIT ? `LIMIT ${LIMIT}` : ''}
  `;
  const r = await query(sql, params);
  console.log(`[email] candidates=${r.rowCount}`);

  let totalFound = 0, totalOrgs = 0, withAny = 0;
  for (const o of r.rows) {
    totalOrgs++;
    try {
      const stat = await processOrg(o);
      totalFound += stat.inserted || 0;
      if (stat.inserted) withAny++;
      if (totalOrgs % 25 === 0) {
        console.log(`[email] orgs=${totalOrgs} with_any=${withAny} emails_inserted=${totalFound}`);
      }
    } catch (e) {
      console.error(`[email] error org=${o.id} (${o.name}): ${e.message}`);
    }
  }

  console.log(`[email] done. orgs=${totalOrgs} with_any=${withAny} emails_inserted=${totalFound}`);
  await pool.end();
}

main().catch(async (err) => {
  console.error('[email] fatal:', err);
  try { await pool.end(); } catch (_) {}
  process.exit(1);
});