`scripts/enrich-contacts-browser.mjs`

119 lines
#!/usr/bin/env node
/*
 * enrich-contacts-browser.mjs — deep contact enrichment with a REAL browser +
 * a LOCAL model. Opens each business website in headless Chromium (renders JS
 * that the HTTP crawler can't see), visits a contact/about page, and logs:
 *   • email      (DOM mailto first; local Ollama model deobfuscates "name [at] domain" text if none)
 *   • instagram  (real profile link from the site — direct, not a search)
 *   • linkedin   (company/in profile link)
 *   • phone      (tel: link)
 *
 * Layers on top of a prospects-<slug>.json (from crawl-csv-emails.py): reads it,
 * enriches each business in place, writes it back. Local + free ($0). Nothing
 * is uploaded to Constant Contact.
 *
 *   node scripts/enrich-contacts-browser.mjs public/data/prospects-sf-architects.json [--limit N] [--workers 6] [--model qwen2.5:latest]
 */
import { readFileSync, writeFileSync } from 'node:fs';
import { createRequire } from 'node:module';

// Reuse an existing Playwright install (browsers already cached in ms-playwright).
const require = createRequire(import.meta.url);
let chromium;
for (const p of ['/Users/macstudio3/Projects/Designer-Wallcoverings/node_modules/playwright',
                 '/Users/macstudio3/Projects/astek-landing/node_modules/playwright',
                 'playwright']) {
  try { ({ chromium } = require(p)); break; } catch {}
}
if (!chromium) { console.error('playwright not found'); process.exit(1); }

const args = process.argv.slice(2);
const FILE = args.find(a => !a.startsWith('--'));
const opt = (k, d) => { const i = args.indexOf('--' + k); return i >= 0 ? args[i + 1] : d; };
const LIMIT = Number(opt('limit', 0));
const WORKERS = Number(opt('workers', 6));
const MODEL = opt('model', 'qwen2.5:latest');
const OLLAMA = 'http://127.0.0.1:11434/api/generate';

const data = JSON.parse(readFileSync(FILE, 'utf8'));
let biz = data.businesses || [];
if (LIMIT) biz = biz.slice(0, LIMIT);

const EMAIL_RE = /[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}/g;
const JUNK = /(sentry|wix\.com|wixpress|godaddy|squarespace|example\.|domain\.com|yourdomain|\.png|\.jpg|\.svg|sentry\.io|@2x)/i;
const norm = u => { try { return new URL(u.startsWith('http') ? u : 'http://' + u).href; } catch { return null; } };
const host = u => { try { return new URL(u).hostname.replace(/^www\./, ''); } catch { return ''; } };

async function ollamaEmail(text, siteHost) {
  try {
    const r = await fetch(OLLAMA, { method: 'POST', headers: { 'Content-Type': 'application/json' },
      body: JSON.stringify({ model: MODEL, think: false, stream: false,
        prompt: `From this business website contact text, extract the single best PUBLIC contact email address (prefer one on the @${siteHost} domain, or info@/hello@/contact@). If a real email is present, reply with ONLY that email. If none, reply with exactly NONE.\n\n---\n${text.slice(0, 3500)}\n---` }) });
    const j = await r.json();
    const m = (j.response || '').match(EMAIL_RE);
    return m && !JUNK.test(m[0]) ? m[0].toLowerCase() : '';
  } catch { return ''; }
}

async function grabLinks(page) {
  return page.$$eval('a[href]', as => as.map(a => a.getAttribute('href') || '').filter(Boolean)).catch(() => []);
}
function pickSocial(hrefs, re) {
  for (const h of hrefs) { const m = h.match(re); if (m) return m[0].split('?')[0].replace(/\/$/, ''); }
  return '';
}

async function enrichOne(ctx, b) {
  const url = norm(b.website); if (!url) return;
  const siteHost = host(url);
  const page = await ctx.newPage();
  try {
    await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 15000 });
    let hrefs = await grabLinks(page);
    // hop to a contact/about page if present
    const contact = hrefs.find(h => /contact|about|connect|team/i.test(h));
    let text = await page.evaluate(() => document.body ? document.body.innerText : '').catch(() => '');
    if (contact) {
      try { await page.goto(new URL(contact, url).href, { waitUntil: 'domcontentloaded', timeout: 12000 });
            hrefs = hrefs.concat(await grabLinks(page));
            text += '\n' + await page.evaluate(() => document.body ? document.body.innerText : '').catch(() => ''); } catch {}
    }
    // deterministic first
    const ig = pickSocial(hrefs, /https?:\/\/(?:www\.)?instagram\.com\/(?!p\/|explore|accounts)[A-Za-z0-9_.]+/i);
    const li = pickSocial(hrefs, /https?:\/\/(?:www\.)?linkedin\.com\/(?:company|in)\/[A-Za-z0-9_%\-]+/i);
    const tel = pickSocial(hrefs, /tel:\+?[0-9().\-\s]{7,}/i);
    let email = '';
    const mailto = hrefs.find(h => /^mailto:/i.test(h));
    if (mailto) { const e = mailto.replace(/^mailto:/i, '').split('?')[0].trim(); if (EMAIL_RE.test(e) && !JUNK.test(e)) email = e.toLowerCase(); }
    if (!email) { const m = (text.match(EMAIL_RE) || []).filter(e => !JUNK.test(e)); if (m.length) email = m[0].toLowerCase(); }
    if (!email) email = await ollamaEmail(text, siteHost);   // local-model fallback for obfuscated emails
    if (ig) b.instagram = ig;
    if (li) b.linkedin = li;
    if (tel) b.phone = b.phone || tel.replace(/^tel:/i, '').trim();
    if (email && !b.email) b.email = email;
    if (email) b.emails = Array.from(new Set([email, ...(b.emails || [])]));
  } catch { /* unreachable site */ } finally { await page.close().catch(() => {}); }
}

const browser = await chromium.launch({ headless: true });
const ctx = await browser.newContext({ userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/124 Safari/537.36' });
let done = 0, ig = 0, li = 0, mail = 0;
const queue = biz.slice();
async function worker() {
  while (queue.length) {
    const b = queue.shift();
    await enrichOne(ctx, b);
    done++; if (b.instagram) ig++; if (b.linkedin) li++; if (b.email) mail++;
    if (done % 10 === 0 || done === biz.length) process.stdout.write(`\r  ${done}/${biz.length} · ${mail} email · ${ig} instagram · ${li} linkedin  `);
  }
}
await Promise.all(Array.from({ length: WORKERS }, worker));
await browser.close();

data.with_email = (data.businesses || []).filter(b => b.email).length;
data.with_instagram = (data.businesses || []).filter(b => b.instagram).length;
data.with_linkedin = (data.businesses || []).filter(b => b.linkedin).length;
data.enriched = new Date().toISOString().slice(0, 10);
writeFileSync(FILE, JSON.stringify(data));
console.log(`\n✓ ${FILE} — ${data.with_email} email · ${data.with_instagram} instagram · ${data.with_linkedin} linkedin  [browser + ${MODEL}, $0 local]`);