`scripts/enrich-domain.js`

102 lines
// Contractor enrichment via DOMAIN-GUESSING (READ-ONLY writes to cslb_raw, $0 local).
// The right max-it architecture: instead of one rate-limited search engine, guess each
// contractor's OWN candidate domains and HTTP-check them — every request hits a DIFFERENT
// host, so there's no shared rate limit and it parallelizes freely. A local Ollama model
// verifies the fetched page actually belongs to the contractor + extracts email.
//
// Usage: node scripts/enrich-domain.js [--loop] [--batch=N] [--shard=i/N] [--ollama=URL] [--model=M] [--all]
import { q, pool } from '../src/db.js';

const flags = new Set(process.argv.slice(2).filter(a => a.startsWith('--') && !a.includes('=')));
const kv = Object.fromEntries(process.argv.slice(2).filter(a => a.includes('=')).map(a => a.slice(2).split('=')));
const OLLAMA = kv.ollama || 'http://127.0.0.1:11434';
const MODEL = kv.model || 'qwen2.5:latest';
const BATCH = Number(kv.batch || 50);
const LA_ONLY = !flags.has('--all');
const [SHARD_I, SHARD_N] = (kv.shard || '0/1').split('/').map(Number);
const SHARD = SHARD_N > 1 ? `AND (abs(hashtext(c."LicenseNo")) % ${SHARD_N}) = ${SHARD_I}` : '';
const sleep = ms => new Promise(r => setTimeout(r, ms));

// business name -> candidate domains (their own hosts — no shared rate limit)
function candidates(name) {
  const base = String(name).toLowerCase().replace(/&/g, 'and').replace(/[^a-z0-9 ]/g, ' ').replace(/\s+/g, ' ').trim();
  const noSuffix = base.replace(/\b(inc|llc|corp|corporation|company|co|ltd|lp|the|construction|development|builders|building|group|enterprises|services)\b/g, ' ').replace(/\s+/g, ' ').trim();
  const compact = s => s.replace(/\s+/g, '');
  const words = base.split(' ').filter(Boolean);
  const set = new Set();
  set.add(compact(base));                        // fullnameconstruction
  if (noSuffix && noSuffix !== base) set.add(compact(noSuffix)); // fullname
  if (words.length >= 2) set.add(compact(words.slice(0, 2).join(''))); // firsttwo
  if (words.length >= 2) set.add(words.map(w => w[0]).join(''));       // initials
  const doms = [];
  for (const s of set) if (s.length >= 3 && s.length <= 40) for (const tld of ['.com', '.net']) doms.push(s + tld);
  return [...new Set(doms)].slice(0, 8);
}

async function tryDomain(dom) {
  try {
    const res = await fetch('https://' + dom, { headers: { 'User-Agent': 'Mozilla/5.0' }, signal: AbortSignal.timeout(6000), redirect: 'follow' });
    if (!res.ok) return null;
    const ct = res.headers.get('content-type') || '';
    if (!/html/.test(ct)) return null;
    const html = (await res.text()).slice(0, 40000);
    const text = html.replace(/<script[\s\S]*?<\/script>/gi, ' ').replace(/<style[\s\S]*?<\/style>/gi, ' ').replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim().slice(0, 1400);
    const email = (html.match(/[a-z0-9._%+-]+@[a-z0-9.-]+\.[a-z]{2,}/i) || [])[0] || null;
    return { url: 'https://' + dom, text, email };
  } catch { return null; }
}

async function verify(c, hits) {
  const prompt = `Which of these websites (if any) is the OFFICIAL site of this contractor? Only match if the page clearly is this business (name and, ideally, city/phone appear).
Contractor: ${c.BusinessName}, ${c.City} CA ${c.ZIPCode}, phone ${c.BusinessPhone || 'n/a'}.
${hits.map((h, i) => `${i + 1}. ${h.url}\n   ${h.text.slice(0, 600)}`).join('\n')}
Return ONLY strict JSON: {"index": number|null (1-based, or null if none match), "confidence": "high"|"low"|"none"}`;
  try {
    const res = await fetch(OLLAMA + '/api/generate', { method: 'POST', headers: { 'Content-Type': 'application/json' },
      body: JSON.stringify({ model: MODEL, prompt, stream: false, format: 'json', options: { temperature: 0 } }) });
    const d = await res.json();
    const j = JSON.parse(d.response);
    const pick = j.index && hits[j.index - 1];
    return pick ? { website: pick.url, email: pick.email, confidence: j.confidence || 'low' } : { website: null, email: null, confidence: 'none' };
  } catch { return { website: null, email: null, confidence: 'none' }; }
}

async function pick() {
  const laFilter = LA_ONLY ? `AND "County"='Los Angeles'` : '';
  return (await q(`
    SELECT c."LicenseNo", c."BusinessName", c."City", c."ZIPCode", c."BusinessPhone"
    FROM cslb_raw c
    WHERE c."PrimaryStatus"='CLEAR' AND c."BusinessName" IS NOT NULL AND c.enriched_at IS NULL ${laFilter} ${SHARD}
    ORDER BY c."Classifications(s)" LIKE 'B%' DESC LIMIT ${BATCH}`)).rows;
}

async function enrichOne(c) {
  let fields = { website: null, email: null, linkedin: null, confidence: 'none' };
  try {
    const doms = candidates(c.BusinessName);
    const hits = (await Promise.all(doms.map(tryDomain))).filter(Boolean);
    if (hits.length) { const v = await verify(c, hits); if (v.website) fields = { ...fields, ...v }; }
  } catch (e) { fields.confidence = 'error:' + e.message.slice(0, 30); }
  await q(`UPDATE cslb_raw SET website=$2, email=COALESCE($3,email), enrich_confidence=$4, enrich_source='domain-guess+local-llm', enriched_at=now() WHERE "LicenseNo"=$1`,
    [c.LicenseNo, fields.website, fields.email, fields.confidence]);
  return fields;
}

async function main() {
  let done = 0, found = 0;
  for (;;) {
    const batch = await pick();
    if (!batch.length) { console.log(`\n✔ complete — no unenriched left (${LA_ONLY ? 'LA County' : 'all'})`); break; }
    for (const c of batch) {
      const f = await enrichOne(c);
      done++; if (f.website) found++;
      if (f.website) console.log(`  [${done}] ${c.BusinessName.slice(0, 32).padEnd(32)} → ${f.website} (${f.confidence})`);
    }
    const rem = Number((await q(`SELECT count(*) c FROM cslb_raw WHERE "PrimaryStatus"='CLEAR' AND enriched_at IS NULL ${LA_ONLY ? `AND "County"='Los Angeles'` : ''}`)).rows[0].c);
    console.log(`— ${done} processed, ${found} sites, ~${rem.toLocaleString()} left — $0`);
    if (!flags.has('--loop')) break;
  }
  console.log(`\nTotal: ${done} processed, ${found} websites. $0 (domain-guess + local model).`);
}
main().catch(e => { console.error('enrich-domain error:', e.message); process.exitCode = 1; }).finally(() => pool.end());