`src/enrich/discover_websites.ts`

266 lines
/**
 * Website discovery via DNS + name verification.
 *
 * For every law firm without a `website`, generate candidate domains from the
 * firm name (stripped of legal suffixes), DNS-resolve, then HTTP-GET the
 * homepage and confirm the firm name actually appears in the title/body before
 * accepting.
 *
 * No API keys, no third-party services — just public DNS + a single GET per
 * candidate. Conservative: name must match before we attribute a website.
 *
 * Once a website is set, the existing src/enrich/firm_website_contacts.ts
 * enricher (which only operates on firms WITH a website) can pull phone/email
 * from the contact pages.
 *
 * CLI:
 *   npx tsx src/enrich/discover_websites.ts            # full pass
 *   npx tsx src/enrich/discover_websites.ts --smoke    # first 10 firms
 *   npx tsx src/enrich/discover_websites.ts --limit 100
 */
import 'dotenv/config';
import { promises as dns } from 'node:dns';
import { pool, query } from '../db/pool.ts';

const SOURCE_NAME = 'DNS website discovery';
const RATE_DELAY_MS = Number(process.env.WEBSITE_DISCOVERY_DELAY_MS || 50);
const USER_AGENT = process.env.USER_AGENT
  || 'LawyerDirectoryBuilder/0.1 (research; contact: steveabramsdesigns@gmail.com)';

type Firm = { id: number; name: string };

const argv = process.argv.slice(2);
const SMOKE = argv.includes('--smoke');
const LIMIT_IDX = argv.indexOf('--limit');
const LIMIT = LIMIT_IDX >= 0 ? Number(argv[LIMIT_IDX + 1]) : (SMOKE ? 10 : 0);

// Strip these suffix tokens before forming candidates.
const SUFFIX_TOKENS = new Set([
  'llp', 'llc', 'pllc', 'pc', 'pa', 'lp', 'inc', 'ltd',
  'esq', 'esquire', 'chartered',
  'and', '&',
  'law', 'laws', 'legal', 'attorney', 'attorneys', 'lawyer', 'lawyers',
  'office', 'offices', 'group', 'firm', 'associates', 'partners', 'partnership',
  'a', 'the', 'of', 'at',
]);

// Skip these — too generic / would yield mass false positives.
const NAME_BLOCKLIST = [
  /^law office$/i, /^law offices$/i, /^attorney$/i, /^we the people$/i,
  /^preferred$/i, /^the law (office|firm)$/i, /^attorneys at law$/i,
];

function tokens(name: string): string[] {
  return name
    .toLowerCase()
    .replace(/&/g, ' and ')
    .replace(/[^a-z0-9\s]/g, ' ')
    .split(/\s+/)
    .filter(Boolean);
}

function meaningfulTokens(name: string): string[] {
  return tokens(name).filter(t => !SUFFIX_TOKENS.has(t) && t.length >= 2);
}

function candidateDomains(name: string): string[] {
  const out = new Set<string>();
  const meaningful = meaningfulTokens(name);
  if (meaningful.length === 0) return [];

  // 1. join all meaningful tokens
  out.add(meaningful.join('') + '.com');

  // 2. join meaningful + "law"
  out.add(meaningful.join('') + 'law.com');

  // 3. first 2 meaningful tokens
  if (meaningful.length >= 2) {
    out.add(meaningful.slice(0, 2).join('') + '.com');
    out.add(meaningful.slice(0, 2).join('') + 'law.com');
  }

  // 4. surname + "law" — for 2+ token names, the LAST meaningful token is
  //    the more reliable surname signal (firm names rarely lead with a surname
  //    and "DavidLaw" type matches generic-firm domains too easily).
  if (meaningful.length >= 2 && meaningful[meaningful.length - 1].length >= 5) {
    out.add(meaningful[meaningful.length - 1] + 'law.com');
  }

  // 5. initials + "law" (3+ tokens only — avoids 2-letter false positives)
  if (meaningful.length >= 3) {
    const initials = meaningful.map(t => t[0]).join('');
    if (initials.length >= 3) out.add(initials + 'law.com');
  }

  // Filter out anything < 6 chars total (e.g. "abc.com")
  return [...out].filter(d => d.length >= 7);
}

async function ensureSource(): Promise<number> {
  await query(`
    INSERT INTO sources (source_name, source_type, base_url, terms_notes, allowed_method, rate_limit_rps)
    VALUES ($1, 'directory', '(public DNS + per-firm websites)',
      'DNS resolution + single HTTP GET per candidate. Robots-respected per host.',
      'crawl', 5.0)
    ON CONFLICT (source_name) DO NOTHING
  `, [SOURCE_NAME]);
  const r = await query<{ id: number }>(`SELECT id FROM sources WHERE source_name = $1`, [SOURCE_NAME]);
  return r.rows[0].id;
}

async function startJob(sourceId: number, label: string): Promise<number> {
  const r = await query<{ id: number }>(`
    INSERT INTO scrape_jobs (source_id, job_label, status, started_at)
    VALUES ($1, $2, 'running', NOW()) RETURNING id
  `, [sourceId, label]);
  return r.rows[0].id;
}

async function finishJob(jobId: number, fields: Record<string, unknown>) {
  const sets: string[] = []; const params: unknown[] = []; let i = 1;
  for (const [k, v] of Object.entries(fields)) { sets.push(`${k} = $${i++}`); params.push(v); }
  sets.push(`finished_at = NOW()`); params.push(jobId);
  await query(`UPDATE scrape_jobs SET ${sets.join(', ')} WHERE id = $${i}`, params);
}

async function loadFirms(): Promise<Firm[]> {
  const lim = LIMIT > 0 ? `LIMIT ${LIMIT}` : '';
  const r = await query<Firm>(`
    SELECT id, name FROM organizations
    WHERE type='law_firm' AND website IS NULL
    ORDER BY id
    ${lim}
  `);
  return r.rows.filter(f => !NAME_BLOCKLIST.some(rx => rx.test(f.name)));
}

async function dnsExists(domain: string): Promise<boolean> {
  try {
    const records = await dns.resolve4(domain);
    return Array.isArray(records) && records.length > 0;
  } catch {
    try {
      const recs = await dns.resolve6(domain);
      return Array.isArray(recs) && recs.length > 0;
    } catch { return false; }
  }
}

async function fetchHomepage(domain: string): Promise<{ ok: boolean; html: string; finalUrl: string }> {
  const { fetch } = await import('undici');
  for (const proto of ['https', 'http']) {
    try {
      const url = `${proto}://${domain}/`;
      const r = await fetch(url, {
        method: 'GET',
        headers: { 'User-Agent': USER_AGENT, Accept: 'text/html,application/xhtml+xml' },
        signal: AbortSignal.timeout(8000),
        redirect: 'follow',
      });
      if (r.ok && r.headers.get('content-type')?.includes('text/html')) {
        const html = (await r.text()).slice(0, 20000);
        return { ok: true, html, finalUrl: r.url };
      }
    } catch { /* try next proto */ }
  }
  return { ok: false, html: '', finalUrl: '' };
}

function nameMatchesPage(firmName: string, html: string): boolean {
  const meaningful = meaningfulTokens(firmName);
  if (meaningful.length === 0) return false;
  const lower = html.toLowerCase();
  // For multi-token names: at least 2 meaningful tokens must appear together
  // (adjacent or as a joined string). Single-word matches against "law" pages
  // produce too many false positives (any "Smith Law" firm matches any other
  // "Smith"-themed law site).
  if (meaningful.length >= 2) {
    const firstTwo = meaningful.slice(0, 2);
    if (lower.includes(firstTwo.join(' '))) return true;
    if (lower.includes(firstTwo.join(''))) return true;
    // Also allow last+first or any adjacent pair (firm name may be reversed on site)
    for (let i = 0; i < meaningful.length - 1; i++) {
      if (lower.includes(meaningful[i] + ' ' + meaningful[i + 1])) return true;
    }
    return false;
  }
  // Single-token firm name: require both the token AND the firm-name length
  // to be substantial, plus a legal-services keyword nearby.
  const only = meaningful[0];
  if (only.length >= 6 && /\b(law|attorney|legal|esq|counsel)\b/.test(lower) && lower.includes(only)) return true;
  return false;
}

async function discover(firm: Firm): Promise<{ website: string; matched: string } | null> {
  const candidates = candidateDomains(firm.name);
  for (const c of candidates) {
    if (!(await dnsExists(c))) continue;
    const r = await fetchHomepage(c);
    if (!r.ok) continue;
    if (nameMatchesPage(firm.name, r.html)) {
      return { website: r.finalUrl || `https://${c}/`, matched: c };
    }
  }
  return null;
}

const CONCURRENCY = Number(process.env.DISCOVERY_CONCURRENCY || 20);

async function main() {
  const sourceId = await ensureSource();
  const label = SMOKE ? 'website-discovery:smoke' : (LIMIT > 0 ? `website-discovery:limit-${LIMIT}` : 'website-discovery:la-firms');
  const jobId = await startJob(sourceId, label);

  const firms = await loadFirms();
  console.log(`[disc] scanning ${firms.length} firms (concurrency=${CONCURRENCY})…`);

  let seen = 0, found = 0, errs = 0;

  async function processOne(firm: Firm) {
    seen++;
    try {
      const r = await discover(firm);
      if (r) {
        await query(
          `UPDATE organizations SET website = $2, updated_at = NOW() WHERE id = $1 AND website IS NULL`,
          [firm.id, r.website]);
        found++;
        console.log(`[disc] ✓ #${firm.id}  ${firm.name.slice(0,50).padEnd(50)} → ${r.website}`);
      }
    } catch {
      errs++;
    }
    if (seen % 200 === 0) console.log(`[disc] ${seen}/${firms.length}  found=${found}  errs=${errs}`);
  }

  try {
    // Parallelism via N concurrent workers pulling from a shared queue.
    let cursor = 0;
    const workers = Array.from({ length: CONCURRENCY }, async () => {
      while (true) {
        const idx = cursor++;
        if (idx >= firms.length) break;
        await processOne(firms[idx]);
      }
    });
    await Promise.all(workers);
  } finally {
    await finishJob(jobId, {
      status: 'completed',
      records_found: seen,
      records_updated: found,
      records_skipped: seen - found,
      error_message: errs > 0 ? `${errs} per-firm errors` : null,
    });
  }
  console.log(`[disc] done. seen=${seen} found=${found} errs=${errs}`);
  await pool.end();
}

main().catch(async (err) => {
  console.error('[disc] fatal:', err);
  try { await pool.end(); } catch {}
  process.exit(1);
});