`src/crawl/firm_front_page.ts`

217 lines
/**
 * Firm front-page crawler — Playwright chromium, concurrency 5.
 *
 * For firm_site rows with a url and crawl_status IS NULL: check robots.txt
 * (skip domains that disallow all for *), load the homepage (15s timeout),
 * record http_status + <title> + IDX listing-search heuristic + screenshot,
 * and extract mailto:/tel:/contact-page links into firm_contacts.
 *
 * Run:  npm run crawl:firms                 # default 100
 *       tsx src/crawl/firm_front_page.ts -- --limit=250
 *
 * $0 — local Playwright, public homepages only.
 */
import 'dotenv/config';
import { mkdirSync } from 'node:fs';
import { join } from 'node:path';
import { chromium, type Browser } from 'playwright';
import { pool, query } from '../../db/pool.ts';

const ROOT = new URL('../..', import.meta.url).pathname;
const SCREEN_DIR = join(ROOT, 'data', 'screenshots');
mkdirSync(SCREEN_DIR, { recursive: true });

const CONCURRENCY = parseInt(process.env.CRAWL_CONCURRENCY || '5', 10);
const TIMEOUT_MS = parseInt(process.env.CRAWL_TIMEOUT_MS || '15000', 10);
const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36 usrealestate/0.1 (front-page-audit; +local)';

interface SiteRow { site_id: number; firm_id: number; url: string }

const robotsCache = new Map<string, boolean>(); // host -> allowed

async function robotsAllows(url: string): Promise<boolean> {
  let h: string;
  try { h = new URL(url).hostname; } catch { return false; }
  const cached = robotsCache.get(h);
  if (cached !== undefined) return cached;
  let allowed = true;
  try {
    const res = await fetch(`https://${h}/robots.txt`, {
      headers: { 'User-Agent': UA },
      signal: AbortSignal.timeout(8000),
      redirect: 'follow',
    });
    if (res.ok) {
      const txt = await res.text();
      // Only honor the wildcard group; we just need "is the homepage off-limits".
      const lines = txt.split(/\r?\n/).map(l => l.replace(/#.*$/, '').trim());
      let inStar = false;
      for (const line of lines) {
        const ua = line.match(/^user-agent:\s*(.+)$/i);
        if (ua) { inStar = ua[1].trim() === '*'; continue; }
        if (!inStar) continue;
        const dis = line.match(/^disallow:\s*(.*)$/i);
        if (dis && dis[1].trim() === '/') { allowed = false; break; }
      }
    }
  } catch {} // unreachable robots.txt -> assume allowed (public homepage)
  robotsCache.set(h, allowed);
  return allowed;
}

interface Capture {
  httpStatus: number | null;
  title: string | null;
  hasIdx: boolean | null;
  screenshotPath: string | null;
  emails: string[];
  phones: string[];
  contactUrls: string[];
  finalUrl: string | null;
  error: string | null;
}

async function captureOne(browser: Browser, s: SiteRow): Promise<Capture> {
  const out: Capture = { httpStatus: null, title: null, hasIdx: null, screenshotPath: null, emails: [], phones: [], contactUrls: [], finalUrl: null, error: null };
  const ctx = await browser.newContext({
    ignoreHTTPSErrors: true,
    userAgent: UA,
    viewport: { width: 1280, height: 900 },
  });
  const page = await ctx.newPage();
  try {
    const resp = await page.goto(s.url, { waitUntil: 'domcontentloaded', timeout: TIMEOUT_MS });
    if (!resp) { out.error = 'no response'; return out; }
    await page.waitForTimeout(1500); // let lazy-rendered nav/search widgets appear
    out.httpStatus = resp.status();
    out.finalUrl = page.url();
    out.title = ((await page.title()) || '').trim().slice(0, 500) || null;

    // Passed as a string so tsx doesn't inject __name() helpers into the page.
    const meta: any = await page.evaluate(`(function(){
      var text = (document.body && document.body.innerText || '').slice(0, 40000);
      var idxText = /search homes|search properties|property search|properties for sale|homes for sale|view listings|our listings|all listings|featured listings|find your (dream )?home|idx/i.test(text);
      var links = Array.prototype.slice.call(document.querySelectorAll('a[href]'));
      var emails = [], phones = [], contacts = [];
      var idxHref = false;
      for (var i = 0; i < links.length; i++) {
        var href = links[i].getAttribute('href') || '';
        if (/^mailto:/i.test(href)) {
          var em = href.replace(/^mailto:/i, '').split('?')[0].trim().toLowerCase();
          if (em && emails.indexOf(em) < 0) emails.push(em);
        } else if (/^tel:/i.test(href)) {
          var ph = href.replace(/^tel:/i, '').split('?')[0].trim();
          if (ph && phones.indexOf(ph) < 0) phones.push(ph);
        } else {
          try {
            var u = new URL(links[i].href, location.href);
            if (/\\/(contact|contact-us|contactus)\\b/i.test(u.pathname) && u.hostname === location.hostname) {
              if (contacts.indexOf(u.href) < 0) contacts.push(u.href);
            }
            if (/idx|property-search|propertysearch|search.*(home|propert|listing)|listings?\\b|homes-for-sale/i.test(u.pathname + u.search)) idxHref = true;
          } catch (e) {}
        }
      }
      var idxWidget = !!document.querySelector('[class*="idx" i], [id*="idx" i], [class*="property-search" i], input[placeholder*="search" i][placeholder*="address" i], input[placeholder*="city" i]');
      return { hasIdx: !!(idxText || idxHref || idxWidget),
               emails: emails.slice(0, 10), phones: phones.slice(0, 10), contacts: contacts.slice(0, 3) };
    })()`);

    out.hasIdx = meta.hasIdx;
    out.emails = meta.emails;
    out.phones = meta.phones;
    out.contactUrls = meta.contacts;

    const shotName = `firm-${s.firm_id}.jpg`;
    await page.screenshot({ path: join(SCREEN_DIR, shotName), type: 'jpeg', quality: 70, fullPage: false });
    out.screenshotPath = `screenshots/${shotName}`;
  } catch (e: any) {
    out.error = (e?.message || String(e)).slice(0, 300);
  } finally {
    await ctx.close().catch(() => {});
  }
  return out;
}

async function persist(s: SiteRow, c: Capture) {
  const status = c.error ? 'error:' + c.error.slice(0, 120) : 'ok';
  await query(
    `UPDATE firm_site SET http_status = $2, title = $3, has_idx_listings = $4,
            screenshot_path = $5, crawl_status = $6, crawled_at = NOW()
      WHERE id = $1`,
    [s.site_id, c.httpStatus, c.title, c.hasIdx, c.screenshotPath, status],
  );
  const src = c.finalUrl || s.url;
  const rows: [string, string][] = [
    ...c.emails.map(v => ['email', v] as [string, string]),
    ...c.phones.map(v => ['phone', v] as [string, string]),
    ...c.contactUrls.map(v => ['contact_url', v] as [string, string]),
  ];
  for (const [kind, value] of rows) {
    await query(
      `INSERT INTO firm_contacts (firm_id, kind, value, source_url) VALUES ($1, $2, $3, $4)
       ON CONFLICT (firm_id, kind, value) DO NOTHING`,
      [s.firm_id, kind, value.slice(0, 500), src.slice(0, 600)],
    );
  }
  return rows.length;
}

async function main() {
  const argLimit = process.argv.find(a => a.startsWith('--limit='));
  const limit = argLimit ? parseInt(argLimit.split('=')[1], 10) : 100;

  const run = await query<{ id: number }>(
    `INSERT INTO ingest_runs (source, notes) VALUES ('firm_crawl', $1) RETURNING id`,
    [`front-page crawl, batch limit ${limit}, concurrency ${CONCURRENCY}`],
  );
  const runId = run.rows[0].id;

  const r = await query<SiteRow>(`
    SELECT s.id AS site_id, s.firm_id, s.url
      FROM firm_site s JOIN firm f ON f.id = s.firm_id
     WHERE s.url IS NOT NULL AND s.crawl_status IS NULL
     ORDER BY f.agent_count DESC NULLS LAST, s.id
     LIMIT $1
  `, [limit]);
  const queue = r.rows.slice();
  console.log(`[crawl] run #${runId} · ${queue.length} sites · concurrency ${CONCURRENCY} · timeout ${TIMEOUT_MS}ms`);

  const browser = await chromium.launch({ headless: true });
  let done = 0, ok = 0, fail = 0, robots = 0, contacts = 0, idx = 0;

  await Promise.all(Array.from({ length: CONCURRENCY }).map(async () => {
    while (queue.length) {
      const s = queue.shift()!;
      if (!(await robotsAllows(s.url))) {
        robots++;
        await query(`UPDATE firm_site SET crawl_status = 'robots_disallow', crawled_at = NOW() WHERE id = $1`, [s.site_id]);
        console.log(`[crawl] ⛔ robots disallow: ${s.url}`);
        continue;
      }
      const c = await captureOne(browser, s);
      contacts += await persist(s, c);
      done++;
      if (c.error) fail++; else { ok++; if (c.hasIdx) idx++; }
      if (done % 10 === 0 || !queue.length) {
        console.log(`[crawl] ${done} done · ok=${ok} fail=${fail} robots=${robots} idx=${idx} contacts=${contacts} · last: ${s.url} ${c.error ? '⚠ ' + c.error.slice(0, 60) : '(' + c.httpStatus + ')'}`);
      }
    }
  }));

  await browser.close();
  await query(
    `UPDATE ingest_runs SET finished_at = NOW(), status = 'ok', rows_upserted = $2, rows_skipped = $3,
            notes = notes || $4 WHERE id = $1`,
    [runId, ok, fail + robots, ` · ${ok} ok, ${fail} errors, ${robots} robots-skipped, ${idx} idx, ${contacts} contacts`],
  );
  console.log(`[crawl] done · ${ok} ok · ${fail} errors · ${robots} robots-skipped · ${idx} with IDX · ${contacts} contact rows`);
  await pool.end();
}

main().catch(async (e) => {
  console.error('[crawl]', e);
  try { await pool.end(); } catch {}
  process.exit(1);
});