`src/crawl/front_page.ts`

237 lines
/**
 * Front-page crawler — lacountyeats.
 *
 * For every facility with a website on file (facility_enrichment.website),
 * fetch the front page with Playwright headless Chromium, persist the raw
 * HTML to disk, and INSERT one row into front_page_audits. The HTML files
 * feed Pass 1 of the ad-social-tracker skill (regex pixel detection, no LLM).
 *
 * - Concurrency ≤ 5 (task spec — Mac2 has 4 other agents running)
 * - 5s navigation timeout (task spec)
 * - 30-day staleness gate (skip facilities audited within last 30 days)
 * - Respects robots.txt with a per-host fetch + 1h memory cache
 * - User-Agent: LACountyEatsBot/1.0
 *
 * Run:
 *   tsx src/crawl/front_page.ts --limit=50    # smoke
 *   tsx src/crawl/front_page.ts --limit=1000  # batch
 */
import 'dotenv/config';
import { mkdirSync, writeFileSync } from 'node:fs';
import { join } from 'node:path';
import { chromium, type Browser } from 'playwright';
import { pool, query } from '../../db/pool.ts';

const ROOT = new URL('../..', import.meta.url).pathname;
const RAW_DIR = join(ROOT, 'data', 'raw');
mkdirSync(RAW_DIR, { recursive: true });

const CONCURRENCY = Math.min(parseInt(process.env.CRAWL_CONCURRENCY || '5', 10), 5);
const TIMEOUT_MS = parseInt(process.env.CRAWL_TIMEOUT_MS || '5000', 10);
const STALE_DAYS = parseInt(process.env.CRAWL_STALE_DAYS || '30', 10);
const USER_AGENT = 'LACountyEatsBot/1.0 (+https://lacountyeats.com; contact info@lacountyeats.com)';

interface FacilityRow {
  facility_id: string;
  name: string;
  website: string;
}

async function pickQueue(limit: number): Promise<FacilityRow[]> {
  const r = await query<FacilityRow>(
    `
    SELECT f.facility_id, f.name, fe.website
    FROM facility f
    JOIN facility_enrichment fe ON fe.facility_id = f.facility_id
    WHERE fe.website IS NOT NULL
      AND fe.website ~ '^https?://'
      AND NOT EXISTS (
        SELECT 1 FROM front_page_audits a
        WHERE a.facility_id = f.facility_id
          AND a.audited_at > now() - ($2::int || ' days')::interval
      )
    ORDER BY f.facility_id
    LIMIT $1
    `,
    [limit, STALE_DAYS],
  );
  return r.rows;
}

// ── robots.txt: minimal allow/deny check, per-host cache (1h)
type RobotsRule = { allow: string[]; disallow: string[] };
const robotsCache = new Map<string, { rules: RobotsRule; ts: number }>();
const ROBOTS_TTL_MS = 60 * 60 * 1000;

function parseRobots(txt: string): RobotsRule {
  const rules: RobotsRule = { allow: [], disallow: [] };
  let applies = false;
  for (const raw of txt.split(/\r?\n/)) {
    const line = raw.replace(/#.*$/, '').trim();
    if (!line) continue;
    const m = line.match(/^([A-Za-z-]+)\s*:\s*(.*)$/);
    if (!m) continue;
    const [, key, val] = m;
    const k = key.toLowerCase();
    if (k === 'user-agent') {
      const ua = val.trim().toLowerCase();
      applies = ua === '*' || ua.includes('lacountyeatsbot');
    } else if (applies && k === 'disallow' && val) {
      rules.disallow.push(val.trim());
    } else if (applies && k === 'allow' && val) {
      rules.allow.push(val.trim());
    }
  }
  return rules;
}

async function isAllowedByRobots(target: string): Promise<boolean> {
  let u: URL;
  try {
    u = new URL(target);
  } catch {
    return false;
  }
  const host = u.origin;
  const cached = robotsCache.get(host);
  let rules: RobotsRule;
  if (cached && Date.now() - cached.ts < ROBOTS_TTL_MS) {
    rules = cached.rules;
  } else {
    try {
      const res = await fetch(host + '/robots.txt', {
        headers: { 'User-Agent': USER_AGENT },
        signal: AbortSignal.timeout(4000),
      });
      // Treat any non-2xx as "no robots" → allow.
      rules = res.ok ? parseRobots(await res.text()) : { allow: [], disallow: [] };
    } catch {
      rules = { allow: [], disallow: [] };
    }
    robotsCache.set(host, { rules, ts: Date.now() });
  }
  // Longest-match wins between allow/disallow.
  const path = u.pathname + u.search;
  let bestAllow = -1;
  let bestDisallow = -1;
  for (const p of rules.allow) if (p && path.startsWith(p) && p.length > bestAllow) bestAllow = p.length;
  for (const p of rules.disallow) if (p && path.startsWith(p) && p.length > bestDisallow) bestDisallow = p.length;
  if (bestDisallow === -1) return true;
  return bestAllow >= bestDisallow;
}

interface Capture {
  http_status: number | null;
  fetch_ms: number | null;
  raw_html_path: string | null;
  error_message: string | null;
  fetched_url: string;
}

async function captureOne(browser: Browser, f: FacilityRow): Promise<Capture> {
  const out: Capture = {
    http_status: null,
    fetch_ms: null,
    raw_html_path: null,
    error_message: null,
    fetched_url: f.website,
  };

  if (!(await isAllowedByRobots(f.website))) {
    out.error_message = 'blocked_by_robots';
    return out;
  }

  const ctx = await browser.newContext({
    ignoreHTTPSErrors: true,
    userAgent: USER_AGENT,
    viewport: { width: 1280, height: 900 },
  });
  const page = await ctx.newPage();
  const startedAt = Date.now();
  try {
    const resp = await page.goto(f.website, { waitUntil: 'load', timeout: TIMEOUT_MS });
    out.fetch_ms = Date.now() - startedAt;
    if (!resp) {
      out.error_message = 'no_response';
      return out;
    }
    out.http_status = resp.status();
    out.fetched_url = page.url();
    const html = await page.content();
    const ts = Date.now();
    const rel = `raw/facility-${f.facility_id}-${ts}.html`;
    writeFileSync(join(RAW_DIR, `facility-${f.facility_id}-${ts}.html`), html, 'utf8');
    out.raw_html_path = rel;
  } catch (e: unknown) {
    out.error_message = (e instanceof Error ? e.message : String(e)).slice(0, 500);
    out.fetch_ms = Date.now() - startedAt;
  } finally {
    await ctx.close().catch(() => {});
  }
  return out;
}

async function insertAudit(facilityId: string, url: string, c: Capture) {
  await query(
    `
    INSERT INTO front_page_audits
      (facility_id, url, audited_at, raw_html_path, http_status, fetch_ms, error_message)
    VALUES ($1, $2, now(), $3, $4, $5, $6)
    `,
    [facilityId, url, c.raw_html_path, c.http_status, c.fetch_ms, c.error_message],
  );
}

async function main() {
  const argLimit = process.argv.find((a) => a.startsWith('--limit='));
  const limit = argLimit ? parseInt(argLimit.split('=')[1], 10) : 1000;

  const queue = await pickQueue(limit);
  console.log(
    `[crawl] ${queue.length} facilities in queue · concurrency ${CONCURRENCY} · timeout ${TIMEOUT_MS}ms · stale ${STALE_DAYS}d`,
  );
  if (queue.length === 0) {
    console.log('[crawl] nothing to do');
    await pool.end();
    return;
  }

  const browser = await chromium.launch({ headless: true });
  const startedAt = Date.now();
  let done = 0;
  let ok = 0;
  let fail = 0;

  const workQueue = queue.slice();
  await Promise.all(
    Array.from({ length: CONCURRENCY }).map(async () => {
      while (workQueue.length) {
        const f = workQueue.shift()!;
        const c = await captureOne(browser, f);
        await insertAudit(f.facility_id, f.website, c);
        done++;
        if (c.raw_html_path) ok++;
        else fail++;
        if (done % 10 === 0 || done === queue.length) {
          const tail = c.raw_html_path
            ? `last: ${f.name} (${c.http_status || '?'}) ${c.fetch_ms || 0}ms`
            : `last fail: ${f.name} → ${(c.error_message || '').slice(0, 80)}`;
          console.log(`[crawl] ${done}/${queue.length}  ok=${ok} fail=${fail}  · ${tail}`);
        }
      }
    }),
  );

  await browser.close();
  const elapsed = Math.round((Date.now() - startedAt) / 1000);
  console.log('');
  console.log(`[crawl] done in ${elapsed}s · ${ok} ok · ${fail} failed`);
  await pool.end();
}

main().catch((e) => {
  console.error('[crawl]', e);
  process.exit(1);
});