`agents/crawler-agent/crawlers/front-page.js`

237 lines
/**
 * Front-page HTML capture for organizations with a `website`.
 *
 * Drives ad-social-tracker Pass 1 (regex pixel detection) downstream via
 * src/enrich/ad_signals.js. Internal sales-targeting only — admin-gated
 * output. We pull only the public marketing front page (one URL per org),
 * never deeper paths, never anything resembling patient/PHI surface.
 *
 * Behavior:
 *   - Selects orgs with website IS NOT NULL/empty AND no front_page_audits row
 *     in last 30 days.
 *   - Playwright headless chromium, 5s nav timeout, single browser, single
 *     context, max 5 parallel pages.
 *   - Saves page.content() to data/raw/org-<id>-<ts>.html.
 *   - INSERTs front_page_audits row per attempt (success OR failure — failures
 *     get raw_html_path=NULL, http_status=null/code).
 *   - Skips robots.txt-disallowed paths.
 *   - User-agent: DoctorDirectoryBot/1.0
 *   - HIPAA: only the website URL declared by the org. Never widen scope.
 */
const { chromium } = require('playwright');
const fs = require('node:fs');
const path = require('node:path');
const { URL } = require('node:url');
const robotsParser = require('robots-parser');
const { pool, query } = require('../../shared/db');

const UA = 'DoctorDirectoryBot/1.0 (+https://drvouch.local/bot)';
const NAV_TIMEOUT_MS = 5000;
const CONCURRENCY = parseInt(process.env.CRAWL_CONCURRENCY || '5', 10);
const LIMIT = parseInt(process.env.CRAWL_LIMIT || '500', 10);
const STALE_DAYS = 30;
const RAW_DIR = path.resolve(__dirname, '../../../data/raw');
const REPO_ROOT = path.resolve(__dirname, '../../..');

fs.mkdirSync(RAW_DIR, { recursive: true });

// In-memory robots.txt cache per origin
const robotsCache = new Map();

function normalizeUrl(raw) {
  if (!raw) return null;
  let s = String(raw).trim();
  if (!s) return null;
  if (!/^https?:\/\//i.test(s)) s = 'http://' + s;
  try {
    const u = new URL(s);
    return u.toString();
  } catch {
    return null;
  }
}

async function getRobots(origin) {
  if (robotsCache.has(origin)) return robotsCache.get(origin);
  const robotsUrl = origin + '/robots.txt';
  let parser;
  try {
    const res = await fetch(robotsUrl, {
      method: 'GET',
      headers: { 'user-agent': UA },
      signal: AbortSignal.timeout(4000),
    });
    const body = res.ok ? await res.text() : '';
    parser = robotsParser(robotsUrl, body);
  } catch {
    // No robots → allowed by default (common for small sites)
    parser = robotsParser(robotsUrl, '');
  }
  robotsCache.set(origin, parser);
  return parser;
}

async function fetchOrgs(limit) {
  const sql = `
    SELECT o.id, o.name, o.website
    FROM organizations o
    WHERE o.website IS NOT NULL
      AND o.website <> ''
      AND o.opted_out = false
      AND NOT EXISTS (
        SELECT 1 FROM front_page_audits a
        WHERE a.organization_id = o.id
          AND a.audited_at > now() - INTERVAL '${STALE_DAYS} days'
      )
    ORDER BY o.id
    LIMIT $1
  `;
  const r = await query(sql, [limit]);
  return r.rows;
}

async function crawlOne(context, org) {
  const url = normalizeUrl(org.website);
  if (!url) {
    await query(
      `INSERT INTO front_page_audits (organization_id, url, raw_html_path, http_status, fetch_ms)
       VALUES ($1, $2, NULL, NULL, NULL)`,
      [org.id, org.website || '']
    );
    return { id: org.id, ok: false, reason: 'bad_url' };
  }

  let origin;
  try {
    origin = new URL(url).origin;
  } catch {
    return { id: org.id, ok: false, reason: 'bad_url' };
  }

  // robots.txt check
  try {
    const robots = await getRobots(origin);
    if (!robots.isAllowed(url, 'DoctorDirectoryBot')) {
      await query(
        `INSERT INTO front_page_audits (organization_id, url, raw_html_path, http_status, fetch_ms)
         VALUES ($1, $2, NULL, $3, NULL)`,
        [org.id, url, 999] // 999 = robots-disallowed sentinel
      );
      return { id: org.id, ok: false, reason: 'robots_disallow' };
    }
  } catch {
    // continue — robots fetch failure is not fatal
  }

  const t0 = Date.now();
  const page = await context.newPage();
  let httpStatus = null;
  let rawPath = null;
  let ok = false;
  let reason = null;

  try {
    page.setDefaultNavigationTimeout(NAV_TIMEOUT_MS);
    const resp = await page.goto(url, { waitUntil: 'domcontentloaded', timeout: NAV_TIMEOUT_MS });
    httpStatus = resp ? resp.status() : null;
    if (resp && resp.ok()) {
      const html = await page.content();
      const ts = Date.now();
      const rel = path.join('data', 'raw', `org-${org.id}-${ts}.html`);
      const abs = path.join(REPO_ROOT, rel);
      fs.writeFileSync(abs, html, 'utf8');
      rawPath = rel;
      ok = true;
    } else {
      reason = `http_${httpStatus}`;
    }
  } catch (e) {
    reason = (e && e.message) ? e.message.slice(0, 80) : 'nav_error';
  } finally {
    try { await page.close(); } catch {}
  }

  const fetchMs = Date.now() - t0;
  await query(
    `INSERT INTO front_page_audits (organization_id, url, raw_html_path, http_status, fetch_ms)
     VALUES ($1, $2, $3, $4, $5)`,
    [org.id, url, rawPath, httpStatus, fetchMs]
  );
  return { id: org.id, ok, reason, httpStatus, fetchMs };
}

async function runPool(items, worker, concurrency) {
  let i = 0;
  let active = 0;
  let done = 0;
  const results = [];
  return new Promise((resolve) => {
    const next = () => {
      while (active < concurrency && i < items.length) {
        const idx = i++;
        active++;
        Promise.resolve(worker(items[idx]))
          .then((r) => { results[idx] = r; })
          .catch((e) => { results[idx] = { ok: false, reason: e?.message || 'err' }; })
          .finally(() => {
            active--;
            done++;
            if (done % 25 === 0) {
              console.log(`[front-page] progress ${done}/${items.length}`);
            }
            if (done === items.length) resolve(results);
            else next();
          });
      }
    };
    if (items.length === 0) resolve(results);
    else next();
  });
}

async function main() {
  const limit = LIMIT;
  console.log(`[front-page] fetching up to ${limit} orgs (stale > ${STALE_DAYS}d)…`);
  const orgs = await fetchOrgs(limit);
  console.log(`[front-page] ${orgs.length} orgs to crawl, concurrency=${CONCURRENCY}`);
  if (orgs.length === 0) {
    await pool.end();
    return;
  }

  const browser = await chromium.launch({ headless: true });
  const context = await browser.newContext({
    userAgent: UA,
    javaScriptEnabled: true,
    bypassCSP: true,
    ignoreHTTPSErrors: true,
    viewport: { width: 1280, height: 900 },
  });
  // Block heavy media — we only need the HTML/script tags for ad-pixel detection
  await context.route('**/*', (route) => {
    const t = route.request().resourceType();
    if (t === 'image' || t === 'media' || t === 'font') return route.abort();
    return route.continue();
  });

  const results = await runPool(orgs, (o) => crawlOne(context, o), CONCURRENCY);

  await context.close();
  await browser.close();

  const ok = results.filter((r) => r && r.ok).length;
  const robotsBlocked = results.filter((r) => r && r.reason === 'robots_disallow').length;
  const errs = results.length - ok - robotsBlocked;
  console.log(`[front-page] done · ok=${ok} robots_blocked=${robotsBlocked} errors=${errs}`);
  await pool.end();
}

if (require.main === module) {
  main().catch((e) => {
    console.error('[front-page] fatal', e);
    process.exit(1);
  });
}

module.exports = { crawlOne, normalizeUrl };