`agents/crawler-agent/crawlers/ucla-health.js`

56 lines
/**
 * UCLA Health provider directory crawler — Stage 5.
 *
 * Public, robots-aware. Polite rate-limit (0.5 rps) via lib/compliance.
 * Discovers provider profile URLs from the main directory and extracts each.
 */
const { fetchCompliant, setHostRateLimit } = require('../../shared/compliance');
const { extractProviderCards, extractProviderProfile } = require('../framework/extractor');

const SOURCE_NAME = 'UCLA Health (uclahealth.org)';
const BASE        = 'https://www.uclahealth.org';
const ENTRY       = `${BASE}/providers`;

setHostRateLimit('www.uclahealth.org', 0.5);

async function fetchHtml(url) {
  const res = await fetchCompliant(url, { accept: 'text/html' });
  if (!res.ok) throw new Error(`HTTP ${res.status} ${url}`);
  return res.text();
}

async function* listProviderUrls() {
  // The main directory is paginated; UCLA uses ?page= or ?p= depending on view.
  for (let page = 1; page <= 200; page++) {
    const url = `${ENTRY}?page=${page}`;
    const html = await fetchHtml(url);
    const cards = extractProviderCards(html, url);
    if (cards.length === 0) return;
    for (const c of cards) {
      if (c.profile_url) yield c.profile_url;
    }
  }
}

async function crawl({ onProvider, maxPages = Infinity } = {}) {
  let count = 0;
  for await (const url of listProviderUrls()) {
    if (count >= maxPages) break;
    try {
      const html = await fetchHtml(url);
      const profile = extractProviderProfile(html, url);
      profile.hospital_system = SOURCE_NAME;
      await onProvider(profile);
    } catch (e) {
      if (e.code === 'CAPTCHA_DETECTED') {
        console.error(`[ucla] CAPTCHA — aborting per compliance`); break;
      }
      console.error(`[ucla] error ${url}: ${e.message}`);
    }
    count++;
  }
  return count;
}

module.exports = { SOURCE_NAME, crawl };