`agents/crawler-agent/framework/extractor.js`

144 lines
/**
 * Generic provider-page extractor utilities used by hospital crawlers.
 * Pure parsing helpers — no I/O, no DB.
 */
const cheerio = require('cheerio');

function textOf($el) {
  return ($el.text() || '').replace(/\s+/g, ' ').trim() || null;
}

function attrFirst($el, ...names) {
  for (const n of names) {
    const v = $el.attr(n);
    if (v) return v.trim();
  }
  return null;
}

/**
 * Extract a flat list of doctor cards from a directory listing page.
 * Tries common selectors first, then JSON-LD, then schema.org microdata.
 * Returns an array of { name, specialty, location, phone, profile_url, image_url }.
 */
function extractProviderCards(html, baseUrl) {
  const $ = cheerio.load(html);
  const results = [];

  // 1. JSON-LD blocks with @type Physician / Person.
  $('script[type="application/ld+json"]').each((_, s) => {
    try {
      const j = JSON.parse($(s).text());
      const arr = Array.isArray(j) ? j : [j];
      for (const obj of arr) {
        if (!obj) continue;
        const t = (obj['@type'] || '').toLowerCase();
        if (t === 'physician' || t === 'person' || t === 'medicalbusiness') {
          results.push({
            name: obj.name || obj.alternateName,
            specialty: obj.medicalSpecialty || obj.knowsAbout,
            phone: obj.telephone,
            profile_url: obj.url ? new URL(obj.url, baseUrl).href : null,
            image_url: obj.image,
            address: obj.address && (obj.address.streetAddress || obj.address),
          });
        }
      }
    } catch (_) { /* ignore */ }
  });

  // 2. schema.org microdata.
  $('[itemtype*="schema.org/Physician"], [itemtype*="schema.org/Person"]').each((_, el) => {
    const $el = $(el);
    const name = textOf($el.find('[itemprop="name"]').first()) || textOf($el.find('h2,h3,.name').first());
    const specialty = textOf($el.find('[itemprop="medicalSpecialty"], .specialty').first());
    const profile_url = $el.find('a[itemprop="url"], a').first().attr('href');
    const image_url = $el.find('img[itemprop="image"], img').first().attr('src');
    if (name) results.push({
      name, specialty,
      profile_url: profile_url ? new URL(profile_url, baseUrl).href : null,
      image_url: image_url ? new URL(image_url, baseUrl).href : null,
    });
  });

  // 3. Common card patterns.
  const cardSelectors = [
    'article.physician-card',
    'article.provider-card',
    'div.provider, div.physician, div.doctor-card, div.search-result',
    'li.provider-card, li.doctor-card',
  ];
  for (const sel of cardSelectors) {
    $(sel).each((_, el) => {
      const $el = $(el);
      const name = textOf($el.find('h2, h3, .name, .provider-name').first());
      if (!name) return;
      const specialty = textOf($el.find('.specialty, .specialties, .provider-specialty').first());
      const phone = textOf($el.find('.phone, [data-phone]').first());
      const profile_url = $el.find('a').first().attr('href');
      const image_url = $el.find('img').first().attr('src');
      const location = textOf($el.find('.location, .address, .provider-location').first());
      results.push({
        name, specialty, phone, location,
        profile_url: profile_url ? new URL(profile_url, baseUrl).href : null,
        image_url: image_url ? new URL(image_url, baseUrl).href : null,
      });
    });
    if (results.length > 0) break;
  }

  // De-dupe by (name, profile_url).
  const seen = new Set();
  return results.filter(r => {
    const k = `${(r.name||'').toLowerCase()}|${r.profile_url||''}`;
    if (seen.has(k)) return false;
    seen.add(k);
    return Boolean(r.name);
  });
}

/**
 * Parse an individual provider profile page for richer fields.
 */
function extractProviderProfile(html, url) {
  const $ = cheerio.load(html);
  const out = { profile_url: url, source_url: url };

  // JSON-LD wins when present.
  $('script[type="application/ld+json"]').each((_, s) => {
    try {
      const j = JSON.parse($(s).text());
      const arr = Array.isArray(j) ? j : [j];
      for (const obj of arr) {
        const t = (obj?.['@type'] || '').toLowerCase();
        if (t === 'physician' || t === 'person') {
          out.name        = out.name        || obj.name;
          out.specialty   = out.specialty   || obj.medicalSpecialty;
          out.phone       = out.phone       || obj.telephone;
          out.image_url   = out.image_url   || obj.image;
          out.email       = out.email       || obj.email;
          out.bio         = out.bio         || obj.description;
          out.npi         = out.npi         || obj.identifier?.value;
          if (obj.alumniOf) {
            const a = Array.isArray(obj.alumniOf) ? obj.alumniOf[0] : obj.alumniOf;
            out.medical_school = out.medical_school || (typeof a === 'string' ? a : a?.name);
          }
        }
      }
    } catch (_) { /* ignore */ }
  });

  // Heuristic fallbacks.
  out.name        ||= textOf($('h1').first());
  out.specialty   ||= textOf($('.specialty, [itemprop="medicalSpecialty"]').first());
  out.phone       ||= textOf($('a[href^="tel:"]').first()) || $('a[href^="tel:"]').first().attr('href')?.replace('tel:','');
  out.email       ||= ($('a[href^="mailto:"]').first().attr('href') || '').replace('mailto:','') || null;
  out.bio         ||= textOf($('.bio, .biography, .provider-bio, [itemprop="description"]').first());
  out.image_url   ||= $('.provider-photo img, .doctor-photo img, [itemprop="image"]').first().attr('src');
  out.medical_school ||= textOf($('.education, .schooling').filter((_,e)=>/medical|md|school/i.test($(e).text())).first());

  return out;
}

module.exports = { extractProviderCards, extractProviderProfile };