`scrapers/scrape-html.js`

224 lines
'use strict';
const fetch = require('node-fetch');
const cheerio = require('cheerio');
const crypto = require('crypto');
const { upsertHeadline, logScrape } = require('./db');
const { SCRAPE_SOURCES } = require('./sources');
const { fetchViaBrowserbase } = require('./fetch-browserbase');

// Use a real browser UA — many sites 403/404 a bot UA
const BROWSER_UA =
  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 ' +
  '(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';

const FETCH_OPTS = {
  timeout: 25000,
  redirect: 'follow',
  headers: {
    'User-Agent': BROWSER_UA,
    'Accept':
      'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
    'Accept-Language': 'en-US,en;q=0.5',
  },
};

function makeUid(source, url) {
  return crypto.createHash('sha1').update(source + '|' + url).digest('hex').slice(0, 16);
}

function isBreaking(title) {
  return /breaking|emergency|urgent|just in/i.test(title);
}
function isDeveloping(title) {
  return /developing|exclusive|update:|report:/i.test(title);
}

function resolveUrl(href, baseUrl) {
  try {
    return new URL(href, baseUrl).href;
  } catch {
    return null;
  }
}

function cleanTitle(text) {
  return String(text || '')
    .replace(/[\r\n\t]/g, ' ')
    .replace(/\s+/g, ' ')
    .trim()
    .slice(0, 200);
}

function slugToTitle(url) {
  try {
    const u = new URL(url);
    const last = u.pathname.split('/').filter(Boolean).pop() || '';
    return last
      .replace(/[-_]+/g, ' ')
      .replace(/\.\w+$/, '')
      .replace(/\b\w/g, c => c.toUpperCase())
      .slice(0, 100);
  } catch {
    return '';
  }
}

async function fetchHtml(url, useBrowserbase = false) {
  if (useBrowserbase) {
    return fetchViaBrowserbase(url);
  }
  const res = await fetch(url, FETCH_OPTS);
  if (!res.ok) throw new Error(`HTTP ${res.status}`);
  return res.text();
}

// ─── Sitemap parsing (Cole & Son uses this) ─────────────────────────────────
function parseSitemapXml(xml, src) {
  // Returns array of { title, url }
  const $ = cheerio.load(xml, { xmlMode: true });
  const items = [];

  $('url').each((_, el) => {
    const loc = $(el).find('loc').first().text().trim();
    if (!loc) return;
    if (src.linkFilter && !src.linkFilter.test(loc)) return;
    if (src.excludeFilter && src.excludeFilter.test(loc)) return;

    let title = '';
    if (src.sitemapTitleFromImage) {
      // <image:title> child gives a human-readable title
      const imgTitle = $(el).find('image\\:title, title').first().text().trim();
      if (imgTitle) title = imgTitle;
    }
    if (!title) title = slugToTitle(loc);
    if (!title || title.length < 4) return;

    const lastmod = $(el).find('lastmod').first().text().trim();
    items.push({ title, url: loc, lastmod });
  });

  // Sort newest first if we have lastmod
  items.sort((a, b) => (b.lastmod || '').localeCompare(a.lastmod || ''));
  return items;
}

async function parseSitemapIndex(xml, src) {
  // For Scalamandre: pick newest child sitemaps that match filter, then read entries
  const $ = cheerio.load(xml, { xmlMode: true });
  const childSitemaps = [];
  $('sitemap').each((_, el) => {
    const loc = $(el).find('loc').first().text().trim();
    const lastmod = $(el).find('lastmod').first().text().trim();
    if (!loc) return;
    if (src.sitemapFilter && !src.sitemapFilter.test(loc)) return;
    childSitemaps.push({ loc, lastmod });
  });
  childSitemaps.sort((a, b) => (b.lastmod || '').localeCompare(a.lastmod || ''));

  // Limit to first 2 child sitemaps for cost
  const items = [];
  for (const cs of childSitemaps.slice(0, 2)) {
    try {
      const childXml = await fetchHtml(cs.loc);
      items.push(...parseSitemapXml(childXml, src));
    } catch (e) {
      console.error(`[sitemap-index] ${src.key} child ${cs.loc} failed: ${e.message}`);
    }
  }
  return items;
}

// ─── HTML scrape with selector + filters ────────────────────────────────────
function parseHtml(html, src) {
  const $ = cheerio.load(html);
  const seen = new Set();
  const items = [];

  $(src.selector).each((_, el) => {
    const href = $(el).attr('href');
    if (!href) return;
    if (href.startsWith('#')) return;

    const absUrl = resolveUrl(href, src.baseUrl);
    if (!absUrl) return;
    if (seen.has(absUrl)) return;
    if (src.linkFilter && !src.linkFilter.test(absUrl)) return;
    if (src.excludeFilter && src.excludeFilter.test(absUrl)) return;
    seen.add(absUrl);

    // Title: prefer anchor text, then <img alt>, then URL slug
    let title = cleanTitle($(el).text());
    if (!title || title.length < 5) {
      title = cleanTitle($(el).find('img').attr('alt') || '');
    }
    if (!title || title.length < 5) {
      title = slugToTitle(absUrl);
    }
    if (!title || title.length < 5) return;

    items.push({ title, url: absUrl });
  });

  return items;
}

async function scrapeSite(src) {
  let items = [];
  let errorMsg = null;

  try {
    const html = await fetchHtml(src.url, !!src.browserbase);

    if (src.parser === 'sitemap') {
      items = parseSitemapXml(html, src);
    } else if (src.parser === 'sitemap-index') {
      items = await parseSitemapIndex(html, src);
    } else {
      items = parseHtml(html, src);
    }
  } catch (err) {
    errorMsg = err.message;
    console.error(`[html] ${src.key} failed: ${err.message}`);
  }

  // Cap at 6 most recent per source (per spec)
  items = items.slice(0, 6);

  let added = 0;
  for (const it of items) {
    const uid = makeUid(src.key, it.url);
    added += upsertHeadline({
      uid,
      title: it.title,
      url: it.url,
      source: src.name,
      category: src.category,
      pub_date: null,
      is_breaking: isBreaking(it.title) ? 1 : 0,
      is_developing: isDeveloping(it.title) ? 1 : 0,
      is_top: 0,
    });
  }

  logScrape(src.key, added, errorMsg);
  return added;
}

async function scrapeAllHtml(category = null) {
  const sources = category
    ? SCRAPE_SOURCES.filter(s => s.category === category)
    : SCRAPE_SOURCES;

  const results = [];
  for (const src of sources) {
    const count = await scrapeSite(src);
    console.log(`[html] ${src.name}: +${count}`);
    results.push({ source: src.key, added: count });
    // Polite delay; Browserbase sources need a longer beat
    await new Promise(r => setTimeout(r, src.browserbase ? 1500 : 500));
  }
  return results;
}

module.exports = { scrapeAllHtml, scrapeSite };